Merge branch 'main' into debug-dns

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

.devcontainer/Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:1.21-bullseye
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends\
+        gettext-base=0.21-4 \ 
+        iptables=1.8.7-1 \
+        libgl1-mesa-dev=20.3.5-1 \
+        xorg-dev=1:7.7+22 \
+        libayatana-appindicator3-dev=0.5.5-2+deb11u2 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && go install -v golang.org/x/tools/gopls@latest
+
+
+WORKDIR /app

.devcontainer/devcontainer.json

@@ -0,0 +1,20 @@
+{
+	"name": "NetBird",
+	"build": {
+		"context": "..",
+		"dockerfile": "Dockerfile"
+	},
+	"features": {
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
+		"ghcr.io/devcontainers/features/go:1": {
+			"version": "1.21"
+		}
+	},
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+	"capAdd": [
+		"NET_ADMIN",
+		"SYS_ADMIN",
+		"SYS_RESOURCE"
+	],
+	"privileged": true
+}

.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.go]
+indent_style = tab

.gitattributes

@@ -0,0 +1 @@
+*.go text eol=lf

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [netbirdio]

.github/ISSUE_TEMPLATE/bug-issue-report.md

@@ -2,15 +2,17 @@
 name: Bug/Issue report
 about: Create a report to help us improve
 title: ''
-labels: ''
+labels: ['triage-needed']
 assignees: ''
 
 ---
 
 **Describe the problem**
+
 A clear and concise description of what the problem is.
 
 **To Reproduce**
+
 Steps to reproduce the behavior:
 1. Go to '...'
 2. Click on '....'
@@ -18,13 +20,30 @@ Steps to reproduce the behavior:
 4. See error
 
 **Expected behavior**
+
 A clear and concise description of what you expected to happen.
 
-**NetBird status -d output:**
+**Are you using NetBird Cloud?**
-If applicable, add the output of the `netbird status -d` command
+
+Please specify whether you use NetBird Cloud or self-host NetBird's control plane.
+
+**NetBird version**
+
+`netbird version`
+
+**NetBird status -dA output:**
+
+If applicable, add the `netbird status -dA' command output.
+
+**Do you face any (non-mobile) client issues?**
+
+Please provide the file created by `netbird debug for 1m -AS`.
+We advise reviewing the anonymized files for any remaining PII.
 
 **Screenshots**
+
 If applicable, add screenshots to help explain your problem.
 
 **Additional context**
+
 Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,7 +2,7 @@
 name: Feature request
 about: Suggest an idea for this project
 title: ''
-labels: ''
+labels: ['feature-request']
 assignees: ''
 
 ---

.github/workflows/golang-test-darwin.yml

@@ -6,27 +6,40 @@ on:
       - main
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
+    strategy:
+      matrix:
+        store: ['sqlite']
     runs-on: macos-latest
     steps:
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.19.x
+          go-version: "1.23.x"
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Cache Go modules
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: macos-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             macos-go-
 
+      - name: Install libpcap
+        run: brew install libpcap
+
       - name: Install modules
         run: go mod tidy
 
+      - name: check git status
+        run: git --no-pager diff --exit-code
+
       - name: Test
-        run: go test -exec 'sudo --preserve-env=CI' -timeout 5m -p 1 ./...
+        run: NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 ./...

.github/workflows/golang-test-freebsd.yml

@@ -0,0 +1,46 @@
+
+name: Test Code FreeBSD
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Test in FreeBSD
+        id: test
+        uses: vmactions/freebsd-vm@v1
+        with:
+          usesh: true
+          copyback: false
+          release: "14.1"
+          prepare: |
+            pkg install -y go
+
+          # -x - to print all executed commands
+          # -e - to faile on first error
+          run: |
+            set -e -x
+            time go build -o netbird client/main.go
+            # check all component except management, since we do not support management server on freebsd
+            time go test -timeout 1m -failfast ./base62/...
+            # NOTE: without -p1 `client/internal/dns` will fail becasue of `listen udp4 :33100: bind: address already in use`
+            time go test -timeout 8m -failfast -p 1 ./client/...
+            time go test -timeout 1m -failfast ./dns/...
+            time go test -timeout 1m -failfast ./encryption/...
+            time go test -timeout 1m -failfast ./formatter/...
+            time go test -timeout 1m -failfast ./client/iface/...
+            time go test -timeout 1m -failfast ./route/...
+            time go test -timeout 1m -failfast ./sharedsock/...
+            time go test -timeout 1m -failfast ./signal/...
+            time go test -timeout 1m -failfast ./util/...
+            time go test -timeout 1m -failfast ./version/...

.github/workflows/golang-test-linux.yml

@@ -6,21 +6,27 @@ on:
       - main
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     strategy:
+      fail-fast: false
       matrix:
-        arch: ['386','amd64']
+        arch: [ '386','amd64' ]
-    runs-on: ubuntu-latest
+        store: [ 'sqlite', 'postgres']
+    runs-on: ubuntu-22.04
     steps:
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.19.x
+          go-version: "1.23.x"
 
 
       - name: Cache Go modules
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -28,28 +34,40 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
+
+      - name: Install 32-bit libpcap
+        if: matrix.arch == '386'
+        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
 
       - name: Install modules
         run: go mod tidy
 
+      - name: check git status
+        run: git --no-pager diff --exit-code
+
       - name: Test
-        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} go test -exec 'sudo --preserve-env=CI' -timeout 5m -p 1 ./...
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 10m -p 1 ./...
 
-  test_client_on_docker:
+  benchmark:
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [ '386','amd64' ]
+        store: [ 'sqlite', 'postgres' ]
+    runs-on: ubuntu-22.04
     steps:
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.19.x
+          go-version: "1.23.x"
 
 
       - name: Cache Go modules
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -57,36 +75,92 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
+
+      - name: Install 32-bit libpcap
+        if: matrix.arch == '386'
+        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
 
       - name: Install modules
         run: go mod tidy
 
-      - name: Generate Iface Test bin
+      - name: check git status
-        run: go test -c -o iface-testing.bin ./iface/...
+        run: git --no-pager diff --exit-code
+
+      - name: Test
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -run=^$ -bench=. -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 10m -p 1 ./...
+
+  test_client_on_docker:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23.x"
+
+      - name: Cache Go modules
+        uses: actions/cache@v4
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
+
+      - name: Install modules
+        run: go mod tidy
+
+      - name: check git status
+        run: git --no-pager diff --exit-code
+
+      - name: Generate Shared Sock Test bin
+        run: CGO_ENABLED=0 go test -c -o sharedsock-testing.bin ./sharedsock
 
       - name: Generate RouteManager Test bin
-        run: go test -c -o routemanager-testing.bin ./client/internal/routemanager/...
+        run: CGO_ENABLED=0 go test -c -o routemanager-testing.bin  ./client/internal/routemanager
+
+      - name: Generate SystemOps Test bin
+        run: CGO_ENABLED=1 go test -c -o systemops-testing.bin -tags netgo -ldflags '-w -extldflags "-static -ldbus-1 -lpcap"' ./client/internal/routemanager/systemops
+
+      - name: Generate nftables Manager Test bin
+        run: CGO_ENABLED=0 go test -c -o nftablesmanager-testing.bin ./client/firewall/nftables/...
 
       - name: Generate Engine Test bin
-        run: go test -c -o engine-testing.bin ./client/internal
+        run: CGO_ENABLED=1 go test -c -o engine-testing.bin ./client/internal
 
       - name: Generate Peer Test bin
-        run: go test -c -o peer-testing.bin ./client/internal/peer/...
+        run: CGO_ENABLED=0 go test -c -o peer-testing.bin ./client/internal/peer/
 
       - run: chmod +x *testing.bin
 
+      - name: Run Shared Sock tests in docker
+        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/sharedsock --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/sharedsock-testing.bin -test.timeout 5m -test.parallel 1
+
       - name: Run Iface tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/iface --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/iface-testing.bin -test.timeout 5m -test.parallel 1
+        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/netbird -v /tmp/cache:/tmp/cache -v /tmp/modcache:/tmp/modcache -w /netbird -e GOCACHE=/tmp/cache -e GOMODCACHE=/tmp/modcache -e CGO_ENABLED=0 golang:1.23-alpine go test  -test.timeout 5m -test.parallel 1 ./client/iface/...
 
       - name: Run RouteManager tests in docker
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/routemanager --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/routemanager-testing.bin  -test.timeout 5m -test.parallel 1
 
-      - name: Run Engine tests in docker
+      - name: Run SystemOps tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/engine-testing.bin  -test.timeout 5m -test.parallel 1
+        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/routemanager/systemops --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/systemops-testing.bin  -test.timeout 5m -test.parallel 1
+
+      - name: Run nftables Manager tests in docker
+        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/firewall --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/nftablesmanager-testing.bin  -test.timeout 5m -test.parallel 1
+
+      - name: Run Engine tests in docker with file store
+        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_ENGINE="jsonfile" --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/engine-testing.bin  -test.timeout 5m -test.parallel 1
+
+      - name: Run Engine tests in docker with sqlite store
+        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_ENGINE="sqlite" --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/engine-testing.bin  -test.timeout 5m -test.parallel 1
 
       - name: Run Peer tests in docker
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/peer  --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/peer-testing.bin  -test.timeout 5m -test.parallel 1

.github/workflows/golang-test-windows.yml

@@ -6,47 +6,47 @@ on:
       - main
   pull_request:
 
+env:
+  downloadPath: '${{ github.workspace }}\temp'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
+  cancel-in-progress: true
+
 jobs:
-  pre:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - run: bash -x wireguard_nt.sh
-        working-directory: client
-
-      - uses: actions/upload-artifact@v2
-        with:
-          name: syso
-          path: client/*.syso
-          retention-days: 1
-
   test:
-    needs: pre
     runs-on: windows-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
+        id: go
         with:
-          go-version: 1.19.x
+          go-version: "1.23.x"
 
-      - uses: actions/cache@v2
+      - name: Download wintun
+        uses: carlosperate/download-file-action@v2
+        id: download-wintun
         with:
-          path: |
+          file-url: https://pkgs.netbird.io/wintun/wintun-0.14.1.zip
-            %LocalAppData%\go-build
+          file-name: wintun.zip
-            ~\go\pkg\mod
+          location: ${{ env.downloadPath }}
-            ~\AppData\Local\go-build
+          sha256: '07c256185d6ee3652e09fa55c0b673e2624b565e02c4b9091c79ca7d2f24ef51'
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
 
-      - uses: actions/download-artifact@v2
+      - name: Decompressing wintun files
-        with:
+        run: tar -zvxf "${{ steps.download-wintun.outputs.file-path }}" -C ${{ env.downloadPath }}
-          name: syso
-          path: iface\
 
-      - name: Test
+      - run: mv ${{ env.downloadPath }}/wintun/bin/amd64/wintun.dll 'C:\Windows\System32\'
-        run: go test -tags=load_wgnt_from_rsrc -timeout 5m -p 1 ./...
+
+      - run: choco install -y sysinternals --ignore-checksums
+      - run: choco install -y mingw
+
+      - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOMODCACHE=C:\Users\runneradmin\go\pkg\mod
+      - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOCACHE=C:\Users\runneradmin\AppData\Local\go-build
+
+      - name: test
+        run: PsExec64 -s -w ${{ github.workspace }} cmd.exe /c "C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe test -timeout 10m -p 1 ./... > test-out.txt 2>&1"
+      - name: test output
+        if: ${{ always() }}
+        run: Get-Content test-out.txt

.github/workflows/golangci-lint.yml

@@ -1,18 +1,52 @@
 name: golangci-lint
 on: [pull_request]
+
+permissions:
+  contents: read
+  pull-requests: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
+  cancel-in-progress: true
+
 jobs:
-  golangci:
+  codespell:
-    name: lint
+    name: codespell
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: codespell
+        uses: codespell-project/actions-codespell@v2
+        with:
+          ignore_words_list: erro,clienta,hastable,iif,groupd
+          skip: go.mod,go.sum
+          only_warn: 1
+  golangci:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-latest]
+    name: lint
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Check for duplicate constants
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          ! awk '/const \(/,/)/{print $0}' management/server/activity/codes.go | grep -o '= [0-9]*' | sort | uniq -d | grep .
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.19.x
+          go-version: "1.23.x"
+          cache: false
       - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev
+        if: matrix.os == 'ubuntu-latest'
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev libpcap-dev
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
         with:
-          args: --timeout=6m
+          version: latest
+          args: --timeout=12m

.github/workflows/install-script-test.yml

@@ -0,0 +1,37 @@
+name: Test installation
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - "release_files/install.sh"
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
+  cancel-in-progress: true
+jobs:
+  test-install-script:
+    strategy:
+      fail-fast: false
+      max-parallel: 2
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        skip_ui_mode: [true, false]
+        install_binary: [true, false]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: run install script
+        env:
+          SKIP_UI_APP: ${{ matrix.skip_ui_mode }}
+          USE_BIN_INSTALL: ${{ matrix.install_binary }}
+          GITHUB_TOKEN: ${{ secrets.RO_API_CALLER_TOKEN }}
+        run: |
+          [ "$SKIP_UI_APP" == "false" ] && export XDG_CURRENT_DESKTOP="none"
+          cat release_files/install.sh | sh -x
+
+      - name: check cli binary
+        run: command -v netbird

.github/workflows/install-test-darwin.yml

@@ -1,58 +0,0 @@
-name: Test installation Darwin
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - "release_files/install.sh"
-
-jobs:
-  install-cli-only:
-    runs-on: macos-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Rename brew package
-        if: ${{ matrix.check_bin_install }}
-        run: mv /opt/homebrew/bin/brew /opt/homebrew/bin/brew.bak
-
-      - name: Run install script
-        run: |
-          sh ./release_files/install.sh
-        env:
-          SKIP_UI_APP: true
-
-      - name: Run tests
-        run: |
-          if ! command -v netbird &> /dev/null; then
-            echo "Error: netbird is not installed"
-            exit 1
-          fi
-  install-all:
-    runs-on: macos-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Rename brew package
-        if: ${{ matrix.check_bin_install }}
-        run: mv /opt/homebrew/bin/brew /opt/homebrew/bin/brew.bak
-
-      - name: Run install script
-        run: |
-          sh ./release_files/install.sh
-
-      - name: Run tests
-        run: |
-          if ! command -v netbird &> /dev/null; then
-              echo "Error: netbird is not installed"
-              exit 1
-          fi
-
-          if [[ $(mdfind "kMDItemContentType == 'com.apple.application-bundle' && kMDItemFSName == '*NetBird UI.app'") ]]; then
-            echo "Error: NetBird UI is not installed"
-            exit 1
-          fi

.github/workflows/install-test-linux.yml

@@ -1,36 +0,0 @@
-name: Test installation Linux
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - "release_files/install.sh"
-
-jobs:
-  install-cli-only:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        check_bin_install: [true, false]
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Rename apt package
-        if: ${{ matrix.check_bin_install }}
-        run: |
-          sudo mv /usr/bin/apt /usr/bin/apt.bak
-          sudo mv /usr/bin/apt-get /usr/bin/apt-get.bak
-
-      - name: Run install script
-        run: |
-          sh ./release_files/install.sh
-
-      - name: Run tests
-        run: |
-          if ! command -v netbird &> /dev/null; then
-            echo "Error: netbird is not installed"
-            exit 1
-          fi

.github/workflows/mobile-build-validation.yml

@@ -0,0 +1,65 @@
+name: Mobile build validation
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
+  cancel-in-progress: true
+
+jobs:
+  android_build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23.x"
+      - name: Setup Android SDK
+        uses: android-actions/setup-android@v3
+        with:
+          cmdline-tools-version: 8512546
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          java-version: "11"
+          distribution: "adopt"
+      - name: NDK Cache
+        id: ndk-cache
+        uses: actions/cache@v4
+        with:
+          path: /usr/local/lib/android/sdk/ndk
+          key: ndk-cache-23.1.7779620
+      - name: Setup NDK
+        run: /usr/local/lib/android/sdk/cmdline-tools/7.0/bin/sdkmanager --install "ndk;23.1.7779620"
+      - name: install gomobile
+        run: go install golang.org/x/mobile/cmd/gomobile@v0.0.0-20240404231514-09dbf07665ed
+      - name: gomobile init
+        run: gomobile init
+      - name: build android netbird lib
+        run: PATH=$PATH:$(go env GOPATH) gomobile bind -o $GITHUB_WORKSPACE/netbird.aar -javapkg=io.netbird.gomobile -ldflags="-X golang.zx2c4.com/wireguard/ipc.socketDirectory=/data/data/io.netbird.client/cache/wireguard -X github.com/netbirdio/netbird/version.version=buildtest"  $GITHUB_WORKSPACE/client/android
+        env:
+          CGO_ENABLED: 0
+          ANDROID_NDK_HOME: /usr/local/lib/android/sdk/ndk/23.1.7779620
+  ios_build:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23.x"
+      - name: install gomobile
+        run: go install golang.org/x/mobile/cmd/gomobile@v0.0.0-20240404231514-09dbf07665ed
+      - name: gomobile init
+        run: gomobile init
+      - name: build iOS netbird lib
+        run: PATH=$PATH:$(go env GOPATH) gomobile bind -target=ios -bundleid=io.netbird.framework -ldflags="-X github.com/netbirdio/netbird/version.version=buildtest" -o ./NetBirdSDK.xcframework ./client/ios/NetBirdSDK
+        env:
+          CGO_ENABLED: 0

.github/workflows/release.yml

@@ -3,100 +3,141 @@ name: Release
 on:
   push:
     tags:
-      - 'v*'
+      - "v*"
     branches:
       - main
   pull_request:
 
 env:
-  SIGN_PIPE_VER: "v0.0.5"
+  SIGN_PIPE_VER: "v0.0.17"
-  GORELEASER_VER: "v1.14.1"
+  GORELEASER_VER: "v2.3.2"
+  PRODUCT_NAME: "NetBird"
+  COPYRIGHT: "Wiretrustee UG (haftungsbeschreankt)"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
+  cancel-in-progress: true
 
 jobs:
   release:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
+    env:
+      flags: ""
     steps:
-      -
+      - name: Parse semver string
-        name: Checkout
+        id: semver_parser
-        uses: actions/checkout@v2
+        uses: booxmedialtd/ws-action-parse-semver@v1
+        with:
+          input_string: ${{ (startsWith(github.ref, 'refs/tags/v') && github.ref) || 'refs/tags/v0.0.0' }}
+          version_extractor_regex: '\/v(.*)$'
+
+      - if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+        run: echo "flags=--snapshot" >> $GITHUB_ENV
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # It is required for GoReleaser to work properly
-
+      - name: Set up Go
-      - name: Generate syso with DLL
+        uses: actions/setup-go@v5
-        run: bash -x wireguard_nt.sh
-        working-directory: client
-      -
-        name: Set up Go
-        uses: actions/setup-go@v2
         with:
-          go-version: 1.19
+          go-version: "1.23"
-      -
+          cache: false
-        name: Cache Go modules
+      - name: Cache Go modules
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
-          path: ~/go/pkg/mod
+          path: |
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+            ~/go/pkg/mod
+            ~/.cache/go-build
+          key: ${{ runner.os }}-go-releaser-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go-
+            ${{ runner.os }}-go-releaser-
-      -
+      - name: Install modules
-        name: Install modules
         run: go mod tidy
-      -
+      - name: check git status
-        name: check git status
         run: git --no-pager diff --exit-code
-      -
+      - name: Set up QEMU
-        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
-        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
-      -
+        uses: docker/setup-buildx-action@v2
-        name: Set up Docker Buildx
+      - name: Login to Docker hub
-        uses: docker/setup-buildx-action@v1
-      -
-        name: Login to Docker hub
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v1
         with:
-          username: netbirdio
+          username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
       - name: Install OS build dependencies
         run: sudo apt update && sudo apt install -y -q gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
-      -
+
-        name: Run GoReleaser
+      - name: Install goversioninfo
-        uses: goreleaser/goreleaser-action@v2
+        run: go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@233067e
+      - name: Generate windows syso amd64
+        run: goversioninfo  -icon client/ui/netbird.ico -manifest client/manifest.xml -product-name ${{ env.PRODUCT_NAME }} -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/resources_windows_amd64.syso
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v4
         with:
           version: ${{ env.GORELEASER_VER }}
-          args: release --rm-dist
+          args: release --clean ${{ env.flags }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }}
           UPLOAD_DEBIAN_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }}
           UPLOAD_YUM_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }}
-      -
+      - name: upload non tags for debug purposes
-        name: upload non tags for debug purposes
+        uses: actions/upload-artifact@v4
-        uses: actions/upload-artifact@v2
         with:
           name: release
           path: dist/
           retention-days: 3
+      - name: upload linux packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: linux-packages
+          path: dist/netbird_linux**
+          retention-days: 3
+      - name: upload windows packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: windows-packages
+          path: dist/netbird_windows**
+          retention-days: 3
+      - name: upload macos packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: macos-packages
+          path: dist/netbird_darwin**
+          retention-days: 3
 
   release_ui:
     runs-on: ubuntu-latest
     steps:
+      - name: Parse semver string
+        id: semver_parser
+        uses: booxmedialtd/ws-action-parse-semver@v1
+        with:
+          input_string: ${{ (startsWith(github.ref, 'refs/tags/v') && github.ref) || 'refs/tags/v0.0.0' }}
+          version_extractor_regex: '\/v(.*)$'
+
+      - if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+        run: echo "flags=--snapshot" >> $GITHUB_ENV
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # It is required for GoReleaser to work properly
 
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.19
+          go-version: "1.23"
+          cache: false
       - name: Cache Go modules
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
-          path: ~/go/pkg/mod
+          path: |
-          key: ${{ runner.os }}-ui-go-${{ hashFiles('**/go.sum') }}
+            ~/go/pkg/mod
+            ~/.cache/go-build
+          key: ${{ runner.os }}-ui-go-releaser-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-ui-go-
+            ${{ runner.os }}-ui-go-releaser-
 
       - name: Install modules
         run: go mod tidy
@@ -105,93 +146,81 @@ jobs:
         run: git --no-pager diff --exit-code
 
       - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-mingw-w64-x86-64
+        run: sudo apt update && sudo apt install -y -q libappindicator3-dev gir1.2-appindicator3-0.1 libxxf86vm-dev gcc-mingw-w64-x86-64
-      - name: Install rsrc
+      - name: Install goversioninfo
-        run: go install github.com/akavel/rsrc@v0.10.2
+        run: go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@233067e
-      - name: Generate windows rsrc
+      - name: Generate windows syso amd64
-        run: rsrc -arch amd64 -ico client/ui/netbird.ico -manifest client/ui/manifest.xml -o client/ui/resources_windows_amd64.syso
+        run: goversioninfo -64 -icon client/ui/netbird.ico -manifest client/ui/manifest.xml -product-name ${{ env.PRODUCT_NAME }}-"UI" -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/ui/resources_windows_amd64.syso
+
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v2
+        uses: goreleaser/goreleaser-action@v4
         with:
           version: ${{ env.GORELEASER_VER }}
-          args: release --config .goreleaser_ui.yaml --rm-dist
+          args: release --config .goreleaser_ui.yaml --clean ${{ env.flags }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }}
           UPLOAD_DEBIAN_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }}
           UPLOAD_YUM_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }}
       - name: upload non tags for debug purposes
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: release-ui
           path: dist/
           retention-days: 3
 
   release_ui_darwin:
-    runs-on: macos-11
+    runs-on: macos-latest
     steps:
-      -
+      - if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
-        name: Checkout
+        run: echo "flags=--snapshot" >> $GITHUB_ENV
-        uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # It is required for GoReleaser to work properly
-      -
+      - name: Set up Go
-        name: Set up Go
+        uses: actions/setup-go@v5
-        uses: actions/setup-go@v2
         with:
-          go-version: 1.19
+          go-version: "1.23"
-      -
+          cache: false
-        name: Cache Go modules
+      - name: Cache Go modules
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
-          path: ~/go/pkg/mod
+          path: |
-          key: ${{ runner.os }}-ui-go-${{ hashFiles('**/go.sum') }}
+            ~/go/pkg/mod
+            ~/.cache/go-build
+          key: ${{ runner.os }}-ui-go-releaser-darwin-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-ui-go-
+            ${{ runner.os }}-ui-go-releaser-darwin-
-      -
+      - name: Install modules
-        name: Install modules
         run: go mod tidy
-      -
+      - name: check git status
-        name: Run GoReleaser
+        run: git --no-pager diff --exit-code
+      - name: Run GoReleaser
         id: goreleaser
-        uses: goreleaser/goreleaser-action@v2
+        uses: goreleaser/goreleaser-action@v4
         with:
           version: ${{ env.GORELEASER_VER }}
-          args: release --config .goreleaser_ui_darwin.yaml --rm-dist
+          args: release --config .goreleaser_ui_darwin.yaml --clean ${{ env.flags }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      -
+      - name: upload non tags for debug purposes
-        name: upload non tags for debug purposes
+        uses: actions/upload-artifact@v4
-        uses: actions/upload-artifact@v2
         with:
           name: release-ui-darwin
           path: dist/
           retention-days: 3
 
-  trigger_windows_signer:
+  trigger_signer:
     runs-on: ubuntu-latest
-    needs: [release,release_ui]
+    needs: [release, release_ui, release_ui_darwin]
     if: startsWith(github.ref, 'refs/tags/')
     steps:
-      - name: Trigger Windows binaries sign pipeline
+      - name: Trigger binaries sign pipelines
         uses: benc-uk/workflow-dispatch@v1
         with:
-          workflow: Sign windows bin and installer
+          workflow: Sign bin and installer
           repo: netbirdio/sign-pipelines
           ref: ${{ env.SIGN_PIPE_VER }}
           token: ${{ secrets.SIGN_GITHUB_TOKEN }}
-          inputs: '{ "tag": "${{ github.ref }}" }'
+          inputs: '{ "tag": "${{ github.ref }}", "skipRelease": false }'
-
-  trigger_darwin_signer:
-    runs-on: ubuntu-latest
-    needs: release_ui_darwin
-    if: startsWith(github.ref, 'refs/tags/')
-    steps:
-      - name: Trigger Darwin App binaries sign pipeline
-        uses: benc-uk/workflow-dispatch@v1
-        with:
-          workflow: Sign darwin ui app with dispatch
-          repo: netbirdio/sign-pipelines
-          ref: ${{ env.SIGN_PIPE_VER }}
-          token: ${{ secrets.SIGN_GITHUB_TOKEN }}
-          inputs: '{ "tag": "${{ github.ref }}" }'

.github/workflows/sync-main.yml

@@ -0,0 +1,22 @@
+name: sync main
+
+on:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
+  cancel-in-progress: true
+
+jobs:
+  trigger_sync_main:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger main branch sync
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: sync-main.yml
+          repo: ${{ secrets.UPSTREAM_REPO }}
+          token: ${{ secrets.NC_GITHUB_TOKEN }}
+          inputs: '{ "sha": "${{ github.sha }}" }'

.github/workflows/sync-tag.yml

@@ -0,0 +1,23 @@
+name: sync tag
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
+  cancel-in-progress: true
+
+jobs:
+  trigger_sync_tag:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger release tag sync
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: sync-tag.yml
+          ref: main
+          repo: ${{ secrets.UPSTREAM_REPO }}
+          token: ${{ secrets.NC_GITHUB_TOKEN }}
+          inputs: '{ "tag": "${{ github.ref_name }}" }'

.github/workflows/test-docker-compose-linux.yml

@@ -1,95 +0,0 @@
-name: Test Docker Compose Linux
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Install jq
-        run: sudo apt-get install -y jq
-
-      - name: Install curl
-        run: sudo apt-get install -y curl
-
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.19.x
-
-      - name: Cache Go modules
-        uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: cp setup.env
-        run: cp infrastructure_files/tests/setup.env infrastructure_files/
-
-      - name: run configure
-        working-directory: infrastructure_files
-        run: bash -x configure.sh
-        env:
-          CI_NETBIRD_DOMAIN: localhost
-          CI_NETBIRD_AUTH_CLIENT_ID: testing.client.id
-          CI_NETBIRD_AUTH_AUDIENCE: testing.ci
-          CI_NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT: https://example.eu.auth0.com/.well-known/openid-configuration
-          CI_NETBIRD_USE_AUTH0: true
-
-      - name: check values
-        working-directory: infrastructure_files
-        env:
-          CI_NETBIRD_DOMAIN: localhost
-          CI_NETBIRD_AUTH_CLIENT_ID: testing.client.id
-          CI_NETBIRD_AUTH_AUDIENCE: testing.ci
-          CI_NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT: https://example.eu.auth0.com/.well-known/openid-configuration
-          CI_NETBIRD_USE_AUTH0: true
-          CI_NETBIRD_AUTH_SUPPORTED_SCOPES: "openid profile email offline_access api email_verified"
-          CI_NETBIRD_AUTH_AUTHORITY: https://example.eu.auth0.com/
-          CI_NETBIRD_AUTH_JWT_CERTS: https://example.eu.auth0.com/.well-known/jwks.json
-          CI_NETBIRD_AUTH_TOKEN_ENDPOINT: https://example.eu.auth0.com/oauth/token
-          CI_NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT: https://example.eu.auth0.com/oauth/device/code
-          CI_NETBIRD_AUTH_REDIRECT_URI: "/peers"
-          CI_NETBIRD_TOKEN_SOURCE: "idToken"
-          CI_NETBIRD_AUTH_USER_ID_CLAIM: "email"
-          CI_NETBIRD_AUTH_DEVICE_AUTH_AUDIENCE: "super"
-          CI_NETBIRD_AUTH_DEVICE_AUTH_SCOPE: "openid email"
-
-        run: |
-          grep AUTH_CLIENT_ID docker-compose.yml | grep $CI_NETBIRD_AUTH_CLIENT_ID
-          grep AUTH_AUTHORITY docker-compose.yml | grep $CI_NETBIRD_AUTH_AUTHORITY
-          grep AUTH_AUDIENCE docker-compose.yml | grep $CI_NETBIRD_AUTH_AUDIENCE
-          grep AUTH_SUPPORTED_SCOPES docker-compose.yml | grep "$CI_NETBIRD_AUTH_SUPPORTED_SCOPES"
-          grep USE_AUTH0 docker-compose.yml | grep $CI_NETBIRD_USE_AUTH0
-          grep NETBIRD_MGMT_API_ENDPOINT docker-compose.yml | grep "$CI_NETBIRD_DOMAIN:33073"
-          grep AUTH_REDIRECT_URI docker-compose.yml | grep $CI_NETBIRD_AUTH_REDIRECT_URI
-          grep AUTH_SILENT_REDIRECT_URI docker-compose.yml | egrep 'AUTH_SILENT_REDIRECT_URI=$'
-          grep LETSENCRYPT_DOMAIN docker-compose.yml | egrep 'LETSENCRYPT_DOMAIN=$'
-          grep NETBIRD_TOKEN_SOURCE docker-compose.yml | grep $CI_NETBIRD_TOKEN_SOURCE
-          grep AuthUserIDClaim management.json | grep $CI_NETBIRD_AUTH_USER_ID_CLAIM
-          grep -A 1 ProviderConfig management.json | grep Audience | grep $CI_NETBIRD_AUTH_DEVICE_AUTH_AUDIENCE
-          grep Scope management.json | grep "$CI_NETBIRD_AUTH_DEVICE_AUTH_SCOPE"
-          grep UseIDToken management.json | grep false
-
-      - name: run docker compose up
-        working-directory: infrastructure_files
-        run: | 
-          docker-compose up -d
-          sleep 5
-          docker-compose ps
-          docker-compose logs --tail=20
-
-      - name: test running containers
-        run: |
-          count=$(docker compose ps --format json | jq '.[] | select(.Project | contains("infrastructure_files")) | .State' | grep -c running)
-          test $count -eq 4
-        working-directory: infrastructure_files

.github/workflows/test-infrastructure-files.yml

@@ -0,0 +1,287 @@
+name: Test Infrastructure files
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - 'infrastructure_files/**'
+      - '.github/workflows/test-infrastructure-files.yml'
+      - 'management/cmd/**'
+      - 'signal/cmd/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
+  cancel-in-progress: true
+
+jobs:
+  test-docker-compose:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        store: [ 'sqlite', 'postgres' ]
+    services:
+      postgres:
+        image: ${{ (matrix.store == 'postgres') && 'postgres' || '' }}
+        env:
+          POSTGRES_USER: netbird
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: netbird
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+        ports:
+          - 5432:5432
+    steps:
+      - name: Set Database Connection String
+        run: |
+          if [ "${{ matrix.store }}" == "postgres" ]; then
+            echo "NETBIRD_STORE_ENGINE_POSTGRES_DSN=host=$(hostname -I | awk '{print $1}') user=netbird password=postgres dbname=netbird port=5432" >> $GITHUB_ENV
+          else
+            echo "NETBIRD_STORE_ENGINE_POSTGRES_DSN==" >> $GITHUB_ENV
+          fi
+
+      - name: Install jq
+        run: sudo apt-get install -y jq
+
+      - name: Install curl
+        run: sudo apt-get install -y curl
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23.x"
+
+      - name: Cache Go modules
+        uses: actions/cache@v4
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: cp setup.env
+        run: cp infrastructure_files/tests/setup.env infrastructure_files/
+
+      - name: run configure
+        working-directory: infrastructure_files
+        run: bash -x configure.sh
+        env:
+          CI_NETBIRD_DOMAIN: localhost
+          CI_NETBIRD_AUTH_CLIENT_ID: testing.client.id
+          CI_NETBIRD_AUTH_CLIENT_SECRET: testing.client.secret
+          CI_NETBIRD_AUTH_AUDIENCE: testing.ci
+          CI_NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT: https://example.eu.auth0.com/.well-known/openid-configuration
+          CI_NETBIRD_USE_AUTH0: true
+          CI_NETBIRD_MGMT_IDP: "none"
+          CI_NETBIRD_IDP_MGMT_CLIENT_ID: testing.client.id
+          CI_NETBIRD_IDP_MGMT_CLIENT_SECRET: testing.client.secret
+          CI_NETBIRD_AUTH_SUPPORTED_SCOPES: "openid profile email offline_access api email_verified"
+          CI_NETBIRD_STORE_CONFIG_ENGINE: ${{ matrix.store }}
+          NETBIRD_STORE_ENGINE_POSTGRES_DSN: ${{ env.NETBIRD_STORE_ENGINE_POSTGRES_DSN }}
+          CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false
+
+      - name: check values
+        working-directory: infrastructure_files/artifacts
+        env:
+          CI_NETBIRD_DOMAIN: localhost
+          CI_NETBIRD_AUTH_CLIENT_ID: testing.client.id
+          CI_NETBIRD_AUTH_CLIENT_SECRET: testing.client.secret
+          CI_NETBIRD_AUTH_AUDIENCE: testing.ci
+          CI_NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT: https://example.eu.auth0.com/.well-known/openid-configuration
+          CI_NETBIRD_USE_AUTH0: true
+          CI_NETBIRD_AUTH_SUPPORTED_SCOPES: "openid profile email offline_access api email_verified"
+          CI_NETBIRD_AUTH_AUTHORITY: https://example.eu.auth0.com/
+          CI_NETBIRD_AUTH_JWT_CERTS: https://example.eu.auth0.com/.well-known/jwks.json
+          CI_NETBIRD_AUTH_TOKEN_ENDPOINT: https://example.eu.auth0.com/oauth/token
+          CI_NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT: https://example.eu.auth0.com/oauth/device/code
+          CI_NETBIRD_AUTH_PKCE_AUTHORIZATION_ENDPOINT: https://example.eu.auth0.com/authorize
+          CI_NETBIRD_AUTH_REDIRECT_URI: "/peers"
+          CI_NETBIRD_TOKEN_SOURCE: "idToken"
+          CI_NETBIRD_AUTH_USER_ID_CLAIM: "email"
+          CI_NETBIRD_AUTH_DEVICE_AUTH_AUDIENCE: "super"
+          CI_NETBIRD_AUTH_DEVICE_AUTH_SCOPE: "openid email"
+          CI_NETBIRD_MGMT_IDP: "none"
+          CI_NETBIRD_IDP_MGMT_CLIENT_ID: testing.client.id
+          CI_NETBIRD_IDP_MGMT_CLIENT_SECRET: testing.client.secret
+          CI_NETBIRD_SIGNAL_PORT: 12345
+          CI_NETBIRD_STORE_CONFIG_ENGINE: ${{ matrix.store }}
+          NETBIRD_STORE_ENGINE_POSTGRES_DSN: '${{ env.NETBIRD_STORE_ENGINE_POSTGRES_DSN }}$'
+          CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false
+          CI_NETBIRD_TURN_EXTERNAL_IP: "1.2.3.4"
+
+        run: |
+          set -x
+          grep AUTH_CLIENT_ID docker-compose.yml | grep $CI_NETBIRD_AUTH_CLIENT_ID
+          grep AUTH_CLIENT_SECRET docker-compose.yml | grep $CI_NETBIRD_AUTH_CLIENT_SECRET
+          grep AUTH_AUTHORITY docker-compose.yml | grep $CI_NETBIRD_AUTH_AUTHORITY
+          grep AUTH_AUDIENCE docker-compose.yml | grep $CI_NETBIRD_AUTH_AUDIENCE
+          grep AUTH_SUPPORTED_SCOPES docker-compose.yml | grep "$CI_NETBIRD_AUTH_SUPPORTED_SCOPES"
+          grep USE_AUTH0 docker-compose.yml | grep $CI_NETBIRD_USE_AUTH0
+          grep NETBIRD_MGMT_API_ENDPOINT docker-compose.yml | grep "$CI_NETBIRD_DOMAIN:33073"
+          grep AUTH_REDIRECT_URI docker-compose.yml | grep $CI_NETBIRD_AUTH_REDIRECT_URI
+          grep AUTH_SILENT_REDIRECT_URI docker-compose.yml | egrep 'AUTH_SILENT_REDIRECT_URI=$'
+          grep $CI_NETBIRD_SIGNAL_PORT docker-compose.yml | grep ':80'
+          grep LETSENCRYPT_DOMAIN docker-compose.yml | egrep 'LETSENCRYPT_DOMAIN=$'
+          grep NETBIRD_TOKEN_SOURCE docker-compose.yml | grep $CI_NETBIRD_TOKEN_SOURCE
+          grep AuthUserIDClaim management.json | grep $CI_NETBIRD_AUTH_USER_ID_CLAIM
+          grep -A 3 DeviceAuthorizationFlow management.json | grep -A 1 ProviderConfig | grep Audience | grep $CI_NETBIRD_AUTH_DEVICE_AUTH_AUDIENCE
+          grep -A 3 DeviceAuthorizationFlow management.json | grep -A 1 ProviderConfig | grep Audience | grep $CI_NETBIRD_AUTH_DEVICE_AUTH_AUDIENCE
+          grep Engine management.json  | grep "$CI_NETBIRD_STORE_CONFIG_ENGINE"
+          grep IdpSignKeyRefreshEnabled management.json | grep "$CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH"
+          grep UseIDToken management.json | grep false
+          grep -A 1 IdpManagerConfig management.json | grep ManagerType | grep $CI_NETBIRD_MGMT_IDP
+          grep -A 3 IdpManagerConfig management.json | grep -A 1 ClientConfig | grep Issuer | grep $CI_NETBIRD_AUTH_AUTHORITY
+          grep -A 4 IdpManagerConfig management.json | grep -A 2 ClientConfig | grep TokenEndpoint | grep $CI_NETBIRD_AUTH_TOKEN_ENDPOINT
+          grep -A 5 IdpManagerConfig management.json | grep -A 3 ClientConfig | grep ClientID | grep $CI_NETBIRD_IDP_MGMT_CLIENT_ID
+          grep -A 6 IdpManagerConfig management.json | grep -A 4 ClientConfig | grep ClientSecret | grep $CI_NETBIRD_IDP_MGMT_CLIENT_SECRET
+          grep -A 7 IdpManagerConfig management.json | grep -A 5 ClientConfig | grep GrantType | grep client_credentials
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep Audience | grep $CI_NETBIRD_AUTH_AUDIENCE
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep ClientID | grep $CI_NETBIRD_AUTH_CLIENT_ID
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep ClientSecret | grep $CI_NETBIRD_AUTH_CLIENT_SECRET
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep AuthorizationEndpoint | grep $CI_NETBIRD_AUTH_PKCE_AUTHORIZATION_ENDPOINT
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep TokenEndpoint | grep $CI_NETBIRD_AUTH_TOKEN_ENDPOINT
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep Scope | grep "$CI_NETBIRD_AUTH_SUPPORTED_SCOPES"
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep -A 3 RedirectURLs | grep "http://localhost:53000"
+          grep "external-ip" turnserver.conf | grep $CI_NETBIRD_TURN_EXTERNAL_IP
+          grep NETBIRD_STORE_ENGINE_POSTGRES_DSN docker-compose.yml | egrep "$NETBIRD_STORE_ENGINE_POSTGRES_DSN"
+          # check relay values
+          grep "NB_EXPOSED_ADDRESS=$CI_NETBIRD_DOMAIN:33445" docker-compose.yml
+          grep "NB_LISTEN_ADDRESS=:33445" docker-compose.yml
+          grep '33445:33445' docker-compose.yml
+          grep -A 10 'relay:' docker-compose.yml | egrep 'NB_AUTH_SECRET=.+$'
+          grep -A 7 Relay management.json | grep "rel://$CI_NETBIRD_DOMAIN:33445"
+          grep -A 7 Relay management.json | egrep '"Secret": ".+"'
+
+      - name: Install modules
+        run: go mod tidy
+
+      - name: check git status
+        run: git --no-pager diff --exit-code
+
+      - name: Build management binary
+        working-directory: management
+        run: CGO_ENABLED=1 go build -o netbird-mgmt main.go
+
+      - name: Build management docker image
+        working-directory: management
+        run: |
+          docker build -t netbirdio/management:latest .
+
+      - name: Build signal binary
+        working-directory: signal
+        run: CGO_ENABLED=0 go build -o netbird-signal main.go
+
+      - name: Build signal docker image
+        working-directory: signal
+        run: |
+          docker build -t netbirdio/signal:latest .
+
+      - name: Build relay binary
+        working-directory: relay
+        run: CGO_ENABLED=0 go build -o netbird-relay main.go
+
+      - name: Build relay docker image
+        working-directory: relay
+        run: |
+          docker build -t netbirdio/relay:latest .
+
+      - name: run docker compose up
+        working-directory: infrastructure_files/artifacts
+        run: |
+          docker compose up -d
+          sleep 5
+          docker compose ps
+          docker compose logs --tail=20
+
+      - name: test running containers
+        run: |
+          count=$(docker compose ps --format json | jq '. | select(.Name | contains("artifacts")) | .State' | grep -c running)
+          test $count -eq 5 || docker compose logs
+        working-directory: infrastructure_files/artifacts
+
+      - name: test geolocation databases
+        working-directory: infrastructure_files/artifacts
+        run: |
+          sleep 30
+          docker compose exec management ls -l /var/lib/netbird/ | grep -i GeoLite2-City_[0-9]*.mmdb
+          docker compose exec management ls -l /var/lib/netbird/ | grep -i geonames_[0-9]*.db
+
+  test-getting-started-script:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install jq
+        run: sudo apt-get install -y jq
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: run script with Zitadel PostgreSQL
+        run: NETBIRD_DOMAIN=use-ip bash -x infrastructure_files/getting-started-with-zitadel.sh
+
+      - name: test Caddy file gen postgres
+        run: test -f Caddyfile
+
+      - name: test docker-compose file gen postgres
+        run: test -f docker-compose.yml
+
+      - name: test management.json file gen postgres
+        run: test -f management.json
+
+      - name: test turnserver.conf file gen postgres
+        run: |
+          set -x
+          test -f turnserver.conf
+          grep external-ip turnserver.conf
+
+      - name: test zitadel.env file gen postgres
+        run: test -f zitadel.env
+
+      - name: test dashboard.env file gen postgres
+        run: test -f dashboard.env
+
+      - name: test relay.env file gen postgres
+        run: test -f relay.env
+
+      - name: test zdb.env file gen postgres
+        run: test -f zdb.env
+
+      - name: Postgres run cleanup
+        run: |
+          docker compose down --volumes --rmi all
+          rm -rf docker-compose.yml Caddyfile zitadel.env dashboard.env machinekey/zitadel-admin-sa.token turnserver.conf management.json zdb.env
+
+      - name: run script with Zitadel CockroachDB
+        run: bash -x infrastructure_files/getting-started-with-zitadel.sh
+        env:
+          NETBIRD_DOMAIN: use-ip
+          ZITADEL_DATABASE: cockroach
+
+      - name: test Caddy file gen CockroachDB
+        run: test -f Caddyfile
+
+      - name: test docker-compose file gen CockroachDB
+        run: test -f docker-compose.yml
+
+      - name: test management.json file gen CockroachDB
+        run: test -f management.json
+
+      - name: test turnserver.conf file gen CockroachDB
+        run: |
+          set -x
+          test -f turnserver.conf
+          grep external-ip turnserver.conf
+
+      - name: test zitadel.env file gen CockroachDB
+        run: test -f zitadel.env
+
+      - name: test dashboard.env file gen CockroachDB
+        run: test -f dashboard.env
+
+      - name: test relay.env file gen CockroachDB
+        run: test -f relay.env

.github/workflows/update-docs.yml

@@ -0,0 +1,22 @@
+name: update docs
+
+on:
+  push:
+    tags:
+      - 'v*'
+    paths:
+      - 'management/server/http/api/openapi.yml'
+
+jobs:
+  trigger_docs_api_update:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+      - name: Trigger API pages generation
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: generate api pages
+          repo: netbirdio/docs
+          ref: "refs/heads/main"
+          token: ${{ secrets.SIGN_GITHUB_TOKEN }}
+          inputs: '{ "tag": "${{ github.ref }}" }'

.gitignore

@@ -6,9 +6,26 @@ bin/
 .env
 conf.json
 http-cmds.sh
-infrastructure_files/management.json
+setup.env
-infrastructure_files/docker-compose.yml
+infrastructure_files/**/Caddyfile
+infrastructure_files/**/dashboard.env
+infrastructure_files/**/zitadel.env
+infrastructure_files/**/management.json
+infrastructure_files/**/management-*.json
+infrastructure_files/**/docker-compose.yml
+infrastructure_files/**/openid-configuration.json
+infrastructure_files/**/turnserver.conf
+infrastructure_files/**/management.json.bkp.**
+infrastructure_files/**/management-*.json.bkp.**
+infrastructure_files/**/docker-compose.yml.bkp.**
+infrastructure_files/**/openid-configuration.json.bkp.**
+infrastructure_files/**/turnserver.conf.bkp.**
+management/management
+client/client
+client/client.exe
 *.syso
 client/.distfiles/
 infrastructure_files/setup.env
+infrastructure_files/setup-*.env
 .vscode
+.DS_Store

.golangci.yaml

@@ -0,0 +1,139 @@
+run:
+  # Timeout for analysis, e.g. 30s, 5m.
+  # Default: 1m
+  timeout: 6m
+
+# This file contains only configs which differ from defaults.
+# All possible options can be found here https://github.com/golangci/golangci-lint/blob/master/.golangci.reference.yml
+linters-settings:
+  errcheck:
+    # Report about not checking of errors in type assertions: `a := b.(MyStruct)`.
+    # Such cases aren't reported by default.
+    # Default: false
+    check-type-assertions: false
+
+  gosec:
+    includes:
+      - G101 # Look for hard coded credentials
+      #- G102 # Bind to all interfaces
+      - G103 # Audit the use of unsafe block
+      - G104 # Audit errors not checked
+      - G106 # Audit the use of ssh.InsecureIgnoreHostKey
+      #- G107 # Url provided to HTTP request as taint input
+      - G108 # Profiling endpoint automatically exposed on /debug/pprof
+      - G109 # Potential Integer overflow made by strconv.Atoi result conversion to int16/32
+      - G110 # Potential DoS vulnerability via decompression bomb
+      - G111 # Potential directory traversal
+      #- G112 # Potential slowloris attack
+      - G113 # Usage of Rat.SetString in math/big with an overflow (CVE-2022-23772)
+      #- G114 # Use of net/http serve function that has no support for setting timeouts
+      - G201 # SQL query construction using format string
+      - G202 # SQL query construction using string concatenation
+      - G203 # Use of unescaped data in HTML templates
+      #- G204 # Audit use of command execution
+      - G301 # Poor file permissions used when creating a directory
+      - G302 # Poor file permissions used with chmod
+      - G303 # Creating tempfile using a predictable path
+      - G304 # File path provided as taint input
+      - G305 # File traversal when extracting zip/tar archive
+      - G306 # Poor file permissions used when writing to a new file
+      - G307 # Poor file permissions used when creating a file with os.Create
+      #- G401 # Detect the usage of DES, RC4, MD5 or SHA1
+      #- G402 # Look for bad TLS connection settings
+      - G403 # Ensure minimum RSA key length of 2048 bits
+      #- G404 # Insecure random number source (rand)
+      #- G501 # Import blocklist: crypto/md5
+      - G502 # Import blocklist: crypto/des
+      - G503 # Import blocklist: crypto/rc4
+      - G504 # Import blocklist: net/http/cgi
+      #- G505 # Import blocklist: crypto/sha1
+      - G601 # Implicit memory aliasing of items from a range statement
+      - G602 # Slice access out of bounds
+
+  gocritic:
+    disabled-checks:
+      - commentFormatting
+      - captLocal
+      - deprecatedComment
+
+  govet:
+    # Enable all analyzers.
+    # Default: false
+    enable-all: false
+    enable:
+      - nilness
+
+  revive:
+    rules:
+      - name: exported
+        severity: warning
+        disabled: false
+        arguments:
+          - "checkPrivateReceivers"
+          - "sayRepetitiveInsteadOfStutters"
+  tenv:
+    # The option `all` will run against whole test files (`_test.go`) regardless of method/function signatures.
+    # Otherwise, only methods that take `*testing.T`, `*testing.B`, and `testing.TB` as arguments are checked.
+    # Default: false
+    all: true
+
+linters:
+  disable-all: true
+  enable:
+    ## enabled by default
+    - errcheck # checking for unchecked errors, these unchecked errors can be critical bugs in some cases
+    - gosimple # specializes in simplifying a code
+    - govet # reports suspicious constructs, such as Printf calls whose arguments do not align with the format string
+    - ineffassign # detects when assignments to existing variables are not used
+    - staticcheck # is a go vet on steroids, applying a ton of static analysis checks
+    - tenv # Tenv is analyzer that detects using os.Setenv instead of t.Setenv since Go1.17.
+    - typecheck # like the front-end of a Go compiler, parses and type-checks Go code
+    - unused # checks for unused constants, variables, functions and types
+    ## disable by default but the have interesting results so lets add them
+    - bodyclose # checks whether HTTP response body is closed successfully
+    - dupword # dupword checks for duplicate words in the source code
+    - durationcheck # durationcheck checks for two durations multiplied together
+    - forbidigo # forbidigo forbids identifiers
+    - gocritic # provides diagnostics that check for bugs, performance and style issues
+    - gosec # inspects source code for security problems
+    - mirror # mirror reports wrong mirror patterns of bytes/strings usage
+    - misspell # misspess finds commonly misspelled English words in comments
+    - nilerr # finds the code that returns nil even if it checks that the error is not nil
+    - nilnil # checks that there is no simultaneous return of nil error and an invalid value
+    - predeclared # predeclared finds code that shadows one of Go's predeclared identifiers
+    - revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint.
+    - sqlclosecheck # checks that sql.Rows and sql.Stmt are closed
+    - thelper # thelper detects Go test helpers without t.Helper() call and checks the consistency of test helpers.
+    - wastedassign # wastedassign finds wasted assignment statements
+issues:
+  # Maximum count of issues with the same text.
+  # Set to 0 to disable.
+  # Default: 3
+  max-same-issues: 5
+
+  exclude-rules:
+    # allow fmt
+    - path: management/cmd/root\.go
+      linters: forbidigo
+    - path: signal/cmd/root\.go
+      linters: forbidigo
+    - path: sharedsock/filter\.go
+      linters:
+      - unused
+    - path: client/firewall/iptables/rule\.go
+      linters:
+      - unused
+    - path: test\.go
+      linters:
+      - mirror
+      - gosec
+    - path: mock\.go
+      linters:
+      - nilnil
+    # Exclude specific deprecation warnings for grpc methods
+    - linters:
+       - staticcheck
+      text: "grpc.DialContext is deprecated"
+    - linters:
+        - staticcheck
+      text: "grpc.WithBlock is deprecated"

.goreleaser.yaml

@@ -1,3 +1,5 @@
+version: 2
+
 project_name: netbird
 builds:
   - id: netbird
@@ -12,11 +14,7 @@ builds:
       - arm
       - amd64
       - arm64
-      - mips
       - 386
-    gomips:
-      - hardfloat
-      - softfloat
     ignore:
       - goos: windows
         goarch: arm64
@@ -26,19 +24,39 @@ builds:
         goarch: 386
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
+    tags:
+      - load_wgnt_from_rsrc
+
+  - id: netbird-static
+    dir: client
+    binary: netbird
+    env: [CGO_ENABLED=0]
+    goos:
+      - linux
+    goarch:
+      - mips
+      - mipsle
+      - mips64
+      - mips64le
+    gomips:
+      - hardfloat
+      - softfloat
+    ldflags:
+      - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
+    mod_timestamp: "{{ .CommitTimestamp }}"
     tags:
       - load_wgnt_from_rsrc
 
   - id: netbird-mgmt
     dir: management
     env:
-    - CGO_ENABLED=1
+      - CGO_ENABLED=1
-    - >-
+      - >-
-      {{- if eq .Runtime.Goos "linux" }}
+        {{- if eq .Runtime.Goos "linux" }}
-        {{- if eq .Arch "arm64"}}CC=aarch64-linux-gnu-gcc{{- end }}
+          {{- if eq .Arch "arm64"}}CC=aarch64-linux-gnu-gcc{{- end }}
-        {{- if eq .Arch "arm"}}CC=arm-linux-gnueabihf-gcc{{- end }}
+          {{- if eq .Arch "arm"}}CC=arm-linux-gnueabihf-gcc{{- end }}
-      {{- end }}
+        {{- end }}
     binary: netbird-mgmt
     goos:
       - linux
@@ -48,7 +66,7 @@ builds:
       - arm
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
 
   - id: netbird-signal
     dir: signal
@@ -62,14 +80,31 @@ builds:
       - arm
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
+
+  - id: netbird-relay
+    dir: relay
+    env: [CGO_ENABLED=0]
+    binary: netbird-relay
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+      - arm
+    ldflags:
+      - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
+    mod_timestamp: "{{ .CommitTimestamp }}"
+
+universal_binaries:
+  - id: netbird
 
 archives:
   - builds:
       - netbird
+      - netbird-static
 
 nfpms:
-
   - maintainer: Netbird <dev@netbird.io>
     description: Netbird client.
     homepage: https://netbird.io/
@@ -144,6 +179,52 @@ dockers:
       - "--label=org.opencontainers.image.revision={{.FullCommit}}"
       - "--label=org.opencontainers.image.version={{.Version}}"
       - "--label=maintainer=dev@netbird.io"
+  - image_templates:
+      - netbirdio/relay:{{ .Version }}-amd64
+    ids:
+      - netbird-relay
+    goarch: amd64
+    use: buildx
+    dockerfile: relay/Dockerfile
+    build_flag_templates:
+      - "--platform=linux/amd64"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=maintainer=dev@netbird.io"
+  - image_templates:
+      - netbirdio/relay:{{ .Version }}-arm64v8
+    ids:
+      - netbird-relay
+    goarch: arm64
+    use: buildx
+    dockerfile: relay/Dockerfile
+    build_flag_templates:
+      - "--platform=linux/arm64"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=maintainer=dev@netbird.io"
+  - image_templates:
+      - netbirdio/relay:{{ .Version }}-arm
+    ids:
+      - netbird-relay
+    goarch: arm
+    goarm: 6
+    use: buildx
+    dockerfile: relay/Dockerfile
+    build_flag_templates:
+      - "--platform=linux/arm"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=maintainer=dev@netbird.io"
   - image_templates:
       - netbirdio/signal:{{ .Version }}-amd64
     ids:
@@ -296,6 +377,18 @@ docker_manifests:
       - netbirdio/netbird:{{ .Version }}-arm
       - netbirdio/netbird:{{ .Version }}-amd64
 
+  - name_template: netbirdio/relay:{{ .Version }}
+    image_templates:
+      - netbirdio/relay:{{ .Version }}-arm64v8
+      - netbirdio/relay:{{ .Version }}-arm
+      - netbirdio/relay:{{ .Version }}-amd64
+
+  - name_template: netbirdio/relay:latest
+    image_templates:
+      - netbirdio/relay:{{ .Version }}-arm64v8
+      - netbirdio/relay:{{ .Version }}-arm
+      - netbirdio/relay:{{ .Version }}-amd64
+
   - name_template: netbirdio/signal:{{ .Version }}
     image_templates:
       - netbirdio/signal:{{ .Version }}-arm64v8
@@ -327,10 +420,9 @@ docker_manifests:
       - netbirdio/management:{{ .Version }}-debug-amd64
 
 brews:
-  -
+  - ids:
-    ids:
       - default
-    tap:
+    repository:
       owner: netbirdio
       name: homebrew-tap
       token: "{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}"
@@ -347,7 +439,7 @@ brews:
 uploads:
   - name: debian
     ids:
-    - netbird-deb
+      - netbird-deb
     mode: archive
     target: https://pkgs.wiretrustee.com/debian/pool/{{ .ArtifactName }};deb.distribution=stable;deb.component=main;deb.architecture={{ if .Arm }}armhf{{ else }}{{ .Arch }}{{ end }};deb.package=
     username: dev@wiretrustee.com
@@ -360,3 +452,13 @@ uploads:
     target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }}
     username: dev@wiretrustee.com
     method: PUT
+
+checksum:
+  extra_files:
+    - glob: ./infrastructure_files/getting-started-with-zitadel.sh
+    - glob: ./release_files/install.sh
+
+release:
+  extra_files:
+    - glob: ./infrastructure_files/getting-started-with-zitadel.sh
+    - glob: ./release_files/install.sh

.goreleaser_ui.yaml

@@ -1,3 +1,5 @@
+version: 2
+
 project_name: netbird-ui
 builds:
   - id: netbird-ui
@@ -11,7 +13,7 @@ builds:
       - amd64
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
 
   - id: netbird-ui-windows
     dir: client/ui
@@ -26,7 +28,7 @@ builds:
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
       - -H windowsgui
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
 
 archives:
   - id: linux-arch
@@ -39,7 +41,6 @@ archives:
       - netbird-ui-windows
 
 nfpms:
-
   - maintainer: Netbird <dev@netbird.io>
     description: Netbird client UI.
     homepage: https://netbird.io/
@@ -52,12 +53,9 @@ nfpms:
     contents:
       - src: client/ui/netbird.desktop
         dst: /usr/share/applications/netbird.desktop
-      - src: client/ui/disconnected.png
+      - src: client/ui/netbird-systemtray-connected.png
         dst: /usr/share/pixmaps/netbird.png
     dependencies:
-      - libayatana-appindicator3-1
-      - libgtk-3-dev
-      - libappindicator3-dev
       - netbird
 
   - maintainer: Netbird <dev@netbird.io>
@@ -72,18 +70,15 @@ nfpms:
     contents:
       - src: client/ui/netbird.desktop
         dst: /usr/share/applications/netbird.desktop
-      - src: client/ui/disconnected.png
+      - src: client/ui/netbird-systemtray-connected.png
         dst: /usr/share/pixmaps/netbird.png
     dependencies:
-      - libayatana-appindicator3-1
-      - libgtk-3-dev
-      - libappindicator3-dev
       - netbird
 
 uploads:
   - name: debian
     ids:
-    - netbird-ui-deb
+      - netbird-ui-deb
     mode: archive
     target: https://pkgs.wiretrustee.com/debian/pool/{{ .ArtifactName }};deb.distribution=stable;deb.component=main;deb.architecture={{ if .Arm }}armhf{{ else }}{{ .Arch }}{{ end }};deb.package=
     username: dev@wiretrustee.com

.goreleaser_ui_darwin.yaml

@@ -1,10 +1,14 @@
+version: 2
+
 project_name: netbird-ui
 builds:
   - id: netbird-ui-darwin
     dir: client/ui
     binary: netbird-ui
-    env: [CGO_ENABLED=1]
+    env:
-
+      - CGO_ENABLED=1
+      - MACOSX_DEPLOYMENT_TARGET=11.0
+      - MACOS_DEPLOYMENT_TARGET=11.0
     goos:
       - darwin
     goarch:
@@ -15,10 +19,13 @@ builds:
       - softfloat
     ldflags:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
-    mod_timestamp: '{{ .CommitTimestamp }}'
+    mod_timestamp: "{{ .CommitTimestamp }}"
     tags:
       - load_wgnt_from_rsrc
 
+universal_binaries:
+  - id: netbird-ui-darwin
+
 archives:
   - builds:
       - netbird-ui-darwin
@@ -26,4 +33,4 @@ archives:
 checksum:
   name_template: "{{ .ProjectName }}_darwin_checksums.txt"
 changelog:
-  skip: true
+  disable: true

CODE_OF_CONDUCT.md

@@ -5,7 +5,7 @@
 We as members, contributors, and leaders pledge to make participation in our
 community a harassment-free experience for everyone, regardless of age, body
 size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
+identity and expression, level of experience, education, socioeconomic status,
 nationality, personal appearance, race, caste, color, religion, or sexual
 identity and orientation.
 

CONTRIBUTING.md

@@ -19,11 +19,11 @@ If you haven't already, join our slack workspace [here](https://join.slack.com/t
     - [Development setup](#development-setup)
         - [Requirements](#requirements)
         - [Local NetBird setup](#local-netbird-setup)
+        - [Dev Container Support](#dev-container-support)
         - [Build and start](#build-and-start)
         - [Test suite](#test-suite)
     - [Checklist before submitting a PR](#checklist-before-submitting-a-pr)
     - [Other project repositories](#other-project-repositories)
-    - [Checklist before submitting a new node](#checklist-before-submitting-a-new-node)
     - [Contributor License Agreement](#contributor-license-agreement)
 
 ## Code of conduct
@@ -70,7 +70,7 @@ dependencies are installed. Here is a short guide on how that can be done.
 
 ### Requirements
 
-#### Go 1.19
+#### Go 1.21
 
 Follow the installation guide from https://go.dev/
 
@@ -96,7 +96,7 @@ They can be executed from the repository root before every push or PR:
 
 **Goreleaser**
 ```shell
-goreleaser --snapshot --rm-dist
+goreleaser build --snapshot --clean
 ```
 **golangci-lint**
 ```shell
@@ -136,18 +136,61 @@ checked out and set up:
    go mod tidy
    ```
 
+### Dev Container Support
+
+If you prefer using a dev container for development, NetBird now includes support for dev containers. 
+Dev containers provide a consistent and isolated development environment, making it easier for contributors to get started quickly. Follow the steps below to set up NetBird in a dev container.
+
+#### 1. Prerequisites:
+
+* Install Docker on your machine: [Docker Installation Guide](https://docs.docker.com/get-docker/)
+* Install Visual Studio Code: [VS Code Installation Guide](https://code.visualstudio.com/download)
+* If you prefer JetBrains Goland please follow this [manual](https://www.jetbrains.com/help/go/connect-to-devcontainer.html)
+
+#### 2. Clone the Repository:
+
+Clone the repository following previous [Local NetBird setup](#local-netbird-setup).
+
+#### 3. Open in project in IDE of your choice:
+
+**VScode**:
+
+Open the project folder in Visual Studio Code:
+
+```bash
+code .
+```
+
+When you open the project in VS Code, it will detect the presence of a dev container configuration.
+Click on the green "Reopen in Container" button in the bottom-right corner of VS Code.
+
+**Goland**:
+
+Open GoLand and select `"File" > "Open"` to open the NetBird project folder.
+GoLand will detect the dev container configuration and prompt you to open the project in the container. Accept the prompt.
+
+#### 4. Wait for the Container to Build:
+
+VsCode or GoLand will use the specified Docker image to build the dev container. This might take some time, depending on your internet connection.
+
+#### 6. Development:
+
+Once the container is built, you can start developing within the dev container. All the necessary dependencies and configurations are set up within the container.
+
+
 ### Build and start
 #### Client
 
-> Windows clients have a Wireguard driver requirement. We provide a bash script that can be executed in WLS 2 with docker support [wireguard_nt.sh](/client/wireguard_nt.sh).
-
 To start NetBird, execute:
 ```
 cd client
-# bash wireguard_nt.sh # if windows
+CGO_ENABLED=0 go build .
-go build .
 ```
 
+> Windows clients have a Wireguard driver requirement. You can download the wintun driver from https://www.wintun.net/builds/wintun-0.14.1.zip, after decompressing, you can copy the file `windtun\bin\ARCH\wintun.dll` to the same path as your binary file or to `C:\Windows\System32\wintun.dll`.
+
+> To test the client GUI application on Windows machines with RDP or vituralized environments (e.g. virtualbox or cloud), you need to download and extract the opengl32.dll from https://fdossena.com/?p=mesa/index.frag next to the built application.
+
 To start NetBird the client in the foreground:
 
 ```
@@ -185,6 +228,42 @@ To start NetBird the management service:
 ./management management --log-level debug --log-file console --config ./management.json
 ```
 
+#### Windows Netbird Installer
+Create dist directory
+```shell
+mkdir -p dist/netbird_windows_amd64
+```
+
+UI client
+```shell
+CC=x86_64-w64-mingw32-gcc CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build -o netbird-ui.exe -ldflags "-s -w -H windowsgui" ./client/ui
+mv netbird-ui.exe ./dist/netbird_windows_amd64/
+```
+
+Client
+```shell
+CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o netbird.exe ./client/
+mv netbird.exe ./dist/netbird_windows_amd64/
+```
+> Windows clients have a Wireguard driver requirement. You can download the wintun driver from https://www.wintun.net/builds/wintun-0.14.1.zip, after decompressing, you can copy the file `windtun\bin\ARCH\wintun.dll` to `./dist/netbird_windows_amd64/`.
+
+NSIS compiler
+- [Windows-nsis]( https://nsis.sourceforge.io/Download)
+- [MacOS-makensis](https://formulae.brew.sh/formula/makensis#default)
+- [Linux-makensis](https://manpages.ubuntu.com/manpages/trusty/man1/makensis.1.html)
+
+NSIS Plugins. Download and move them to the NSIS plugins folder.
+- [EnVar](https://nsis.sourceforge.io/mediawiki/images/7/7f/EnVar_plugin.zip)
+- [ShellExecAsUser](https://nsis.sourceforge.io/mediawiki/images/6/68/ShellExecAsUser_amd64-Unicode.7z)
+
+Windows Installer
+```shell
+export APPVER=0.0.0.1
+makensis -V4 client/installer.nsis
+```
+
+The installer `netbird-installer.exe` will be created in root directory.
+
 ### Test suite
 
 The tests can be started via:
@@ -195,6 +274,8 @@ go test -exec sudo ./...
 ```
 > On Windows use a powershell with administrator privileges
 
+> Non-GTK environments will need the `libayatana-appindicator3-dev` (debian/ubuntu) package installed
+
 ## Checklist before submitting a PR
 As a critical network service and open-source project, we must enforce a few things before submitting the pull-requests:
 - Keep functions as simple as possible, with a single purpose

README.md

@@ -1,6 +1,6 @@
 <p align="center">
- <strong>:hatching_chick: New Release! Peer expiration.</strong>
+ <strong>:hatching_chick: New Release! Device Posture Checks.</strong>
-  <a href="https://github.com/netbirdio/netbird/releases">
+  <a href="https://docs.netbird.io/how-to/manage-posture-checks">
        Learn more
      </a>   
 </p>
@@ -10,25 +10,31 @@
   <img width="234" src="docs/media/logo-full.png"/>
 </p>
   <p>
+   <a href="https://img.shields.io/badge/license-BSD--3-blue)">
+       <img src="https://sonarcloud.io/api/project_badges/measure?project=netbirdio_netbird&metric=alert_status" />
+     </a> 
      <a href="https://github.com/netbirdio/netbird/blob/main/LICENSE">
        <img src="https://img.shields.io/badge/license-BSD--3-blue" />
      </a> 
-   <a href="https://www.codacy.com/gh/netbirdio/netbird/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=netbirdio/netbird&amp;utm_campaign=Badge_Grade"><img src="https://app.codacy.com/project/badge/Grade/e3013d046aec44cdb7462c8673b00976"/></a>
     <br>
-    <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-vrahf41g-ik1v7fV8du6t0RwxSrJ96A">
+    <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-2utg2ncdz-W7LEB6toRBLE1Jca37dYpg">
         <img src="https://img.shields.io/badge/slack-@netbird-red.svg?logo=slack"/>
      </a>  
+     <br>
+    <a href="https://gurubase.io/g/netbird">
+        <img src="https://img.shields.io/badge/Gurubase-Ask%20NetBird%20Guru-006BFF"/>
+     </a>    
   </p>
 </div>
 
 
 <p align="center">
 <strong>
-  Start using NetBird at <a href="https://app.netbird.io/">app.netbird.io</a>
+  Start using NetBird at <a href="https://netbird.io/pricing">netbird.io</a>
   <br/>
   See <a href="https://netbird.io/docs/">Documentation</a>
   <br/>
-   Join our <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-vrahf41g-ik1v7fV8du6t0RwxSrJ96A">Slack channel</a>
+   Join our <a href="https://join.slack.com/t/netbirdio/shared_invite/zt-2utg2ncdz-W7LEB6toRBLE1Jca37dYpg">Slack channel</a>
   <br/>
  
 </strong>
@@ -36,69 +42,86 @@
 
 <br>
 
-**NetBird is an open-source VPN management platform built on top of WireGuard® making it easy to create secure private networks for your organization or home.**
+**NetBird combines a configuration-free peer-to-peer private network and a centralized access control system in a single platform, making it easy to create secure private networks for your organization or home.**
 
-It requires zero configuration effort leaving behind the hassle of opening ports, complex firewall rules, VPN gateways, and so forth.
+**Connect.** NetBird creates a WireGuard-based overlay network that automatically connects your machines over an encrypted tunnel, leaving behind the hassle of opening ports, complex firewall rules, VPN gateways, and so forth.
 
-NetBird uses [NAT traversal techniques](https://en.wikipedia.org/wiki/Interactive_Connectivity_Establishment) to automatically create an overlay peer-to-peer network connecting machines regardless of location (home, office, data center, container, cloud, or edge environments), unifying virtual private network management experience.
+**Secure.** NetBird enables secure remote access by applying granular access policies while allowing you to manage them intuitively from a single place. Works universally on any infrastructure.
 
-**Key features:**
+### Open-Source Network Security in a Single Platform
-- \[x] Automatic IP allocation and network management with a Web UI ([separate repo](https://github.com/netbirdio/dashboard))
-- \[x] Automatic WireGuard peer (machine) discovery and configuration.
-- \[x] Encrypted peer-to-peer connections without a central VPN gateway.
-- \[x] Connection relay fallback in case a peer-to-peer connection is not possible.
-- \[x] Desktop client applications for Linux, MacOS, and Windows (systray).
-- \[x] Multiuser support - sharing network between multiple users.
-- \[x] SSO and MFA support. 
-- \[x] Multicloud and hybrid-cloud support.
-- \[x] Kernel WireGuard usage when possible.
-- \[x] Access Controls - groups & rules.
-- \[x] Remote SSH access without managing SSH keys.
-- \[x] Network Routes.  
-- \[x] Private DNS.
-- \[x] Network Activity Monitoring.
 
-**Coming soon:**
--  \[ ] Mobile clients.
 
-### Secure peer-to-peer VPN with SSO and MFA in minutes
+![netbird_2](https://github.com/netbirdio/netbird/assets/700848/46bc3b73-508d-4a0e-bb9a-f465d68646ab)
 
-https://user-images.githubusercontent.com/700848/197345890-2e2cded5-7b7a-436f-a444-94e80dd24f46.mov
+### NetBird on Lawrence Systems (Video)
+[![Watch the video](https://img.youtube.com/vi/Kwrff6h0rEw/0.jpg)](https://www.youtube.com/watch?v=Kwrff6h0rEw)
 
-**Note**: The `main` branch may be in an *unstable or even broken state* during development. 
+### Key features
-For stable versions, see [releases](https://github.com/netbirdio/netbird/releases).
 
-### Start using NetBird
+| Connectivity                                                                                                                 | Management                                                                                               | Security                                                                                                                              | Automation                                                                                                                               | Platforms                                                                               |
--  Hosted version: [https://app.netbird.io/](https://app.netbird.io/).
+|------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------|
--  See our documentation for [Quickstart Guide](https://netbird.io/docs/getting-started/quickstart).
+| <ul><li> - \[x] Kernel WireGuard </ul></li>                                                                                  | <ul><li> - \[x] [Admin Web UI](https://github.com/netbirdio/dashboard) </ul></li>                        | <ul><li> - \[x] [SSO & MFA support](https://docs.netbird.io/how-to/installation#running-net-bird-with-sso-login) </ul></li>           | <ul><li> - \[x] [Public API](https://docs.netbird.io/api) </ul></li>                                                                     | <ul><li> - \[x] Linux </ul></li>                                                        |
--  If you are looking to self-host NetBird, check our [Self-Hosting Guide](https://netbird.io/docs/getting-started/self-hosting).
+| <ul><li> - \[x] Peer-to-peer connections </ul></li>                                                                          | <ul><li> - \[x] Auto peer discovery and configuration </ul></li>                                         | <ul><li> - \[x] [Access control - groups & rules](https://docs.netbird.io/how-to/manage-network-access) </ul></li>                    | <ul><li> - \[x] [Setup keys for bulk network provisioning](https://docs.netbird.io/how-to/register-machines-using-setup-keys) </ul></li> | <ul><li> - \[x] Mac </ul></li>                                                          |
--  Step-by-step [Installation Guide](https://netbird.io/docs/getting-started/installation) for different platforms.
+| <ul><li> - \[x] Connection relay fallback </ul></li>                                                                         | <ul><li> - \[x] [IdP integrations](https://docs.netbird.io/selfhosted/identity-providers) </ul></li>     | <ul><li> - \[x] [Activity logging](https://docs.netbird.io/how-to/monitor-system-and-network-activity) </ul></li>                     | <ul><li> - \[x] [Self-hosting quickstart script](https://docs.netbird.io/selfhosted/selfhosted-quickstart) </ul></li>                    | <ul><li> - \[x] Windows </ul></li>                                                      |
--  Web UI [repository](https://github.com/netbirdio/dashboard).
+| <ul><li> - \[x] [Routes to external networks](https://docs.netbird.io/how-to/routing-traffic-to-private-networks) </ul></li> | <ul><li> - \[x] [Private DNS](https://docs.netbird.io/how-to/manage-dns-in-your-network) </ul></li>      | <ul><li> - \[x] [Device posture checks](https://docs.netbird.io/how-to/manage-posture-checks) </ul></li>                              | <ul><li> - \[x] IdP groups sync with JWT </ul></li>                                                                                      | <ul><li> - \[x] Android </ul></li>                                                      |
--  5 min [demo video](https://youtu.be/Tu9tPsUWaY0) on YouTube.
+| <ul><li> - \[x] NAT traversal with BPF </ul></li>                                                                            | <ul><li> - \[x] [Multiuser support](https://docs.netbird.io/how-to/add-users-to-your-network) </ul></li> | <ul><li> -  \[x] Peer-to-peer encryption </ul></li>                                                                                   |                                                                                                                                          | <ul><li> - \[x] iOS </ul></li>                                                          |
+|                                                                                                                              |                                                                                                          | <ul><li> - \[x] [Quantum-resistance with Rosenpass](https://netbird.io/knowledge-hub/the-first-quantum-resistant-mesh-vpn) </ul></li> |                                                                                                                                          | <ul><li> - \[x] OpenWRT </ul></li>                                                      |
+|                                                                                                                              |                                                                                                          | <ui><li> - \[x] [Periodic re-authentication](https://docs.netbird.io/how-to/enforce-periodic-user-authentication)</ul></li>           |                                                                                                                                          | <ul><li> - \[x] [Serverless](https://docs.netbird.io/how-to/netbird-on-faas) </ul></li> |
+|                                                                                                                              |                                                                                                          |                                                                                                                                       |                                                                                                                                          | <ul><li> - \[x] Docker </ul></li>                                                       |
 
+### Quickstart with NetBird Cloud
+
+- Download and install NetBird at [https://app.netbird.io/install](https://app.netbird.io/install)
+- Follow the steps to sign-up with Google, Microsoft, GitHub or your email address.
+- Check NetBird [admin UI](https://app.netbird.io/).
+- Add more machines.
+
+### Quickstart with self-hosted NetBird
+
+> This is the quickest way to try self-hosted NetBird. It should take around 5 minutes to get started if you already have a public domain and a VM.
+Follow the [Advanced guide with a custom identity provider](https://docs.netbird.io/selfhosted/selfhosted-guide#advanced-guide-with-a-custom-identity-provider) for installations with different IDPs.
+
+**Infrastructure requirements:**
+- A Linux VM with at least **1CPU** and **2GB** of memory.
+- The VM should be publicly accessible on TCP ports **80** and **443** and UDP ports: **3478**, **49152-65535**.
+- **Public domain** name pointing to the VM.
+
+**Software requirements:**
+- Docker installed on the VM with the docker-compose plugin ([Docker installation guide](https://docs.docker.com/engine/install/)) or docker with docker-compose in version 2 or higher.
+- [jq](https://jqlang.github.io/jq/) installed. In most distributions
+  Usually available in the official repositories and can be installed with `sudo apt install jq` or `sudo yum install jq`
+- [curl](https://curl.se/) installed.
+  Usually available in the official repositories and can be installed with `sudo apt install curl` or `sudo yum install curl`
+
+**Steps**
+- Download and run the installation script:
+```bash
+export NETBIRD_DOMAIN=netbird.example.com; curl -fsSL https://github.com/netbirdio/netbird/releases/latest/download/getting-started-with-zitadel.sh | bash
+```
+- Once finished, you can manage the resources via `docker-compose`
 
 ### A bit on NetBird internals
 -  Every machine in the network runs [NetBird Agent (or Client)](client/) that manages WireGuard.
 -  Every agent connects to [Management Service](management/) that holds network state, manages peer IPs, and distributes network updates to agents (peers).
 -  NetBird agent uses WebRTC ICE implemented in [pion/ice library](https://github.com/pion/ice) to discover connection candidates when establishing a peer-to-peer connection between machines.
--  Connection candidates are discovered with a help of [STUN](https://en.wikipedia.org/wiki/STUN) servers.
+-  Connection candidates are discovered with the help of [STUN](https://en.wikipedia.org/wiki/STUN) servers.
 -  Agents negotiate a connection through [Signal Service](signal/) passing p2p encrypted messages with candidates.
--  Sometimes the NAT traversal is unsuccessful due to strict NATs (e.g. mobile carrier-grade NAT) and p2p connection isn't possible. When this occurs the system falls back to a relay server called [TURN](https://en.wikipedia.org/wiki/Traversal_Using_Relays_around_NAT), and a secure WireGuard tunnel is established via the TURN server. 
+-  Sometimes the NAT traversal is unsuccessful due to strict NATs (e.g. mobile carrier-grade NAT) and a p2p connection isn't possible. When this occurs the system falls back to a relay server called [TURN](https://en.wikipedia.org/wiki/Traversal_Using_Relays_around_NAT), and a secure WireGuard tunnel is established via the TURN server. 
  
 [Coturn](https://github.com/coturn/coturn) is the one that has been successfully used for STUN and TURN in NetBird setups.
 
 <p float="left" align="middle">
-  <img src="https://netbird.io/docs/img/architecture/high-level-dia.png" width="700"/>
+  <img src="https://docs.netbird.io/docs-static/img/architecture/high-level-dia.png" width="700"/>
 </p>
 
-See a complete [architecture overview](https://netbird.io/docs/overview/architecture) for details.
+See a complete [architecture overview](https://docs.netbird.io/about-netbird/how-netbird-works#architecture) for details.
-
-### Roadmap
--  [Public Roadmap](https://github.com/netbirdio/netbird/projects/2)
 
 ### Community projects
--  [NetBird on OpenWRT](https://github.com/messense/openwrt-netbird)
 -  [NetBird installer script](https://github.com/physk/netbird-installer)
+-  [NetBird ansible collection by Dominion Solutions](https://galaxy.ansible.com/ui/repo/published/dominion_solutions/netbird/)
+
+**Note**: The `main` branch may be in an *unstable or even broken state* during development.
+For stable versions, see [releases](https://github.com/netbirdio/netbird/releases).
 
 ### Support acknowledgement
 
@@ -107,7 +130,7 @@ In November 2022, NetBird joined the [StartUpSecure program](https://www.forschu
 ![CISPA_Logo_BLACK_EN_RZ_RGB (1)](https://user-images.githubusercontent.com/700848/203091324-c6d311a0-22b5-4b05-a288-91cbc6cdcc46.png)
 
 ### Testimonials
-We use open-source technologies like [WireGuard®](https://www.wireguard.com/), [Pion ICE (WebRTC)](https://github.com/pion/ice), and [Coturn](https://github.com/coturn/coturn). We very much appreciate the work these guys are doing and we'd greatly appreciate if you could support them in any way (e.g. giving a star or a contribution).
+We use open-source technologies like [WireGuard®](https://www.wireguard.com/), [Pion ICE (WebRTC)](https://github.com/pion/ice), [Coturn](https://github.com/coturn/coturn), and [Rosenpass](https://rosenpass.eu). We very much appreciate the work these guys are doing and we'd greatly appreciate if you could support them in any way (e.g., by giving a star or a contribution).
 
 ### Legal
  _WireGuard_ and the _WireGuard_ logo are [registered trademarks](https://www.wireguard.com/trademark-policy/) of Jason A. Donenfeld.

base62/base62.go

@@ -0,0 +1,58 @@
+package base62
+
+import (
+	"fmt"
+	"math"
+	"strings"
+)
+
+const (
+	alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+	base     = uint32(len(alphabet))
+)
+
+// Encode encodes a uint32 value to a base62 string.
+func Encode(num uint32) string {
+	if num == 0 {
+		return string(alphabet[0])
+	}
+
+	var encoded strings.Builder
+
+	for num > 0 {
+		remainder := num % base
+		encoded.WriteByte(alphabet[remainder])
+		num /= base
+	}
+
+	// Reverse the encoded string
+	encodedString := encoded.String()
+	reversed := reverse(encodedString)
+	return reversed
+}
+
+// Decode decodes a base62 string to a uint32 value.
+func Decode(encoded string) (uint32, error) {
+	var decoded uint32
+	strLen := len(encoded)
+
+	for i, char := range encoded {
+		index := strings.IndexRune(alphabet, char)
+		if index < 0 {
+			return 0, fmt.Errorf("invalid character: %c", char)
+		}
+
+		decoded += uint32(index) * uint32(math.Pow(float64(base), float64(strLen-i-1)))
+	}
+
+	return decoded, nil
+}
+
+// Reverse a string.
+func reverse(s string) string {
+	runes := []rune(s)
+	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
+		runes[i], runes[j] = runes[j], runes[i]
+	}
+	return string(runes)
+}

base62/base62_test.go

@@ -0,0 +1,31 @@
+package base62
+
+import (
+	"testing"
+)
+
+func TestEncodeDecode(t *testing.T) {
+	tests := []struct {
+		num uint32
+	}{
+		{0},
+		{1},
+		{42},
+		{12345},
+		{99999},
+		{123456789},
+	}
+
+	for _, tt := range tests {
+		encoded := Encode(tt.num)
+		decoded, err := Decode(encoded)
+
+		if err != nil {
+			t.Errorf("Decode error: %v", err)
+		}
+
+		if decoded != tt.num {
+			t.Errorf("Decode(%v) = %v, want %v", encoded, decoded, tt.num)
+		}
+	}
+}

client/Dockerfile

@@ -1,7 +1,5 @@
-FROM gcr.io/distroless/base:debug
+FROM alpine:3.20
+RUN apk add --no-cache ca-certificates iptables ip6tables
 ENV NB_FOREGROUND_MODE=true
-ENV PATH=/sbin:/usr/sbin:/bin:/usr/bin:/busybox
+ENTRYPOINT [ "/usr/local/bin/netbird","up"]
-SHELL ["/busybox/sh","-c"]
+COPY netbird /usr/local/bin/netbird
-RUN sed -i -E 's/(^root:.+)\/sbin\/nologin/\1\/busybox\/sh/g' /etc/passwd
-ENTRYPOINT [ "/go/bin/netbird","up"]
-COPY netbird /go/bin/netbird

client/android/client.go

@@ -1,3 +1,5 @@
+//go:build android
+
 package android
 
 import (
@@ -6,12 +8,15 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"github.com/netbirdio/netbird/client/iface/device"
 	"github.com/netbirdio/netbird/client/internal"
+	"github.com/netbirdio/netbird/client/internal/dns"
+	"github.com/netbirdio/netbird/client/internal/listener"
 	"github.com/netbirdio/netbird/client/internal/peer"
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 	"github.com/netbirdio/netbird/client/system"
 	"github.com/netbirdio/netbird/formatter"
-	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/util/net"
 )
 
 // ConnectionListener export internal Listener for mobile
@@ -21,12 +26,22 @@ type ConnectionListener interface {
 
 // TunAdapter export internal TunAdapter for mobile
 type TunAdapter interface {
-	iface.TunAdapter
+	device.TunAdapter
 }
 
 // IFaceDiscover export internal IFaceDiscover for mobile
 type IFaceDiscover interface {
-	stdnet.IFaceDiscover
+	stdnet.ExternalIFaceDiscover
+}
+
+// NetworkChangeListener export internal NetworkChangeListener for mobile
+type NetworkChangeListener interface {
+	listener.NetworkChangeListener
+}
+
+// DnsReadyListener export internal dns ReadyListener for mobile
+type DnsReadyListener interface {
+	dns.ReadyListener
 }
 
 func init() {
@@ -35,32 +50,34 @@ func init() {
 
 // Client struct manage the life circle of background service
 type Client struct {
-	cfgFile       string
+	cfgFile               string
-	tunAdapter    iface.TunAdapter
+	tunAdapter            device.TunAdapter
-	iFaceDiscover IFaceDiscover
+	iFaceDiscover         IFaceDiscover
-	recorder      *peer.Status
+	recorder              *peer.Status
-	ctxCancel     context.CancelFunc
+	ctxCancel             context.CancelFunc
-	ctxCancelLock *sync.Mutex
+	ctxCancelLock         *sync.Mutex
-	deviceName    string
+	deviceName            string
+	uiVersion             string
+	networkChangeListener listener.NetworkChangeListener
 }
 
 // NewClient instantiate a new Client
-func NewClient(cfgFile, deviceName string, tunAdapter TunAdapter, iFaceDiscover IFaceDiscover) *Client {
+func NewClient(cfgFile, deviceName string, uiVersion string, tunAdapter TunAdapter, iFaceDiscover IFaceDiscover, networkChangeListener NetworkChangeListener) *Client {
-	lvl, _ := log.ParseLevel("trace")
+	net.SetAndroidProtectSocketFn(tunAdapter.ProtectSocket)
-	log.SetLevel(lvl)
-
 	return &Client{
-		cfgFile:       cfgFile,
+		cfgFile:               cfgFile,
-		deviceName:    deviceName,
+		deviceName:            deviceName,
-		tunAdapter:    tunAdapter,
+		uiVersion:             uiVersion,
-		iFaceDiscover: iFaceDiscover,
+		tunAdapter:            tunAdapter,
-		recorder:      peer.NewRecorder(""),
+		iFaceDiscover:         iFaceDiscover,
-		ctxCancelLock: &sync.Mutex{},
+		recorder:              peer.NewRecorder(""),
+		ctxCancelLock:         &sync.Mutex{},
+		networkChangeListener: networkChangeListener,
 	}
 }
 
 // Run start the internal client. It is a blocker function
-func (c *Client) Run(urlOpener URLOpener) error {
+func (c *Client) Run(urlOpener URLOpener, dns *DNSList, dnsReadyListener DnsReadyListener) error {
 	cfg, err := internal.UpdateOrCreateConfig(internal.ConfigInput{
 		ConfigPath: c.cfgFile,
 	})
@@ -68,10 +85,14 @@ func (c *Client) Run(urlOpener URLOpener) error {
 		return err
 	}
 	c.recorder.UpdateManagementAddress(cfg.ManagementURL.String())
+	c.recorder.UpdateRosenpass(cfg.RosenpassEnabled, cfg.RosenpassPermissive)
 
 	var ctx context.Context
 	//nolint
 	ctxWithValues := context.WithValue(context.Background(), system.DeviceNameCtxKey, c.deviceName)
+	//nolint
+	ctxWithValues = context.WithValue(ctxWithValues, system.UiVersionCtxKey, c.uiVersion)
+
 	c.ctxCancelLock.Lock()
 	ctx, c.ctxCancel = context.WithCancel(ctxWithValues)
 	defer c.ctxCancel()
@@ -85,7 +106,34 @@ func (c *Client) Run(urlOpener URLOpener) error {
 
 	// todo do not throw error in case of cancelled context
 	ctx = internal.CtxInitState(ctx)
-	return internal.RunClient(ctx, cfg, c.recorder, c.tunAdapter, c.iFaceDiscover)
+	connectClient := internal.NewConnectClient(ctx, cfg, c.recorder)
+	return connectClient.RunOnAndroid(c.tunAdapter, c.iFaceDiscover, c.networkChangeListener, dns.items, dnsReadyListener)
+}
+
+// RunWithoutLogin we apply this type of run function when the backed has been started without UI (i.e. after reboot).
+// In this case make no sense handle registration steps.
+func (c *Client) RunWithoutLogin(dns *DNSList, dnsReadyListener DnsReadyListener) error {
+	cfg, err := internal.UpdateOrCreateConfig(internal.ConfigInput{
+		ConfigPath: c.cfgFile,
+	})
+	if err != nil {
+		return err
+	}
+	c.recorder.UpdateManagementAddress(cfg.ManagementURL.String())
+	c.recorder.UpdateRosenpass(cfg.RosenpassEnabled, cfg.RosenpassPermissive)
+
+	var ctx context.Context
+	//nolint
+	ctxWithValues := context.WithValue(context.Background(), system.DeviceNameCtxKey, c.deviceName)
+	c.ctxCancelLock.Lock()
+	ctx, c.ctxCancel = context.WithCancel(ctxWithValues)
+	defer c.ctxCancel()
+	c.ctxCancelLock.Unlock()
+
+	// todo do not throw error in case of cancelled context
+	ctx = internal.CtxInitState(ctx)
+	connectClient := internal.NewConnectClient(ctx, cfg, c.recorder)
+	return connectClient.RunOnAndroid(c.tunAdapter, c.iFaceDiscover, c.networkChangeListener, dns.items, dnsReadyListener)
 }
 
 // Stop the internal client and free the resources
@@ -99,6 +147,16 @@ func (c *Client) Stop() {
 	c.ctxCancel()
 }
 
+// SetTraceLogLevel configure the logger to trace level
+func (c *Client) SetTraceLogLevel() {
+	log.SetLevel(log.TraceLevel)
+}
+
+// SetInfoLogLevel configure the logger to info level
+func (c *Client) SetInfoLogLevel() {
+	log.SetLevel(log.InfoLevel)
+}
+
 // PeersList return with the list of the PeerInfos
 func (c *Client) PeersList() *PeerInfoArray {
 
@@ -110,14 +168,23 @@ func (c *Client) PeersList() *PeerInfoArray {
 			p.IP,
 			p.FQDN,
 			p.ConnStatus.String(),
-			p.Direct,
 		}
 		peerInfos[n] = pi
 	}
-
 	return &PeerInfoArray{items: peerInfos}
 }
 
+// OnUpdatedHostDNS update the DNS servers addresses for root zones
+func (c *Client) OnUpdatedHostDNS(list *DNSList) error {
+	dnsServer, err := dns.GetServerDns()
+	if err != nil {
+		return err
+	}
+
+	dnsServer.OnUpdatedHostDNSServer(list.items)
+	return nil
+}
+
 // SetConnectionListener set the network connection listener
 func (c *Client) SetConnectionListener(listener ConnectionListener) {
 	c.recorder.SetConnectionListener(listener)

client/android/dns_list.go

@@ -0,0 +1,26 @@
+package android
+
+import "fmt"
+
+// DNSList is a wrapper of []string
+type DNSList struct {
+	items []string
+}
+
+// Add new DNS address to the collection
+func (array *DNSList) Add(s string) {
+	array.items = append(array.items, s)
+}
+
+// Get return an element of the collection
+func (array *DNSList) Get(i int) (string, error) {
+	if i >= len(array.items) || i < 0 {
+		return "", fmt.Errorf("out of range")
+	}
+	return array.items[i], nil
+}
+
+// Size return with the size of the collection
+func (array *DNSList) Size() int {
+	return len(array.items)
+}

client/android/dns_list_test.go

@@ -0,0 +1,24 @@
+package android
+
+import "testing"
+
+func TestDNSList_Get(t *testing.T) {
+	l := DNSList{
+		items: make([]string, 1),
+	}
+
+	_, err := l.Get(0)
+	if err != nil {
+		t.Errorf("invalid error: %s", err)
+	}
+
+	_, err = l.Get(-1)
+	if err == nil {
+		t.Errorf("expected error but got nil")
+	}
+
+	_, err = l.Get(1)
+	if err == nil {
+		t.Errorf("expected error but got nil")
+	}
+}

client/android/gomobile.go

@@ -0,0 +1,5 @@
+package android
+
+import _ "golang.org/x/mobile/bind"
+
+// to keep our CI/CD that checks go.mod and go.sum files happy, we need to import the package above

client/android/login.go

@@ -6,15 +6,14 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff/v4"
-
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc/codes"
 	gstatus "google.golang.org/grpc/status"
 
 	"github.com/netbirdio/netbird/client/cmd"
-	"github.com/netbirdio/netbird/client/system"
-
 	"github.com/netbirdio/netbird/client/internal"
+	"github.com/netbirdio/netbird/client/internal/auth"
+	"github.com/netbirdio/netbird/client/system"
 )
 
 // SSOListener is async listener for mobile framework
@@ -85,11 +84,21 @@ func (a *Auth) SaveConfigIfSSOSupported(listener SSOListener) {
 func (a *Auth) saveConfigIfSSOSupported() (bool, error) {
 	supportsSSO := true
 	err := a.withBackOff(a.ctx, func() (err error) {
-		_, err = internal.GetDeviceAuthorizationFlowInfo(a.ctx, a.config.PrivateKey, a.config.ManagementURL)
+		_, err = internal.GetPKCEAuthorizationFlowInfo(a.ctx, a.config.PrivateKey, a.config.ManagementURL, nil)
-		if s, ok := gstatus.FromError(err); ok && s.Code() == codes.NotFound {
+		if s, ok := gstatus.FromError(err); ok && (s.Code() == codes.NotFound || s.Code() == codes.Unimplemented) {
-			supportsSSO = false
+			_, err = internal.GetDeviceAuthorizationFlowInfo(a.ctx, a.config.PrivateKey, a.config.ManagementURL)
-			err = nil
+			s, ok := gstatus.FromError(err)
+			if !ok {
+				return err
+			}
+			if s.Code() == codes.NotFound || s.Code() == codes.Unimplemented {
+				supportsSSO = false
+				err = nil
+			}
+
+			return err
 		}
+
 		return err
 	})
 
@@ -183,35 +192,23 @@ func (a *Auth) login(urlOpener URLOpener) error {
 	return nil
 }
 
-func (a *Auth) foregroundGetTokenInfo(urlOpener URLOpener) (*internal.TokenInfo, error) {
+func (a *Auth) foregroundGetTokenInfo(urlOpener URLOpener) (*auth.TokenInfo, error) {
-	providerConfig, err := internal.GetDeviceAuthorizationFlowInfo(a.ctx, a.config.PrivateKey, a.config.ManagementURL)
+	oAuthFlow, err := auth.NewOAuthFlow(a.ctx, a.config, false)
 	if err != nil {
-		s, ok := gstatus.FromError(err)
+		return nil, err
-		if ok && s.Code() == codes.NotFound {
-			return nil, fmt.Errorf("no SSO provider returned from management. " +
-				"If you are using hosting Netbird see documentation at " +
-				"https://github.com/netbirdio/netbird/tree/main/management for details")
-		} else if ok && s.Code() == codes.Unimplemented {
-			return nil, fmt.Errorf("the management server, %s, does not support SSO providers, "+
-				"please update your servver or use Setup Keys to login", a.config.ManagementURL)
-		} else {
-			return nil, fmt.Errorf("getting device authorization flow info failed with error: %v", err)
-		}
 	}
 
-	hostedClient := internal.NewHostedDeviceFlow(providerConfig.ProviderConfig)
+	flowInfo, err := oAuthFlow.RequestAuthInfo(context.TODO())
-
-	flowInfo, err := hostedClient.RequestDeviceCode(context.TODO())
 	if err != nil {
-		return nil, fmt.Errorf("getting a request device code failed: %v", err)
+		return nil, fmt.Errorf("getting a request OAuth flow info failed: %v", err)
 	}
 
 	go urlOpener.Open(flowInfo.VerificationURIComplete)
 
-	waitTimeout := time.Duration(flowInfo.ExpiresIn)
+	waitTimeout := time.Duration(flowInfo.ExpiresIn) * time.Second
-	waitCTX, cancel := context.WithTimeout(a.ctx, waitTimeout*time.Second)
+	waitCTX, cancel := context.WithTimeout(a.ctx, waitTimeout)
 	defer cancel()
-	tokenInfo, err := hostedClient.WaitToken(waitCTX, flowInfo)
+	tokenInfo, err := oAuthFlow.WaitToken(waitCTX, flowInfo)
 	if err != nil {
 		return nil, fmt.Errorf("waiting for browser login failed: %v", err)
 	}

client/android/peer_notifier.go

@@ -5,7 +5,6 @@ type PeerInfo struct {
 	IP         string
 	FQDN       string
 	ConnStatus string // Todo replace to enum
-	Direct     bool
 }
 
 // PeerInfoCollection made for Java layer to get non default types as collection

client/android/preferences_test.go

@@ -57,11 +57,11 @@ func TestPreferences_ReadUncommitedValues(t *testing.T) {
 	p.SetManagementURL(exampleString)
 	resp, err = p.GetManagementURL()
 	if err != nil {
-		t.Fatalf("failed to read managmenet url: %s", err)
+		t.Fatalf("failed to read management url: %s", err)
 	}
 
 	if resp != exampleString {
-		t.Errorf("unexpected managemenet url: %s", resp)
+		t.Errorf("unexpected management url: %s", resp)
 	}
 
 	p.SetPreSharedKey(exampleString)
@@ -102,11 +102,11 @@ func TestPreferences_Commit(t *testing.T) {
 
 	resp, err = p.GetManagementURL()
 	if err != nil {
-		t.Fatalf("failed to read managmenet url: %s", err)
+		t.Fatalf("failed to read management url: %s", err)
 	}
 
 	if resp != exampleURL {
-		t.Errorf("unexpected managemenet url: %s", resp)
+		t.Errorf("unexpected management url: %s", resp)
 	}
 
 	resp, err = p.GetPreSharedKey()

client/anonymize/anonymize.go

@@ -0,0 +1,241 @@
+package anonymize
+
+import (
+	"crypto/rand"
+	"fmt"
+	"math/big"
+	"net"
+	"net/netip"
+	"net/url"
+	"regexp"
+	"slices"
+	"strings"
+)
+
+const anonTLD = ".domain"
+
+type Anonymizer struct {
+	ipAnonymizer     map[netip.Addr]netip.Addr
+	domainAnonymizer map[string]string
+	currentAnonIPv4  netip.Addr
+	currentAnonIPv6  netip.Addr
+	startAnonIPv4    netip.Addr
+	startAnonIPv6    netip.Addr
+}
+
+func DefaultAddresses() (netip.Addr, netip.Addr) {
+	// 192.51.100.0, 100::
+	return netip.AddrFrom4([4]byte{198, 51, 100, 0}), netip.AddrFrom16([16]byte{0x01})
+}
+
+func NewAnonymizer(startIPv4, startIPv6 netip.Addr) *Anonymizer {
+	return &Anonymizer{
+		ipAnonymizer:     map[netip.Addr]netip.Addr{},
+		domainAnonymizer: map[string]string{},
+		currentAnonIPv4:  startIPv4,
+		currentAnonIPv6:  startIPv6,
+		startAnonIPv4:    startIPv4,
+		startAnonIPv6:    startIPv6,
+	}
+}
+
+func (a *Anonymizer) AnonymizeIP(ip netip.Addr) netip.Addr {
+	if ip.IsLoopback() ||
+		ip.IsLinkLocalUnicast() ||
+		ip.IsLinkLocalMulticast() ||
+		ip.IsInterfaceLocalMulticast() ||
+		ip.IsPrivate() ||
+		ip.IsUnspecified() ||
+		ip.IsMulticast() ||
+		isWellKnown(ip) ||
+		a.isInAnonymizedRange(ip) {
+
+		return ip
+	}
+
+	if _, ok := a.ipAnonymizer[ip]; !ok {
+		if ip.Is4() {
+			a.ipAnonymizer[ip] = a.currentAnonIPv4
+			a.currentAnonIPv4 = a.currentAnonIPv4.Next()
+		} else {
+			a.ipAnonymizer[ip] = a.currentAnonIPv6
+			a.currentAnonIPv6 = a.currentAnonIPv6.Next()
+		}
+	}
+	return a.ipAnonymizer[ip]
+}
+
+// isInAnonymizedRange checks if an IP is within the range of already assigned anonymized IPs
+func (a *Anonymizer) isInAnonymizedRange(ip netip.Addr) bool {
+	if ip.Is4() && ip.Compare(a.startAnonIPv4) >= 0 && ip.Compare(a.currentAnonIPv4) <= 0 {
+		return true
+	} else if !ip.Is4() && ip.Compare(a.startAnonIPv6) >= 0 && ip.Compare(a.currentAnonIPv6) <= 0 {
+		return true
+	}
+	return false
+}
+
+func (a *Anonymizer) AnonymizeIPString(ip string) string {
+	addr, err := netip.ParseAddr(ip)
+	if err != nil {
+		return ip
+	}
+
+	return a.AnonymizeIP(addr).String()
+}
+
+func (a *Anonymizer) AnonymizeDomain(domain string) string {
+	baseDomain := domain
+	hasDot := strings.HasSuffix(domain, ".")
+	if hasDot {
+		baseDomain = domain[:len(domain)-1]
+	}
+
+	if strings.HasSuffix(baseDomain, "netbird.io") ||
+		strings.HasSuffix(baseDomain, "netbird.selfhosted") ||
+		strings.HasSuffix(baseDomain, "netbird.cloud") ||
+		strings.HasSuffix(baseDomain, "netbird.stage") ||
+		strings.HasSuffix(baseDomain, anonTLD) {
+		return domain
+	}
+
+	parts := strings.Split(baseDomain, ".")
+	if len(parts) < 2 {
+		return domain
+	}
+
+	baseForLookup := parts[len(parts)-2] + "." + parts[len(parts)-1]
+
+	anonymized, ok := a.domainAnonymizer[baseForLookup]
+	if !ok {
+		anonymizedBase := "anon-" + generateRandomString(5) + anonTLD
+		a.domainAnonymizer[baseForLookup] = anonymizedBase
+		anonymized = anonymizedBase
+	}
+
+	result := strings.Replace(baseDomain, baseForLookup, anonymized, 1)
+	if hasDot {
+		result += "."
+	}
+	return result
+}
+
+func (a *Anonymizer) AnonymizeURI(uri string) string {
+	u, err := url.Parse(uri)
+	if err != nil {
+		return uri
+	}
+
+	var anonymizedHost string
+	if u.Opaque != "" {
+		host, port, err := net.SplitHostPort(u.Opaque)
+		if err == nil {
+			anonymizedHost = fmt.Sprintf("%s:%s", a.AnonymizeDomain(host), port)
+		} else {
+			anonymizedHost = a.AnonymizeDomain(u.Opaque)
+		}
+		u.Opaque = anonymizedHost
+	} else if u.Host != "" {
+		host, port, err := net.SplitHostPort(u.Host)
+		if err == nil {
+			anonymizedHost = fmt.Sprintf("%s:%s", a.AnonymizeDomain(host), port)
+		} else {
+			anonymizedHost = a.AnonymizeDomain(u.Host)
+		}
+		u.Host = anonymizedHost
+	}
+	return u.String()
+}
+
+func (a *Anonymizer) AnonymizeString(str string) string {
+	ipv4Regex := regexp.MustCompile(`\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b`)
+	ipv6Regex := regexp.MustCompile(`\b([0-9a-fA-F:]+:+[0-9a-fA-F]{0,4})(?:%[0-9a-zA-Z]+)?(?:\/[0-9]{1,3})?(?::[0-9]{1,5})?\b`)
+
+	str = ipv4Regex.ReplaceAllStringFunc(str, a.AnonymizeIPString)
+	str = ipv6Regex.ReplaceAllStringFunc(str, a.AnonymizeIPString)
+
+	for domain, anonDomain := range a.domainAnonymizer {
+		str = strings.ReplaceAll(str, domain, anonDomain)
+	}
+
+	str = a.AnonymizeSchemeURI(str)
+	str = a.AnonymizeDNSLogLine(str)
+
+	return str
+}
+
+// AnonymizeSchemeURI finds and anonymizes URIs with ws, wss, rel, rels, stun, stuns, turn, and turns schemes.
+func (a *Anonymizer) AnonymizeSchemeURI(text string) string {
+	re := regexp.MustCompile(`(?i)\b(wss?://|rels?://|stuns?:|turns?:|https?://)\S+\b`)
+
+	return re.ReplaceAllStringFunc(text, a.AnonymizeURI)
+}
+
+// AnonymizeDNSLogLine anonymizes domain names in DNS log entries by replacing them with a random string.
+func (a *Anonymizer) AnonymizeDNSLogLine(logEntry string) string {
+	domainPattern := `dns\.Question{Name:"([^"]+)",`
+	domainRegex := regexp.MustCompile(domainPattern)
+
+	return domainRegex.ReplaceAllStringFunc(logEntry, func(match string) string {
+		parts := strings.Split(match, `"`)
+		if len(parts) >= 2 {
+			domain := parts[1]
+			if strings.HasSuffix(domain, anonTLD) {
+				return match
+			}
+			randomDomain := generateRandomString(10) + anonTLD
+			return strings.Replace(match, domain, randomDomain, 1)
+		}
+		return match
+	})
+}
+
+// AnonymizeRoute anonymizes a route string by replacing IP addresses with anonymized versions and
+// domain names with random strings.
+func (a *Anonymizer) AnonymizeRoute(route string) string {
+	prefix, err := netip.ParsePrefix(route)
+	if err == nil {
+		ip := a.AnonymizeIPString(prefix.Addr().String())
+		return fmt.Sprintf("%s/%d", ip, prefix.Bits())
+	}
+	domains := strings.Split(route, ", ")
+	for i, domain := range domains {
+		domains[i] = a.AnonymizeDomain(domain)
+	}
+	return strings.Join(domains, ", ")
+}
+
+func isWellKnown(addr netip.Addr) bool {
+	wellKnown := []string{
+		"8.8.8.8", "8.8.4.4", // Google DNS IPv4
+		"2001:4860:4860::8888", "2001:4860:4860::8844", // Google DNS IPv6
+		"1.1.1.1", "1.0.0.1", // Cloudflare DNS IPv4
+		"2606:4700:4700::1111", "2606:4700:4700::1001", // Cloudflare DNS IPv6
+		"9.9.9.9", "149.112.112.112", // Quad9 DNS IPv4
+		"2620:fe::fe", "2620:fe::9", // Quad9 DNS IPv6
+
+		"128.0.0.0", "8000::", // 2nd split subnet for default routes
+	}
+
+	if slices.Contains(wellKnown, addr.String()) {
+		return true
+	}
+
+	cgnatRangeStart := netip.AddrFrom4([4]byte{100, 64, 0, 0})
+	cgnatRange := netip.PrefixFrom(cgnatRangeStart, 10)
+
+	return cgnatRange.Contains(addr)
+}
+
+func generateRandomString(length int) string {
+	const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+	result := make([]byte, length)
+	for i := range result {
+		num, err := rand.Int(rand.Reader, big.NewInt(int64(len(letters))))
+		if err != nil {
+			continue
+		}
+		result[i] = letters[num.Int64()]
+	}
+	return string(result)
+}

client/anonymize/anonymize_test.go

@@ -0,0 +1,249 @@
+package anonymize_test
+
+import (
+	"net/netip"
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/client/anonymize"
+)
+
+func TestAnonymizeIP(t *testing.T) {
+	startIPv4 := netip.MustParseAddr("198.51.100.0")
+	startIPv6 := netip.MustParseAddr("100::")
+	anonymizer := anonymize.NewAnonymizer(startIPv4, startIPv6)
+
+	tests := []struct {
+		name   string
+		ip     string
+		expect string
+	}{
+		{"Well known", "8.8.8.8", "8.8.8.8"},
+		{"First Public IPv4", "1.2.3.4", "198.51.100.0"},
+		{"Second Public IPv4", "4.3.2.1", "198.51.100.1"},
+		{"Repeated IPv4", "1.2.3.4", "198.51.100.0"},
+		{"Private IPv4", "192.168.1.1", "192.168.1.1"},
+		{"First Public IPv6", "2607:f8b0:4005:805::200e", "100::"},
+		{"Second Public IPv6", "a::b", "100::1"},
+		{"Repeated IPv6", "2607:f8b0:4005:805::200e", "100::"},
+		{"Private IPv6", "fe80::1", "fe80::1"},
+		{"In Range IPv4", "198.51.100.2", "198.51.100.2"},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ip := netip.MustParseAddr(tc.ip)
+			anonymizedIP := anonymizer.AnonymizeIP(ip)
+			if anonymizedIP.String() != tc.expect {
+				t.Errorf("%s: expected %s, got %s", tc.name, tc.expect, anonymizedIP)
+			}
+		})
+	}
+}
+
+func TestAnonymizeDNSLogLine(t *testing.T) {
+	anonymizer := anonymize.NewAnonymizer(netip.Addr{}, netip.Addr{})
+	testLog := `2024-04-23T20:01:11+02:00 TRAC client/internal/dns/local.go:25: received question: dns.Question{Name:"example.com", Qtype:0x1c, Qclass:0x1}`
+
+	result := anonymizer.AnonymizeDNSLogLine(testLog)
+	require.NotEqual(t, testLog, result)
+	assert.NotContains(t, result, "example.com")
+}
+
+func TestAnonymizeDomain(t *testing.T) {
+	anonymizer := anonymize.NewAnonymizer(netip.Addr{}, netip.Addr{})
+	tests := []struct {
+		name            string
+		domain          string
+		expectPattern   string
+		shouldAnonymize bool
+	}{
+		{
+			"General Domain",
+			"example.com",
+			`^anon-[a-zA-Z0-9]+\.domain$`,
+			true,
+		},
+		{
+			"Domain with Trailing Dot",
+			"example.com.",
+			`^anon-[a-zA-Z0-9]+\.domain.$`,
+			true,
+		},
+		{
+			"Subdomain",
+			"sub.example.com",
+			`^sub\.anon-[a-zA-Z0-9]+\.domain$`,
+			true,
+		},
+		{
+			"Subdomain with Trailing Dot",
+			"sub.example.com.",
+			`^sub\.anon-[a-zA-Z0-9]+\.domain.$`,
+			true,
+		},
+		{
+			"Protected Domain",
+			"netbird.io",
+			`^netbird\.io$`,
+			false,
+		},
+		{
+			"Protected Domain with Trailing Dot",
+			"netbird.io.",
+			`^netbird\.io.$`,
+			false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			result := anonymizer.AnonymizeDomain(tc.domain)
+			if tc.shouldAnonymize {
+				assert.Regexp(t, tc.expectPattern, result, "The anonymized domain should match the expected pattern")
+				assert.NotContains(t, result, tc.domain, "The original domain should not be present in the result")
+			} else {
+				assert.Equal(t, tc.domain, result, "Protected domains should not be anonymized")
+			}
+		})
+	}
+}
+
+func TestAnonymizeURI(t *testing.T) {
+	anonymizer := anonymize.NewAnonymizer(netip.Addr{}, netip.Addr{})
+	tests := []struct {
+		name  string
+		uri   string
+		regex string
+	}{
+		{
+			"HTTP URI with Port",
+			"http://example.com:80/path",
+			`^http://anon-[a-zA-Z0-9]+\.domain:80/path$`,
+		},
+		{
+			"HTTP URI without Port",
+			"http://example.com/path",
+			`^http://anon-[a-zA-Z0-9]+\.domain/path$`,
+		},
+		{
+			"Opaque URI with Port",
+			"stun:example.com:80?transport=udp",
+			`^stun:anon-[a-zA-Z0-9]+\.domain:80\?transport=udp$`,
+		},
+		{
+			"Opaque URI without Port",
+			"stun:example.com?transport=udp",
+			`^stun:anon-[a-zA-Z0-9]+\.domain\?transport=udp$`,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			result := anonymizer.AnonymizeURI(tc.uri)
+			assert.Regexp(t, regexp.MustCompile(tc.regex), result, "URI should match expected pattern")
+			require.NotContains(t, result, "example.com", "Original domain should not be present")
+		})
+	}
+}
+
+func TestAnonymizeSchemeURI(t *testing.T) {
+	anonymizer := anonymize.NewAnonymizer(netip.Addr{}, netip.Addr{})
+	tests := []struct {
+		name   string
+		input  string
+		expect string
+	}{
+		{"STUN URI in text", "Connection made via stun:example.com", `Connection made via stun:anon-[a-zA-Z0-9]+\.domain`},
+		{"STUNS URI in message", "Secure connection to stuns:example.com:443", `Secure connection to stuns:anon-[a-zA-Z0-9]+\.domain:443`},
+		{"TURN URI in log", "Failed attempt turn:some.example.com:3478?transport=tcp: retrying", `Failed attempt turn:some.anon-[a-zA-Z0-9]+\.domain:3478\?transport=tcp: retrying`},
+		{"TURNS URI in message", "Secure connection to turns:example.com:5349", `Secure connection to turns:anon-[a-zA-Z0-9]+\.domain:5349`},
+		{"HTTP URI in text", "Visit http://example.com for more", `Visit http://anon-[a-zA-Z0-9]+\.domain for more`},
+		{"HTTPS URI in CAPS", "Visit HTTPS://example.com for more", `Visit https://anon-[a-zA-Z0-9]+\.domain for more`},
+		{"HTTPS URI in message", "Visit https://example.com for more", `Visit https://anon-[a-zA-Z0-9]+\.domain for more`},
+		{"WS URI in log", "Connection established to ws://example.com:8080", `Connection established to ws://anon-[a-zA-Z0-9]+\.domain:8080`},
+		{"WSS URI in message", "Secure connection to wss://example.com", `Secure connection to wss://anon-[a-zA-Z0-9]+\.domain`},
+		{"Rel URI in text", "Relaying to rel://example.com", `Relaying to rel://anon-[a-zA-Z0-9]+\.domain`},
+		{"Rels URI in message", "Relaying to rels://example.com", `Relaying to rels://anon-[a-zA-Z0-9]+\.domain`},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			result := anonymizer.AnonymizeSchemeURI(tc.input)
+			assert.Regexp(t, tc.expect, result, "The anonymized output should match expected pattern")
+			require.NotContains(t, result, "example.com", "Original domain should not be present")
+		})
+	}
+}
+
+func TestAnonymizString_MemorizedDomain(t *testing.T) {
+	anonymizer := anonymize.NewAnonymizer(netip.Addr{}, netip.Addr{})
+	domain := "example.com"
+	anonymizedDomain := anonymizer.AnonymizeDomain(domain)
+
+	sampleString := "This is a test string including the domain example.com which should be anonymized."
+
+	firstPassResult := anonymizer.AnonymizeString(sampleString)
+	secondPassResult := anonymizer.AnonymizeString(firstPassResult)
+
+	assert.Contains(t, firstPassResult, anonymizedDomain, "The domain should be anonymized in the first pass")
+	assert.NotContains(t, firstPassResult, domain, "The original domain should not appear in the first pass output")
+
+	assert.Equal(t, firstPassResult, secondPassResult, "The second pass should not further anonymize the string")
+}
+
+func TestAnonymizeString_DoubleURI(t *testing.T) {
+	anonymizer := anonymize.NewAnonymizer(netip.Addr{}, netip.Addr{})
+	domain := "example.com"
+	anonymizedDomain := anonymizer.AnonymizeDomain(domain)
+
+	sampleString := "Check out our site at https://example.com for more info."
+
+	firstPassResult := anonymizer.AnonymizeString(sampleString)
+	secondPassResult := anonymizer.AnonymizeString(firstPassResult)
+
+	assert.Contains(t, firstPassResult, "https://"+anonymizedDomain, "The URI should be anonymized in the first pass")
+	assert.NotContains(t, firstPassResult, "https://example.com", "The original URI should not appear in the first pass output")
+
+	assert.Equal(t, firstPassResult, secondPassResult, "The second pass should not further anonymize the URI")
+}
+
+func TestAnonymizeString_IPAddresses(t *testing.T) {
+	anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses())
+	tests := []struct {
+		name   string
+		input  string
+		expect string
+	}{
+		{
+			name:   "IPv4 Address",
+			input:  "Error occurred at IP 122.138.1.1",
+			expect: "Error occurred at IP 198.51.100.0",
+		},
+		{
+			name:   "IPv6 Address",
+			input:  "Access attempted from 2001:db8::ff00:42",
+			expect: "Access attempted from 100::",
+		},
+		{
+			name:   "IPv6 Address with Port",
+			input:  "Access attempted from [2001:db8::ff00:42]:8080",
+			expect: "Access attempted from [100::]:8080",
+		},
+		{
+			name:   "Both IPv4 and IPv6",
+			input:  "IPv4: 142.108.0.1 and IPv6: 2001:db8::ff00:43",
+			expect: "IPv4: 198.51.100.1 and IPv6: 100::1",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			result := anonymizer.AnonymizeString(tc.input)
+			assert.Equal(t, tc.expect, result, "IP addresses should be anonymized correctly")
+		})
+	}
+}

client/cmd/debug.go

@@ -0,0 +1,325 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/status"
+
+	"github.com/netbirdio/netbird/client/internal"
+	"github.com/netbirdio/netbird/client/proto"
+	"github.com/netbirdio/netbird/client/server"
+)
+
+const errCloseConnection = "Failed to close connection: %v"
+
+var debugCmd = &cobra.Command{
+	Use:   "debug",
+	Short: "Debugging commands",
+	Long:  "Provides commands for debugging and logging control within the Netbird daemon.",
+}
+
+var debugBundleCmd = &cobra.Command{
+	Use:     "bundle",
+	Example: "  netbird debug bundle",
+	Short:   "Create a debug bundle",
+	Long:    "Generates a compressed archive of the daemon's logs and status for debugging purposes.",
+	RunE:    debugBundle,
+}
+
+var logCmd = &cobra.Command{
+	Use:   "log",
+	Short: "Manage logging for the Netbird daemon",
+	Long:  `Commands to manage logging settings for the Netbird daemon, including ICE, gRPC, and general log levels.`,
+}
+
+var logLevelCmd = &cobra.Command{
+	Use:   "level <level>",
+	Short: "Set the logging level for this session",
+	Long: `Sets the logging level for the current session. This setting is temporary and will revert to the default on daemon restart.
+Available log levels are:
+  panic:   for panic level, highest level of severity
+  fatal:   for fatal level errors that cause the program to exit
+  error:   for error conditions
+  warn:    for warning conditions
+  info:    for informational messages
+  debug:   for debug-level messages
+  trace:   for trace-level messages, which include more fine-grained information than debug`,
+	Args: cobra.ExactArgs(1),
+	RunE: setLogLevel,
+}
+
+var forCmd = &cobra.Command{
+	Use:     "for <time>",
+	Short:   "Run debug logs for a specified duration and create a debug bundle",
+	Long:    `Sets the logging level to trace, runs for the specified duration, and then generates a debug bundle.`,
+	Example: "  netbird debug for 5m",
+	Args:    cobra.ExactArgs(1),
+	RunE:    runForDuration,
+}
+
+var persistenceCmd = &cobra.Command{
+	Use:     "persistence [on|off]",
+	Short:   "Set network map memory persistence",
+	Long:    `Configure whether the latest network map should persist in memory. When enabled, the last known network map will be kept in memory.`,
+	Example: "  netbird debug persistence on",
+	Args:    cobra.ExactArgs(1),
+	RunE:    setNetworkMapPersistence,
+}
+
+func debugBundle(cmd *cobra.Command, _ []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	client := proto.NewDaemonServiceClient(conn)
+	resp, err := client.DebugBundle(cmd.Context(), &proto.DebugBundleRequest{
+		Anonymize:  anonymizeFlag,
+		Status:     getStatusOutput(cmd),
+		SystemInfo: debugSystemInfoFlag,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to bundle debug: %v", status.Convert(err).Message())
+	}
+
+	cmd.Println(resp.GetPath())
+
+	return nil
+}
+
+func setLogLevel(cmd *cobra.Command, args []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	client := proto.NewDaemonServiceClient(conn)
+	level := server.ParseLogLevel(args[0])
+	if level == proto.LogLevel_UNKNOWN {
+		return fmt.Errorf("unknown log level: %s. Available levels are: panic, fatal, error, warn, info, debug, trace\n", args[0])
+	}
+
+	_, err = client.SetLogLevel(cmd.Context(), &proto.SetLogLevelRequest{
+		Level: level,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to set log level: %v", status.Convert(err).Message())
+	}
+
+	cmd.Println("Log level set successfully to", args[0])
+	return nil
+}
+
+func runForDuration(cmd *cobra.Command, args []string) error {
+	duration, err := time.ParseDuration(args[0])
+	if err != nil {
+		return fmt.Errorf("invalid duration format: %v", err)
+	}
+
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	client := proto.NewDaemonServiceClient(conn)
+
+	stat, err := client.Status(cmd.Context(), &proto.StatusRequest{})
+	if err != nil {
+		return fmt.Errorf("failed to get status: %v", status.Convert(err).Message())
+	}
+
+	stateWasDown := stat.Status != string(internal.StatusConnected) && stat.Status != string(internal.StatusConnecting)
+
+	initialLogLevel, err := client.GetLogLevel(cmd.Context(), &proto.GetLogLevelRequest{})
+	if err != nil {
+		return fmt.Errorf("failed to get log level: %v", status.Convert(err).Message())
+	}
+
+	if stateWasDown {
+		if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
+			return fmt.Errorf("failed to up: %v", status.Convert(err).Message())
+		}
+		cmd.Println("Netbird up")
+		time.Sleep(time.Second * 10)
+	}
+
+	initialLevelTrace := initialLogLevel.GetLevel() >= proto.LogLevel_TRACE
+	if !initialLevelTrace {
+		_, err = client.SetLogLevel(cmd.Context(), &proto.SetLogLevelRequest{
+			Level: proto.LogLevel_TRACE,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to set log level to TRACE: %v", status.Convert(err).Message())
+		}
+		cmd.Println("Log level set to trace.")
+	}
+
+	if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
+		return fmt.Errorf("failed to down: %v", status.Convert(err).Message())
+	}
+	cmd.Println("Netbird down")
+
+	time.Sleep(1 * time.Second)
+
+	// Enable network map persistence before bringing the service up
+	if _, err := client.SetNetworkMapPersistence(cmd.Context(), &proto.SetNetworkMapPersistenceRequest{
+		Enabled: true,
+	}); err != nil {
+		return fmt.Errorf("failed to enable network map persistence: %v", status.Convert(err).Message())
+	}
+
+	if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
+		return fmt.Errorf("failed to up: %v", status.Convert(err).Message())
+	}
+	cmd.Println("Netbird up")
+
+	time.Sleep(3 * time.Second)
+
+	headerPostUp := fmt.Sprintf("----- Netbird post-up - Timestamp: %s", time.Now().Format(time.RFC3339))
+	statusOutput := fmt.Sprintf("%s\n%s", headerPostUp, getStatusOutput(cmd))
+
+	if waitErr := waitForDurationOrCancel(cmd.Context(), duration, cmd); waitErr != nil {
+		return waitErr
+	}
+	cmd.Println("\nDuration completed")
+
+	cmd.Println("Creating debug bundle...")
+
+	headerPreDown := fmt.Sprintf("----- Netbird pre-down - Timestamp: %s - Duration: %s", time.Now().Format(time.RFC3339), duration)
+	statusOutput = fmt.Sprintf("%s\n%s\n%s", statusOutput, headerPreDown, getStatusOutput(cmd))
+
+	resp, err := client.DebugBundle(cmd.Context(), &proto.DebugBundleRequest{
+		Anonymize:  anonymizeFlag,
+		Status:     statusOutput,
+		SystemInfo: debugSystemInfoFlag,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to bundle debug: %v", status.Convert(err).Message())
+	}
+
+	// Disable network map persistence after creating the debug bundle
+	if _, err := client.SetNetworkMapPersistence(cmd.Context(), &proto.SetNetworkMapPersistenceRequest{
+		Enabled: false,
+	}); err != nil {
+		return fmt.Errorf("failed to disable network map persistence: %v", status.Convert(err).Message())
+	}
+
+	if stateWasDown {
+		if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
+			return fmt.Errorf("failed to down: %v", status.Convert(err).Message())
+		}
+		cmd.Println("Netbird down")
+	}
+
+	if !initialLevelTrace {
+		if _, err := client.SetLogLevel(cmd.Context(), &proto.SetLogLevelRequest{Level: initialLogLevel.GetLevel()}); err != nil {
+			return fmt.Errorf("failed to restore log level: %v", status.Convert(err).Message())
+		}
+		cmd.Println("Log level restored to", initialLogLevel.GetLevel())
+	}
+
+	cmd.Println(resp.GetPath())
+
+	return nil
+}
+
+func setNetworkMapPersistence(cmd *cobra.Command, args []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	persistence := strings.ToLower(args[0])
+	if persistence != "on" && persistence != "off" {
+		return fmt.Errorf("invalid persistence value: %s. Use 'on' or 'off'", args[0])
+	}
+
+	client := proto.NewDaemonServiceClient(conn)
+	_, err = client.SetNetworkMapPersistence(cmd.Context(), &proto.SetNetworkMapPersistenceRequest{
+		Enabled: persistence == "on",
+	})
+	if err != nil {
+		return fmt.Errorf("failed to set network map persistence: %v", status.Convert(err).Message())
+	}
+
+	cmd.Printf("Network map persistence set to: %s\n", persistence)
+	return nil
+}
+
+func getStatusOutput(cmd *cobra.Command) string {
+	var statusOutputString string
+	statusResp, err := getStatus(cmd.Context())
+	if err != nil {
+		cmd.PrintErrf("Failed to get status: %v\n", err)
+	} else {
+		statusOutputString = parseToFullDetailSummary(convertToStatusOutputOverview(statusResp))
+	}
+	return statusOutputString
+}
+
+func waitForDurationOrCancel(ctx context.Context, duration time.Duration, cmd *cobra.Command) error {
+	ticker := time.NewTicker(1 * time.Second)
+	defer ticker.Stop()
+
+	startTime := time.Now()
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				elapsed := time.Since(startTime)
+				if elapsed >= duration {
+					return
+				}
+				remaining := duration - elapsed
+				cmd.Printf("\rRemaining time: %s", formatDuration(remaining))
+			}
+		}
+	}()
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-done:
+		return nil
+	}
+}
+
+func formatDuration(d time.Duration) string {
+	d = d.Round(time.Second)
+	h := d / time.Hour
+	d %= time.Hour
+	m := d / time.Minute
+	d %= time.Minute
+	s := d / time.Second
+	return fmt.Sprintf("%02d:%02d:%02d", h, m, s)
+}

client/cmd/down.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"context"
-	"github.com/netbirdio/netbird/util"
 	"time"
 
+	"github.com/netbirdio/netbird/util"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
@@ -25,7 +26,7 @@ var downCmd = &cobra.Command{
 			return err
 		}
 
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*7)
 		defer cancel()
 
 		conn, err := DialClientGRPCServer(ctx, daemonAddr)
@@ -41,6 +42,8 @@ var downCmd = &cobra.Command{
 			log.Errorf("call service down method: %v", err)
 			return err
 		}
+
+		cmd.Println("Disconnected")
 		return nil
 	},
 }

client/cmd/login.go

@@ -3,18 +3,20 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/skratchdot/open-golang/open"
+	"github.com/spf13/cobra"
 	"google.golang.org/grpc/codes"
 	gstatus "google.golang.org/grpc/status"
 
-	"github.com/netbirdio/netbird/util"
-
-	"github.com/spf13/cobra"
-
 	"github.com/netbirdio/netbird/client/internal"
+	"github.com/netbirdio/netbird/client/internal/auth"
 	"github.com/netbirdio/netbird/client/proto"
+	"github.com/netbirdio/netbird/client/system"
+	"github.com/netbirdio/netbird/util"
 )
 
 var loginCmd = &cobra.Command{
@@ -32,6 +34,16 @@ var loginCmd = &cobra.Command{
 
 		ctx := internal.CtxInitState(context.Background())
 
+		if hostName != "" {
+			// nolint
+			ctx = context.WithValue(ctx, system.DeviceNameCtxKey, hostName)
+		}
+
+		providedSetupKey, err := getSetupKey()
+		if err != nil {
+			return err
+		}
+
 		// workaround to run without service
 		if logFile == "console" {
 			err = handleRebrand(cmd)
@@ -39,19 +51,23 @@ var loginCmd = &cobra.Command{
 				return err
 			}
 
-			config, err := internal.UpdateOrCreateConfig(internal.ConfigInput{
+			ic := internal.ConfigInput{
 				ManagementURL: managementURL,
 				AdminURL:      adminURL,
 				ConfigPath:    configPath,
-				PreSharedKey:  &preSharedKey,
+			}
-			})
+			if rootCmd.PersistentFlags().Changed(preSharedKeyFlag) {
+				ic.PreSharedKey = &preSharedKey
+			}
+
+			config, err := internal.UpdateOrCreateConfig(ic)
 			if err != nil {
 				return fmt.Errorf("get config file: %v", err)
 			}
 
-			config, _ = internal.UpdateOldManagementPort(ctx, config, configPath)
+			config, _ = internal.UpdateOldManagementURL(ctx, config, configPath)
 
-			err = foregroundLogin(ctx, cmd, config, setupKey)
+			err = foregroundLogin(ctx, cmd, config, providedSetupKey)
 			if err != nil {
 				return fmt.Errorf("foreground login failed: %v", err)
 			}
@@ -70,9 +86,14 @@ var loginCmd = &cobra.Command{
 		client := proto.NewDaemonServiceClient(conn)
 
 		loginRequest := proto.LoginRequest{
-			SetupKey:      setupKey,
+			SetupKey:             providedSetupKey,
-			PreSharedKey:  preSharedKey,
+			ManagementUrl:        managementURL,
-			ManagementUrl: managementURL,
+			IsLinuxDesktopClient: isLinuxRunningDesktop(),
+			Hostname:             hostName,
+		}
+
+		if rootCmd.PersistentFlags().Changed(preSharedKeyFlag) {
+			loginRequest.OptionalPreSharedKey = &preSharedKey
 		}
 
 		var loginErr error
@@ -100,9 +121,9 @@ var loginCmd = &cobra.Command{
 		}
 
 		if loginResp.NeedsSSOLogin {
-			openURL(cmd, loginResp.VerificationURIComplete)
+			openURL(cmd, loginResp.VerificationURIComplete, loginResp.UserCode)
 
-			_, err = client.WaitSSOLogin(ctx, &proto.WaitSSOLoginRequest{UserCode: loginResp.UserCode})
+			_, err = client.WaitSSOLogin(ctx, &proto.WaitSSOLoginRequest{UserCode: loginResp.UserCode, Hostname: hostName})
 			if err != nil {
 				return fmt.Errorf("waiting sso login failed with: %v", err)
 			}
@@ -138,13 +159,21 @@ func foregroundLogin(ctx context.Context, cmd *cobra.Command, config *internal.C
 		jwtToken = tokenInfo.GetTokenToUse()
 	}
 
+	var lastError error
+
 	err = WithBackOff(func() error {
 		err := internal.Login(ctx, config, setupKey, jwtToken)
 		if s, ok := gstatus.FromError(err); ok && (s.Code() == codes.InvalidArgument || s.Code() == codes.PermissionDenied) {
+			lastError = err
 			return nil
 		}
 		return err
 	})
+
+	if lastError != nil {
+		return fmt.Errorf("login failed: %v", lastError)
+	}
+
 	if err != nil {
 		return fmt.Errorf("backoff cycle failed: %v", err)
 	}
@@ -152,40 +181,24 @@ func foregroundLogin(ctx context.Context, cmd *cobra.Command, config *internal.C
 	return nil
 }
 
-func foregroundGetTokenInfo(ctx context.Context, cmd *cobra.Command, config *internal.Config) (*internal.TokenInfo, error) {
+func foregroundGetTokenInfo(ctx context.Context, cmd *cobra.Command, config *internal.Config) (*auth.TokenInfo, error) {
-	providerConfig, err := internal.GetDeviceAuthorizationFlowInfo(ctx, config.PrivateKey, config.ManagementURL)
+	oAuthFlow, err := auth.NewOAuthFlow(ctx, config, isLinuxRunningDesktop())
 	if err != nil {
-		s, ok := gstatus.FromError(err)
+		return nil, err
-		if ok && s.Code() == codes.NotFound {
-			return nil, fmt.Errorf("no SSO provider returned from management. " +
-				"If you are using hosting Netbird see documentation at " +
-				"https://github.com/netbirdio/netbird/tree/main/management for details")
-		} else if ok && s.Code() == codes.Unimplemented {
-			mgmtURL := managementURL
-			if mgmtURL == "" {
-				mgmtURL = internal.DefaultManagementURL
-			}
-			return nil, fmt.Errorf("the management server, %s, does not support SSO providers, "+
-				"please update your servver or use Setup Keys to login", mgmtURL)
-		} else {
-			return nil, fmt.Errorf("getting device authorization flow info failed with error: %v", err)
-		}
 	}
 
-	hostedClient := internal.NewHostedDeviceFlow(providerConfig.ProviderConfig)
+	flowInfo, err := oAuthFlow.RequestAuthInfo(context.TODO())
-
-	flowInfo, err := hostedClient.RequestDeviceCode(context.TODO())
 	if err != nil {
-		return nil, fmt.Errorf("getting a request device code failed: %v", err)
+		return nil, fmt.Errorf("getting a request OAuth flow info failed: %v", err)
 	}
 
-	openURL(cmd, flowInfo.VerificationURIComplete)
+	openURL(cmd, flowInfo.VerificationURIComplete, flowInfo.UserCode)
 
-	waitTimeout := time.Duration(flowInfo.ExpiresIn)
+	waitTimeout := time.Duration(flowInfo.ExpiresIn) * time.Second
-	waitCTX, c := context.WithTimeout(context.TODO(), waitTimeout*time.Second)
+	waitCTX, c := context.WithTimeout(context.TODO(), waitTimeout)
 	defer c()
 
-	tokenInfo, err := hostedClient.WaitToken(waitCTX, flowInfo)
+	tokenInfo, err := oAuthFlow.WaitToken(waitCTX, flowInfo)
 	if err != nil {
 		return nil, fmt.Errorf("waiting for browser login failed: %v", err)
 	}
@@ -193,12 +206,23 @@ func foregroundGetTokenInfo(ctx context.Context, cmd *cobra.Command, config *int
 	return &tokenInfo, nil
 }
 
-func openURL(cmd *cobra.Command, verificationURIComplete string) {
+func openURL(cmd *cobra.Command, verificationURIComplete, userCode string) {
-	err := open.Run(verificationURIComplete)
+	var codeMsg string
-	cmd.Printf("Please do the SSO login in your browser. \n" +
+	if userCode != "" && !strings.Contains(verificationURIComplete, userCode) {
+		codeMsg = fmt.Sprintf("and enter the code %s to authenticate.", userCode)
+	}
+
+	cmd.Println("Please do the SSO login in your browser. \n" +
 		"If your browser didn't open automatically, use this URL to log in:\n\n" +
-		" " + verificationURIComplete + " \n\n")
+		verificationURIComplete + " " + codeMsg)
-	if err != nil {
+	cmd.Println("")
-		cmd.Printf("Alternatively, you may want to use a setup key, see:\n\n https://www.netbird.io/docs/overview/setup-keys\n")
+	if err := open.Run(verificationURIComplete); err != nil {
+		cmd.Println("\nAlternatively, you may want to use a setup key, see:\n\n" +
+			"https://docs.netbird.io/how-to/register-machines-using-setup-keys")
 	}
 }
+
+// isLinuxRunningDesktop checks if a Linux OS is running desktop environment
+func isLinuxRunningDesktop() bool {
+	return os.Getenv("DESKTOP_SESSION") != "" || os.Getenv("XDG_CURRENT_DESKTOP") != ""
+}

client/cmd/login_test.go

@@ -5,8 +5,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/internal"
-	"github.com/netbirdio/netbird/iface"
 	"github.com/netbirdio/netbird/util"
 )
 

client/cmd/pprof.go

@@ -0,0 +1,33 @@
+//go:build pprof
+// +build pprof
+
+package cmd
+
+import (
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func init() {
+	addr := pprofAddr()
+	go pprof(addr)
+}
+
+func pprofAddr() string {
+	listenAddr := os.Getenv("NB_PPROF_ADDR")
+	if listenAddr == "" {
+		return "localhost:6969"
+	}
+
+	return listenAddr
+}
+
+func pprof(listenAddr string) {
+	log.Infof("listening pprof on: %s\n", listenAddr)
+	if err := http.ListenAndServe(listenAddr, nil); err != nil {
+		log.Fatalf("Failed to start pprof: %v", err)
+	}
+}

client/cmd/root.go

@@ -25,8 +25,19 @@ import (
 )
 
 const (
-	externalIPMapFlag  = "external-ip-map"
+	externalIPMapFlag       = "external-ip-map"
-	dnsResolverAddress = "dns-resolver-address"
+	dnsResolverAddress      = "dns-resolver-address"
+	enableRosenpassFlag     = "enable-rosenpass"
+	rosenpassPermissiveFlag = "rosenpass-permissive"
+	preSharedKeyFlag        = "preshared-key"
+	interfaceNameFlag       = "interface-name"
+	wireguardPortFlag       = "wireguard-port"
+	networkMonitorFlag      = "network-monitor"
+	disableAutoConnectFlag  = "disable-auto-connect"
+	serverSSHAllowedFlag    = "allow-server-ssh"
+	extraIFaceBlackListFlag = "extra-iface-blacklist"
+	dnsRouteIntervalFlag    = "dns-router-interval"
+	systemInfoFlag          = "system-info"
 )
 
 var (
@@ -45,10 +56,25 @@ var (
 	managementURL           string
 	adminURL                string
 	setupKey                string
+	setupKeyPath            string
+	hostName                string
 	preSharedKey            string
 	natExternalIPs          []string
 	customDNSAddress        string
-	rootCmd                 = &cobra.Command{
+	rosenpassEnabled        bool
+	rosenpassPermissive     bool
+	serverSSHAllowed        bool
+	interfaceName           string
+	wireguardPort           uint16
+	networkMonitor          bool
+	serviceName             string
+	autoConnectDisabled     bool
+	extraIFaceBlackList     []string
+	anonymizeFlag           bool
+	debugSystemInfoFlag     bool
+	dnsRouteInterval        time.Duration
+
+	rootCmd = &cobra.Command{
 		Use:          "netbird",
 		Short:        "",
 		Long:         "",
@@ -68,12 +94,15 @@ func init() {
 	oldDefaultConfigPathDir = "/etc/wiretrustee/"
 	oldDefaultLogFileDir = "/var/log/wiretrustee/"
 
-	if runtime.GOOS == "windows" {
+	switch runtime.GOOS {
+	case "windows":
 		defaultConfigPathDir = os.Getenv("PROGRAMDATA") + "\\Netbird\\"
 		defaultLogFileDir = os.Getenv("PROGRAMDATA") + "\\Netbird\\"
 
 		oldDefaultConfigPathDir = os.Getenv("PROGRAMDATA") + "\\Wiretrustee\\"
 		oldDefaultLogFileDir = os.Getenv("PROGRAMDATA") + "\\Wiretrustee\\"
+	case "freebsd":
+		defaultConfigPathDir = "/var/db/netbird/"
 	}
 
 	defaultConfigPath = defaultConfigPathDir + "config.json"
@@ -86,14 +115,26 @@ func init() {
 	if runtime.GOOS == "windows" {
 		defaultDaemonAddr = "tcp://127.0.0.1:41731"
 	}
+
+	defaultServiceName := "netbird"
+	if runtime.GOOS == "windows" {
+		defaultServiceName = "Netbird"
+	}
+
 	rootCmd.PersistentFlags().StringVar(&daemonAddr, "daemon-addr", defaultDaemonAddr, "Daemon service address to serve CLI requests [unix|tcp]://[path|host:port]")
 	rootCmd.PersistentFlags().StringVarP(&managementURL, "management-url", "m", "", fmt.Sprintf("Management Service URL [http|https]://[host]:[port] (default \"%s\")", internal.DefaultManagementURL))
 	rootCmd.PersistentFlags().StringVar(&adminURL, "admin-url", "", fmt.Sprintf("Admin Panel URL [http|https]://[host]:[port] (default \"%s\")", internal.DefaultAdminURL))
+	rootCmd.PersistentFlags().StringVarP(&serviceName, "service", "s", defaultServiceName, "Netbird system service name")
 	rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", defaultConfigPath, "Netbird config file location")
 	rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", "info", "sets Netbird log level")
-	rootCmd.PersistentFlags().StringVar(&logFile, "log-file", defaultLogFile, "sets Netbird log path. If console is specified the the log will be output to stdout")
+	rootCmd.PersistentFlags().StringVar(&logFile, "log-file", defaultLogFile, "sets Netbird log path. If console is specified the log will be output to stdout. If syslog is specified the log will be sent to syslog daemon.")
 	rootCmd.PersistentFlags().StringVarP(&setupKey, "setup-key", "k", "", "Setup key obtained from the Management Service Dashboard (used to register peer)")
-	rootCmd.PersistentFlags().StringVar(&preSharedKey, "preshared-key", "", "Sets Wireguard PreSharedKey property. If set, then only peers that have the same key can communicate.")
+	rootCmd.PersistentFlags().StringVar(&setupKeyPath, "setup-key-file", "", "The path to a setup key obtained from the Management Service Dashboard (used to register peer) This is ignored if the setup-key flag is provided.")
+	rootCmd.MarkFlagsMutuallyExclusive("setup-key", "setup-key-file")
+	rootCmd.PersistentFlags().StringVar(&preSharedKey, preSharedKeyFlag, "", "Sets Wireguard PreSharedKey property. If set, then only peers that have the same key can communicate.")
+	rootCmd.PersistentFlags().StringVarP(&hostName, "hostname", "n", "", "Sets a custom hostname for the device")
+	rootCmd.PersistentFlags().BoolVarP(&anonymizeFlag, "anonymize", "A", false, "anonymize IP addresses and non-netbird.io domains in logs and status output")
+
 	rootCmd.AddCommand(serviceCmd)
 	rootCmd.AddCommand(upCmd)
 	rootCmd.AddCommand(downCmd)
@@ -101,8 +142,21 @@ func init() {
 	rootCmd.AddCommand(loginCmd)
 	rootCmd.AddCommand(versionCmd)
 	rootCmd.AddCommand(sshCmd)
+	rootCmd.AddCommand(routesCmd)
+	rootCmd.AddCommand(debugCmd)
+
 	serviceCmd.AddCommand(runCmd, startCmd, stopCmd, restartCmd) // service control commands are subcommands of service
 	serviceCmd.AddCommand(installCmd, uninstallCmd)              // service installer commands are subcommands of service
+
+	routesCmd.AddCommand(routesListCmd)
+	routesCmd.AddCommand(routesSelectCmd, routesDeselectCmd)
+
+	debugCmd.AddCommand(debugBundleCmd)
+	debugCmd.AddCommand(logCmd)
+	logCmd.AddCommand(logLevelCmd)
+	debugCmd.AddCommand(forCmd)
+	debugCmd.AddCommand(persistenceCmd)
+
 	upCmd.PersistentFlags().StringSliceVar(&natExternalIPs, externalIPMapFlag, nil,
 		`Sets external IPs maps between local addresses and interfaces.`+
 			`You can specify a comma-separated list with a single IP and IP/IP or IP/Interface Name. `+
@@ -116,6 +170,12 @@ func init() {
 			`An empty string "" clears the previous configuration. `+
 			`E.g. --dns-resolver-address 127.0.0.1:5053 or --dns-resolver-address ""`,
 	)
+	upCmd.PersistentFlags().BoolVar(&rosenpassEnabled, enableRosenpassFlag, false, "[Experimental] Enable Rosenpass feature. If enabled, the connection will be post-quantum secured via Rosenpass.")
+	upCmd.PersistentFlags().BoolVar(&rosenpassPermissive, rosenpassPermissiveFlag, false, "[Experimental] Enable Rosenpass in permissive mode to allow this peer to accept WireGuard connections without requiring Rosenpass functionality from peers that do not have Rosenpass enabled.")
+	upCmd.PersistentFlags().BoolVar(&serverSSHAllowed, serverSSHAllowedFlag, false, "Allow SSH server on peer. If enabled, the SSH server will be permitted")
+	upCmd.PersistentFlags().BoolVar(&autoConnectDisabled, disableAutoConnectFlag, false, "Disables auto-connect feature. If enabled, then the client won't connect automatically when the service starts.")
+
+	debugCmd.PersistentFlags().BoolVarP(&debugSystemInfoFlag, systemInfoFlag, "S", false, "Adds system information to the debug bundle")
 }
 
 // SetupCloseHandler handles SIGTERM signal and exits with success
@@ -166,7 +226,7 @@ func FlagNameToEnvVar(cmdFlag string, prefix string) string {
 	return prefix + upper
 }
 
-// DialClientGRPCServer returns client connection to the dameno server.
+// DialClientGRPCServer returns client connection to the daemon server.
 func DialClientGRPCServer(ctx context.Context, addr string) (*grpc.ClientConn, error) {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*3)
 	defer cancel()
@@ -197,6 +257,21 @@ var CLIBackOffSettings = &backoff.ExponentialBackOff{
 	Clock:               backoff.SystemClock,
 }
 
+func getSetupKey() (string, error) {
+	if setupKeyPath != "" && setupKey == "" {
+		return getSetupKeyFromFile(setupKeyPath)
+	}
+	return setupKey, nil
+}
+
+func getSetupKeyFromFile(setupKeyPath string) (string, error) {
+	data, err := os.ReadFile(setupKeyPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to read setup key file: %v", err)
+	}
+	return strings.TrimSpace(string(data)), nil
+}
+
 func handleRebrand(cmd *cobra.Command) error {
 	var err error
 	if logFile == defaultLogFile {
@@ -306,3 +381,17 @@ func migrateToNetbird(oldPath, newPath string) bool {
 
 	return true
 }
+
+func getClient(cmd *cobra.Command) (*grpc.ClientConn, error) {
+	SetFlagsFromEnvVars(rootCmd)
+	cmd.SetOut(cmd.OutOrStdout())
+
+	conn, err := DialClientGRPCServer(cmd.Context(), daemonAddr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to daemon error: %v\n"+
+			"If the daemon is not running please run: "+
+			"\nnetbird service install \nnetbird service start\n", err)
+	}
+
+	return conn, nil
+}

client/cmd/root_test.go

@@ -4,6 +4,10 @@ import (
 	"fmt"
 	"io"
 	"testing"
+
+	"github.com/spf13/cobra"
+
+	"github.com/netbirdio/netbird/client/iface"
 )
 
 func TestInitCommands(t *testing.T) {
@@ -34,3 +38,44 @@ func TestInitCommands(t *testing.T) {
 		})
 	}
 }
+
+func TestSetFlagsFromEnvVars(t *testing.T) {
+	var cmd = &cobra.Command{
+		Use:          "netbird",
+		Long:         "test",
+		SilenceUsage: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			SetFlagsFromEnvVars(cmd)
+		},
+	}
+
+	cmd.PersistentFlags().StringSliceVar(&natExternalIPs, externalIPMapFlag, nil,
+		`comma separated list of external IPs to map to the Wireguard interface`)
+	cmd.PersistentFlags().StringVar(&interfaceName, interfaceNameFlag, iface.WgInterfaceDefault, "Wireguard interface name")
+	cmd.PersistentFlags().BoolVar(&rosenpassEnabled, enableRosenpassFlag, false, "Enable Rosenpass feature Rosenpass.")
+	cmd.PersistentFlags().Uint16Var(&wireguardPort, wireguardPortFlag, iface.DefaultWgPort, "Wireguard interface listening port")
+
+	t.Setenv("NB_EXTERNAL_IP_MAP", "abc,dec")
+	t.Setenv("NB_INTERFACE_NAME", "test-name")
+	t.Setenv("NB_ENABLE_ROSENPASS", "true")
+	t.Setenv("NB_WIREGUARD_PORT", "10000")
+	err := cmd.Execute()
+	if err != nil {
+		t.Fatalf("expected no error while running netbird command, got %v", err)
+	}
+	if len(natExternalIPs) != 2 {
+		t.Errorf("expected 2 external ips, got %d", len(natExternalIPs))
+	}
+	if natExternalIPs[0] != "abc" || natExternalIPs[1] != "dec" {
+		t.Errorf("expected abc,dec, got %s,%s", natExternalIPs[0], natExternalIPs[1])
+	}
+	if interfaceName != "test-name" {
+		t.Errorf("expected test-name, got %s", interfaceName)
+	}
+	if !rosenpassEnabled {
+		t.Errorf("expected rosenpassEnabled to be true, got false")
+	}
+	if wireguardPort != 10000 {
+		t.Errorf("expected wireguardPort to be 10000, got %d", wireguardPort)
+	}
+}

client/cmd/route.go

@@ -0,0 +1,174 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/status"
+
+	"github.com/netbirdio/netbird/client/proto"
+)
+
+var appendFlag bool
+
+var routesCmd = &cobra.Command{
+	Use:   "routes",
+	Short: "Manage network routes",
+	Long:  `Commands to list, select, or deselect network routes.`,
+}
+
+var routesListCmd = &cobra.Command{
+	Use:     "list",
+	Aliases: []string{"ls"},
+	Short:   "List routes",
+	Example: "  netbird routes list",
+	Long:    "List all available network routes.",
+	RunE:    routesList,
+}
+
+var routesSelectCmd = &cobra.Command{
+	Use:     "select route...|all",
+	Short:   "Select routes",
+	Long:    "Select a list of routes by identifiers or 'all' to clear all selections and to accept all (including new) routes.\nDefault mode is replace, use -a to append to already selected routes.",
+	Example: "  netbird routes select all\n  netbird routes select route1 route2\n  netbird routes select -a route3",
+	Args:    cobra.MinimumNArgs(1),
+	RunE:    routesSelect,
+}
+
+var routesDeselectCmd = &cobra.Command{
+	Use:     "deselect route...|all",
+	Short:   "Deselect routes",
+	Long:    "Deselect previously selected routes by identifiers or 'all' to disable accepting any routes.",
+	Example: "  netbird routes deselect all\n  netbird routes deselect route1 route2",
+	Args:    cobra.MinimumNArgs(1),
+	RunE:    routesDeselect,
+}
+
+func init() {
+	routesSelectCmd.PersistentFlags().BoolVarP(&appendFlag, "append", "a", false, "Append to current route selection instead of replacing")
+}
+
+func routesList(cmd *cobra.Command, _ []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client := proto.NewDaemonServiceClient(conn)
+	resp, err := client.ListRoutes(cmd.Context(), &proto.ListRoutesRequest{})
+	if err != nil {
+		return fmt.Errorf("failed to list routes: %v", status.Convert(err).Message())
+	}
+
+	if len(resp.Routes) == 0 {
+		cmd.Println("No routes available.")
+		return nil
+	}
+
+	printRoutes(cmd, resp)
+
+	return nil
+}
+
+func printRoutes(cmd *cobra.Command, resp *proto.ListRoutesResponse) {
+	cmd.Println("Available Routes:")
+	for _, route := range resp.Routes {
+		printRoute(cmd, route)
+	}
+}
+
+func printRoute(cmd *cobra.Command, route *proto.Route) {
+	selectedStatus := getSelectedStatus(route)
+	domains := route.GetDomains()
+
+	if len(domains) > 0 {
+		printDomainRoute(cmd, route, domains, selectedStatus)
+	} else {
+		printNetworkRoute(cmd, route, selectedStatus)
+	}
+}
+
+func getSelectedStatus(route *proto.Route) string {
+	if route.GetSelected() {
+		return "Selected"
+	}
+	return "Not Selected"
+}
+
+func printDomainRoute(cmd *cobra.Command, route *proto.Route, domains []string, selectedStatus string) {
+	cmd.Printf("\n  - ID: %s\n    Domains: %s\n    Status: %s\n", route.GetID(), strings.Join(domains, ", "), selectedStatus)
+	resolvedIPs := route.GetResolvedIPs()
+
+	if len(resolvedIPs) > 0 {
+		printResolvedIPs(cmd, domains, resolvedIPs)
+	} else {
+		cmd.Printf("    Resolved IPs: -\n")
+	}
+}
+
+func printNetworkRoute(cmd *cobra.Command, route *proto.Route, selectedStatus string) {
+	cmd.Printf("\n  - ID: %s\n    Network: %s\n    Status: %s\n", route.GetID(), route.GetNetwork(), selectedStatus)
+}
+
+func printResolvedIPs(cmd *cobra.Command, domains []string, resolvedIPs map[string]*proto.IPList) {
+	cmd.Printf("    Resolved IPs:\n")
+	for _, domain := range domains {
+		if ipList, exists := resolvedIPs[domain]; exists {
+			cmd.Printf("      [%s]: %s\n", domain, strings.Join(ipList.GetIps(), ", "))
+		}
+	}
+}
+
+func routesSelect(cmd *cobra.Command, args []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client := proto.NewDaemonServiceClient(conn)
+	req := &proto.SelectRoutesRequest{
+		RouteIDs: args,
+	}
+
+	if len(args) == 1 && args[0] == "all" {
+		req.All = true
+	} else if appendFlag {
+		req.Append = true
+	}
+
+	if _, err := client.SelectRoutes(cmd.Context(), req); err != nil {
+		return fmt.Errorf("failed to select routes: %v", status.Convert(err).Message())
+	}
+
+	cmd.Println("Routes selected successfully.")
+
+	return nil
+}
+
+func routesDeselect(cmd *cobra.Command, args []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client := proto.NewDaemonServiceClient(conn)
+	req := &proto.SelectRoutesRequest{
+		RouteIDs: args,
+	}
+
+	if len(args) == 1 && args[0] == "all" {
+		req.All = true
+	}
+
+	if _, err := client.DeselectRoutes(cmd.Context(), req); err != nil {
+		return fmt.Errorf("failed to deselect routes: %v", status.Convert(err).Message())
+	}
+
+	cmd.Println("Routes deselected successfully.")
+
+	return nil
+}

client/cmd/service.go

@@ -2,7 +2,7 @@ package cmd
 
 import (
 	"context"
-	"runtime"
+	"sync"
 
 	"github.com/kardianos/service"
 	log "github.com/sirupsen/logrus"
@@ -10,12 +10,15 @@ import (
 	"google.golang.org/grpc"
 
 	"github.com/netbirdio/netbird/client/internal"
+	"github.com/netbirdio/netbird/client/server"
 )
 
 type program struct {
-	ctx    context.Context
+	ctx              context.Context
-	cancel context.CancelFunc
+	cancel           context.CancelFunc
-	serv   *grpc.Server
+	serv             *grpc.Server
+	serverInstance   *server.Server
+	serverInstanceMu sync.Mutex
 }
 
 func newProgram(ctx context.Context, cancel context.CancelFunc) *program {
@@ -24,12 +27,8 @@ func newProgram(ctx context.Context, cancel context.CancelFunc) *program {
 }
 
 func newSVCConfig() *service.Config {
-	name := "netbird"
-	if runtime.GOOS == "windows" {
-		name = "Netbird"
-	}
 	return &service.Config{
-		Name:        name,
+		Name:        serviceName,
 		DisplayName: "Netbird",
 		Description: "A WireGuard-based mesh network that connects your devices into a single private network.",
 		Option:      make(service.KeyValue),

client/cmd/service_controller.go

@@ -11,11 +11,12 @@ import (
 	"github.com/kardianos/service"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc"
+
 	"github.com/netbirdio/netbird/client/proto"
 	"github.com/netbirdio/netbird/client/server"
 	"github.com/netbirdio/netbird/util"
-	"github.com/spf13/cobra"
-	"google.golang.org/grpc"
 )
 
 func (p *program) Start(svc service.Service) error {
@@ -60,6 +61,10 @@ func (p *program) Start(svc service.Service) error {
 		}
 		proto.RegisterDaemonServiceServer(p.serv, serverInstance)
 
+		p.serverInstanceMu.Lock()
+		p.serverInstance = serverInstance
+		p.serverInstanceMu.Unlock()
+
 		log.Printf("started daemon server: %v", split[1])
 		if err := p.serv.Serve(listen); err != nil {
 			log.Errorf("failed to serve daemon requests: %v", err)
@@ -69,6 +74,16 @@ func (p *program) Start(svc service.Service) error {
 }
 
 func (p *program) Stop(srv service.Service) error {
+	p.serverInstanceMu.Lock()
+	if p.serverInstance != nil {
+		in := new(proto.DownRequest)
+		_, err := p.serverInstance.Down(p.ctx, in)
+		if err != nil {
+			log.Errorf("failed to stop daemon: %v", err)
+		}
+	}
+	p.serverInstanceMu.Unlock()
+
 	p.cancel()
 
 	if p.serv != nil {
@@ -109,7 +124,6 @@ var runCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		cmd.Printf("Netbird service is running")
 		return nil
 	},
 }

client/cmd/service_installer.go

@@ -31,6 +31,8 @@ var installCmd = &cobra.Command{
 			configPath,
 			"--log-level",
 			logLevel,
+			"--daemon-addr",
+			daemonAddr,
 		}
 
 		if managementURL != "" {
@@ -64,6 +66,10 @@ var installCmd = &cobra.Command{
 			}
 		}
 
+		if runtime.GOOS == "windows" {
+			svcConfig.Option["OnFailure"] = "restart"
+		}
+
 		ctx, cancel := context.WithCancel(cmd.Context())
 
 		s, err := newSVC(newProgram(ctx, cancel), svcConfig)
@@ -77,6 +83,7 @@ var installCmd = &cobra.Command{
 			cmd.PrintErrln(err)
 			return err
 		}
+
 		cmd.Println("Netbird service has been installed")
 		return nil
 	},
@@ -106,7 +113,7 @@ var uninstallCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		cmd.Println("Netbird has been uninstalled")
+		cmd.Println("Netbird service has been uninstalled")
 		return nil
 	},
 }

client/cmd/ssh.go

@@ -24,7 +24,7 @@ var (
 )
 
 var sshCmd = &cobra.Command{
-	Use: "ssh",
+	Use: "ssh [user@]host",
 	Args: func(cmd *cobra.Command, args []string) error {
 		if len(args) < 1 {
 			return errors.New("requires a host argument")
@@ -73,7 +73,8 @@ var sshCmd = &cobra.Command{
 		go func() {
 			// blocking
 			if err := runSSH(sshctx, host, []byte(config.SSHKey), cmd); err != nil {
-				log.Print(err)
+				log.Debug(err)
+				os.Exit(1)
 			}
 			cancel()
 		}()
@@ -92,12 +93,10 @@ func runSSH(ctx context.Context, addr string, pemKey []byte, cmd *cobra.Command)
 	c, err := nbssh.DialWithKey(fmt.Sprintf("%s:%d", addr, port), user, pemKey)
 	if err != nil {
 		cmd.Printf("Error: %v\n", err)
-		cmd.Printf("Couldn't connect. " +
+		cmd.Printf("Couldn't connect. Please check the connection status or if the ssh server is enabled on the other peer" +
-			"You might be disconnected from the NetBird network, or the NetBird agent isn't running.\n" +
+			"\nYou can verify the connection by running:\n\n" +
-			"Run the status command: \n\n" +
+			" netbird status\n\n")
-			" netbird status\n\n" +
+		return err
-			"It might also be that the SSH server is disabled on the agent you are trying to connect to.\n")
-		return nil
 	}
 	go func() {
 		<-ctx.Done()

client/cmd/state.go

@@ -0,0 +1,181 @@
+package cmd
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/status"
+
+	"github.com/netbirdio/netbird/client/proto"
+)
+
+var (
+	allFlag bool
+)
+
+var stateCmd = &cobra.Command{
+	Use:   "state",
+	Short: "Manage daemon state",
+	Long:  "Provides commands for managing and inspecting the Netbird daemon state.",
+}
+
+var stateListCmd = &cobra.Command{
+	Use:     "list",
+	Aliases: []string{"ls"},
+	Short:   "List all stored states",
+	Long:    "Lists all registered states with their status and basic information.",
+	Example: "  netbird state list",
+	RunE:    stateList,
+}
+
+var stateCleanCmd = &cobra.Command{
+	Use:   "clean [state-name]",
+	Short: "Clean stored states",
+	Long: `Clean specific state or all states. The daemon must not be running.
+This will perform cleanup operations and remove the state.`,
+	Example: `  netbird state clean dns_state
+  netbird state clean --all`,
+	RunE: stateClean,
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		// Check mutual exclusivity between --all flag and state-name argument
+		if allFlag && len(args) > 0 {
+			return fmt.Errorf("cannot specify both --all flag and state name")
+		}
+		if !allFlag && len(args) != 1 {
+			return fmt.Errorf("requires a state name argument or --all flag")
+		}
+		return nil
+	},
+}
+
+var stateDeleteCmd = &cobra.Command{
+	Use:   "delete [state-name]",
+	Short: "Delete stored states",
+	Long: `Delete specific state or all states from storage. The daemon must not be running.
+This will remove the state without performing any cleanup operations.`,
+	Example: `  netbird state delete dns_state
+  netbird state delete --all`,
+	RunE: stateDelete,
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		// Check mutual exclusivity between --all flag and state-name argument
+		if allFlag && len(args) > 0 {
+			return fmt.Errorf("cannot specify both --all flag and state name")
+		}
+		if !allFlag && len(args) != 1 {
+			return fmt.Errorf("requires a state name argument or --all flag")
+		}
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(stateCmd)
+	stateCmd.AddCommand(stateListCmd, stateCleanCmd, stateDeleteCmd)
+
+	stateCleanCmd.Flags().BoolVarP(&allFlag, "all", "a", false, "Clean all states")
+	stateDeleteCmd.Flags().BoolVarP(&allFlag, "all", "a", false, "Delete all states")
+}
+
+func stateList(cmd *cobra.Command, _ []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	client := proto.NewDaemonServiceClient(conn)
+	resp, err := client.ListStates(cmd.Context(), &proto.ListStatesRequest{})
+	if err != nil {
+		return fmt.Errorf("failed to list states: %v", status.Convert(err).Message())
+	}
+
+	cmd.Printf("\nStored states:\n\n")
+	for _, state := range resp.States {
+		cmd.Printf("- %s\n", state.Name)
+	}
+
+	return nil
+}
+
+func stateClean(cmd *cobra.Command, args []string) error {
+	var stateName string
+	if !allFlag {
+		stateName = args[0]
+	}
+
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	client := proto.NewDaemonServiceClient(conn)
+	resp, err := client.CleanState(cmd.Context(), &proto.CleanStateRequest{
+		StateName: stateName,
+		All:       allFlag,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to clean state: %v", status.Convert(err).Message())
+	}
+
+	if resp.CleanedStates == 0 {
+		cmd.Println("No states were cleaned")
+		return nil
+	}
+
+	if allFlag {
+		cmd.Printf("Successfully cleaned %d states\n", resp.CleanedStates)
+	} else {
+		cmd.Printf("Successfully cleaned state %q\n", stateName)
+	}
+
+	return nil
+}
+
+func stateDelete(cmd *cobra.Command, args []string) error {
+	var stateName string
+	if !allFlag {
+		stateName = args[0]
+	}
+
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	client := proto.NewDaemonServiceClient(conn)
+	resp, err := client.DeleteState(cmd.Context(), &proto.DeleteStateRequest{
+		StateName: stateName,
+		All:       allFlag,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to delete state: %v", status.Convert(err).Message())
+	}
+
+	if resp.DeletedStates == 0 {
+		cmd.Println("No states were deleted")
+		return nil
+	}
+
+	if allFlag {
+		cmd.Printf("Successfully deleted %d states\n", resp.DeletedStates)
+	} else {
+		cmd.Printf("Successfully deleted state %q\n", stateName)
+	}
+
+	return nil
+}

client/cmd/status.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"net"
 	"net/netip"
+	"os"
+	"runtime"
 	"sort"
 	"strings"
 	"time"
@@ -14,6 +16,7 @@ import (
 	"google.golang.org/grpc/status"
 	"gopkg.in/yaml.v3"
 
+	"github.com/netbirdio/netbird/client/anonymize"
 	"github.com/netbirdio/netbird/client/internal"
 	"github.com/netbirdio/netbird/client/internal/peer"
 	"github.com/netbirdio/netbird/client/proto"
@@ -22,14 +25,21 @@ import (
 )
 
 type peerStateDetailOutput struct {
-	FQDN             string           `json:"fqdn" yaml:"fqdn"`
+	FQDN                   string           `json:"fqdn" yaml:"fqdn"`
-	IP               string           `json:"netbirdIp" yaml:"netbirdIp"`
+	IP                     string           `json:"netbirdIp" yaml:"netbirdIp"`
-	PubKey           string           `json:"publicKey" yaml:"publicKey"`
+	PubKey                 string           `json:"publicKey" yaml:"publicKey"`
-	Status           string           `json:"status" yaml:"status"`
+	Status                 string           `json:"status" yaml:"status"`
-	LastStatusUpdate time.Time        `json:"lastStatusUpdate" yaml:"lastStatusUpdate"`
+	LastStatusUpdate       time.Time        `json:"lastStatusUpdate" yaml:"lastStatusUpdate"`
-	ConnType         string           `json:"connectionType" yaml:"connectionType"`
+	ConnType               string           `json:"connectionType" yaml:"connectionType"`
-	Direct           bool             `json:"direct" yaml:"direct"`
+	IceCandidateType       iceCandidateType `json:"iceCandidateType" yaml:"iceCandidateType"`
-	IceCandidateType iceCandidateType `json:"iceCandidateType" yaml:"iceCandidateType"`
+	IceCandidateEndpoint   iceCandidateType `json:"iceCandidateEndpoint" yaml:"iceCandidateEndpoint"`
+	RelayAddress           string           `json:"relayAddress" yaml:"relayAddress"`
+	LastWireguardHandshake time.Time        `json:"lastWireguardHandshake" yaml:"lastWireguardHandshake"`
+	TransferReceived       int64            `json:"transferReceived" yaml:"transferReceived"`
+	TransferSent           int64            `json:"transferSent" yaml:"transferSent"`
+	Latency                time.Duration    `json:"latency" yaml:"latency"`
+	RosenpassEnabled       bool             `json:"quantumResistance" yaml:"quantumResistance"`
+	Routes                 []string         `json:"routes" yaml:"routes"`
 }
 
 type peersStateOutput struct {
@@ -41,11 +51,25 @@ type peersStateOutput struct {
 type signalStateOutput struct {
 	URL       string `json:"url" yaml:"url"`
 	Connected bool   `json:"connected" yaml:"connected"`
+	Error     string `json:"error" yaml:"error"`
 }
 
 type managementStateOutput struct {
 	URL       string `json:"url" yaml:"url"`
 	Connected bool   `json:"connected" yaml:"connected"`
+	Error     string `json:"error" yaml:"error"`
+}
+
+type relayStateOutputDetail struct {
+	URI       string `json:"uri" yaml:"uri"`
+	Available bool   `json:"available" yaml:"available"`
+	Error     string `json:"error" yaml:"error"`
+}
+
+type relayStateOutput struct {
+	Total     int                      `json:"total" yaml:"total"`
+	Available int                      `json:"available" yaml:"available"`
+	Details   []relayStateOutputDetail `json:"details" yaml:"details"`
 }
 
 type iceCandidateType struct {
@@ -53,26 +77,40 @@ type iceCandidateType struct {
 	Remote string `json:"remote" yaml:"remote"`
 }
 
+type nsServerGroupStateOutput struct {
+	Servers []string `json:"servers" yaml:"servers"`
+	Domains []string `json:"domains" yaml:"domains"`
+	Enabled bool     `json:"enabled" yaml:"enabled"`
+	Error   string   `json:"error" yaml:"error"`
+}
+
 type statusOutputOverview struct {
-	Peers           peersStateOutput      `json:"peers" yaml:"peers"`
+	Peers               peersStateOutput           `json:"peers" yaml:"peers"`
-	CliVersion      string                `json:"cliVersion" yaml:"cliVersion"`
+	CliVersion          string                     `json:"cliVersion" yaml:"cliVersion"`
-	DaemonVersion   string                `json:"daemonVersion" yaml:"daemonVersion"`
+	DaemonVersion       string                     `json:"daemonVersion" yaml:"daemonVersion"`
-	ManagementState managementStateOutput `json:"management" yaml:"management"`
+	ManagementState     managementStateOutput      `json:"management" yaml:"management"`
-	SignalState     signalStateOutput     `json:"signal" yaml:"signal"`
+	SignalState         signalStateOutput          `json:"signal" yaml:"signal"`
-	IP              string                `json:"netbirdIp" yaml:"netbirdIp"`
+	Relays              relayStateOutput           `json:"relays" yaml:"relays"`
-	PubKey          string                `json:"publicKey" yaml:"publicKey"`
+	IP                  string                     `json:"netbirdIp" yaml:"netbirdIp"`
-	KernelInterface bool                  `json:"usesKernelInterface" yaml:"usesKernelInterface"`
+	PubKey              string                     `json:"publicKey" yaml:"publicKey"`
-	FQDN            string                `json:"fqdn" yaml:"fqdn"`
+	KernelInterface     bool                       `json:"usesKernelInterface" yaml:"usesKernelInterface"`
+	FQDN                string                     `json:"fqdn" yaml:"fqdn"`
+	RosenpassEnabled    bool                       `json:"quantumResistance" yaml:"quantumResistance"`
+	RosenpassPermissive bool                       `json:"quantumResistancePermissive" yaml:"quantumResistancePermissive"`
+	Routes              []string                   `json:"routes" yaml:"routes"`
+	NSServerGroups      []nsServerGroupStateOutput `json:"dnsServers" yaml:"dnsServers"`
 }
 
 var (
-	detailFlag   bool
+	detailFlag           bool
-	ipv4Flag     bool
+	ipv4Flag             bool
-	jsonFlag     bool
+	jsonFlag             bool
-	yamlFlag     bool
+	yamlFlag             bool
-	ipsFilter    []string
+	ipsFilter            []string
-	statusFilter string
+	prefixNamesFilter    []string
-	ipsFilterMap map[string]struct{}
+	statusFilter         string
+	ipsFilterMap         map[string]struct{}
+	prefixNamesFilterMap map[string]struct{}
 )
 
 var statusCmd = &cobra.Command{
@@ -83,12 +121,14 @@ var statusCmd = &cobra.Command{
 
 func init() {
 	ipsFilterMap = make(map[string]struct{})
+	prefixNamesFilterMap = make(map[string]struct{})
 	statusCmd.PersistentFlags().BoolVarP(&detailFlag, "detail", "d", false, "display detailed status information in human-readable format")
 	statusCmd.PersistentFlags().BoolVar(&jsonFlag, "json", false, "display detailed status information in json format")
 	statusCmd.PersistentFlags().BoolVar(&yamlFlag, "yaml", false, "display detailed status information in yaml format")
 	statusCmd.PersistentFlags().BoolVar(&ipv4Flag, "ipv4", false, "display only NetBird IPv4 of this peer, e.g., --ipv4 will output 100.64.0.33")
 	statusCmd.MarkFlagsMutuallyExclusive("detail", "json", "yaml", "ipv4")
 	statusCmd.PersistentFlags().StringSliceVar(&ipsFilter, "filter-by-ips", []string{}, "filters the detailed output by a list of one or more IPs, e.g., --filter-by-ips 100.64.0.100,100.64.0.200")
+	statusCmd.PersistentFlags().StringSliceVar(&prefixNamesFilter, "filter-by-names", []string{}, "filters the detailed output by a list of one or more peer FQDN or hostnames, e.g., --filter-by-names peer-a,peer-b.netbird.cloud")
 	statusCmd.PersistentFlags().StringVar(&statusFilter, "filter-by-status", "", "filters the detailed output by connection status(connected|disconnected), e.g., --filter-by-status connected")
 }
 
@@ -107,11 +147,11 @@ func statusFunc(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed initializing log %v", err)
 	}
 
-	ctx := internal.CtxInitState(context.Background())
+	ctx := internal.CtxInitState(cmd.Context())
 
-	resp, _ := getStatus(ctx, cmd)
+	resp, err := getStatus(ctx)
 	if err != nil {
-		return nil
+		return err
 	}
 
 	if resp.GetStatus() == string(internal.StatusNeedsLogin) || resp.GetStatus() == string(internal.StatusLoginFailed) {
@@ -120,7 +160,7 @@ func statusFunc(cmd *cobra.Command, args []string) error {
 			" netbird up \n\n"+
 			"If you are running a self-hosted version and no SSO provider has been configured in your Management Server,\n"+
 			"you can use a setup-key:\n\n netbird up --management-url <YOUR_MANAGEMENT_URL> --setup-key <YOUR_SETUP_KEY>\n\n"+
-			"More info: https://www.netbird.io/docs/overview/setup-keys\n\n",
+			"More info: https://docs.netbird.io/how-to/register-machines-using-setup-keys\n\n",
 			resp.GetStatus(),
 		)
 		return nil
@@ -133,7 +173,7 @@ func statusFunc(cmd *cobra.Command, args []string) error {
 
 	outputInformationHolder := convertToStatusOutputOverview(resp)
 
-	statusOutputString := ""
+	var statusOutputString string
 	switch {
 	case detailFlag:
 		statusOutputString = parseToFullDetailSummary(outputInformationHolder)
@@ -142,7 +182,7 @@ func statusFunc(cmd *cobra.Command, args []string) error {
 	case yamlFlag:
 		statusOutputString, err = parseToYAML(outputInformationHolder)
 	default:
-		statusOutputString = parseGeneralSummary(outputInformationHolder, false)
+		statusOutputString = parseGeneralSummary(outputInformationHolder, false, false, false)
 	}
 
 	if err != nil {
@@ -154,7 +194,7 @@ func statusFunc(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func getStatus(ctx context.Context, cmd *cobra.Command) (*proto.StatusResponse, error) {
+func getStatus(ctx context.Context) (*proto.StatusResponse, error) {
 	conn, err := DialClientGRPCServer(ctx, daemonAddr)
 	if err != nil {
 		return nil, fmt.Errorf("failed to connect to daemon error: %v\n"+
@@ -163,7 +203,7 @@ func getStatus(ctx context.Context, cmd *cobra.Command) (*proto.StatusResponse,
 	}
 	defer conn.Close()
 
-	resp, err := proto.NewDaemonServiceClient(conn).Status(cmd.Context(), &proto.StatusRequest{GetFullPeerStatus: true})
+	resp, err := proto.NewDaemonServiceClient(conn).Status(ctx, &proto.StatusRequest{GetFullPeerStatus: true})
 	if err != nil {
 		return nil, fmt.Errorf("status failed: %v", status.Convert(err).Message())
 	}
@@ -172,8 +212,12 @@ func getStatus(ctx context.Context, cmd *cobra.Command) (*proto.StatusResponse,
 }
 
 func parseFilters() error {
+
 	switch strings.ToLower(statusFilter) {
 	case "", "disconnected", "connected":
+		if strings.ToLower(statusFilter) != "" {
+			enableDetailFlagWhenFilterFlag()
+		}
 	default:
 		return fmt.Errorf("wrong status filter, should be one of connected|disconnected, got: %s", statusFilter)
 	}
@@ -185,11 +229,26 @@ func parseFilters() error {
 				return fmt.Errorf("got an invalid IP address in the filter: address %s, error %s", addr, err)
 			}
 			ipsFilterMap[addr] = struct{}{}
+			enableDetailFlagWhenFilterFlag()
 		}
 	}
+
+	if len(prefixNamesFilter) > 0 {
+		for _, name := range prefixNamesFilter {
+			prefixNamesFilterMap[strings.ToLower(name)] = struct{}{}
+		}
+		enableDetailFlagWhenFilterFlag()
+	}
+
 	return nil
 }
 
+func enableDetailFlagWhenFilterFlag() {
+	if !detailFlag && !jsonFlag && !yamlFlag {
+		detailFlag = true
+	}
+}
+
 func convertToStatusOutputOverview(resp *proto.StatusResponse) statusOutputOverview {
 	pbFullStatus := resp.GetFullStatus()
 
@@ -197,51 +256,116 @@ func convertToStatusOutputOverview(resp *proto.StatusResponse) statusOutputOverv
 	managementOverview := managementStateOutput{
 		URL:       managementState.GetURL(),
 		Connected: managementState.GetConnected(),
+		Error:     managementState.Error,
 	}
 
 	signalState := pbFullStatus.GetSignalState()
 	signalOverview := signalStateOutput{
 		URL:       signalState.GetURL(),
 		Connected: signalState.GetConnected(),
+		Error:     signalState.Error,
 	}
 
+	relayOverview := mapRelays(pbFullStatus.GetRelays())
 	peersOverview := mapPeers(resp.GetFullStatus().GetPeers())
 
 	overview := statusOutputOverview{
-		Peers:           peersOverview,
+		Peers:               peersOverview,
-		CliVersion:      version.NetbirdVersion(),
+		CliVersion:          version.NetbirdVersion(),
-		DaemonVersion:   resp.GetDaemonVersion(),
+		DaemonVersion:       resp.GetDaemonVersion(),
-		ManagementState: managementOverview,
+		ManagementState:     managementOverview,
-		SignalState:     signalOverview,
+		SignalState:         signalOverview,
-		IP:              pbFullStatus.GetLocalPeerState().GetIP(),
+		Relays:              relayOverview,
-		PubKey:          pbFullStatus.GetLocalPeerState().GetPubKey(),
+		IP:                  pbFullStatus.GetLocalPeerState().GetIP(),
-		KernelInterface: pbFullStatus.GetLocalPeerState().GetKernelInterface(),
+		PubKey:              pbFullStatus.GetLocalPeerState().GetPubKey(),
-		FQDN:            pbFullStatus.GetLocalPeerState().GetFqdn(),
+		KernelInterface:     pbFullStatus.GetLocalPeerState().GetKernelInterface(),
+		FQDN:                pbFullStatus.GetLocalPeerState().GetFqdn(),
+		RosenpassEnabled:    pbFullStatus.GetLocalPeerState().GetRosenpassEnabled(),
+		RosenpassPermissive: pbFullStatus.GetLocalPeerState().GetRosenpassPermissive(),
+		Routes:              pbFullStatus.GetLocalPeerState().GetRoutes(),
+		NSServerGroups:      mapNSGroups(pbFullStatus.GetDnsServers()),
+	}
+
+	if anonymizeFlag {
+		anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses())
+		anonymizeOverview(anonymizer, &overview)
 	}
 
 	return overview
 }
 
+func mapRelays(relays []*proto.RelayState) relayStateOutput {
+	var relayStateDetail []relayStateOutputDetail
+
+	var relaysAvailable int
+	for _, relay := range relays {
+		available := relay.GetAvailable()
+		relayStateDetail = append(relayStateDetail,
+			relayStateOutputDetail{
+				URI:       relay.URI,
+				Available: available,
+				Error:     relay.GetError(),
+			},
+		)
+
+		if available {
+			relaysAvailable++
+		}
+	}
+
+	return relayStateOutput{
+		Total:     len(relays),
+		Available: relaysAvailable,
+		Details:   relayStateDetail,
+	}
+}
+
+func mapNSGroups(servers []*proto.NSGroupState) []nsServerGroupStateOutput {
+	mappedNSGroups := make([]nsServerGroupStateOutput, 0, len(servers))
+	for _, pbNsGroupServer := range servers {
+		mappedNSGroups = append(mappedNSGroups, nsServerGroupStateOutput{
+			Servers: pbNsGroupServer.GetServers(),
+			Domains: pbNsGroupServer.GetDomains(),
+			Enabled: pbNsGroupServer.GetEnabled(),
+			Error:   pbNsGroupServer.GetError(),
+		})
+	}
+	return mappedNSGroups
+}
+
 func mapPeers(peers []*proto.PeerState) peersStateOutput {
 	var peersStateDetail []peerStateDetailOutput
-	localICE := ""
-	remoteICE := ""
-	connType := ""
 	peersConnected := 0
 	for _, pbPeerState := range peers {
+		localICE := ""
+		remoteICE := ""
+		localICEEndpoint := ""
+		remoteICEEndpoint := ""
+		relayServerAddress := ""
+		connType := ""
+		lastHandshake := time.Time{}
+		transferReceived := int64(0)
+		transferSent := int64(0)
+
 		isPeerConnected := pbPeerState.ConnStatus == peer.StatusConnected.String()
 		if skipDetailByFilters(pbPeerState, isPeerConnected) {
 			continue
 		}
 		if isPeerConnected {
-			peersConnected = peersConnected + 1
+			peersConnected++
 
 			localICE = pbPeerState.GetLocalIceCandidateType()
 			remoteICE = pbPeerState.GetRemoteIceCandidateType()
+			localICEEndpoint = pbPeerState.GetLocalIceCandidateEndpoint()
+			remoteICEEndpoint = pbPeerState.GetRemoteIceCandidateEndpoint()
 			connType = "P2P"
 			if pbPeerState.Relayed {
 				connType = "Relayed"
 			}
+			relayServerAddress = pbPeerState.GetRelayAddress()
+			lastHandshake = pbPeerState.GetLastWireguardHandshake().AsTime().Local()
+			transferReceived = pbPeerState.GetBytesRx()
+			transferSent = pbPeerState.GetBytesTx()
 		}
 
 		timeLocal := pbPeerState.GetConnStatusUpdate().AsTime().Local()
@@ -249,14 +373,24 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 			IP:               pbPeerState.GetIP(),
 			PubKey:           pbPeerState.GetPubKey(),
 			Status:           pbPeerState.GetConnStatus(),
-			LastStatusUpdate: timeLocal.UTC(),
+			LastStatusUpdate: timeLocal,
 			ConnType:         connType,
-			Direct:           pbPeerState.GetDirect(),
 			IceCandidateType: iceCandidateType{
 				Local:  localICE,
 				Remote: remoteICE,
 			},
-			FQDN: pbPeerState.GetFqdn(),
+			IceCandidateEndpoint: iceCandidateType{
+				Local:  localICEEndpoint,
+				Remote: remoteICEEndpoint,
+			},
+			RelayAddress:           relayServerAddress,
+			FQDN:                   pbPeerState.GetFqdn(),
+			LastWireguardHandshake: lastHandshake,
+			TransferReceived:       transferReceived,
+			TransferSent:           transferSent,
+			Latency:                pbPeerState.GetLatency().AsDuration(),
+			RosenpassEnabled:       pbPeerState.GetRosenpassEnabled(),
+			Routes:                 pbPeerState.GetRoutes(),
 		}
 
 		peersStateDetail = append(peersStateDetail, peerState)
@@ -306,22 +440,31 @@ func parseToYAML(overview statusOutputOverview) (string, error) {
 	return string(yamlBytes), nil
 }
 
-func parseGeneralSummary(overview statusOutputOverview, showURL bool) string {
+func parseGeneralSummary(overview statusOutputOverview, showURL bool, showRelays bool, showNameServers bool) string {
-
+	var managementConnString string
-	managementConnString := "Disconnected"
 	if overview.ManagementState.Connected {
 		managementConnString = "Connected"
 		if showURL {
 			managementConnString = fmt.Sprintf("%s to %s", managementConnString, overview.ManagementState.URL)
 		}
+	} else {
+		managementConnString = "Disconnected"
+		if overview.ManagementState.Error != "" {
+			managementConnString = fmt.Sprintf("%s, reason: %s", managementConnString, overview.ManagementState.Error)
+		}
 	}
 
-	signalConnString := "Disconnected"
+	var signalConnString string
 	if overview.SignalState.Connected {
 		signalConnString = "Connected"
 		if showURL {
 			signalConnString = fmt.Sprintf("%s to %s", signalConnString, overview.SignalState.URL)
 		}
+	} else {
+		signalConnString = "Disconnected"
+		if overview.SignalState.Error != "" {
+			signalConnString = fmt.Sprintf("%s, reason: %s", signalConnString, overview.SignalState.Error)
+		}
 	}
 
 	interfaceTypeString := "Userspace"
@@ -333,32 +476,107 @@ func parseGeneralSummary(overview statusOutputOverview, showURL bool) string {
 		interfaceIP = "N/A"
 	}
 
+	var relaysString string
+	if showRelays {
+		for _, relay := range overview.Relays.Details {
+			available := "Available"
+			reason := ""
+			if !relay.Available {
+				available = "Unavailable"
+				reason = fmt.Sprintf(", reason: %s", relay.Error)
+			}
+			relaysString += fmt.Sprintf("\n  [%s] is %s%s", relay.URI, available, reason)
+		}
+	} else {
+		relaysString = fmt.Sprintf("%d/%d Available", overview.Relays.Available, overview.Relays.Total)
+	}
+
+	routes := "-"
+	if len(overview.Routes) > 0 {
+		sort.Strings(overview.Routes)
+		routes = strings.Join(overview.Routes, ", ")
+	}
+
+	var dnsServersString string
+	if showNameServers {
+		for _, nsServerGroup := range overview.NSServerGroups {
+			enabled := "Available"
+			if !nsServerGroup.Enabled {
+				enabled = "Unavailable"
+			}
+			errorString := ""
+			if nsServerGroup.Error != "" {
+				errorString = fmt.Sprintf(", reason: %s", nsServerGroup.Error)
+				errorString = strings.TrimSpace(errorString)
+			}
+
+			domainsString := strings.Join(nsServerGroup.Domains, ", ")
+			if domainsString == "" {
+				domainsString = "." // Show "." for the default zone
+			}
+			dnsServersString += fmt.Sprintf(
+				"\n  [%s] for [%s] is %s%s",
+				strings.Join(nsServerGroup.Servers, ", "),
+				domainsString,
+				enabled,
+				errorString,
+			)
+		}
+	} else {
+		dnsServersString = fmt.Sprintf("%d/%d Available", countEnabled(overview.NSServerGroups), len(overview.NSServerGroups))
+	}
+
+	rosenpassEnabledStatus := "false"
+	if overview.RosenpassEnabled {
+		rosenpassEnabledStatus = "true"
+		if overview.RosenpassPermissive {
+			rosenpassEnabledStatus = "true (permissive)" //nolint:gosec
+		}
+	}
+
 	peersCountString := fmt.Sprintf("%d/%d Connected", overview.Peers.Connected, overview.Peers.Total)
 
+	goos := runtime.GOOS
+	goarch := runtime.GOARCH
+	goarm := ""
+	if goarch == "arm" {
+		goarm = fmt.Sprintf(" (ARMv%s)", os.Getenv("GOARM"))
+	}
+
 	summary := fmt.Sprintf(
-		"Daemon version: %s\n"+
+		"OS: %s\n"+
+			"Daemon version: %s\n"+
 			"CLI version: %s\n"+
 			"Management: %s\n"+
 			"Signal: %s\n"+
+			"Relays: %s\n"+
+			"Nameservers: %s\n"+
 			"FQDN: %s\n"+
 			"NetBird IP: %s\n"+
 			"Interface type: %s\n"+
+			"Quantum resistance: %s\n"+
+			"Routes: %s\n"+
 			"Peers count: %s\n",
+		fmt.Sprintf("%s/%s%s", goos, goarch, goarm),
 		overview.DaemonVersion,
 		version.NetbirdVersion(),
 		managementConnString,
 		signalConnString,
+		relaysString,
+		dnsServersString,
 		overview.FQDN,
 		interfaceIP,
 		interfaceTypeString,
+		rosenpassEnabledStatus,
+		routes,
 		peersCountString,
 	)
 	return summary
 }
 
 func parseToFullDetailSummary(overview statusOutputOverview) string {
-	parsedPeersString := parsePeers(overview.Peers)
+	parsedPeersString := parsePeers(overview.Peers, overview.RosenpassEnabled, overview.RosenpassPermissive)
-	summary := parseGeneralSummary(overview, true)
+	summary := parseGeneralSummary(overview, true, true, true)
 
 	return fmt.Sprintf(
 		"Peers detail:"+
@@ -369,7 +587,7 @@ func parseToFullDetailSummary(overview statusOutputOverview) string {
 	)
 }
 
-func parsePeers(peers peersStateOutput) string {
+func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bool) string {
 	var (
 		peersString = ""
 	)
@@ -386,6 +604,39 @@ func parsePeers(peers peersStateOutput) string {
 			remoteICE = peerState.IceCandidateType.Remote
 		}
 
+		localICEEndpoint := "-"
+		if peerState.IceCandidateEndpoint.Local != "" {
+			localICEEndpoint = peerState.IceCandidateEndpoint.Local
+		}
+
+		remoteICEEndpoint := "-"
+		if peerState.IceCandidateEndpoint.Remote != "" {
+			remoteICEEndpoint = peerState.IceCandidateEndpoint.Remote
+		}
+
+		rosenpassEnabledStatus := "false"
+		if rosenpassEnabled {
+			if peerState.RosenpassEnabled {
+				rosenpassEnabledStatus = "true"
+			} else {
+				if rosenpassPermissive {
+					rosenpassEnabledStatus = "false (remote didn't enable quantum resistance)"
+				} else {
+					rosenpassEnabledStatus = "false (connection won't work without a permissive mode)"
+				}
+			}
+		} else {
+			if peerState.RosenpassEnabled {
+				rosenpassEnabledStatus = "false (connection might not work without a remote permissive mode)"
+			}
+		}
+
+		routes := "-"
+		if len(peerState.Routes) > 0 {
+			sort.Strings(peerState.Routes)
+			routes = strings.Join(peerState.Routes, ", ")
+		}
+
 		peerString := fmt.Sprintf(
 			"\n %s:\n"+
 				"  NetBird IP: %s\n"+
@@ -393,21 +644,35 @@ func parsePeers(peers peersStateOutput) string {
 				"  Status: %s\n"+
 				"  -- detail --\n"+
 				"  Connection type: %s\n"+
-				"  Direct: %t\n"+
 				"  ICE candidate (Local/Remote): %s/%s\n"+
-				"  Last connection update: %s\n",
+				"  ICE candidate endpoints (Local/Remote): %s/%s\n"+
+				"  Relay server address: %s\n"+
+				"  Last connection update: %s\n"+
+				"  Last WireGuard handshake: %s\n"+
+				"  Transfer status (received/sent) %s/%s\n"+
+				"  Quantum resistance: %s\n"+
+				"  Routes: %s\n"+
+				"  Latency: %s\n",
 			peerState.FQDN,
 			peerState.IP,
 			peerState.PubKey,
 			peerState.Status,
 			peerState.ConnType,
-			peerState.Direct,
 			localICE,
 			remoteICE,
-			peerState.LastStatusUpdate.Format("2006-01-02 15:04:05"),
+			localICEEndpoint,
+			remoteICEEndpoint,
+			peerState.RelayAddress,
+			timeAgo(peerState.LastStatusUpdate),
+			timeAgo(peerState.LastWireguardHandshake),
+			toIEC(peerState.TransferReceived),
+			toIEC(peerState.TransferSent),
+			rosenpassEnabledStatus,
+			routes,
+			peerState.Latency.String(),
 		)
 
-		peersString = peersString + peerString
+		peersString += peerString
 	}
 	return peersString
 }
@@ -415,6 +680,7 @@ func parsePeers(peers peersStateOutput) string {
 func skipDetailByFilters(peerState *proto.PeerState, isConnected bool) bool {
 	statusEval := false
 	ipEval := false
+	nameEval := true
 
 	if statusFilter != "" {
 		lowerStatusFilter := strings.ToLower(statusFilter)
@@ -431,5 +697,162 @@ func skipDetailByFilters(peerState *proto.PeerState, isConnected bool) bool {
 			ipEval = true
 		}
 	}
-	return statusEval || ipEval
+
+	if len(prefixNamesFilter) > 0 {
+		for prefixNameFilter := range prefixNamesFilterMap {
+			if strings.HasPrefix(peerState.Fqdn, prefixNameFilter) {
+				nameEval = false
+				break
+			}
+		}
+	} else {
+		nameEval = false
+	}
+
+	return statusEval || ipEval || nameEval
+}
+
+func toIEC(b int64) string {
+	const unit = 1024
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB",
+		float64(b)/float64(div), "KMGTPE"[exp])
+}
+
+func countEnabled(dnsServers []nsServerGroupStateOutput) int {
+	count := 0
+	for _, server := range dnsServers {
+		if server.Enabled {
+			count++
+		}
+	}
+	return count
+}
+
+// timeAgo returns a string representing the duration since the provided time in a human-readable format.
+func timeAgo(t time.Time) string {
+	if t.IsZero() || t.Equal(time.Unix(0, 0)) {
+		return "-"
+	}
+	duration := time.Since(t)
+	switch {
+	case duration < time.Second:
+		return "Now"
+	case duration < time.Minute:
+		seconds := int(duration.Seconds())
+		if seconds == 1 {
+			return "1 second ago"
+		}
+		return fmt.Sprintf("%d seconds ago", seconds)
+	case duration < time.Hour:
+		minutes := int(duration.Minutes())
+		seconds := int(duration.Seconds()) % 60
+		if minutes == 1 {
+			if seconds == 1 {
+				return "1 minute, 1 second ago"
+			} else if seconds > 0 {
+				return fmt.Sprintf("1 minute, %d seconds ago", seconds)
+			}
+			return "1 minute ago"
+		}
+		if seconds > 0 {
+			return fmt.Sprintf("%d minutes, %d seconds ago", minutes, seconds)
+		}
+		return fmt.Sprintf("%d minutes ago", minutes)
+	case duration < 24*time.Hour:
+		hours := int(duration.Hours())
+		minutes := int(duration.Minutes()) % 60
+		if hours == 1 {
+			if minutes == 1 {
+				return "1 hour, 1 minute ago"
+			} else if minutes > 0 {
+				return fmt.Sprintf("1 hour, %d minutes ago", minutes)
+			}
+			return "1 hour ago"
+		}
+		if minutes > 0 {
+			return fmt.Sprintf("%d hours, %d minutes ago", hours, minutes)
+		}
+		return fmt.Sprintf("%d hours ago", hours)
+	}
+
+	days := int(duration.Hours()) / 24
+	hours := int(duration.Hours()) % 24
+	if days == 1 {
+		if hours == 1 {
+			return "1 day, 1 hour ago"
+		} else if hours > 0 {
+			return fmt.Sprintf("1 day, %d hours ago", hours)
+		}
+		return "1 day ago"
+	}
+	if hours > 0 {
+		return fmt.Sprintf("%d days, %d hours ago", days, hours)
+	}
+	return fmt.Sprintf("%d days ago", days)
+}
+
+func anonymizePeerDetail(a *anonymize.Anonymizer, peer *peerStateDetailOutput) {
+	peer.FQDN = a.AnonymizeDomain(peer.FQDN)
+	if localIP, port, err := net.SplitHostPort(peer.IceCandidateEndpoint.Local); err == nil {
+		peer.IceCandidateEndpoint.Local = fmt.Sprintf("%s:%s", a.AnonymizeIPString(localIP), port)
+	}
+	if remoteIP, port, err := net.SplitHostPort(peer.IceCandidateEndpoint.Remote); err == nil {
+		peer.IceCandidateEndpoint.Remote = fmt.Sprintf("%s:%s", a.AnonymizeIPString(remoteIP), port)
+	}
+
+	peer.RelayAddress = a.AnonymizeURI(peer.RelayAddress)
+
+	for i, route := range peer.Routes {
+		peer.Routes[i] = a.AnonymizeIPString(route)
+	}
+
+	for i, route := range peer.Routes {
+		peer.Routes[i] = a.AnonymizeRoute(route)
+	}
+}
+
+func anonymizeOverview(a *anonymize.Anonymizer, overview *statusOutputOverview) {
+	for i, peer := range overview.Peers.Details {
+		peer := peer
+		anonymizePeerDetail(a, &peer)
+		overview.Peers.Details[i] = peer
+	}
+
+	overview.ManagementState.URL = a.AnonymizeURI(overview.ManagementState.URL)
+	overview.ManagementState.Error = a.AnonymizeString(overview.ManagementState.Error)
+	overview.SignalState.URL = a.AnonymizeURI(overview.SignalState.URL)
+	overview.SignalState.Error = a.AnonymizeString(overview.SignalState.Error)
+
+	overview.IP = a.AnonymizeIPString(overview.IP)
+	for i, detail := range overview.Relays.Details {
+		detail.URI = a.AnonymizeURI(detail.URI)
+		detail.Error = a.AnonymizeString(detail.Error)
+		overview.Relays.Details[i] = detail
+	}
+
+	for i, nsGroup := range overview.NSServerGroups {
+		for j, domain := range nsGroup.Domains {
+			overview.NSServerGroups[i].Domains[j] = a.AnonymizeDomain(domain)
+		}
+		for j, ns := range nsGroup.Servers {
+			host, port, err := net.SplitHostPort(ns)
+			if err == nil {
+				overview.NSServerGroups[i].Servers[j] = fmt.Sprintf("%s:%s", a.AnonymizeIPString(host), port)
+			}
+		}
+	}
+
+	for i, route := range overview.Routes {
+		overview.Routes[i] = a.AnonymizeRoute(route)
+	}
+
+	overview.FQDN = a.AnonymizeDomain(overview.FQDN)
 }

client/cmd/status_test.go

@@ -1,56 +1,123 @@
 package cmd
 
 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"runtime"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/netbirdio/netbird/client/proto"
 	"github.com/netbirdio/netbird/version"
 )
 
+func init() {
+	loc, err := time.LoadLocation("UTC")
+	if err != nil {
+		panic(err)
+	}
+
+	time.Local = loc
+}
+
 var resp = &proto.StatusResponse{
 	Status: "Connected",
 	FullStatus: &proto.FullStatus{
 		Peers: []*proto.PeerState{
 			{
-				IP:                     "192.168.178.101",
+				IP:                         "192.168.178.101",
-				PubKey:                 "Pubkey1",
+				PubKey:                     "Pubkey1",
-				Fqdn:                   "peer-1.awesome-domain.com",
+				Fqdn:                       "peer-1.awesome-domain.com",
-				ConnStatus:             "Connected",
+				ConnStatus:                 "Connected",
-				ConnStatusUpdate:       timestamppb.New(time.Date(2001, time.Month(1), 1, 1, 1, 1, 0, time.UTC)),
+				ConnStatusUpdate:           timestamppb.New(time.Date(2001, time.Month(1), 1, 1, 1, 1, 0, time.UTC)),
-				Relayed:                false,
+				Relayed:                    false,
-				Direct:                 true,
+				LocalIceCandidateType:      "",
-				LocalIceCandidateType:  "",
+				RemoteIceCandidateType:     "",
-				RemoteIceCandidateType: "",
+				LocalIceCandidateEndpoint:  "",
+				RemoteIceCandidateEndpoint: "",
+				LastWireguardHandshake:     timestamppb.New(time.Date(2001, time.Month(1), 1, 1, 1, 2, 0, time.UTC)),
+				BytesRx:                    200,
+				BytesTx:                    100,
+				Routes: []string{
+					"10.1.0.0/24",
+				},
+				Latency: durationpb.New(time.Duration(10000000)),
 			},
 			{
-				IP:                     "192.168.178.102",
+				IP:                         "192.168.178.102",
-				PubKey:                 "Pubkey2",
+				PubKey:                     "Pubkey2",
-				Fqdn:                   "peer-2.awesome-domain.com",
+				Fqdn:                       "peer-2.awesome-domain.com",
-				ConnStatus:             "Connected",
+				ConnStatus:                 "Connected",
-				ConnStatusUpdate:       timestamppb.New(time.Date(2002, time.Month(2), 2, 2, 2, 2, 0, time.UTC)),
+				ConnStatusUpdate:           timestamppb.New(time.Date(2002, time.Month(2), 2, 2, 2, 2, 0, time.UTC)),
-				Relayed:                true,
+				Relayed:                    true,
-				Direct:                 false,
+				LocalIceCandidateType:      "relay",
-				LocalIceCandidateType:  "relay",
+				RemoteIceCandidateType:     "prflx",
-				RemoteIceCandidateType: "prflx",
+				LocalIceCandidateEndpoint:  "10.0.0.1:10001",
+				RemoteIceCandidateEndpoint: "10.0.10.1:10002",
+				LastWireguardHandshake:     timestamppb.New(time.Date(2002, time.Month(2), 2, 2, 2, 3, 0, time.UTC)),
+				BytesRx:                    2000,
+				BytesTx:                    1000,
+				Latency:                    durationpb.New(time.Duration(10000000)),
 			},
 		},
 		ManagementState: &proto.ManagementState{
 			URL:       "my-awesome-management.com:443",
 			Connected: true,
+			Error:     "",
 		},
 		SignalState: &proto.SignalState{
 			URL:       "my-awesome-signal.com:443",
 			Connected: true,
+			Error:     "",
+		},
+		Relays: []*proto.RelayState{
+			{
+				URI:       "stun:my-awesome-stun.com:3478",
+				Available: true,
+				Error:     "",
+			},
+			{
+				URI:       "turns:my-awesome-turn.com:443?transport=tcp",
+				Available: false,
+				Error:     "context: deadline exceeded",
+			},
 		},
 		LocalPeerState: &proto.LocalPeerState{
 			IP:              "192.168.178.100/16",
 			PubKey:          "Some-Pub-Key",
 			KernelInterface: true,
 			Fqdn:            "some-localhost.awesome-domain.com",
+			Routes: []string{
+				"10.10.0.0/24",
+			},
+		},
+		DnsServers: []*proto.NSGroupState{
+			{
+				Servers: []string{
+					"8.8.8.8:53",
+				},
+				Domains: nil,
+				Enabled: true,
+				Error:   "",
+			},
+			{
+				Servers: []string{
+					"1.1.1.1:53",
+					"2.2.2.2:53",
+				},
+				Domains: []string{
+					"example.com",
+					"example.net",
+				},
+				Enabled: false,
+				Error:   "timeout",
+			},
 		},
 	},
 	DaemonVersion: "0.14.1",
@@ -68,11 +135,21 @@ var overview = statusOutputOverview{
 				Status:           "Connected",
 				LastStatusUpdate: time.Date(2001, 1, 1, 1, 1, 1, 0, time.UTC),
 				ConnType:         "P2P",
-				Direct:           true,
 				IceCandidateType: iceCandidateType{
 					Local:  "",
 					Remote: "",
 				},
+				IceCandidateEndpoint: iceCandidateType{
+					Local:  "",
+					Remote: "",
+				},
+				LastWireguardHandshake: time.Date(2001, 1, 1, 1, 1, 2, 0, time.UTC),
+				TransferReceived:       200,
+				TransferSent:           100,
+				Routes: []string{
+					"10.1.0.0/24",
+				},
+				Latency: time.Duration(10000000),
 			},
 			{
 				IP:               "192.168.178.102",
@@ -81,11 +158,18 @@ var overview = statusOutputOverview{
 				Status:           "Connected",
 				LastStatusUpdate: time.Date(2002, 2, 2, 2, 2, 2, 0, time.UTC),
 				ConnType:         "Relayed",
-				Direct:           false,
 				IceCandidateType: iceCandidateType{
 					Local:  "relay",
 					Remote: "prflx",
 				},
+				IceCandidateEndpoint: iceCandidateType{
+					Local:  "10.0.0.1:10001",
+					Remote: "10.0.10.1:10002",
+				},
+				LastWireguardHandshake: time.Date(2002, 2, 2, 2, 2, 3, 0, time.UTC),
+				TransferReceived:       2000,
+				TransferSent:           1000,
+				Latency:                time.Duration(10000000),
 			},
 		},
 	},
@@ -94,15 +178,58 @@ var overview = statusOutputOverview{
 	ManagementState: managementStateOutput{
 		URL:       "my-awesome-management.com:443",
 		Connected: true,
+		Error:     "",
 	},
 	SignalState: signalStateOutput{
 		URL:       "my-awesome-signal.com:443",
 		Connected: true,
+		Error:     "",
+	},
+	Relays: relayStateOutput{
+		Total:     2,
+		Available: 1,
+		Details: []relayStateOutputDetail{
+			{
+				URI:       "stun:my-awesome-stun.com:3478",
+				Available: true,
+				Error:     "",
+			},
+			{
+				URI:       "turns:my-awesome-turn.com:443?transport=tcp",
+				Available: false,
+				Error:     "context: deadline exceeded",
+			},
+		},
 	},
 	IP:              "192.168.178.100/16",
 	PubKey:          "Some-Pub-Key",
 	KernelInterface: true,
 	FQDN:            "some-localhost.awesome-domain.com",
+	NSServerGroups: []nsServerGroupStateOutput{
+		{
+			Servers: []string{
+				"8.8.8.8:53",
+			},
+			Domains: nil,
+			Enabled: true,
+			Error:   "",
+		},
+		{
+			Servers: []string{
+				"1.1.1.1:53",
+				"2.2.2.2:53",
+			},
+			Domains: []string{
+				"example.com",
+				"example.net",
+			},
+			Enabled: false,
+			Error:   "timeout",
+		},
+	},
+	Routes: []string{
+		"10.10.0.0/24",
+	},
 }
 
 func TestConversionFromFullStatusToOutputOverview(t *testing.T) {
@@ -136,158 +263,309 @@ func TestSortingOfPeers(t *testing.T) {
 }
 
 func TestParsingToJSON(t *testing.T) {
-	json, _ := parseToJSON(overview)
+	jsonString, _ := parseToJSON(overview)
 
 	//@formatter:off
-	expectedJSON := "{\"" +
+	expectedJSONString := `
-		"peers\":" +
+        {
-		"{" +
+          "peers": {
-		"\"total\":2," +
+            "total": 2,
-		"\"connected\":2," +
+            "connected": 2,
-		"\"details\":" +
+            "details": [
-		"[" +
+              {
-		"{" +
+                "fqdn": "peer-1.awesome-domain.com",
-		"\"fqdn\":\"peer-1.awesome-domain.com\"," +
+                "netbirdIp": "192.168.178.101",
-		"\"netbirdIp\":\"192.168.178.101\"," +
+                "publicKey": "Pubkey1",
-		"\"publicKey\":\"Pubkey1\"," +
+                "status": "Connected",
-		"\"status\":\"Connected\"," +
+                "lastStatusUpdate": "2001-01-01T01:01:01Z",
-		"\"lastStatusUpdate\":\"2001-01-01T01:01:01Z\"," +
+                "connectionType": "P2P",
-		"\"connectionType\":\"P2P\"," +
+                "iceCandidateType": {
-		"\"direct\":true," +
+                  "local": "",
-		"\"iceCandidateType\":" +
+                  "remote": ""
-		"{" +
+                },
-		"\"local\":\"\"," +
+                "iceCandidateEndpoint": {
-		"\"remote\":\"\"" +
+                  "local": "",
-		"}" +
+                  "remote": ""
-		"}," +
+                },
-		"{" +
+				"relayAddress": "",
-		"\"fqdn\":\"peer-2.awesome-domain.com\"," +
+                "lastWireguardHandshake": "2001-01-01T01:01:02Z",
-		"\"netbirdIp\":\"192.168.178.102\"," +
+                "transferReceived": 200,
-		"\"publicKey\":\"Pubkey2\"," +
+                "transferSent": 100,
-		"\"status\":\"Connected\"," +
+				"latency": 10000000,
-		"\"lastStatusUpdate\":\"2002-02-02T02:02:02Z\"," +
+                "quantumResistance": false,
-		"\"connectionType\":\"Relayed\"," +
+                "routes": [
-		"\"direct\":false," +
+                  "10.1.0.0/24"
-		"\"iceCandidateType\":" +
+                ]
-		"{" +
+              },
-		"\"local\":\"relay\"," +
+              {
-		"\"remote\":\"prflx\"" +
+                "fqdn": "peer-2.awesome-domain.com",
-		"}" +
+                "netbirdIp": "192.168.178.102",
-		"}" +
+                "publicKey": "Pubkey2",
-		"]" +
+                "status": "Connected",
-		"}," +
+                "lastStatusUpdate": "2002-02-02T02:02:02Z",
-		"\"cliVersion\":\"development\"," +
+                "connectionType": "Relayed",
-		"\"daemonVersion\":\"0.14.1\"," +
+                "iceCandidateType": {
-		"\"management\":" +
+                  "local": "relay",
-		"{" +
+                  "remote": "prflx"
-		"\"url\":\"my-awesome-management.com:443\"," +
+                },
-		"\"connected\":true" +
+                "iceCandidateEndpoint": {
-		"}," +
+                  "local": "10.0.0.1:10001",
-		"\"signal\":" +
+                  "remote": "10.0.10.1:10002"
-		"{\"" +
+                },
-		"url\":\"my-awesome-signal.com:443\"," +
+				"relayAddress": "",
-		"\"connected\":true" +
+                "lastWireguardHandshake": "2002-02-02T02:02:03Z",
-		"}," +
+                "transferReceived": 2000,
-		"\"netbirdIp\":\"192.168.178.100/16\"," +
+                "transferSent": 1000,
-		"\"publicKey\":\"Some-Pub-Key\"," +
+				"latency": 10000000,
-		"\"usesKernelInterface\":true," +
+                "quantumResistance": false,
-		"\"fqdn\":\"some-localhost.awesome-domain.com\"" +
+                "routes": null
-		"}"
+              }
+            ]
+          },
+          "cliVersion": "development",
+          "daemonVersion": "0.14.1",
+          "management": {
+            "url": "my-awesome-management.com:443",
+            "connected": true,
+            "error": ""
+          },
+          "signal": {
+            "url": "my-awesome-signal.com:443",
+            "connected": true,
+            "error": ""
+          },
+          "relays": {
+            "total": 2,
+            "available": 1,
+            "details": [
+              {
+                "uri": "stun:my-awesome-stun.com:3478",
+                "available": true,
+                "error": ""
+              },
+              {
+                "uri": "turns:my-awesome-turn.com:443?transport=tcp",
+                "available": false,
+                "error": "context: deadline exceeded"
+              }
+            ]
+          },
+          "netbirdIp": "192.168.178.100/16",
+          "publicKey": "Some-Pub-Key",
+          "usesKernelInterface": true,
+          "fqdn": "some-localhost.awesome-domain.com",
+          "quantumResistance": false,
+          "quantumResistancePermissive": false,
+          "routes": [
+            "10.10.0.0/24"
+          ],
+          "dnsServers": [
+            {
+              "servers": [
+                "8.8.8.8:53"
+              ],
+              "domains": null,
+              "enabled": true,
+              "error": ""
+            },
+            {
+              "servers": [
+                "1.1.1.1:53",
+                "2.2.2.2:53"
+              ],
+              "domains": [
+                "example.com",
+                "example.net"
+              ],
+              "enabled": false,
+              "error": "timeout"
+            }
+          ]
+        }`
 	// @formatter:on
 
-	assert.Equal(t, expectedJSON, json)
+	var expectedJSON bytes.Buffer
+	require.NoError(t, json.Compact(&expectedJSON, []byte(expectedJSONString)))
+
+	assert.Equal(t, expectedJSON.String(), jsonString)
 }
 
 func TestParsingToYAML(t *testing.T) {
 	yaml, _ := parseToYAML(overview)
 
-	expectedYAML := "peers:\n" +
+	expectedYAML :=
-		"    total: 2\n" +
+		`peers:
-		"    connected: 2\n" +
+    total: 2
-		"    details:\n" +
+    connected: 2
-		"        - fqdn: peer-1.awesome-domain.com\n" +
+    details:
-		"          netbirdIp: 192.168.178.101\n" +
+        - fqdn: peer-1.awesome-domain.com
-		"          publicKey: Pubkey1\n" +
+          netbirdIp: 192.168.178.101
-		"          status: Connected\n" +
+          publicKey: Pubkey1
-		"          lastStatusUpdate: 2001-01-01T01:01:01Z\n" +
+          status: Connected
-		"          connectionType: P2P\n" +
+          lastStatusUpdate: 2001-01-01T01:01:01Z
-		"          direct: true\n" +
+          connectionType: P2P
-		"          iceCandidateType:\n" +
+          iceCandidateType:
-		"            local: \"\"\n" +
+            local: ""
-		"            remote: \"\"\n" +
+            remote: ""
-		"        - fqdn: peer-2.awesome-domain.com\n" +
+          iceCandidateEndpoint:
-		"          netbirdIp: 192.168.178.102\n" +
+            local: ""
-		"          publicKey: Pubkey2\n" +
+            remote: ""
-		"          status: Connected\n" +
+          relayAddress: ""
-		"          lastStatusUpdate: 2002-02-02T02:02:02Z\n" +
+          lastWireguardHandshake: 2001-01-01T01:01:02Z
-		"          connectionType: Relayed\n" +
+          transferReceived: 200
-		"          direct: false\n" +
+          transferSent: 100
-		"          iceCandidateType:\n" +
+          latency: 10ms
-		"            local: relay\n" +
+          quantumResistance: false
-		"            remote: prflx\n" +
+          routes:
-		"cliVersion: development\n" +
+            - 10.1.0.0/24
-		"daemonVersion: 0.14.1\n" +
+        - fqdn: peer-2.awesome-domain.com
-		"management:\n" +
+          netbirdIp: 192.168.178.102
-		"    url: my-awesome-management.com:443\n" +
+          publicKey: Pubkey2
-		"    connected: true\n" +
+          status: Connected
-		"signal:\n" +
+          lastStatusUpdate: 2002-02-02T02:02:02Z
-		"    url: my-awesome-signal.com:443\n" +
+          connectionType: Relayed
-		"    connected: true\n" +
+          iceCandidateType:
-		"netbirdIp: 192.168.178.100/16\n" +
+            local: relay
-		"publicKey: Some-Pub-Key\n" +
+            remote: prflx
-		"usesKernelInterface: true\n" +
+          iceCandidateEndpoint:
-		"fqdn: some-localhost.awesome-domain.com\n"
+            local: 10.0.0.1:10001
+            remote: 10.0.10.1:10002
+          relayAddress: ""
+          lastWireguardHandshake: 2002-02-02T02:02:03Z
+          transferReceived: 2000
+          transferSent: 1000
+          latency: 10ms
+          quantumResistance: false
+          routes: []
+cliVersion: development
+daemonVersion: 0.14.1
+management:
+    url: my-awesome-management.com:443
+    connected: true
+    error: ""
+signal:
+    url: my-awesome-signal.com:443
+    connected: true
+    error: ""
+relays:
+    total: 2
+    available: 1
+    details:
+        - uri: stun:my-awesome-stun.com:3478
+          available: true
+          error: ""
+        - uri: turns:my-awesome-turn.com:443?transport=tcp
+          available: false
+          error: 'context: deadline exceeded'
+netbirdIp: 192.168.178.100/16
+publicKey: Some-Pub-Key
+usesKernelInterface: true
+fqdn: some-localhost.awesome-domain.com
+quantumResistance: false
+quantumResistancePermissive: false
+routes:
+    - 10.10.0.0/24
+dnsServers:
+    - servers:
+        - 8.8.8.8:53
+      domains: []
+      enabled: true
+      error: ""
+    - servers:
+        - 1.1.1.1:53
+        - 2.2.2.2:53
+      domains:
+        - example.com
+        - example.net
+      enabled: false
+      error: timeout
+`
 
 	assert.Equal(t, expectedYAML, yaml)
 }
 
 func TestParsingToDetail(t *testing.T) {
+	// Calculate time ago based on the fixture dates
+	lastConnectionUpdate1 := timeAgo(overview.Peers.Details[0].LastStatusUpdate)
+	lastHandshake1 := timeAgo(overview.Peers.Details[0].LastWireguardHandshake)
+	lastConnectionUpdate2 := timeAgo(overview.Peers.Details[1].LastStatusUpdate)
+	lastHandshake2 := timeAgo(overview.Peers.Details[1].LastWireguardHandshake)
+
 	detail := parseToFullDetailSummary(overview)
 
-	expectedDetail := "Peers detail:\n" +
+	expectedDetail := fmt.Sprintf(
-		" peer-1.awesome-domain.com:\n" +
+		`Peers detail:
-		"  NetBird IP: 192.168.178.101\n" +
+ peer-1.awesome-domain.com:
-		"  Public key: Pubkey1\n" +
+  NetBird IP: 192.168.178.101
-		"  Status: Connected\n" +
+  Public key: Pubkey1
-		"  -- detail --\n" +
+  Status: Connected
-		"  Connection type: P2P\n" +
+  -- detail --
-		"  Direct: true\n" +
+  Connection type: P2P
-		"  ICE candidate (Local/Remote): -/-\n" +
+  ICE candidate (Local/Remote): -/-
-		"  Last connection update: 2001-01-01 01:01:01\n" +
+  ICE candidate endpoints (Local/Remote): -/-
-		"\n" +
+  Relay server address: 
-		" peer-2.awesome-domain.com:\n" +
+  Last connection update: %s
-		"  NetBird IP: 192.168.178.102\n" +
+  Last WireGuard handshake: %s
-		"  Public key: Pubkey2\n" +
+  Transfer status (received/sent) 200 B/100 B
-		"  Status: Connected\n" +
+  Quantum resistance: false
-		"  -- detail --\n" +
+  Routes: 10.1.0.0/24
-		"  Connection type: Relayed\n" +
+  Latency: 10ms
-		"  Direct: false\n" +
+
-		"  ICE candidate (Local/Remote): relay/prflx\n" +
+ peer-2.awesome-domain.com:
-		"  Last connection update: 2002-02-02 02:02:02\n" +
+  NetBird IP: 192.168.178.102
-		"\n" +
+  Public key: Pubkey2
-		"Daemon version: 0.14.1\n" +
+  Status: Connected
-		"CLI version: development\n" +
+  -- detail --
-		"Management: Connected to my-awesome-management.com:443\n" +
+  Connection type: Relayed
-		"Signal: Connected to my-awesome-signal.com:443\n" +
+  ICE candidate (Local/Remote): relay/prflx
-		"FQDN: some-localhost.awesome-domain.com\n" +
+  ICE candidate endpoints (Local/Remote): 10.0.0.1:10001/10.0.10.1:10002
-		"NetBird IP: 192.168.178.100/16\n" +
+  Relay server address: 
-		"Interface type: Kernel\n" +
+  Last connection update: %s
-		"Peers count: 2/2 Connected\n"
+  Last WireGuard handshake: %s
+  Transfer status (received/sent) 2.0 KiB/1000 B
+  Quantum resistance: false
+  Routes: -
+  Latency: 10ms
+
+OS: %s/%s
+Daemon version: 0.14.1
+CLI version: %s
+Management: Connected to my-awesome-management.com:443
+Signal: Connected to my-awesome-signal.com:443
+Relays: 
+  [stun:my-awesome-stun.com:3478] is Available
+  [turns:my-awesome-turn.com:443?transport=tcp] is Unavailable, reason: context: deadline exceeded
+Nameservers: 
+  [8.8.8.8:53] for [.] is Available
+  [1.1.1.1:53, 2.2.2.2:53] for [example.com, example.net] is Unavailable, reason: timeout
+FQDN: some-localhost.awesome-domain.com
+NetBird IP: 192.168.178.100/16
+Interface type: Kernel
+Quantum resistance: false
+Routes: 10.10.0.0/24
+Peers count: 2/2 Connected
+`, lastConnectionUpdate1, lastHandshake1, lastConnectionUpdate2, lastHandshake2, runtime.GOOS, runtime.GOARCH, overview.CliVersion)
 
 	assert.Equal(t, expectedDetail, detail)
 }
 
 func TestParsingToShortVersion(t *testing.T) {
-	shortVersion := parseGeneralSummary(overview, false)
+	shortVersion := parseGeneralSummary(overview, false, false, false)
 
-	expectedString := "Daemon version: 0.14.1\n" +
+	expectedString := fmt.Sprintf("OS: %s/%s", runtime.GOOS, runtime.GOARCH) + `
-		"CLI version: development\n" +
+Daemon version: 0.14.1
-		"Management: Connected\n" +
+CLI version: development
-		"Signal: Connected\n" +
+Management: Connected
-		"FQDN: some-localhost.awesome-domain.com\n" +
+Signal: Connected
-		"NetBird IP: 192.168.178.100/16\n" +
+Relays: 1/2 Available
-		"Interface type: Kernel\n" +
+Nameservers: 1/2 Available
-		"Peers count: 2/2 Connected\n"
+FQDN: some-localhost.awesome-domain.com
+NetBird IP: 192.168.178.100/16
+Interface type: Kernel
+Quantum resistance: false
+Routes: 10.10.0.0/24
+Peers count: 2/2 Connected
+`
 
 	assert.Equal(t, expectedString, shortVersion)
 }
@@ -299,3 +577,31 @@ func TestParsingOfIP(t *testing.T) {
 
 	assert.Equal(t, "192.168.178.123\n", parsedIP)
 }
+
+func TestTimeAgo(t *testing.T) {
+	now := time.Now()
+
+	cases := []struct {
+		name     string
+		input    time.Time
+		expected string
+	}{
+		{"Now", now, "Now"},
+		{"Seconds ago", now.Add(-10 * time.Second), "10 seconds ago"},
+		{"One minute ago", now.Add(-1 * time.Minute), "1 minute ago"},
+		{"Minutes and seconds ago", now.Add(-(1*time.Minute + 30*time.Second)), "1 minute, 30 seconds ago"},
+		{"One hour ago", now.Add(-1 * time.Hour), "1 hour ago"},
+		{"Hours and minutes ago", now.Add(-(2*time.Hour + 15*time.Minute)), "2 hours, 15 minutes ago"},
+		{"One day ago", now.Add(-24 * time.Hour), "1 day ago"},
+		{"Multiple days ago", now.Add(-(72*time.Hour + 20*time.Minute)), "3 days ago"},
+		{"Zero time", time.Time{}, "-"},
+		{"Unix zero time", time.Unix(0, 0), "-"},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := timeAgo(tc.input)
+			assert.Equal(t, tc.expected, result, "Failed %s", tc.name)
+		})
+	}
+}

client/cmd/testutil_test.go

@@ -2,52 +2,58 @@ package cmd
 
 import (
 	"context"
-	"github.com/netbirdio/netbird/management/server/activity"
 	"net"
-	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel"
+
+	"github.com/netbirdio/netbird/management/server/activity"
+	"github.com/netbirdio/netbird/management/server/telemetry"
+
 	"github.com/netbirdio/netbird/util"
 
+	"google.golang.org/grpc"
+
+	"github.com/netbirdio/management-integrations/integrations"
+
 	clientProto "github.com/netbirdio/netbird/client/proto"
 	client "github.com/netbirdio/netbird/client/server"
 	mgmtProto "github.com/netbirdio/netbird/management/proto"
 	mgmt "github.com/netbirdio/netbird/management/server"
 	sigProto "github.com/netbirdio/netbird/signal/proto"
 	sig "github.com/netbirdio/netbird/signal/server"
-	"google.golang.org/grpc"
 )
 
 func startTestingServices(t *testing.T) string {
+	t.Helper()
 	config := &mgmt.Config{}
 	_, err := util.ReadJson("../testdata/management.json", config)
 	if err != nil {
 		t.Fatal(err)
 	}
-	testDir := t.TempDir()
-	config.Datadir = testDir
-	err = util.CopyFileContents("../testdata/store.json", filepath.Join(testDir, "store.json"))
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	_, signalLis := startSignal(t)
 	signalAddr := signalLis.Addr().String()
 	config.Signal.URI = signalAddr
 
-	_, mgmLis := startManagement(t, config)
+	_, mgmLis := startManagement(t, config, "../testdata/store.sql")
 	mgmAddr := mgmLis.Addr().String()
 	return mgmAddr
 }
 
 func startSignal(t *testing.T) (*grpc.Server, net.Listener) {
+	t.Helper()
 	lis, err := net.Listen("tcp", ":0")
 	if err != nil {
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	sigProto.RegisterSignalExchangeServer(s, sig.NewServer())
+	srv, err := sig.NewServer(context.Background(), otel.Meter(""))
+	require.NoError(t, err)
+
+	sigProto.RegisterSignalExchangeServer(s, srv)
 	go func() {
 		if err := s.Serve(lis); err != nil {
 			panic(err)
@@ -57,29 +63,37 @@ func startSignal(t *testing.T) (*grpc.Server, net.Listener) {
 	return s, lis
 }
 
-func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Listener) {
+func startManagement(t *testing.T, config *mgmt.Config, testFile string) (*grpc.Server, net.Listener) {
+	t.Helper()
+
 	lis, err := net.Listen("tcp", ":0")
 	if err != nil {
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	store, err := mgmt.NewFileStore(config.Datadir)
+	store, cleanUp, err := mgmt.NewTestStoreFromSQL(context.Background(), testFile, t.TempDir())
 	if err != nil {
 		t.Fatal(err)
 	}
+	t.Cleanup(cleanUp)
 
-	peersUpdateManager := mgmt.NewPeersUpdateManager()
+	peersUpdateManager := mgmt.NewPeersUpdateManager(nil)
 	eventStore := &activity.InMemoryEventStore{}
 	if err != nil {
 		return nil, nil
 	}
-	accountManager, err := mgmt.BuildManager(store, peersUpdateManager, nil, "", "",
+	iv, _ := integrations.NewIntegratedValidator(context.Background(), eventStore)
-		eventStore)
+
+	metrics, err := telemetry.NewDefaultAppMetrics(context.Background())
+	require.NoError(t, err)
+
+	accountManager, err := mgmt.BuildManager(context.Background(), store, peersUpdateManager, nil, "", "netbird.selfhosted", eventStore, nil, false, iv, metrics)
 	if err != nil {
 		t.Fatal(err)
 	}
-	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+
-	mgmtServer, err := mgmt.NewServer(config, accountManager, peersUpdateManager, turnManager, nil)
+	secretsManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay)
+	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -94,8 +108,9 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste
 }
 
 func startClientDaemon(
-	t *testing.T, ctx context.Context, managementURL, configPath string,
+	t *testing.T, ctx context.Context, _, configPath string,
 ) (*grpc.Server, net.Listener) {
+	t.Helper()
 	lis, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		t.Fatal(err)

client/cmd/up.go

@@ -5,16 +5,21 @@ import (
 	"fmt"
 	"net"
 	"net/netip"
+	"runtime"
 	"strings"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc/codes"
 	gstatus "google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/durationpb"
 
+	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/internal"
 	"github.com/netbirdio/netbird/client/internal/peer"
 	"github.com/netbirdio/netbird/client/proto"
+	"github.com/netbirdio/netbird/client/system"
 	"github.com/netbirdio/netbird/util"
 )
 
@@ -35,6 +40,14 @@ var (
 
 func init() {
 	upCmd.PersistentFlags().BoolVarP(&foregroundMode, "foreground-mode", "F", false, "start service in foreground")
+	upCmd.PersistentFlags().StringVar(&interfaceName, interfaceNameFlag, iface.WgInterfaceDefault, "Wireguard interface name")
+	upCmd.PersistentFlags().Uint16Var(&wireguardPort, wireguardPortFlag, iface.DefaultWgPort, "Wireguard interface listening port")
+	upCmd.PersistentFlags().BoolVarP(&networkMonitor, networkMonitorFlag, "N", networkMonitor,
+		`Manage network monitoring. Defaults to true on Windows and macOS, false on Linux. `+
+			`E.g. --network-monitor=false to disable or --network-monitor=true to enable.`,
+	)
+	upCmd.PersistentFlags().StringSliceVar(&extraIFaceBlackList, extraIFaceBlackListFlag, nil, "Extra list of default interfaces to ignore for listening")
+	upCmd.PersistentFlags().DurationVar(&dnsRouteInterval, dnsRouteIntervalFlag, time.Minute, "DNS route update interval")
 }
 
 func upFunc(cmd *cobra.Command, args []string) error {
@@ -55,6 +68,11 @@ func upFunc(cmd *cobra.Command, args []string) error {
 
 	ctx := internal.CtxInitState(cmd.Context())
 
+	if hostName != "" {
+		// nolint
+		ctx = context.WithValue(ctx, system.DeviceNameCtxKey, hostName)
+	}
+
 	if foregroundMode {
 		return runInForegroundMode(ctx, cmd)
 	}
@@ -72,21 +90,76 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error {
 		return err
 	}
 
-	config, err := internal.UpdateOrCreateConfig(internal.ConfigInput{
+	ic := internal.ConfigInput{
-		ManagementURL:    managementURL,
+		ManagementURL:       managementURL,
-		AdminURL:         adminURL,
+		AdminURL:            adminURL,
-		ConfigPath:       configPath,
+		ConfigPath:          configPath,
-		PreSharedKey:     &preSharedKey,
+		NATExternalIPs:      natExternalIPs,
-		NATExternalIPs:   natExternalIPs,
+		CustomDNSAddress:    customDNSAddressConverted,
-		CustomDNSAddress: customDNSAddressConverted,
+		ExtraIFaceBlackList: extraIFaceBlackList,
-	})
+	}
+
+	if cmd.Flag(enableRosenpassFlag).Changed {
+		ic.RosenpassEnabled = &rosenpassEnabled
+	}
+
+	if cmd.Flag(rosenpassPermissiveFlag).Changed {
+		ic.RosenpassPermissive = &rosenpassPermissive
+	}
+
+	if cmd.Flag(serverSSHAllowedFlag).Changed {
+		ic.ServerSSHAllowed = &serverSSHAllowed
+	}
+
+	if cmd.Flag(interfaceNameFlag).Changed {
+		if err := parseInterfaceName(interfaceName); err != nil {
+			return err
+		}
+		ic.InterfaceName = &interfaceName
+	}
+
+	if cmd.Flag(wireguardPortFlag).Changed {
+		p := int(wireguardPort)
+		ic.WireguardPort = &p
+	}
+
+	if cmd.Flag(networkMonitorFlag).Changed {
+		ic.NetworkMonitor = &networkMonitor
+	}
+
+	if rootCmd.PersistentFlags().Changed(preSharedKeyFlag) {
+		ic.PreSharedKey = &preSharedKey
+	}
+
+	if cmd.Flag(disableAutoConnectFlag).Changed {
+		ic.DisableAutoConnect = &autoConnectDisabled
+
+		if autoConnectDisabled {
+			cmd.Println("Autoconnect has been disabled. The client won't connect automatically when the service starts.")
+		}
+
+		if !autoConnectDisabled {
+			cmd.Println("Autoconnect has been enabled. The client will connect automatically when the service starts.")
+		}
+	}
+
+	if cmd.Flag(dnsRouteIntervalFlag).Changed {
+		ic.DNSRouteInterval = &dnsRouteInterval
+	}
+
+	providedSetupKey, err := getSetupKey()
+	if err != nil {
+		return err
+	}
+
+	config, err := internal.UpdateOrCreateConfig(ic)
 	if err != nil {
 		return fmt.Errorf("get config file: %v", err)
 	}
 
-	config, _ = internal.UpdateOldManagementPort(ctx, config, configPath)
+	config, _ = internal.UpdateOldManagementURL(ctx, config, configPath)
 
-	err = foregroundLogin(ctx, cmd, config, setupKey)
+	err = foregroundLogin(ctx, cmd, config, providedSetupKey)
 	if err != nil {
 		return fmt.Errorf("foreground login failed: %v", err)
 	}
@@ -94,11 +167,15 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error {
 	var cancel context.CancelFunc
 	ctx, cancel = context.WithCancel(ctx)
 	SetupCloseHandler(ctx, cancel)
-	return internal.RunClient(ctx, config, peer.NewRecorder(config.ManagementURL.String()), nil, nil)
+
+	r := peer.NewRecorder(config.ManagementURL.String())
+	r.GetFullStatus()
+
+	connectClient := internal.NewConnectClient(ctx, config, r)
+	return connectClient.Run()
 }
 
 func runInDaemonMode(ctx context.Context, cmd *cobra.Command) error {
-
 	customDNSAddressConverted, err := parseCustomDNSAddress(cmd.Flag(dnsResolverAddress).Changed)
 	if err != nil {
 		return err
@@ -113,7 +190,7 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command) error {
 	defer func() {
 		err := conn.Close()
 		if err != nil {
-			log.Warnf("failed closing dameon gRPC client connection %v", err)
+			log.Warnf("failed closing daemon gRPC client connection %v", err)
 			return
 		}
 	}()
@@ -130,14 +207,61 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command) error {
 		return nil
 	}
 
+	providedSetupKey, err := getSetupKey()
+	if err != nil {
+		return err
+	}
+
 	loginRequest := proto.LoginRequest{
-		SetupKey:            setupKey,
+		SetupKey:             providedSetupKey,
-		PreSharedKey:        preSharedKey,
+		ManagementUrl:        managementURL,
-		ManagementUrl:       managementURL,
+		AdminURL:             adminURL,
-		AdminURL:            adminURL,
+		NatExternalIPs:       natExternalIPs,
-		NatExternalIPs:      natExternalIPs,
+		CleanNATExternalIPs:  natExternalIPs != nil && len(natExternalIPs) == 0,
-		CleanNATExternalIPs: natExternalIPs != nil && len(natExternalIPs) == 0,
+		CustomDNSAddress:     customDNSAddressConverted,
-		CustomDNSAddress:    customDNSAddressConverted,
+		IsLinuxDesktopClient: isLinuxRunningDesktop(),
+		Hostname:             hostName,
+		ExtraIFaceBlacklist:  extraIFaceBlackList,
+	}
+
+	if rootCmd.PersistentFlags().Changed(preSharedKeyFlag) {
+		loginRequest.OptionalPreSharedKey = &preSharedKey
+	}
+
+	if cmd.Flag(enableRosenpassFlag).Changed {
+		loginRequest.RosenpassEnabled = &rosenpassEnabled
+	}
+
+	if cmd.Flag(rosenpassPermissiveFlag).Changed {
+		loginRequest.RosenpassPermissive = &rosenpassPermissive
+	}
+
+	if cmd.Flag(serverSSHAllowedFlag).Changed {
+		loginRequest.ServerSSHAllowed = &serverSSHAllowed
+	}
+
+	if cmd.Flag(disableAutoConnectFlag).Changed {
+		loginRequest.DisableAutoConnect = &autoConnectDisabled
+	}
+
+	if cmd.Flag(interfaceNameFlag).Changed {
+		if err := parseInterfaceName(interfaceName); err != nil {
+			return err
+		}
+		loginRequest.InterfaceName = &interfaceName
+	}
+
+	if cmd.Flag(wireguardPortFlag).Changed {
+		wp := int64(wireguardPort)
+		loginRequest.WireguardPort = &wp
+	}
+
+	if cmd.Flag(networkMonitorFlag).Changed {
+		loginRequest.NetworkMonitor = &networkMonitor
+	}
+
+	if cmd.Flag(dnsRouteIntervalFlag).Changed {
+		loginRequest.DnsRouteInterval = durationpb.New(dnsRouteInterval)
 	}
 
 	var loginErr error
@@ -166,9 +290,9 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command) error {
 
 	if loginResp.NeedsSSOLogin {
 
-		openURL(cmd, loginResp.VerificationURIComplete)
+		openURL(cmd, loginResp.VerificationURIComplete, loginResp.UserCode)
 
-		_, err = client.WaitSSOLogin(ctx, &proto.WaitSSOLoginRequest{UserCode: loginResp.UserCode})
+		_, err = client.WaitSSOLogin(ctx, &proto.WaitSSOLoginRequest{UserCode: loginResp.UserCode, Hostname: hostName})
 		if err != nil {
 			return fmt.Errorf("waiting sso login failed with: %v", err)
 		}
@@ -189,11 +313,11 @@ func validateNATExternalIPs(list []string) error {
 
 		subElements := strings.Split(element, "/")
 		if len(subElements) > 2 {
-			return fmt.Errorf("%s is not a valid input for %s. it should be formated as \"String\" or \"String/String\"", element, externalIPMapFlag)
+			return fmt.Errorf("%s is not a valid input for %s. it should be formatted as \"String\" or \"String/String\"", element, externalIPMapFlag)
 		}
 
 		if len(subElements) == 1 && !isValidIP(subElements[0]) {
-			return fmt.Errorf("%s is not a valid input for %s. it should be formated as \"IP\" or \"IP/IP\", or \"IP/Interface Name\"", element, externalIPMapFlag)
+			return fmt.Errorf("%s is not a valid input for %s. it should be formatted as \"IP\" or \"IP/IP\", or \"IP/Interface Name\"", element, externalIPMapFlag)
 		}
 
 		last := 0
@@ -211,6 +335,18 @@ func validateNATExternalIPs(list []string) error {
 	return nil
 }
 
+func parseInterfaceName(name string) error {
+	if runtime.GOOS != "darwin" {
+		return nil
+	}
+
+	if strings.HasPrefix(name, "utun") {
+		return nil
+	}
+
+	return fmt.Errorf("invalid interface name %s. Please use the prefix utun followed by a number on MacOS. e.g., utun1 or utun199", name)
+}
+
 func validateElement(element string) (int, error) {
 	if isValidIP(element) {
 		return ipInputType, nil
@@ -248,7 +384,7 @@ func parseCustomDNSAddress(modified bool) ([]byte, error) {
 	var parsed []byte
 	if modified {
 		if !isValidAddrPort(customDNSAddress) {
-			return nil, fmt.Errorf("%s is invalid, it should be formated as IP:Port string or as an empty string like \"\"", customDNSAddress)
+			return nil, fmt.Errorf("%s is invalid, it should be formatted as IP:Port string or as an empty string like \"\"", customDNSAddress)
 		}
 		if customDNSAddress == "" && logFile != "console" {
 			parsed = []byte("empty")

client/cmd/up_daemon_test.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"os"
 	"testing"
 	"time"
 
@@ -40,6 +41,36 @@ func TestUpDaemon(t *testing.T) {
 		return
 	}
 
+	// Test the setup-key-file flag.
+	tempFile, err := os.CreateTemp("", "setup-key")
+	if err != nil {
+		t.Errorf("could not create temp file, got error %v", err)
+		return
+	}
+	defer os.Remove(tempFile.Name())
+	if _, err := tempFile.Write([]byte("A2C8E62B-38F5-4553-B31E-DD66C696CEBB")); err != nil {
+		t.Errorf("could not write to temp file, got error %v", err)
+		return
+	}
+	if err := tempFile.Close(); err != nil {
+		t.Errorf("unable to close file, got error %v", err)
+	}
+	rootCmd.SetArgs([]string{
+		"login",
+		"--daemon-addr", "tcp://" + cliAddr,
+		"--setup-key-file", tempFile.Name(),
+		"--log-file", "",
+	})
+	if err := rootCmd.Execute(); err != nil {
+		t.Errorf("expected no error while running up command, got %v", err)
+		return
+	}
+	time.Sleep(time.Second * 3)
+	if status, err := state.Status(); err != nil && status != internal.StatusIdle {
+		t.Errorf("wrong status after login: %s, %v", internal.StatusIdle, err)
+		return
+	}
+
 	rootCmd.SetArgs([]string{
 		"up",
 		"--daemon-addr", "tcp://" + cliAddr,

client/errors/errors.go

@@ -0,0 +1,30 @@
+package errors
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/go-multierror"
+)
+
+func formatError(es []error) string {
+	if len(es) == 1 {
+		return fmt.Sprintf("1 error occurred:\n\t* %s", es[0])
+	}
+
+	points := make([]string, len(es))
+	for i, err := range es {
+		points[i] = fmt.Sprintf("* %s", err)
+	}
+
+	return fmt.Sprintf(
+		"%d errors occurred:\n\t%s",
+		len(es), strings.Join(points, "\n\t"))
+}
+
+func FormatErrorOrNil(err *multierror.Error) error {
+	if err != nil {
+		err.ErrorFormat = formatError
+	}
+	return err.ErrorOrNil()
+}

client/firewall/create.go

@@ -0,0 +1,32 @@
+//go:build !linux || android
+
+package firewall
+
+import (
+	"fmt"
+	"runtime"
+
+	log "github.com/sirupsen/logrus"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/firewall/uspfilter"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+)
+
+// NewFirewall creates a firewall manager instance
+func NewFirewall(iface IFaceMapper, _ *statemanager.Manager) (firewall.Manager, error) {
+	if !iface.IsUserspaceBind() {
+		return nil, fmt.Errorf("not implemented for this OS: %s", runtime.GOOS)
+	}
+
+	// use userspace packet filtering firewall
+	fm, err := uspfilter.Create(iface)
+	if err != nil {
+		return nil, err
+	}
+	err = fm.AllowNetbird()
+	if err != nil {
+		log.Warnf("failed to allow netbird interface traffic: %v", err)
+	}
+	return fm, nil
+}

client/firewall/create_linux.go

@@ -0,0 +1,161 @@
+//go:build !android
+
+package firewall
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/google/nftables"
+	log "github.com/sirupsen/logrus"
+
+	nbiptables "github.com/netbirdio/netbird/client/firewall/iptables"
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	nbnftables "github.com/netbirdio/netbird/client/firewall/nftables"
+	"github.com/netbirdio/netbird/client/firewall/uspfilter"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+)
+
+const (
+	// UNKNOWN is the default value for the firewall type for unknown firewall type
+	UNKNOWN FWType = iota
+	// IPTABLES is the value for the iptables firewall type
+	IPTABLES
+	// NFTABLES is the value for the nftables firewall type
+	NFTABLES
+)
+
+// SKIP_NFTABLES_ENV is the environment variable to skip nftables check
+const SKIP_NFTABLES_ENV = "NB_SKIP_NFTABLES_CHECK"
+
+// FWType is the type for the firewall type
+type FWType int
+
+func NewFirewall(iface IFaceMapper, stateManager *statemanager.Manager) (firewall.Manager, error) {
+	// on the linux system we try to user nftables or iptables
+	// in any case, because we need to allow netbird interface traffic
+	// so we use AllowNetbird traffic from these firewall managers
+	// for the userspace packet filtering firewall
+	fm, err := createNativeFirewall(iface, stateManager)
+
+	if !iface.IsUserspaceBind() {
+		return fm, err
+	}
+
+	if err != nil {
+		log.Warnf("failed to create native firewall: %v. Proceeding with userspace", err)
+	}
+	return createUserspaceFirewall(iface, fm)
+}
+
+func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager) (firewall.Manager, error) {
+	fm, err := createFW(iface)
+	if err != nil {
+		return nil, fmt.Errorf("create firewall: %s", err)
+	}
+
+	if err = fm.Init(stateManager); err != nil {
+		return nil, fmt.Errorf("init firewall: %s", err)
+	}
+
+	return fm, nil
+}
+
+func createFW(iface IFaceMapper) (firewall.Manager, error) {
+	switch check() {
+	case IPTABLES:
+		log.Info("creating an iptables firewall manager")
+		return nbiptables.Create(iface)
+	case NFTABLES:
+		log.Info("creating an nftables firewall manager")
+		return nbnftables.Create(iface)
+	default:
+		log.Info("no firewall manager found, trying to use userspace packet filtering firewall")
+		return nil, errors.New("no firewall manager found")
+	}
+}
+
+func createUserspaceFirewall(iface IFaceMapper, fm firewall.Manager) (firewall.Manager, error) {
+	var errUsp error
+	if fm != nil {
+		fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm)
+	} else {
+		fm, errUsp = uspfilter.Create(iface)
+	}
+
+	if errUsp != nil {
+		return nil, fmt.Errorf("create userspace firewall: %s", errUsp)
+	}
+
+	if err := fm.AllowNetbird(); err != nil {
+		log.Errorf("failed to allow netbird interface traffic: %v", err)
+	}
+	return fm, nil
+}
+
+// check returns the firewall type based on common lib checks. It returns UNKNOWN if no firewall is found.
+func check() FWType {
+	useIPTABLES := false
+	var iptablesChains []string
+	ip, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+	if err == nil && isIptablesClientAvailable(ip) {
+		major, minor, _ := ip.GetIptablesVersion()
+		// use iptables when its version is lower than 1.8.0 which doesn't work well with our nftables manager
+		if major < 1 || (major == 1 && minor < 8) {
+			return IPTABLES
+		}
+
+		useIPTABLES = true
+
+		iptablesChains, err = ip.ListChains("filter")
+		if err != nil {
+			log.Errorf("failed to list iptables chains: %s", err)
+			useIPTABLES = false
+		}
+	}
+
+	nf := nftables.Conn{}
+	if chains, err := nf.ListChains(); err == nil && os.Getenv(SKIP_NFTABLES_ENV) != "true" {
+		if !useIPTABLES {
+			return NFTABLES
+		}
+
+		// search for chains where table is filter
+		// if we find one, we assume that nftables manager can be used with iptables
+		for _, chain := range chains {
+			if chain.Table.Name == "filter" {
+				return NFTABLES
+			}
+		}
+
+		// check tables for the following constraints:
+		// 1. there is no chain in nftables for the filter table and there is at least one chain in iptables, we assume that nftables manager can not be used
+		// 2. there is no tables or more than one table, we assume that nftables manager can be used
+		// 3. there is only one table and its name is filter, we assume that nftables manager can not be used, since there was no chain in it
+		// 4. if we find an error we log and continue with iptables check
+		nbTablesList, err := nf.ListTables()
+		switch {
+		case err == nil && len(iptablesChains) > 0:
+			return IPTABLES
+		case err == nil && len(nbTablesList) != 1:
+			return NFTABLES
+		case err == nil && len(nbTablesList) == 1 && nbTablesList[0].Name == "filter":
+			return IPTABLES
+		case err != nil:
+			log.Errorf("failed to list nftables tables on fw manager discovery: %s", err)
+		}
+	}
+
+	if useIPTABLES {
+		return IPTABLES
+	}
+
+	return UNKNOWN
+}
+
+func isIptablesClientAvailable(client *iptables.IPTables) bool {
+	_, err := client.ListChains("filter")
+	return err == nil
+}

client/firewall/firewall.go

@@ -1,57 +0,0 @@
-package firewall
-
-import (
-	"net"
-)
-
-// Rule abstraction should be implemented by each firewall manager
-//
-// Each firewall type for different OS can use different type
-// of the properties to hold data of the created rule
-type Rule interface {
-	// GetRuleID returns the rule id
-	GetRuleID() string
-}
-
-// Direction is the direction of the traffic
-type Direction int
-
-const (
-	// DirectionSrc is the direction of the traffic from the source
-	DirectionSrc Direction = iota
-	// DirectionDst is the direction of the traffic from the destination
-	DirectionDst
-)
-
-// Action is the action to be taken on a rule
-type Action int
-
-const (
-	// ActionAccept is the action to accept a packet
-	ActionAccept Action = iota
-	// ActionDrop is the action to drop a packet
-	ActionDrop
-)
-
-// Manager is the high level abstraction of a firewall manager
-//
-// It declares methods which handle actions required by the
-// Netbird client for ACL and routing functionality
-type Manager interface {
-	// AddFiltering rule to the firewall
-	AddFiltering(
-		ip net.IP,
-		port *Port,
-		direction Direction,
-		action Action,
-		comment string,
-	) (Rule, error)
-
-	// DeleteRule from the firewall by rule definition
-	DeleteRule(rule Rule) error
-
-	// Reset firewall to the default state
-	Reset() error
-
-	// TODO: migrate routemanager firewal actions to this interface
-}

client/firewall/iface.go

@@ -0,0 +1,13 @@
+package firewall
+
+import (
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+// IFaceMapper defines subset methods of interface required for manager
+type IFaceMapper interface {
+	Name() string
+	Address() device.WGAddress
+	IsUserspaceBind() bool
+	SetFilter(device.PacketFilter) error
+}

client/firewall/iptables/acl_linux.go

@@ -0,0 +1,457 @@
+package iptables
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/google/uuid"
+	"github.com/nadoo/ipset"
+	log "github.com/sirupsen/logrus"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+	nbnet "github.com/netbirdio/netbird/util/net"
+)
+
+const (
+	tableName = "filter"
+
+	// rules chains contains the effective ACL rules
+	chainNameInputRules  = "NETBIRD-ACL-INPUT"
+	chainNameOutputRules = "NETBIRD-ACL-OUTPUT"
+)
+
+type aclEntries map[string][][]string
+
+type entry struct {
+	spec     []string
+	position int
+}
+
+type aclManager struct {
+	iptablesClient     *iptables.IPTables
+	wgIface            iFaceMapper
+	routingFwChainName string
+
+	entries         aclEntries
+	optionalEntries map[string][]entry
+	ipsetStore      *ipsetStore
+
+	stateManager *statemanager.Manager
+}
+
+func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper, routingFwChainName string) (*aclManager, error) {
+	m := &aclManager{
+		iptablesClient:     iptablesClient,
+		wgIface:            wgIface,
+		routingFwChainName: routingFwChainName,
+
+		entries:         make(map[string][][]string),
+		optionalEntries: make(map[string][]entry),
+		ipsetStore:      newIpsetStore(),
+	}
+
+	if err := ipset.Init(); err != nil {
+		return nil, fmt.Errorf("init ipset: %w", err)
+	}
+
+	return m, nil
+}
+
+func (m *aclManager) init(stateManager *statemanager.Manager) error {
+	m.stateManager = stateManager
+
+	m.seedInitialEntries()
+	m.seedInitialOptionalEntries()
+
+	if err := m.cleanChains(); err != nil {
+		return fmt.Errorf("clean chains: %w", err)
+	}
+
+	if err := m.createDefaultChains(); err != nil {
+		return fmt.Errorf("create default chains: %w", err)
+	}
+
+	m.updateState()
+
+	return nil
+}
+
+func (m *aclManager) AddPeerFiltering(
+	ip net.IP,
+	protocol firewall.Protocol,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	direction firewall.RuleDirection,
+	action firewall.Action,
+	ipsetName string,
+) ([]firewall.Rule, error) {
+	var dPortVal, sPortVal string
+	if dPort != nil && dPort.Values != nil {
+		// TODO: we support only one port per rule in current implementation of ACLs
+		dPortVal = strconv.Itoa(dPort.Values[0])
+	}
+	if sPort != nil && sPort.Values != nil {
+		sPortVal = strconv.Itoa(sPort.Values[0])
+	}
+
+	var chain string
+	if direction == firewall.RuleDirectionOUT {
+		chain = chainNameOutputRules
+	} else {
+		chain = chainNameInputRules
+	}
+
+	ipsetName = transformIPsetName(ipsetName, sPortVal, dPortVal)
+	specs := filterRuleSpecs(ip, string(protocol), sPortVal, dPortVal, direction, action, ipsetName)
+	if ipsetName != "" {
+		if ipList, ipsetExists := m.ipsetStore.ipset(ipsetName); ipsetExists {
+			if err := ipset.Add(ipsetName, ip.String()); err != nil {
+				return nil, fmt.Errorf("failed to add IP to ipset: %w", err)
+			}
+			// if ruleset already exists it means we already have the firewall rule
+			// so we need to update IPs in the ruleset and return new fw.Rule object for ACL manager.
+			ipList.addIP(ip.String())
+			return []firewall.Rule{&Rule{
+				ruleID:    uuid.New().String(),
+				ipsetName: ipsetName,
+				ip:        ip.String(),
+				chain:     chain,
+				specs:     specs,
+			}}, nil
+		}
+
+		if err := ipset.Flush(ipsetName); err != nil {
+			log.Errorf("flush ipset %s before use it: %s", ipsetName, err)
+		}
+		if err := ipset.Create(ipsetName); err != nil {
+			return nil, fmt.Errorf("failed to create ipset: %w", err)
+		}
+		if err := ipset.Add(ipsetName, ip.String()); err != nil {
+			return nil, fmt.Errorf("failed to add IP to ipset: %w", err)
+		}
+
+		ipList := newIpList(ip.String())
+		m.ipsetStore.addIpList(ipsetName, ipList)
+	}
+
+	ok, err := m.iptablesClient.Exists("filter", chain, specs...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to check rule: %w", err)
+	}
+	if ok {
+		return nil, fmt.Errorf("rule already exists")
+	}
+
+	if err := m.iptablesClient.Append("filter", chain, specs...); err != nil {
+		return nil, err
+	}
+
+	rule := &Rule{
+		ruleID:    uuid.New().String(),
+		specs:     specs,
+		ipsetName: ipsetName,
+		ip:        ip.String(),
+		chain:     chain,
+	}
+
+	m.updateState()
+
+	return []firewall.Rule{rule}, nil
+}
+
+// DeletePeerRule from the firewall by rule definition
+func (m *aclManager) DeletePeerRule(rule firewall.Rule) error {
+	r, ok := rule.(*Rule)
+	if !ok {
+		return fmt.Errorf("invalid rule type")
+	}
+
+	if ipsetList, ok := m.ipsetStore.ipset(r.ipsetName); ok {
+		// delete IP from ruleset IPs list and ipset
+		if _, ok := ipsetList.ips[r.ip]; ok {
+			if err := ipset.Del(r.ipsetName, r.ip); err != nil {
+				return fmt.Errorf("failed to delete ip from ipset: %w", err)
+			}
+			delete(ipsetList.ips, r.ip)
+		}
+
+		// if after delete, set still contains other IPs,
+		// no need to delete firewall rule and we should exit here
+		if len(ipsetList.ips) != 0 {
+			return nil
+		}
+
+		// we delete last IP from the set, that means we need to delete
+		// set itself and associated firewall rule too
+		m.ipsetStore.deleteIpset(r.ipsetName)
+
+		if err := ipset.Destroy(r.ipsetName); err != nil {
+			log.Errorf("delete empty ipset: %v", err)
+		}
+	}
+
+	if err := m.iptablesClient.Delete(tableName, r.chain, r.specs...); err != nil {
+		return fmt.Errorf("failed to delete rule: %s, %v: %w", r.chain, r.specs, err)
+	}
+
+	m.updateState()
+
+	return nil
+}
+
+func (m *aclManager) Reset() error {
+	if err := m.cleanChains(); err != nil {
+		return fmt.Errorf("clean chains: %w", err)
+	}
+
+	m.updateState()
+
+	return nil
+}
+
+// todo write less destructive cleanup mechanism
+func (m *aclManager) cleanChains() error {
+	ok, err := m.iptablesClient.ChainExists(tableName, chainNameOutputRules)
+	if err != nil {
+		log.Debugf("failed to list chains: %s", err)
+		return err
+	}
+	if ok {
+		rules := m.entries["OUTPUT"]
+		for _, rule := range rules {
+			err := m.iptablesClient.DeleteIfExists(tableName, "OUTPUT", rule...)
+			if err != nil {
+				log.Errorf("failed to delete rule: %v, %s", rule, err)
+			}
+		}
+
+		err = m.iptablesClient.ClearAndDeleteChain(tableName, chainNameOutputRules)
+		if err != nil {
+			log.Debugf("failed to clear and delete %s chain: %s", chainNameOutputRules, err)
+			return err
+		}
+	}
+
+	ok, err = m.iptablesClient.ChainExists(tableName, chainNameInputRules)
+	if err != nil {
+		log.Debugf("failed to list chains: %s", err)
+		return err
+	}
+	if ok {
+		for _, rule := range m.entries["INPUT"] {
+			err := m.iptablesClient.DeleteIfExists(tableName, "INPUT", rule...)
+			if err != nil {
+				log.Errorf("failed to delete rule: %v, %s", rule, err)
+			}
+		}
+
+		for _, rule := range m.entries["FORWARD"] {
+			err := m.iptablesClient.DeleteIfExists(tableName, "FORWARD", rule...)
+			if err != nil {
+				log.Errorf("failed to delete rule: %v, %s", rule, err)
+			}
+		}
+
+		err = m.iptablesClient.ClearAndDeleteChain(tableName, chainNameInputRules)
+		if err != nil {
+			log.Debugf("failed to clear and delete %s chain: %s", chainNameInputRules, err)
+			return err
+		}
+	}
+
+	ok, err = m.iptablesClient.ChainExists("mangle", "PREROUTING")
+	if err != nil {
+		return fmt.Errorf("list chains: %w", err)
+	}
+	if ok {
+		for _, rule := range m.entries["PREROUTING"] {
+			err := m.iptablesClient.DeleteIfExists("mangle", "PREROUTING", rule...)
+			if err != nil {
+				log.Errorf("failed to delete rule: %v, %s", rule, err)
+			}
+		}
+	}
+
+	for _, ipsetName := range m.ipsetStore.ipsetNames() {
+		if err := ipset.Flush(ipsetName); err != nil {
+			log.Errorf("flush ipset %q during reset: %v", ipsetName, err)
+		}
+		if err := ipset.Destroy(ipsetName); err != nil {
+			log.Errorf("delete ipset %q during reset: %v", ipsetName, err)
+		}
+		m.ipsetStore.deleteIpset(ipsetName)
+	}
+
+	return nil
+}
+
+func (m *aclManager) createDefaultChains() error {
+	// chain netbird-acl-input-rules
+	if err := m.iptablesClient.NewChain(tableName, chainNameInputRules); err != nil {
+		log.Debugf("failed to create '%s' chain: %s", chainNameInputRules, err)
+		return err
+	}
+
+	// chain netbird-acl-output-rules
+	if err := m.iptablesClient.NewChain(tableName, chainNameOutputRules); err != nil {
+		log.Debugf("failed to create '%s' chain: %s", chainNameOutputRules, err)
+		return err
+	}
+
+	for chainName, rules := range m.entries {
+		for _, rule := range rules {
+			if err := m.iptablesClient.InsertUnique(tableName, chainName, 1, rule...); err != nil {
+				log.Debugf("failed to create input chain jump rule: %s", err)
+				return err
+			}
+		}
+	}
+
+	for chainName, entries := range m.optionalEntries {
+		for _, entry := range entries {
+			if err := m.iptablesClient.InsertUnique(tableName, chainName, entry.position, entry.spec...); err != nil {
+				log.Errorf("failed to insert optional entry %v: %v", entry.spec, err)
+				continue
+			}
+			m.entries[chainName] = append(m.entries[chainName], entry.spec)
+		}
+	}
+	clear(m.optionalEntries)
+
+	return nil
+}
+
+// seedInitialEntries adds default rules to the entries map, rules are inserted on pos 1, hence the order is reversed.
+// We want to make sure our traffic is not dropped by existing rules.
+
+// The existing FORWARD rules/policies decide outbound traffic towards our interface.
+// In case the FORWARD policy is set to "drop", we add an established/related rule to allow return traffic for the inbound rule.
+
+// The OUTPUT chain gets an extra rule to allow traffic to any set up routes, the return traffic is handled by the INPUT related/established rule.
+func (m *aclManager) seedInitialEntries() {
+
+	established := getConntrackEstablished()
+
+	m.appendToEntries("INPUT", []string{"-i", m.wgIface.Name(), "-j", "DROP"})
+	m.appendToEntries("INPUT", []string{"-i", m.wgIface.Name(), "-j", chainNameInputRules})
+	m.appendToEntries("INPUT", append([]string{"-i", m.wgIface.Name()}, established...))
+
+	m.appendToEntries("OUTPUT", []string{"-o", m.wgIface.Name(), "-j", "DROP"})
+	m.appendToEntries("OUTPUT", []string{"-o", m.wgIface.Name(), "-j", chainNameOutputRules})
+	m.appendToEntries("OUTPUT", []string{"-o", m.wgIface.Name(), "!", "-d", m.wgIface.Address().String(), "-j", "ACCEPT"})
+	m.appendToEntries("OUTPUT", append([]string{"-o", m.wgIface.Name()}, established...))
+
+	m.appendToEntries("FORWARD", []string{"-i", m.wgIface.Name(), "-j", "DROP"})
+	m.appendToEntries("FORWARD", []string{"-i", m.wgIface.Name(), "-j", m.routingFwChainName})
+	m.appendToEntries("FORWARD", append([]string{"-o", m.wgIface.Name()}, established...))
+}
+
+func (m *aclManager) seedInitialOptionalEntries() {
+	m.optionalEntries["FORWARD"] = []entry{
+		{
+			spec:     []string{"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), "-j", chainNameInputRules},
+			position: 2,
+		},
+	}
+
+	m.optionalEntries["PREROUTING"] = []entry{
+		{
+			spec:     []string{"-t", "mangle", "-i", m.wgIface.Name(), "-m", "addrtype", "--dst-type", "LOCAL", "-j", "MARK", "--set-mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected)},
+			position: 1,
+		},
+	}
+}
+
+func (m *aclManager) appendToEntries(chainName string, spec []string) {
+	m.entries[chainName] = append(m.entries[chainName], spec)
+}
+
+func (m *aclManager) updateState() {
+	if m.stateManager == nil {
+		return
+	}
+
+	var currentState *ShutdownState
+	if existing := m.stateManager.GetState(currentState); existing != nil {
+		if existingState, ok := existing.(*ShutdownState); ok {
+			currentState = existingState
+		}
+	}
+	if currentState == nil {
+		currentState = &ShutdownState{}
+	}
+
+	currentState.Lock()
+	defer currentState.Unlock()
+
+	currentState.ACLEntries = m.entries
+	currentState.ACLIPsetStore = m.ipsetStore
+
+	if err := m.stateManager.UpdateState(currentState); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+}
+
+// filterRuleSpecs returns the specs of a filtering rule
+func filterRuleSpecs(
+	ip net.IP, protocol string, sPort, dPort string, direction firewall.RuleDirection, action firewall.Action, ipsetName string,
+) (specs []string) {
+	matchByIP := true
+	// don't use IP matching if IP is ip 0.0.0.0
+	if ip.String() == "0.0.0.0" {
+		matchByIP = false
+	}
+	switch direction {
+	case firewall.RuleDirectionIN:
+		if matchByIP {
+			if ipsetName != "" {
+				specs = append(specs, "-m", "set", "--set", ipsetName, "src")
+			} else {
+				specs = append(specs, "-s", ip.String())
+			}
+		}
+	case firewall.RuleDirectionOUT:
+		if matchByIP {
+			if ipsetName != "" {
+				specs = append(specs, "-m", "set", "--set", ipsetName, "dst")
+			} else {
+				specs = append(specs, "-d", ip.String())
+			}
+		}
+	}
+	if protocol != "all" {
+		specs = append(specs, "-p", protocol)
+	}
+	if sPort != "" {
+		specs = append(specs, "--sport", sPort)
+	}
+	if dPort != "" {
+		specs = append(specs, "--dport", dPort)
+	}
+	return append(specs, "-j", actionToStr(action))
+}
+
+func actionToStr(action firewall.Action) string {
+	if action == firewall.ActionAccept {
+		return "ACCEPT"
+	}
+	return "DROP"
+}
+
+func transformIPsetName(ipsetName string, sPort, dPort string) string {
+	switch {
+	case ipsetName == "":
+		return ""
+	case sPort != "" && dPort != "":
+		return ipsetName + "-sport-dport"
+	case sPort != "":
+		return ipsetName + "-sport"
+	case dPort != "":
+		return ipsetName + "-dport"
+	default:
+		return ipsetName
+	}
+}

client/firewall/iptables/manager_linux.go

@@ -1,160 +1,230 @@
 package iptables
 
 import (
+	"context"
 	"fmt"
 	"net"
-	"strconv"
+	"net/netip"
 	"sync"
 
 	"github.com/coreos/go-iptables/iptables"
-	"github.com/google/uuid"
+	"github.com/hashicorp/go-multierror"
+	log "github.com/sirupsen/logrus"
 
-	fw "github.com/netbirdio/netbird/client/firewall"
+	nberrors "github.com/netbirdio/netbird/client/errors"
-)
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
-
+	"github.com/netbirdio/netbird/client/iface"
-const (
+	"github.com/netbirdio/netbird/client/internal/statemanager"
-	// ChainFilterName is the name of the chain that is used for filtering by the Netbird client
-	ChainFilterName = "NETBIRD-ACL"
 )
 
 // Manager of iptables firewall
 type Manager struct {
 	mutex sync.Mutex
 
+	wgIface iFaceMapper
+
 	ipv4Client *iptables.IPTables
-	ipv6Client *iptables.IPTables
+	aclMgr     *aclManager
+	router     *router
+}
+
+// iFaceMapper defines subset methods of interface required for manager
+type iFaceMapper interface {
+	Name() string
+	Address() iface.WGAddress
+	IsUserspaceBind() bool
 }
 
 // Create iptables firewall manager
-func Create() (*Manager, error) {
+func Create(wgIface iFaceMapper) (*Manager, error) {
-	m := &Manager{}
+	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
-
-	// init clients for booth ipv4 and ipv6
-	ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	if err != nil {
-		return nil, fmt.Errorf("iptables is not installed in the system or not supported")
+		return nil, fmt.Errorf("init iptables: %w", err)
 	}
-	m.ipv4Client = ipv4Client
 
-	ipv6Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+	m := &Manager{
+		wgIface:    wgIface,
+		ipv4Client: iptablesClient,
+	}
+
+	m.router, err = newRouter(iptablesClient, wgIface)
 	if err != nil {
-		return nil, fmt.Errorf("ip6tables is not installed in the system or not supported")
+		return nil, fmt.Errorf("create router: %w", err)
 	}
-	m.ipv6Client = ipv6Client
 
-	if err := m.Reset(); err != nil {
+	m.aclMgr, err = newAclManager(iptablesClient, wgIface, chainRTFWD)
-		return nil, fmt.Errorf("failed to reset firewall: %s", err)
+	if err != nil {
+		return nil, fmt.Errorf("create acl manager: %w", err)
 	}
 
 	return m, nil
 }
 
-// AddFiltering rule to the firewall
+func (m *Manager) Init(stateManager *statemanager.Manager) error {
-func (m *Manager) AddFiltering(
+	state := &ShutdownState{
-	ip net.IP,
+		InterfaceState: &InterfaceState{
-	port *fw.Port,
+			NameStr:       m.wgIface.Name(),
-	direction fw.Direction,
+			WGAddress:     m.wgIface.Address(),
-	action fw.Action,
+			UserspaceBind: m.wgIface.IsUserspaceBind(),
-	comment string,
+		},
-) (fw.Rule, error) {
-	m.mutex.Lock()
-	defer m.mutex.Unlock()
-	client := m.client(ip)
-	ok, err := client.ChainExists("filter", ChainFilterName)
-	if err != nil {
-		return nil, fmt.Errorf("failed to check if chain exists: %s", err)
 	}
-	if !ok {
+	stateManager.RegisterState(state)
-		if err := client.NewChain("filter", ChainFilterName); err != nil {
+	if err := stateManager.UpdateState(state); err != nil {
-			return nil, fmt.Errorf("failed to create chain: %s", err)
+		log.Errorf("failed to update state: %v", err)
+	}
+
+	if err := m.router.init(stateManager); err != nil {
+		return fmt.Errorf("router init: %w", err)
+	}
+
+	if err := m.aclMgr.init(stateManager); err != nil {
+		// TODO: cleanup router
+		return fmt.Errorf("acl manager init: %w", err)
+	}
+
+	// persist early to ensure cleanup of chains
+	go func() {
+		if err := stateManager.PersistState(context.Background()); err != nil {
+			log.Errorf("failed to persist state: %v", err)
 		}
-	}
+	}()
-	if port == nil || port.Values == nil || (port.IsRange && len(port.Values) != 2) {
-		return nil, fmt.Errorf("invalid port definition")
-	}
-	pv := strconv.Itoa(port.Values[0])
-	if port.IsRange {
-		pv += ":" + strconv.Itoa(port.Values[1])
-	}
-	specs := m.filterRuleSpecs("filter", ChainFilterName, ip, pv, direction, action, comment)
-	if err := client.AppendUnique("filter", ChainFilterName, specs...); err != nil {
-		return nil, err
-	}
-	rule := &Rule{
-		id:    uuid.New().String(),
-		specs: specs,
-		v6:    ip.To4() == nil,
-	}
-	return rule, nil
-}
 
-// DeleteRule from the firewall by rule definition
-func (m *Manager) DeleteRule(rule fw.Rule) error {
-	m.mutex.Lock()
-	defer m.mutex.Unlock()
-	r, ok := rule.(*Rule)
-	if !ok {
-		return fmt.Errorf("invalid rule type")
-	}
-	client := m.ipv4Client
-	if r.v6 {
-		client = m.ipv6Client
-	}
-	return client.Delete("filter", ChainFilterName, r.specs...)
-}
-
-// Reset firewall to the default state
-func (m *Manager) Reset() error {
-	m.mutex.Lock()
-	defer m.mutex.Unlock()
-	if err := m.reset(m.ipv4Client, "filter", ChainFilterName); err != nil {
-		return fmt.Errorf("clean ipv4 firewall ACL chain: %w", err)
-	}
-	if err := m.reset(m.ipv6Client, "filter", ChainFilterName); err != nil {
-		return fmt.Errorf("clean ipv6 firewall ACL chain: %w", err)
-	}
 	return nil
 }
 
-// reset firewall chain, clear it and drop it
+// AddPeerFiltering adds a rule to the firewall
-func (m *Manager) reset(client *iptables.IPTables, table, chain string) error {
+//
-	ok, err := client.ChainExists(table, chain)
+// Comment will be ignored because some system this feature is not supported
-	if err != nil {
+func (m *Manager) AddPeerFiltering(
-		return fmt.Errorf("failed to check if chain exists: %w", err)
+	ip net.IP,
+	protocol firewall.Protocol,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	direction firewall.RuleDirection,
+	action firewall.Action,
+	ipsetName string,
+	comment string,
+) ([]firewall.Rule, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.aclMgr.AddPeerFiltering(ip, protocol, sPort, dPort, direction, action, ipsetName)
+}
+
+func (m *Manager) AddRouteFiltering(
+	sources []netip.Prefix,
+	destination netip.Prefix,
+	proto firewall.Protocol,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	action firewall.Action,
+) (firewall.Rule, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	if !destination.Addr().Is4() {
+		return nil, fmt.Errorf("unsupported IP version: %s", destination.Addr().String())
 	}
-	if !ok {
+
+	return m.router.AddRouteFiltering(sources, destination, proto, sPort, dPort, action)
+}
+
+// DeletePeerRule from the firewall by rule definition
+func (m *Manager) DeletePeerRule(rule firewall.Rule) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.aclMgr.DeletePeerRule(rule)
+}
+
+func (m *Manager) DeleteRouteRule(rule firewall.Rule) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.router.DeleteRouteRule(rule)
+}
+
+func (m *Manager) IsServerRouteSupported() bool {
+	return true
+}
+
+func (m *Manager) AddNatRule(pair firewall.RouterPair) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.router.AddNatRule(pair)
+}
+
+func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.router.RemoveNatRule(pair)
+}
+
+func (m *Manager) SetLegacyManagement(isLegacy bool) error {
+	return firewall.SetLegacyManagement(m.router, isLegacy)
+}
+
+// Reset firewall to the default state
+func (m *Manager) Reset(stateManager *statemanager.Manager) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	var merr *multierror.Error
+
+	if err := m.aclMgr.Reset(); err != nil {
+		merr = multierror.Append(merr, fmt.Errorf("reset acl manager: %w", err))
+	}
+	if err := m.router.Reset(); err != nil {
+		merr = multierror.Append(merr, fmt.Errorf("reset router: %w", err))
+	}
+
+	// attempt to delete state only if all other operations succeeded
+	if merr == nil {
+		if err := stateManager.DeleteState(&ShutdownState{}); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("delete state: %w", err))
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+// AllowNetbird allows netbird interface traffic
+func (m *Manager) AllowNetbird() error {
+	if !m.wgIface.IsUserspaceBind() {
 		return nil
 	}
-	if err := client.ClearChain(table, ChainFilterName); err != nil {
+
-		return fmt.Errorf("failed to clear chain: %w", err)
+	_, err := m.AddPeerFiltering(
+		net.ParseIP("0.0.0.0"),
+		"all",
+		nil,
+		nil,
+		firewall.RuleDirectionIN,
+		firewall.ActionAccept,
+		"",
+		"",
+	)
+	if err != nil {
+		return fmt.Errorf("failed to allow netbird interface traffic: %w", err)
 	}
-	return client.DeleteChain(table, ChainFilterName)
+	_, err = m.AddPeerFiltering(
+		net.ParseIP("0.0.0.0"),
+		"all",
+		nil,
+		nil,
+		firewall.RuleDirectionOUT,
+		firewall.ActionAccept,
+		"",
+		"",
+	)
+	return err
 }
 
-// filterRuleSpecs returns the specs of a filtering rule
+// Flush doesn't need to be implemented for this manager
-func (m *Manager) filterRuleSpecs(
+func (m *Manager) Flush() error { return nil }
-	table string, chain string, ip net.IP, port string,
-	direction fw.Direction, action fw.Action, comment string,
-) (specs []string) {
-	if direction == fw.DirectionSrc {
-		specs = append(specs, "-s", ip.String())
-	}
-	specs = append(specs, "-p", "tcp", "--dport", port)
-	specs = append(specs, "-j", m.actionToStr(action))
-	return append(specs, "-m", "comment", "--comment", comment)
-}
 
-// client returns corresponding iptables client for the given ip
+func getConntrackEstablished() []string {
-func (m *Manager) client(ip net.IP) *iptables.IPTables {
+	return []string{"-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}
-	if ip.To4() != nil {
-		return m.ipv4Client
-	}
-	return m.ipv6Client
-}
-
-func (m *Manager) actionToStr(action fw.Action) string {
-	if action == fw.ActionAccept {
-		return "ACCEPT"
-	}
-	return "DROP"
 }

client/firewall/iptables/manager_linux_test.go

@@ -1,105 +1,284 @@
 package iptables
 
 import (
+	"fmt"
 	"net"
 	"testing"
+	"time"
 
 	"github.com/coreos/go-iptables/iptables"
-	fw "github.com/netbirdio/netbird/client/firewall"
+	"github.com/stretchr/testify/require"
+
+	fw "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/iface"
 )
 
-func TestNewManager(t *testing.T) {
+var ifaceMock = &iFaceMock{
+	NameFunc: func() string {
+		return "lo"
+	},
+	AddressFunc: func() iface.WGAddress {
+		return iface.WGAddress{
+			IP: net.ParseIP("10.20.0.1"),
+			Network: &net.IPNet{
+				IP:   net.ParseIP("10.20.0.0"),
+				Mask: net.IPv4Mask(255, 255, 255, 0),
+			},
+		}
+	},
+}
+
+// iFaceMapper defines subset methods of interface required for manager
+type iFaceMock struct {
+	NameFunc    func() string
+	AddressFunc func() iface.WGAddress
+}
+
+func (i *iFaceMock) Name() string {
+	if i.NameFunc != nil {
+		return i.NameFunc()
+	}
+	panic("NameFunc is not set")
+}
+
+func (i *iFaceMock) Address() iface.WGAddress {
+	if i.AddressFunc != nil {
+		return i.AddressFunc()
+	}
+	panic("AddressFunc is not set")
+}
+
+func (i *iFaceMock) IsUserspaceBind() bool { return false }
+
+func TestIptablesManager(t *testing.T) {
 	ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 
-	manager, err := Create()
+	// just check on the local interface
-	if err != nil {
+	manager, err := Create(ifaceMock)
-		t.Fatal(err)
+	require.NoError(t, err)
-	}
+	require.NoError(t, manager.Init(nil))
 
-	var rule1 fw.Rule
+	time.Sleep(time.Second)
+
+	defer func() {
+		err := manager.Reset(nil)
+		require.NoError(t, err, "clear the manager state")
+
+		time.Sleep(time.Second)
+	}()
+
+	var rule1 []fw.Rule
 	t.Run("add first rule", func(t *testing.T) {
 		ip := net.ParseIP("10.20.0.2")
-		port := &fw.Port{Proto: fw.PortProtocolTCP, Values: []int{8080}}
+		port := &fw.Port{Values: []int{8080}}
-		rule1, err = manager.AddFiltering(ip, port, fw.DirectionDst, fw.ActionAccept, "accept HTTP traffic")
+		rule1, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept HTTP traffic")
-		if err != nil {
+		require.NoError(t, err, "failed to add rule")
-			t.Errorf("failed to add rule: %v", err)
+
+		for _, r := range rule1 {
+			checkRuleSpecs(t, ipv4Client, chainNameOutputRules, true, r.(*Rule).specs...)
 		}
 
-		checkRuleSpecs(t, ipv4Client, true, rule1.(*Rule).specs...)
 	})
 
-	var rule2 fw.Rule
+	var rule2 []fw.Rule
 	t.Run("add second rule", func(t *testing.T) {
 		ip := net.ParseIP("10.20.0.3")
 		port := &fw.Port{
-			Proto:  fw.PortProtocolTCP,
 			Values: []int{8043: 8046},
 		}
-		rule2, err = manager.AddFiltering(
+		rule2, err = manager.AddPeerFiltering(
-			ip, port, fw.DirectionDst, fw.ActionAccept, "accept HTTPS traffic from ports range")
+			ip, "tcp", port, nil, fw.RuleDirectionIN, fw.ActionAccept, "", "accept HTTPS traffic from ports range")
-		if err != nil {
+		require.NoError(t, err, "failed to add rule")
-			t.Errorf("failed to add rule: %v", err)
-		}
 
-		checkRuleSpecs(t, ipv4Client, true, rule2.(*Rule).specs...)
+		for _, r := range rule2 {
+			rr := r.(*Rule)
+			checkRuleSpecs(t, ipv4Client, rr.chain, true, rr.specs...)
+		}
 	})
 
 	t.Run("delete first rule", func(t *testing.T) {
-		if err := manager.DeleteRule(rule1); err != nil {
+		for _, r := range rule1 {
-			t.Errorf("failed to delete rule: %v", err)
+			err := manager.DeletePeerRule(r)
-		}
+			require.NoError(t, err, "failed to delete rule")
 
-		checkRuleSpecs(t, ipv4Client, false, rule1.(*Rule).specs...)
+			checkRuleSpecs(t, ipv4Client, chainNameOutputRules, false, r.(*Rule).specs...)
+		}
 	})
 
 	t.Run("delete second rule", func(t *testing.T) {
-		if err := manager.DeleteRule(rule2); err != nil {
+		for _, r := range rule2 {
-			t.Errorf("failed to delete rule: %v", err)
+			err := manager.DeletePeerRule(r)
+			require.NoError(t, err, "failed to delete rule")
 		}
 
-		checkRuleSpecs(t, ipv4Client, false, rule2.(*Rule).specs...)
+		require.Empty(t, manager.aclMgr.ipsetStore.ipsets, "rulesets index after removed second rule must be empty")
 	})
 
 	t.Run("reset check", func(t *testing.T) {
 		// add second rule
 		ip := net.ParseIP("10.20.0.3")
-		port := &fw.Port{Proto: fw.PortProtocolUDP, Values: []int{5353}}
+		port := &fw.Port{Values: []int{5353}}
-		_, err = manager.AddFiltering(ip, port, fw.DirectionDst, fw.ActionAccept, "accept Fake DNS traffic")
+		_, err = manager.AddPeerFiltering(ip, "udp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept Fake DNS traffic")
-		if err != nil {
+		require.NoError(t, err, "failed to add rule")
-			t.Errorf("failed to add rule: %v", err)
-		}
 
-		if err := manager.Reset(); err != nil {
+		err = manager.Reset(nil)
-			t.Errorf("failed to reset: %v", err)
+		require.NoError(t, err, "failed to reset")
-		}
 
-		ok, err := ipv4Client.ChainExists("filter", ChainFilterName)
+		ok, err := ipv4Client.ChainExists("filter", chainNameInputRules)
-		if err != nil {
+		require.NoError(t, err, "failed check chain exists")
-			t.Errorf("failed to drop chain: %v", err)
-		}
 
 		if ok {
-			t.Errorf("chain '%v' still exists after Reset", ChainFilterName)
+			require.NoErrorf(t, err, "chain '%v' still exists after Reset", chainNameInputRules)
 		}
 	})
 }
 
-func checkRuleSpecs(t *testing.T, ipv4Client *iptables.IPTables, mustExists bool, rulespec ...string) {
+func TestIptablesManagerIPSet(t *testing.T) {
-	exists, err := ipv4Client.Exists("filter", ChainFilterName, rulespec...)
+	ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
-	if err != nil {
+	require.NoError(t, err)
-		t.Errorf("failed to check rule: %v", err)
+
-		return
+	mock := &iFaceMock{
+		NameFunc: func() string {
+			return "lo"
+		},
+		AddressFunc: func() iface.WGAddress {
+			return iface.WGAddress{
+				IP: net.ParseIP("10.20.0.1"),
+				Network: &net.IPNet{
+					IP:   net.ParseIP("10.20.0.0"),
+					Mask: net.IPv4Mask(255, 255, 255, 0),
+				},
+			}
+		},
 	}
 
-	if !exists && mustExists {
+	// just check on the local interface
-		t.Errorf("rule '%v' does not exist", rulespec)
+	manager, err := Create(mock)
-		return
+	require.NoError(t, err)
+	require.NoError(t, manager.Init(nil))
+
+	time.Sleep(time.Second)
+
+	defer func() {
+		err := manager.Reset(nil)
+		require.NoError(t, err, "clear the manager state")
+
+		time.Sleep(time.Second)
+	}()
+
+	var rule1 []fw.Rule
+	t.Run("add first rule with set", func(t *testing.T) {
+		ip := net.ParseIP("10.20.0.2")
+		port := &fw.Port{Values: []int{8080}}
+		rule1, err = manager.AddPeerFiltering(
+			ip, "tcp", nil, port, fw.RuleDirectionOUT,
+			fw.ActionAccept, "default", "accept HTTP traffic",
+		)
+		require.NoError(t, err, "failed to add rule")
+
+		for _, r := range rule1 {
+			checkRuleSpecs(t, ipv4Client, chainNameOutputRules, true, r.(*Rule).specs...)
+			require.Equal(t, r.(*Rule).ipsetName, "default-dport", "ipset name must be set")
+			require.Equal(t, r.(*Rule).ip, "10.20.0.2", "ipset IP must be set")
+		}
+	})
+
+	var rule2 []fw.Rule
+	t.Run("add second rule", func(t *testing.T) {
+		ip := net.ParseIP("10.20.0.3")
+		port := &fw.Port{
+			Values: []int{443},
+		}
+		rule2, err = manager.AddPeerFiltering(
+			ip, "tcp", port, nil, fw.RuleDirectionIN, fw.ActionAccept,
+			"default", "accept HTTPS traffic from ports range",
+		)
+		for _, r := range rule2 {
+			require.NoError(t, err, "failed to add rule")
+			require.Equal(t, r.(*Rule).ipsetName, "default-sport", "ipset name must be set")
+			require.Equal(t, r.(*Rule).ip, "10.20.0.3", "ipset IP must be set")
+		}
+	})
+
+	t.Run("delete first rule", func(t *testing.T) {
+		for _, r := range rule1 {
+			err := manager.DeletePeerRule(r)
+			require.NoError(t, err, "failed to delete rule")
+
+			require.NotContains(t, manager.aclMgr.ipsetStore.ipsets, r.(*Rule).ruleID, "rule must be removed form the ruleset index")
+		}
+	})
+
+	t.Run("delete second rule", func(t *testing.T) {
+		for _, r := range rule2 {
+			err := manager.DeletePeerRule(r)
+			require.NoError(t, err, "failed to delete rule")
+
+			require.Empty(t, manager.aclMgr.ipsetStore.ipsets, "rulesets index after removed second rule must be empty")
+		}
+	})
+
+	t.Run("reset check", func(t *testing.T) {
+		err = manager.Reset(nil)
+		require.NoError(t, err, "failed to reset")
+	})
+}
+
+func checkRuleSpecs(t *testing.T, ipv4Client *iptables.IPTables, chainName string, mustExists bool, rulespec ...string) {
+	t.Helper()
+	exists, err := ipv4Client.Exists("filter", chainName, rulespec...)
+	require.NoError(t, err, "failed to check rule")
+	require.Falsef(t, !exists && mustExists, "rule '%v' does not exist", rulespec)
+	require.Falsef(t, exists && !mustExists, "rule '%v' exist", rulespec)
+}
+
+func TestIptablesCreatePerformance(t *testing.T) {
+	mock := &iFaceMock{
+		NameFunc: func() string {
+			return "lo"
+		},
+		AddressFunc: func() iface.WGAddress {
+			return iface.WGAddress{
+				IP: net.ParseIP("10.20.0.1"),
+				Network: &net.IPNet{
+					IP:   net.ParseIP("10.20.0.0"),
+					Mask: net.IPv4Mask(255, 255, 255, 0),
+				},
+			}
+		},
 	}
-	if exists && !mustExists {
+
-		t.Errorf("rule '%v' exist", rulespec)
+	for _, testMax := range []int{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000} {
-		return
+		t.Run(fmt.Sprintf("Testing %d rules", testMax), func(t *testing.T) {
+			// just check on the local interface
+			manager, err := Create(mock)
+			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
+			time.Sleep(time.Second)
+
+			defer func() {
+				err := manager.Reset(nil)
+				require.NoError(t, err, "clear the manager state")
+
+				time.Sleep(time.Second)
+			}()
+
+			require.NoError(t, err)
+
+			ip := net.ParseIP("10.20.0.100")
+			start := time.Now()
+			for i := 0; i < testMax; i++ {
+				port := &fw.Port{Values: []int{1000 + i}}
+				if i%2 == 0 {
+					_, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept HTTP traffic")
+				} else {
+					_, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionIN, fw.ActionAccept, "", "accept HTTP traffic")
+				}
+
+				require.NoError(t, err, "failed to add rule")
+			}
+			t.Logf("execution avg per rule: %s", time.Since(start)/time.Duration(testMax))
+		})
 	}
 }

client/firewall/iptables/router_linux.go

@@ -0,0 +1,599 @@
+//go:build !android
+
+package iptables
+
+import (
+	"fmt"
+	"net/netip"
+	"strconv"
+	"strings"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/hashicorp/go-multierror"
+	"github.com/nadoo/ipset"
+	log "github.com/sirupsen/logrus"
+
+	nberrors "github.com/netbirdio/netbird/client/errors"
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/internal/acl/id"
+	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+	nbnet "github.com/netbirdio/netbird/util/net"
+)
+
+// constants needed to manage and create iptable rules
+const (
+	tableFilter             = "filter"
+	tableNat                = "nat"
+	tableMangle             = "mangle"
+	chainPOSTROUTING        = "POSTROUTING"
+	chainPREROUTING         = "PREROUTING"
+	chainRTNAT              = "NETBIRD-RT-NAT"
+	chainRTFWD              = "NETBIRD-RT-FWD"
+	chainRTPRE              = "NETBIRD-RT-PRE"
+	routingFinalForwardJump = "ACCEPT"
+	routingFinalNatJump     = "MASQUERADE"
+
+	jumpPre  = "jump-pre"
+	jumpNat  = "jump-nat"
+	matchSet = "--match-set"
+)
+
+type routeFilteringRuleParams struct {
+	Sources     []netip.Prefix
+	Destination netip.Prefix
+	Proto       firewall.Protocol
+	SPort       *firewall.Port
+	DPort       *firewall.Port
+	Direction   firewall.RuleDirection
+	Action      firewall.Action
+	SetName     string
+}
+
+type routeRules map[string][]string
+
+type ipsetCounter = refcounter.Counter[string, []netip.Prefix, struct{}]
+
+type router struct {
+	iptablesClient   *iptables.IPTables
+	rules            routeRules
+	ipsetCounter     *ipsetCounter
+	wgIface          iFaceMapper
+	legacyManagement bool
+
+	stateManager *statemanager.Manager
+}
+
+func newRouter(iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*router, error) {
+	r := &router{
+		iptablesClient: iptablesClient,
+		rules:          make(map[string][]string),
+		wgIface:        wgIface,
+	}
+
+	r.ipsetCounter = refcounter.New(
+		func(name string, sources []netip.Prefix) (struct{}, error) {
+			return struct{}{}, r.createIpSet(name, sources)
+		},
+		func(name string, _ struct{}) error {
+			return r.deleteIpSet(name)
+		},
+	)
+
+	if err := ipset.Init(); err != nil {
+		return nil, fmt.Errorf("init ipset: %w", err)
+	}
+
+	return r, nil
+}
+
+func (r *router) init(stateManager *statemanager.Manager) error {
+	r.stateManager = stateManager
+
+	if err := r.cleanUpDefaultForwardRules(); err != nil {
+		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
+	}
+
+	if err := r.createContainers(); err != nil {
+		return fmt.Errorf("create containers: %w", err)
+	}
+
+	r.updateState()
+
+	return nil
+}
+
+func (r *router) AddRouteFiltering(
+	sources []netip.Prefix,
+	destination netip.Prefix,
+	proto firewall.Protocol,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	action firewall.Action,
+) (firewall.Rule, error) {
+	ruleKey := id.GenerateRouteRuleKey(sources, destination, proto, sPort, dPort, action)
+	if _, ok := r.rules[string(ruleKey)]; ok {
+		return ruleKey, nil
+	}
+
+	var setName string
+	if len(sources) > 1 {
+		setName = firewall.GenerateSetName(sources)
+		if _, err := r.ipsetCounter.Increment(setName, sources); err != nil {
+			return nil, fmt.Errorf("create or get ipset: %w", err)
+		}
+	}
+
+	params := routeFilteringRuleParams{
+		Sources:     sources,
+		Destination: destination,
+		Proto:       proto,
+		SPort:       sPort,
+		DPort:       dPort,
+		Action:      action,
+		SetName:     setName,
+	}
+
+	rule := genRouteFilteringRuleSpec(params)
+	if err := r.iptablesClient.Append(tableFilter, chainRTFWD, rule...); err != nil {
+		return nil, fmt.Errorf("add route rule: %v", err)
+	}
+
+	r.rules[string(ruleKey)] = rule
+
+	r.updateState()
+
+	return ruleKey, nil
+}
+
+func (r *router) DeleteRouteRule(rule firewall.Rule) error {
+	ruleKey := rule.GetRuleID()
+
+	if rule, exists := r.rules[ruleKey]; exists {
+		setName := r.findSetNameInRule(rule)
+
+		if err := r.iptablesClient.Delete(tableFilter, chainRTFWD, rule...); err != nil {
+			return fmt.Errorf("delete route rule: %v", err)
+		}
+		delete(r.rules, ruleKey)
+
+		if setName != "" {
+			if _, err := r.ipsetCounter.Decrement(setName); err != nil {
+				return fmt.Errorf("failed to remove ipset: %w", err)
+			}
+		}
+	} else {
+		log.Debugf("route rule %s not found", ruleKey)
+	}
+
+	r.updateState()
+
+	return nil
+}
+
+func (r *router) findSetNameInRule(rule []string) string {
+	for i, arg := range rule {
+		if arg == "-m" && i+3 < len(rule) && rule[i+1] == "set" && rule[i+2] == matchSet {
+			return rule[i+3]
+		}
+	}
+	return ""
+}
+
+func (r *router) createIpSet(setName string, sources []netip.Prefix) error {
+	if err := ipset.Create(setName, ipset.OptTimeout(0)); err != nil {
+		return fmt.Errorf("create set %s: %w", setName, err)
+	}
+
+	for _, prefix := range sources {
+		if err := ipset.AddPrefix(setName, prefix); err != nil {
+			return fmt.Errorf("add element to set %s: %w", setName, err)
+		}
+	}
+
+	return nil
+}
+
+func (r *router) deleteIpSet(setName string) error {
+	if err := ipset.Destroy(setName); err != nil {
+		return fmt.Errorf("destroy set %s: %w", setName, err)
+	}
+	return nil
+}
+
+// AddNatRule inserts an iptables rule pair into the nat chain
+func (r *router) AddNatRule(pair firewall.RouterPair) error {
+	if r.legacyManagement {
+		log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination)
+		if err := r.addLegacyRouteRule(pair); err != nil {
+			return fmt.Errorf("add legacy routing rule: %w", err)
+		}
+	}
+
+	if !pair.Masquerade {
+		return nil
+	}
+
+	if err := r.addNatRule(pair); err != nil {
+		return fmt.Errorf("add nat rule: %w", err)
+	}
+
+	if err := r.addNatRule(firewall.GetInversePair(pair)); err != nil {
+		return fmt.Errorf("add inverse nat rule: %w", err)
+	}
+
+	r.updateState()
+
+	return nil
+}
+
+// RemoveNatRule removes an iptables rule pair from forwarding and nat chains
+func (r *router) RemoveNatRule(pair firewall.RouterPair) error {
+	if err := r.removeNatRule(pair); err != nil {
+		return fmt.Errorf("remove nat rule: %w", err)
+	}
+
+	if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil {
+		return fmt.Errorf("remove inverse nat rule: %w", err)
+	}
+
+	if err := r.removeLegacyRouteRule(pair); err != nil {
+		return fmt.Errorf("remove legacy routing rule: %w", err)
+	}
+
+	r.updateState()
+
+	return nil
+}
+
+// addLegacyRouteRule adds a legacy routing rule for mgmt servers pre route acls
+func (r *router) addLegacyRouteRule(pair firewall.RouterPair) error {
+	ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair)
+
+	if err := r.removeLegacyRouteRule(pair); err != nil {
+		return err
+	}
+
+	rule := []string{"-s", pair.Source.String(), "-d", pair.Destination.String(), "-j", routingFinalForwardJump}
+	if err := r.iptablesClient.Append(tableFilter, chainRTFWD, rule...); err != nil {
+		return fmt.Errorf("add legacy forwarding rule %s -> %s: %v", pair.Source, pair.Destination, err)
+	}
+
+	r.rules[ruleKey] = rule
+
+	return nil
+}
+
+func (r *router) removeLegacyRouteRule(pair firewall.RouterPair) error {
+	ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair)
+
+	if rule, exists := r.rules[ruleKey]; exists {
+		if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFWD, rule...); err != nil {
+			return fmt.Errorf("remove legacy forwarding rule %s -> %s: %v", pair.Source, pair.Destination, err)
+		}
+		delete(r.rules, ruleKey)
+	} else {
+		log.Debugf("legacy forwarding rule %s not found", ruleKey)
+	}
+
+	return nil
+}
+
+// GetLegacyManagement returns the current legacy management mode
+func (r *router) GetLegacyManagement() bool {
+	return r.legacyManagement
+}
+
+// SetLegacyManagement sets the route manager to use legacy management mode
+func (r *router) SetLegacyManagement(isLegacy bool) {
+	r.legacyManagement = isLegacy
+}
+
+// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls
+func (r *router) RemoveAllLegacyRouteRules() error {
+	var merr *multierror.Error
+	for k, rule := range r.rules {
+		if !strings.HasPrefix(k, firewall.ForwardingFormatPrefix) {
+			continue
+		}
+		if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFWD, rule...); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %v", err))
+		} else {
+			delete(r.rules, k)
+		}
+	}
+
+	r.updateState()
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+func (r *router) Reset() error {
+	var merr *multierror.Error
+	if err := r.cleanUpDefaultForwardRules(); err != nil {
+		merr = multierror.Append(merr, err)
+	}
+	r.rules = make(map[string][]string)
+
+	if err := r.ipsetCounter.Flush(); err != nil {
+		merr = multierror.Append(merr, err)
+	}
+
+	r.updateState()
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+func (r *router) cleanUpDefaultForwardRules() error {
+	if err := r.cleanJumpRules(); err != nil {
+		return fmt.Errorf("clean jump rules: %w", err)
+	}
+
+	log.Debug("flushing routing related tables")
+	for _, chainInfo := range []struct {
+		chain string
+		table string
+	}{
+		{chainRTFWD, tableFilter},
+		{chainRTNAT, tableNat},
+		{chainRTPRE, tableMangle},
+	} {
+		ok, err := r.iptablesClient.ChainExists(chainInfo.table, chainInfo.chain)
+		if err != nil {
+			return fmt.Errorf("check chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)
+		} else if ok {
+			if err = r.iptablesClient.ClearAndDeleteChain(chainInfo.table, chainInfo.chain); err != nil {
+				return fmt.Errorf("clear and delete chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (r *router) createContainers() error {
+	for _, chainInfo := range []struct {
+		chain string
+		table string
+	}{
+		{chainRTFWD, tableFilter},
+		{chainRTPRE, tableMangle},
+		{chainRTNAT, tableNat},
+	} {
+		if err := r.createAndSetupChain(chainInfo.chain); err != nil {
+			return fmt.Errorf("create chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)
+		}
+	}
+
+	if err := r.insertEstablishedRule(chainRTFWD); err != nil {
+		return fmt.Errorf("insert established rule: %w", err)
+	}
+
+	if err := r.addPostroutingRules(); err != nil {
+		return fmt.Errorf("add static nat rules: %w", err)
+	}
+
+	if err := r.addJumpRules(); err != nil {
+		return fmt.Errorf("add jump rules: %w", err)
+	}
+
+	return nil
+}
+
+func (r *router) addPostroutingRules() error {
+	// First rule for outbound masquerade
+	rule1 := []string{
+		"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade),
+		"!", "-o", "lo",
+		"-j", routingFinalNatJump,
+	}
+	if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule1...); err != nil {
+		return fmt.Errorf("add outbound masquerade rule: %v", err)
+	}
+	r.rules["static-nat-outbound"] = rule1
+
+	// Second rule for return traffic masquerade
+	rule2 := []string{
+		"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn),
+		"-o", r.wgIface.Name(),
+		"-j", routingFinalNatJump,
+	}
+	if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule2...); err != nil {
+		return fmt.Errorf("add return masquerade rule: %v", err)
+	}
+	r.rules["static-nat-return"] = rule2
+
+	return nil
+}
+
+func (r *router) createAndSetupChain(chain string) error {
+	table := r.getTableForChain(chain)
+
+	if err := r.iptablesClient.NewChain(table, chain); err != nil {
+		return fmt.Errorf("failed creating chain %s, error: %v", chain, err)
+	}
+
+	return nil
+}
+
+func (r *router) getTableForChain(chain string) string {
+	switch chain {
+	case chainRTNAT:
+		return tableNat
+	case chainRTPRE:
+		return tableMangle
+	default:
+		return tableFilter
+	}
+}
+
+func (r *router) insertEstablishedRule(chain string) error {
+	establishedRule := getConntrackEstablished()
+
+	err := r.iptablesClient.Insert(tableFilter, chain, 1, establishedRule...)
+	if err != nil {
+		return fmt.Errorf("failed to insert established rule: %v", err)
+	}
+
+	ruleKey := "established-" + chain
+	r.rules[ruleKey] = establishedRule
+
+	return nil
+}
+
+func (r *router) addJumpRules() error {
+	// Jump to NAT chain
+	natRule := []string{"-j", chainRTNAT}
+	if err := r.iptablesClient.Insert(tableNat, chainPOSTROUTING, 1, natRule...); err != nil {
+		return fmt.Errorf("add nat jump rule: %v", err)
+	}
+	r.rules[jumpNat] = natRule
+
+	// Jump to prerouting chain
+	preRule := []string{"-j", chainRTPRE}
+	if err := r.iptablesClient.Insert(tableMangle, chainPREROUTING, 1, preRule...); err != nil {
+		return fmt.Errorf("add prerouting jump rule: %v", err)
+	}
+	r.rules[jumpPre] = preRule
+
+	return nil
+}
+
+func (r *router) cleanJumpRules() error {
+	for _, ruleKey := range []string{jumpNat, jumpPre} {
+		if rule, exists := r.rules[ruleKey]; exists {
+			table := tableNat
+			chain := chainPOSTROUTING
+			if ruleKey == jumpPre {
+				table = tableMangle
+				chain = chainPREROUTING
+			}
+
+			if err := r.iptablesClient.DeleteIfExists(table, chain, rule...); err != nil {
+				return fmt.Errorf("delete rule from chain %s in table %s, err: %v", chain, table, err)
+			}
+			delete(r.rules, ruleKey)
+		}
+	}
+	return nil
+}
+
+func (r *router) addNatRule(pair firewall.RouterPair) error {
+	ruleKey := firewall.GenKey(firewall.NatFormat, pair)
+
+	if rule, exists := r.rules[ruleKey]; exists {
+		if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPRE, rule...); err != nil {
+			return fmt.Errorf("error while removing existing marking rule for %s: %v", pair.Destination, err)
+		}
+		delete(r.rules, ruleKey)
+	}
+
+	markValue := nbnet.PreroutingFwmarkMasquerade
+	if pair.Inverse {
+		markValue = nbnet.PreroutingFwmarkMasqueradeReturn
+	}
+
+	rule := []string{"-i", r.wgIface.Name()}
+	if pair.Inverse {
+		rule = []string{"!", "-i", r.wgIface.Name()}
+	}
+
+	rule = append(rule,
+		"-m", "conntrack",
+		"--ctstate", "NEW",
+		"-s", pair.Source.String(),
+		"-d", pair.Destination.String(),
+		"-j", "MARK", "--set-mark", fmt.Sprintf("%#x", markValue),
+	)
+
+	if err := r.iptablesClient.Append(tableMangle, chainRTPRE, rule...); err != nil {
+		return fmt.Errorf("error while adding marking rule for %s: %v", pair.Destination, err)
+	}
+
+	r.rules[ruleKey] = rule
+	return nil
+}
+
+func (r *router) removeNatRule(pair firewall.RouterPair) error {
+	ruleKey := firewall.GenKey(firewall.NatFormat, pair)
+
+	if rule, exists := r.rules[ruleKey]; exists {
+		if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPRE, rule...); err != nil {
+			return fmt.Errorf("error while removing marking rule for %s: %v", pair.Destination, err)
+		}
+		delete(r.rules, ruleKey)
+	} else {
+		log.Debugf("marking rule %s not found", ruleKey)
+	}
+
+	return nil
+}
+
+func (r *router) updateState() {
+	if r.stateManager == nil {
+		return
+	}
+
+	var currentState *ShutdownState
+	if existing := r.stateManager.GetState(currentState); existing != nil {
+		if existingState, ok := existing.(*ShutdownState); ok {
+			currentState = existingState
+		}
+	}
+	if currentState == nil {
+		currentState = &ShutdownState{}
+	}
+
+	currentState.Lock()
+	defer currentState.Unlock()
+
+	currentState.RouteRules = r.rules
+	currentState.RouteIPsetCounter = r.ipsetCounter
+
+	if err := r.stateManager.UpdateState(currentState); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+}
+
+func genRouteFilteringRuleSpec(params routeFilteringRuleParams) []string {
+	var rule []string
+
+	if params.SetName != "" {
+		rule = append(rule, "-m", "set", matchSet, params.SetName, "src")
+	} else if len(params.Sources) > 0 {
+		source := params.Sources[0]
+		rule = append(rule, "-s", source.String())
+	}
+
+	rule = append(rule, "-d", params.Destination.String())
+
+	if params.Proto != firewall.ProtocolALL {
+		rule = append(rule, "-p", strings.ToLower(string(params.Proto)))
+		rule = append(rule, applyPort("--sport", params.SPort)...)
+		rule = append(rule, applyPort("--dport", params.DPort)...)
+	}
+
+	rule = append(rule, "-j", actionToStr(params.Action))
+
+	return rule
+}
+
+func applyPort(flag string, port *firewall.Port) []string {
+	if port == nil {
+		return nil
+	}
+
+	if port.IsRange && len(port.Values) == 2 {
+		return []string{flag, fmt.Sprintf("%d:%d", port.Values[0], port.Values[1])}
+	}
+
+	if len(port.Values) > 1 {
+		portList := make([]string, len(port.Values))
+		for i, p := range port.Values {
+			portList[i] = strconv.Itoa(p)
+		}
+		return []string{"-m", "multiport", flag, strings.Join(portList, ",")}
+	}
+
+	return []string{flag, strconv.Itoa(port.Values[0])}
+}

client/firewall/iptables/router_linux_test.go

@@ -0,0 +1,376 @@
+//go:build !android
+
+package iptables
+
+import (
+	"fmt"
+	"net/netip"
+	"os/exec"
+	"testing"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/firewall/test"
+	nbnet "github.com/netbirdio/netbird/util/net"
+)
+
+func isIptablesSupported() bool {
+	_, err4 := exec.LookPath("iptables")
+	return err4 == nil
+}
+
+func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) {
+	if !isIptablesSupported() {
+		t.SkipNow()
+	}
+
+	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+	require.NoError(t, err, "failed to init iptables client")
+
+	manager, err := newRouter(iptablesClient, ifaceMock)
+	require.NoError(t, err, "should return a valid iptables manager")
+	require.NoError(t, manager.init(nil))
+
+	defer func() {
+		assert.NoError(t, manager.Reset(), "shouldn't return error")
+	}()
+
+	// Now 5 rules:
+	// 1. established rule in forward chain
+	// 2. jump rule to NAT chain
+	// 3. jump rule to PRE chain
+	// 4. static outbound masquerade rule
+	// 5. static return masquerade rule
+	require.Len(t, manager.rules, 5, "should have created rules map")
+
+	exists, err := manager.iptablesClient.Exists(tableNat, chainPOSTROUTING, "-j", chainRTNAT)
+	require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainPOSTROUTING)
+	require.True(t, exists, "postrouting jump rule should exist")
+
+	exists, err = manager.iptablesClient.Exists(tableMangle, chainPREROUTING, "-j", chainRTPRE)
+	require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainPREROUTING)
+	require.True(t, exists, "prerouting jump rule should exist")
+
+	pair := firewall.RouterPair{
+		ID:          "abc",
+		Source:      netip.MustParsePrefix("100.100.100.1/32"),
+		Destination: netip.MustParsePrefix("100.100.100.0/24"),
+		Masquerade:  true,
+	}
+
+	err = manager.AddNatRule(pair)
+	require.NoError(t, err, "adding NAT rule should not return error")
+
+	err = manager.Reset()
+	require.NoError(t, err, "shouldn't return error")
+}
+
+func TestIptablesManager_AddNatRule(t *testing.T) {
+	if !isIptablesSupported() {
+		t.SkipNow()
+	}
+
+	for _, testCase := range test.InsertRuleTestCases {
+		t.Run(testCase.Name, func(t *testing.T) {
+			iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+			require.NoError(t, err, "failed to init iptables client")
+
+			manager, err := newRouter(iptablesClient, ifaceMock)
+			require.NoError(t, err, "shouldn't return error")
+			require.NoError(t, manager.init(nil))
+
+			defer func() {
+				assert.NoError(t, manager.Reset(), "shouldn't return error")
+			}()
+
+			err = manager.AddNatRule(testCase.InputPair)
+			require.NoError(t, err, "marking rule should be inserted")
+
+			natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair)
+			markingRule := []string{
+				"-i", ifaceMock.Name(),
+				"-m", "conntrack",
+				"--ctstate", "NEW",
+				"-s", testCase.InputPair.Source.String(),
+				"-d", testCase.InputPair.Destination.String(),
+				"-j", "MARK", "--set-mark",
+				fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade),
+			}
+
+			exists, err := iptablesClient.Exists(tableMangle, chainRTPRE, markingRule...)
+			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE)
+			if testCase.InputPair.Masquerade {
+				require.True(t, exists, "marking rule should be created")
+				foundRule, found := manager.rules[natRuleKey]
+				require.True(t, found, "marking rule should exist in the map")
+				require.Equal(t, markingRule, foundRule, "stored marking rule should match")
+			} else {
+				require.False(t, exists, "marking rule should not be created")
+				_, found := manager.rules[natRuleKey]
+				require.False(t, found, "marking rule should not exist in the map")
+			}
+
+			// Check inverse rule
+			inversePair := firewall.GetInversePair(testCase.InputPair)
+			inverseRuleKey := firewall.GenKey(firewall.NatFormat, inversePair)
+			inverseMarkingRule := []string{
+				"!", "-i", ifaceMock.Name(),
+				"-m", "conntrack",
+				"--ctstate", "NEW",
+				"-s", inversePair.Source.String(),
+				"-d", inversePair.Destination.String(),
+				"-j", "MARK", "--set-mark",
+				fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn),
+			}
+
+			exists, err = iptablesClient.Exists(tableMangle, chainRTPRE, inverseMarkingRule...)
+			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE)
+			if testCase.InputPair.Masquerade {
+				require.True(t, exists, "inverse marking rule should be created")
+				foundRule, found := manager.rules[inverseRuleKey]
+				require.True(t, found, "inverse marking rule should exist in the map")
+				require.Equal(t, inverseMarkingRule, foundRule, "stored inverse marking rule should match")
+			} else {
+				require.False(t, exists, "inverse marking rule should not be created")
+				_, found := manager.rules[inverseRuleKey]
+				require.False(t, found, "inverse marking rule should not exist in the map")
+			}
+		})
+	}
+}
+
+func TestIptablesManager_RemoveNatRule(t *testing.T) {
+	if !isIptablesSupported() {
+		t.SkipNow()
+	}
+
+	for _, testCase := range test.RemoveRuleTestCases {
+		t.Run(testCase.Name, func(t *testing.T) {
+			iptablesClient, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+
+			manager, err := newRouter(iptablesClient, ifaceMock)
+			require.NoError(t, err, "shouldn't return error")
+			require.NoError(t, manager.init(nil))
+			defer func() {
+				assert.NoError(t, manager.Reset(), "shouldn't return error")
+			}()
+
+			err = manager.AddNatRule(testCase.InputPair)
+			require.NoError(t, err, "should add NAT rule without error")
+
+			err = manager.RemoveNatRule(testCase.InputPair)
+			require.NoError(t, err, "shouldn't return error")
+
+			natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair)
+			markingRule := []string{
+				"-i", ifaceMock.Name(),
+				"-m", "conntrack",
+				"--ctstate", "NEW",
+				"-s", testCase.InputPair.Source.String(),
+				"-d", testCase.InputPair.Destination.String(),
+				"-j", "MARK", "--set-mark",
+				fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade),
+			}
+
+			exists, err := iptablesClient.Exists(tableMangle, chainRTPRE, markingRule...)
+			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE)
+			require.False(t, exists, "marking rule should not exist")
+
+			_, found := manager.rules[natRuleKey]
+			require.False(t, found, "marking rule should not exist in the manager map")
+
+			// Check inverse rule removal
+			inversePair := firewall.GetInversePair(testCase.InputPair)
+			inverseRuleKey := firewall.GenKey(firewall.NatFormat, inversePair)
+			inverseMarkingRule := []string{
+				"!", "-i", ifaceMock.Name(),
+				"-m", "conntrack",
+				"--ctstate", "NEW",
+				"-s", inversePair.Source.String(),
+				"-d", inversePair.Destination.String(),
+				"-j", "MARK", "--set-mark",
+				fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn),
+			}
+
+			exists, err = iptablesClient.Exists(tableMangle, chainRTPRE, inverseMarkingRule...)
+			require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE)
+			require.False(t, exists, "inverse marking rule should not exist")
+
+			_, found = manager.rules[inverseRuleKey]
+			require.False(t, found, "inverse marking rule should not exist in the map")
+		})
+	}
+}
+
+func TestRouter_AddRouteFiltering(t *testing.T) {
+	if !isIptablesSupported() {
+		t.Skip("iptables not supported on this system")
+	}
+
+	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+	require.NoError(t, err, "Failed to create iptables client")
+
+	r, err := newRouter(iptablesClient, ifaceMock)
+	require.NoError(t, err, "Failed to create router manager")
+	require.NoError(t, r.init(nil))
+
+	defer func() {
+		err := r.Reset()
+		require.NoError(t, err, "Failed to reset router")
+	}()
+
+	tests := []struct {
+		name        string
+		sources     []netip.Prefix
+		destination netip.Prefix
+		proto       firewall.Protocol
+		sPort       *firewall.Port
+		dPort       *firewall.Port
+		direction   firewall.RuleDirection
+		action      firewall.Action
+		expectSet   bool
+	}{
+		{
+			name:        "Basic TCP rule with single source",
+			sources:     []netip.Prefix{netip.MustParsePrefix("192.168.1.0/24")},
+			destination: netip.MustParsePrefix("10.0.0.0/24"),
+			proto:       firewall.ProtocolTCP,
+			sPort:       nil,
+			dPort:       &firewall.Port{Values: []int{80}},
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name: "UDP rule with multiple sources",
+			sources: []netip.Prefix{
+				netip.MustParsePrefix("172.16.0.0/16"),
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+			destination: netip.MustParsePrefix("10.0.0.0/8"),
+			proto:       firewall.ProtocolUDP,
+			sPort:       &firewall.Port{Values: []int{1024, 2048}, IsRange: true},
+			dPort:       nil,
+			direction:   firewall.RuleDirectionOUT,
+			action:      firewall.ActionDrop,
+			expectSet:   true,
+		},
+		{
+			name:        "All protocols rule",
+			sources:     []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")},
+			destination: netip.MustParsePrefix("0.0.0.0/0"),
+			proto:       firewall.ProtocolALL,
+			sPort:       nil,
+			dPort:       nil,
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "ICMP rule",
+			sources:     []netip.Prefix{netip.MustParsePrefix("192.168.0.0/16")},
+			destination: netip.MustParsePrefix("10.0.0.0/8"),
+			proto:       firewall.ProtocolICMP,
+			sPort:       nil,
+			dPort:       nil,
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "TCP rule with multiple source ports",
+			sources:     []netip.Prefix{netip.MustParsePrefix("172.16.0.0/12")},
+			destination: netip.MustParsePrefix("192.168.0.0/16"),
+			proto:       firewall.ProtocolTCP,
+			sPort:       &firewall.Port{Values: []int{80, 443, 8080}},
+			dPort:       nil,
+			direction:   firewall.RuleDirectionOUT,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "UDP rule with single IP and port range",
+			sources:     []netip.Prefix{netip.MustParsePrefix("192.168.1.1/32")},
+			destination: netip.MustParsePrefix("10.0.0.0/24"),
+			proto:       firewall.ProtocolUDP,
+			sPort:       nil,
+			dPort:       &firewall.Port{Values: []int{5000, 5100}, IsRange: true},
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionDrop,
+			expectSet:   false,
+		},
+		{
+			name:        "TCP rule with source and destination ports",
+			sources:     []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")},
+			destination: netip.MustParsePrefix("172.16.0.0/16"),
+			proto:       firewall.ProtocolTCP,
+			sPort:       &firewall.Port{Values: []int{1024, 65535}, IsRange: true},
+			dPort:       &firewall.Port{Values: []int{22}},
+			direction:   firewall.RuleDirectionOUT,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "Drop all incoming traffic",
+			sources:     []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")},
+			destination: netip.MustParsePrefix("192.168.0.0/24"),
+			proto:       firewall.ProtocolALL,
+			sPort:       nil,
+			dPort:       nil,
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionDrop,
+			expectSet:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ruleKey, err := r.AddRouteFiltering(tt.sources, tt.destination, tt.proto, tt.sPort, tt.dPort, tt.action)
+			require.NoError(t, err, "AddRouteFiltering failed")
+
+			// Check if the rule is in the internal map
+			rule, ok := r.rules[ruleKey.GetRuleID()]
+			assert.True(t, ok, "Rule not found in internal map")
+
+			// Log the internal rule
+			t.Logf("Internal rule: %v", rule)
+
+			// Check if the rule exists in iptables
+			exists, err := iptablesClient.Exists(tableFilter, chainRTFWD, rule...)
+			assert.NoError(t, err, "Failed to check rule existence")
+			assert.True(t, exists, "Rule not found in iptables")
+
+			// Verify rule content
+			params := routeFilteringRuleParams{
+				Sources:     tt.sources,
+				Destination: tt.destination,
+				Proto:       tt.proto,
+				SPort:       tt.sPort,
+				DPort:       tt.dPort,
+				Action:      tt.action,
+				SetName:     "",
+			}
+
+			expectedRule := genRouteFilteringRuleSpec(params)
+
+			if tt.expectSet {
+				setName := firewall.GenerateSetName(tt.sources)
+				params.SetName = setName
+				expectedRule = genRouteFilteringRuleSpec(params)
+
+				// Check if the set was created
+				_, exists := r.ipsetCounter.Get(setName)
+				assert.True(t, exists, "IPSet not created")
+			}
+
+			assert.Equal(t, expectedRule, rule, "Rule content mismatch")
+
+			// Clean up
+			err = r.DeleteRouteRule(ruleKey)
+			require.NoError(t, err, "Failed to delete rule")
+		})
+	}
+}

client/firewall/iptables/rule.go

@@ -2,12 +2,15 @@ package iptables
 
 // Rule to handle management of rules
 type Rule struct {
-	id    string
+	ruleID    string
+	ipsetName string
+
 	specs []string
-	v6    bool
+	ip    string
+	chain string
 }
 
 // GetRuleID returns the rule id
 func (r *Rule) GetRuleID() string {
-	return r.id
+	return r.ruleID
 }

client/firewall/iptables/rulestore_linux.go

@@ -0,0 +1,103 @@
+package iptables
+
+import "encoding/json"
+
+type ipList struct {
+	ips map[string]struct{}
+}
+
+func newIpList(ip string) *ipList {
+	ips := make(map[string]struct{})
+	ips[ip] = struct{}{}
+
+	return &ipList{
+		ips: ips,
+	}
+}
+
+func (s *ipList) addIP(ip string) {
+	s.ips[ip] = struct{}{}
+}
+
+// MarshalJSON implements json.Marshaler
+func (s *ipList) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		IPs map[string]struct{} `json:"ips"`
+	}{
+		IPs: s.ips,
+	})
+}
+
+// UnmarshalJSON implements json.Unmarshaler
+func (s *ipList) UnmarshalJSON(data []byte) error {
+	temp := struct {
+		IPs map[string]struct{} `json:"ips"`
+	}{}
+	if err := json.Unmarshal(data, &temp); err != nil {
+		return err
+	}
+	s.ips = temp.IPs
+
+	if temp.IPs == nil {
+		temp.IPs = make(map[string]struct{})
+	}
+
+	return nil
+}
+
+type ipsetStore struct {
+	ipsets map[string]*ipList
+}
+
+func newIpsetStore() *ipsetStore {
+	return &ipsetStore{
+		ipsets: make(map[string]*ipList),
+	}
+}
+
+func (s *ipsetStore) ipset(ipsetName string) (*ipList, bool) {
+	r, ok := s.ipsets[ipsetName]
+	return r, ok
+}
+
+func (s *ipsetStore) addIpList(ipsetName string, list *ipList) {
+	s.ipsets[ipsetName] = list
+}
+
+func (s *ipsetStore) deleteIpset(ipsetName string) {
+	delete(s.ipsets, ipsetName)
+}
+
+func (s *ipsetStore) ipsetNames() []string {
+	names := make([]string, 0, len(s.ipsets))
+	for name := range s.ipsets {
+		names = append(names, name)
+	}
+	return names
+}
+
+// MarshalJSON implements json.Marshaler
+func (s *ipsetStore) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		IPSets map[string]*ipList `json:"ipsets"`
+	}{
+		IPSets: s.ipsets,
+	})
+}
+
+// UnmarshalJSON implements json.Unmarshaler
+func (s *ipsetStore) UnmarshalJSON(data []byte) error {
+	temp := struct {
+		IPSets map[string]*ipList `json:"ipsets"`
+	}{}
+	if err := json.Unmarshal(data, &temp); err != nil {
+		return err
+	}
+	s.ipsets = temp.IPSets
+
+	if temp.IPSets == nil {
+		temp.IPSets = make(map[string]*ipList)
+	}
+
+	return nil
+}

client/firewall/iptables/state_linux.go

@@ -0,0 +1,70 @@
+package iptables
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+type InterfaceState struct {
+	NameStr       string          `json:"name"`
+	WGAddress     iface.WGAddress `json:"wg_address"`
+	UserspaceBind bool            `json:"userspace_bind"`
+}
+
+func (i *InterfaceState) Name() string {
+	return i.NameStr
+}
+
+func (i *InterfaceState) Address() device.WGAddress {
+	return i.WGAddress
+}
+
+func (i *InterfaceState) IsUserspaceBind() bool {
+	return i.UserspaceBind
+}
+
+type ShutdownState struct {
+	sync.Mutex
+
+	InterfaceState *InterfaceState `json:"interface_state,omitempty"`
+
+	RouteRules        routeRules    `json:"route_rules,omitempty"`
+	RouteIPsetCounter *ipsetCounter `json:"route_ipset_counter,omitempty"`
+
+	ACLEntries    aclEntries  `json:"acl_entries,omitempty"`
+	ACLIPsetStore *ipsetStore `json:"acl_ipset_store,omitempty"`
+}
+
+func (s *ShutdownState) Name() string {
+	return "iptables_state"
+}
+
+func (s *ShutdownState) Cleanup() error {
+	ipt, err := Create(s.InterfaceState)
+	if err != nil {
+		return fmt.Errorf("create iptables manager: %w", err)
+	}
+
+	if s.RouteRules != nil {
+		ipt.router.rules = s.RouteRules
+	}
+	if s.RouteIPsetCounter != nil {
+		ipt.router.ipsetCounter.LoadData(s.RouteIPsetCounter)
+	}
+
+	if s.ACLEntries != nil {
+		ipt.aclMgr.entries = s.ACLEntries
+	}
+	if s.ACLIPsetStore != nil {
+		ipt.aclMgr.ipsetStore = s.ACLIPsetStore
+	}
+
+	if err := ipt.Reset(nil); err != nil {
+		return fmt.Errorf("reset iptables manager: %w", err)
+	}
+
+	return nil
+}

client/firewall/manager/firewall.go

@@ -0,0 +1,190 @@
+package manager
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"net/netip"
+	"sort"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+)
+
+const (
+	ForwardingFormatPrefix = "netbird-fwd-"
+	ForwardingFormat       = "netbird-fwd-%s-%t"
+	PreroutingFormat       = "netbird-prerouting-%s-%t"
+	NatFormat              = "netbird-nat-%s-%t"
+)
+
+// Rule abstraction should be implemented by each firewall manager
+//
+// Each firewall type for different OS can use different type
+// of the properties to hold data of the created rule
+type Rule interface {
+	// GetRuleID returns the rule id
+	GetRuleID() string
+}
+
+// RuleDirection is the traffic direction which a rule is applied
+type RuleDirection int
+
+const (
+	// RuleDirectionIN applies to filters that handlers incoming traffic
+	RuleDirectionIN RuleDirection = iota
+	// RuleDirectionOUT applies to filters that handlers outgoing traffic
+	RuleDirectionOUT
+)
+
+// Action is the action to be taken on a rule
+type Action int
+
+const (
+	// ActionAccept is the action to accept a packet
+	ActionAccept Action = iota
+	// ActionDrop is the action to drop a packet
+	ActionDrop
+)
+
+// Manager is the high level abstraction of a firewall manager
+//
+// It declares methods which handle actions required by the
+// Netbird client for ACL and routing functionality
+type Manager interface {
+	Init(stateManager *statemanager.Manager) error
+
+	// AllowNetbird allows netbird interface traffic
+	AllowNetbird() error
+
+	// AddPeerFiltering adds a rule to the firewall
+	//
+	// If comment argument is empty firewall manager should set
+	// rule ID as comment for the rule
+	AddPeerFiltering(
+		ip net.IP,
+		proto Protocol,
+		sPort *Port,
+		dPort *Port,
+		direction RuleDirection,
+		action Action,
+		ipsetName string,
+		comment string,
+	) ([]Rule, error)
+
+	// DeletePeerRule from the firewall by rule definition
+	DeletePeerRule(rule Rule) error
+
+	// IsServerRouteSupported returns true if the firewall supports server side routing operations
+	IsServerRouteSupported() bool
+
+	AddRouteFiltering(source []netip.Prefix, destination netip.Prefix, proto Protocol, sPort *Port, dPort *Port, action Action) (Rule, error)
+
+	// DeleteRouteRule deletes a routing rule
+	DeleteRouteRule(rule Rule) error
+
+	// AddNatRule inserts a routing NAT rule
+	AddNatRule(pair RouterPair) error
+
+	// RemoveNatRule removes a routing NAT rule
+	RemoveNatRule(pair RouterPair) error
+
+	// SetLegacyManagement sets the legacy management mode
+	SetLegacyManagement(legacy bool) error
+
+	// Reset firewall to the default state
+	Reset(stateManager *statemanager.Manager) error
+
+	// Flush the changes to firewall controller
+	Flush() error
+}
+
+func GenKey(format string, pair RouterPair) string {
+	return fmt.Sprintf(format, pair.ID, pair.Inverse)
+}
+
+// LegacyManager defines the interface for legacy management operations
+type LegacyManager interface {
+	RemoveAllLegacyRouteRules() error
+	GetLegacyManagement() bool
+	SetLegacyManagement(bool)
+}
+
+// SetLegacyManagement sets the route manager to use legacy management
+func SetLegacyManagement(router LegacyManager, isLegacy bool) error {
+	oldLegacy := router.GetLegacyManagement()
+
+	if oldLegacy != isLegacy {
+		router.SetLegacyManagement(isLegacy)
+		log.Debugf("Set legacy management to %v", isLegacy)
+	}
+
+	// client reconnected to a newer mgmt, we need to clean up the legacy rules
+	if !isLegacy && oldLegacy {
+		if err := router.RemoveAllLegacyRouteRules(); err != nil {
+			return fmt.Errorf("remove legacy routing rules: %v", err)
+		}
+
+		log.Debugf("Legacy routing rules removed")
+	}
+
+	return nil
+}
+
+// GenerateSetName generates a unique name for an ipset based on the given sources.
+func GenerateSetName(sources []netip.Prefix) string {
+	// sort for consistent naming
+	SortPrefixes(sources)
+
+	var sourcesStr strings.Builder
+	for _, src := range sources {
+		sourcesStr.WriteString(src.String())
+	}
+
+	hash := sha256.Sum256([]byte(sourcesStr.String()))
+	shortHash := hex.EncodeToString(hash[:])[:8]
+
+	return fmt.Sprintf("nb-%s", shortHash)
+}
+
+// MergeIPRanges merges overlapping IP ranges and returns a slice of non-overlapping netip.Prefix
+func MergeIPRanges(prefixes []netip.Prefix) []netip.Prefix {
+	if len(prefixes) == 0 {
+		return prefixes
+	}
+
+	merged := []netip.Prefix{prefixes[0]}
+	for _, prefix := range prefixes[1:] {
+		last := merged[len(merged)-1]
+		if last.Contains(prefix.Addr()) {
+			// If the current prefix is contained within the last merged prefix, skip it
+			continue
+		}
+		if prefix.Contains(last.Addr()) {
+			// If the current prefix contains the last merged prefix, replace it
+			merged[len(merged)-1] = prefix
+		} else {
+			// Otherwise, add the current prefix to the merged list
+			merged = append(merged, prefix)
+		}
+	}
+
+	return merged
+}
+
+// SortPrefixes sorts the given slice of netip.Prefix in place.
+// It sorts first by IP address, then by prefix length (most specific to least specific).
+func SortPrefixes(prefixes []netip.Prefix) {
+	sort.Slice(prefixes, func(i, j int) bool {
+		addrCmp := prefixes[i].Addr().Compare(prefixes[j].Addr())
+		if addrCmp != 0 {
+			return addrCmp < 0
+		}
+
+		// If IP addresses are the same, compare prefix lengths (longer prefixes first)
+		return prefixes[i].Bits() > prefixes[j].Bits()
+	})
+}

client/firewall/manager/firewall_test.go

@@ -0,0 +1,192 @@
+package manager_test
+
+import (
+	"net/netip"
+	"reflect"
+	"regexp"
+	"testing"
+
+	"github.com/netbirdio/netbird/client/firewall/manager"
+)
+
+func TestGenerateSetName(t *testing.T) {
+	t.Run("Different orders result in same hash", func(t *testing.T) {
+		prefixes1 := []netip.Prefix{
+			netip.MustParsePrefix("192.168.1.0/24"),
+			netip.MustParsePrefix("10.0.0.0/8"),
+		}
+		prefixes2 := []netip.Prefix{
+			netip.MustParsePrefix("10.0.0.0/8"),
+			netip.MustParsePrefix("192.168.1.0/24"),
+		}
+
+		result1 := manager.GenerateSetName(prefixes1)
+		result2 := manager.GenerateSetName(prefixes2)
+
+		if result1 != result2 {
+			t.Errorf("Different orders produced different hashes: %s != %s", result1, result2)
+		}
+	})
+
+	t.Run("Result format is correct", func(t *testing.T) {
+		prefixes := []netip.Prefix{
+			netip.MustParsePrefix("192.168.1.0/24"),
+			netip.MustParsePrefix("10.0.0.0/8"),
+		}
+
+		result := manager.GenerateSetName(prefixes)
+
+		matched, err := regexp.MatchString(`^nb-[0-9a-f]{8}$`, result)
+		if err != nil {
+			t.Fatalf("Error matching regex: %v", err)
+		}
+		if !matched {
+			t.Errorf("Result format is incorrect: %s", result)
+		}
+	})
+
+	t.Run("Empty input produces consistent result", func(t *testing.T) {
+		result1 := manager.GenerateSetName([]netip.Prefix{})
+		result2 := manager.GenerateSetName([]netip.Prefix{})
+
+		if result1 != result2 {
+			t.Errorf("Empty input produced inconsistent results: %s != %s", result1, result2)
+		}
+	})
+
+	t.Run("IPv4 and IPv6 mixing", func(t *testing.T) {
+		prefixes1 := []netip.Prefix{
+			netip.MustParsePrefix("192.168.1.0/24"),
+			netip.MustParsePrefix("2001:db8::/32"),
+		}
+		prefixes2 := []netip.Prefix{
+			netip.MustParsePrefix("2001:db8::/32"),
+			netip.MustParsePrefix("192.168.1.0/24"),
+		}
+
+		result1 := manager.GenerateSetName(prefixes1)
+		result2 := manager.GenerateSetName(prefixes2)
+
+		if result1 != result2 {
+			t.Errorf("Different orders of IPv4 and IPv6 produced different hashes: %s != %s", result1, result2)
+		}
+	})
+}
+
+func TestMergeIPRanges(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []netip.Prefix
+		expected []netip.Prefix
+	}{
+		{
+			name:     "Empty input",
+			input:    []netip.Prefix{},
+			expected: []netip.Prefix{},
+		},
+		{
+			name: "Single range",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+			},
+		},
+		{
+			name: "Two non-overlapping ranges",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("10.0.0.0/8"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("10.0.0.0/8"),
+			},
+		},
+		{
+			name: "One range containing another",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/16"),
+				netip.MustParsePrefix("192.168.1.0/24"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+		},
+		{
+			name: "One range containing another (different order)",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+		},
+		{
+			name: "Overlapping ranges",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("192.168.1.128/25"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+			},
+		},
+		{
+			name: "Overlapping ranges (different order)",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.128/25"),
+				netip.MustParsePrefix("192.168.1.0/24"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.0/24"),
+			},
+		},
+		{
+			name: "Multiple overlapping ranges",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/16"),
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("192.168.2.0/24"),
+				netip.MustParsePrefix("192.168.1.128/25"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+		},
+		{
+			name: "Partially overlapping ranges",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/23"),
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("192.168.2.0/25"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("192.168.0.0/23"),
+				netip.MustParsePrefix("192.168.2.0/25"),
+			},
+		},
+		{
+			name: "IPv6 ranges",
+			input: []netip.Prefix{
+				netip.MustParsePrefix("2001:db8::/32"),
+				netip.MustParsePrefix("2001:db8:1::/48"),
+				netip.MustParsePrefix("2001:db8:2::/48"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("2001:db8::/32"),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := manager.MergeIPRanges(tt.input)
+			if !reflect.DeepEqual(result, tt.expected) {
+				t.Errorf("MergeIPRanges() = %v, want %v", result, tt.expected)
+			}
+		})
+	}
+}

client/firewall/manager/port.go

@@ -0,0 +1,46 @@
+package manager
+
+import (
+	"strconv"
+)
+
+// Protocol is the protocol of the port
+type Protocol string
+
+const (
+	// ProtocolTCP is the TCP protocol
+	ProtocolTCP Protocol = "tcp"
+
+	// ProtocolUDP is the UDP protocol
+	ProtocolUDP Protocol = "udp"
+
+	// ProtocolICMP is the ICMP protocol
+	ProtocolICMP Protocol = "icmp"
+
+	// ProtocolALL cover all supported protocols
+	ProtocolALL Protocol = "all"
+
+	// ProtocolUnknown unknown protocol
+	ProtocolUnknown Protocol = "unknown"
+)
+
+// Port of the address for firewall rule
+type Port struct {
+	// IsRange is true Values contains two values, the first is the start port, the second is the end port
+	IsRange bool
+
+	// Values contains one value for single port, multiple values for the list of ports, or two values for the range of ports
+	Values []int
+}
+
+// String interface implementation
+func (p *Port) String() string {
+	var ports string
+	for _, port := range p.Values {
+		if ports != "" {
+			ports += ","
+		}
+		ports += strconv.Itoa(port)
+	}
+	return ports
+}

client/firewall/manager/routerpair.go

@@ -0,0 +1,26 @@
+package manager
+
+import (
+	"net/netip"
+
+	"github.com/netbirdio/netbird/route"
+)
+
+type RouterPair struct {
+	ID          route.ID
+	Source      netip.Prefix
+	Destination netip.Prefix
+	Masquerade  bool
+	Inverse     bool
+}
+
+func GetInversePair(pair RouterPair) RouterPair {
+	return RouterPair{
+		ID: pair.ID,
+		// invert Source/Destination
+		Source:      pair.Destination,
+		Destination: pair.Source,
+		Masquerade:  pair.Masquerade,
+		Inverse:     true,
+	}
+}

client/firewall/nftables/acl_linux.go

@@ -0,0 +1,823 @@
+package nftables
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"net"
+	"net/netip"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
+	"github.com/google/nftables/expr"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	nbnet "github.com/netbirdio/netbird/util/net"
+)
+
+const (
+
+	// rules chains contains the effective ACL rules
+	chainNameInputRules  = "netbird-acl-input-rules"
+	chainNameOutputRules = "netbird-acl-output-rules"
+
+	// filter chains contains the rules that jump to the rules chains
+	chainNameInputFilter   = "netbird-acl-input-filter"
+	chainNameOutputFilter  = "netbird-acl-output-filter"
+	chainNameForwardFilter = "netbird-acl-forward-filter"
+	chainNamePrerouting    = "netbird-rt-prerouting"
+
+	allowNetbirdInputRuleID = "allow Netbird incoming traffic"
+)
+
+const flushError = "flush: %w"
+
+var (
+	anyIP = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+)
+
+type AclManager struct {
+	rConn              *nftables.Conn
+	sConn              *nftables.Conn
+	wgIface            iFaceMapper
+	routingFwChainName string
+
+	workTable        *nftables.Table
+	chainInputRules  *nftables.Chain
+	chainOutputRules *nftables.Chain
+
+	ipsetStore *ipsetStore
+	rules      map[string]*Rule
+}
+
+func newAclManager(table *nftables.Table, wgIface iFaceMapper, routingFwChainName string) (*AclManager, error) {
+	// sConn is used for creating sets and adding/removing elements from them
+	// it's differ then rConn (which does create new conn for each flush operation)
+	// and is permanent. Using same connection for both type of operations
+	// overloads netlink with high amount of rules ( > 10000)
+	sConn, err := nftables.New(nftables.AsLasting())
+	if err != nil {
+		return nil, fmt.Errorf("create nf conn: %w", err)
+	}
+
+	return &AclManager{
+		rConn:              &nftables.Conn{},
+		sConn:              sConn,
+		wgIface:            wgIface,
+		workTable:          table,
+		routingFwChainName: routingFwChainName,
+
+		ipsetStore: newIpsetStore(),
+		rules:      make(map[string]*Rule),
+	}, nil
+}
+
+func (m *AclManager) init(workTable *nftables.Table) error {
+	m.workTable = workTable
+	return m.createDefaultChains()
+}
+
+// AddPeerFiltering rule to the firewall
+//
+// If comment argument is empty firewall manager should set
+// rule ID as comment for the rule
+func (m *AclManager) AddPeerFiltering(
+	ip net.IP,
+	proto firewall.Protocol,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	direction firewall.RuleDirection,
+	action firewall.Action,
+	ipsetName string,
+	comment string,
+) ([]firewall.Rule, error) {
+	var ipset *nftables.Set
+	if ipsetName != "" {
+		var err error
+		ipset, err = m.addIpToSet(ipsetName, ip)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	newRules := make([]firewall.Rule, 0, 2)
+	ioRule, err := m.addIOFiltering(ip, proto, sPort, dPort, direction, action, ipset, comment)
+	if err != nil {
+		return nil, err
+	}
+
+	newRules = append(newRules, ioRule)
+	return newRules, nil
+}
+
+// DeletePeerRule from the firewall by rule definition
+func (m *AclManager) DeletePeerRule(rule firewall.Rule) error {
+	r, ok := rule.(*Rule)
+	if !ok {
+		return fmt.Errorf("invalid rule type")
+	}
+
+	if r.nftSet == nil {
+		err := m.rConn.DelRule(r.nftRule)
+		if err != nil {
+			log.Errorf("failed to delete rule: %v", err)
+		}
+		delete(m.rules, r.GetRuleID())
+		return m.rConn.Flush()
+	}
+
+	ips, ok := m.ipsetStore.ips(r.nftSet.Name)
+	if !ok {
+		err := m.rConn.DelRule(r.nftRule)
+		if err != nil {
+			log.Errorf("failed to delete rule: %v", err)
+		}
+		delete(m.rules, r.GetRuleID())
+		return m.rConn.Flush()
+	}
+	if _, ok := ips[r.ip.String()]; ok {
+		err := m.sConn.SetDeleteElements(r.nftSet, []nftables.SetElement{{Key: r.ip.To4()}})
+		if err != nil {
+			log.Errorf("delete elements for set %q: %v", r.nftSet.Name, err)
+		}
+		if err := m.sConn.Flush(); err != nil {
+			log.Debugf("flush error of set delete element, %s", r.nftSet.Name)
+			return err
+		}
+		m.ipsetStore.DeleteIpFromSet(r.nftSet.Name, r.ip)
+	}
+
+	// if after delete, set still contains other IPs,
+	// no need to delete firewall rule and we should exit here
+	if len(ips) > 0 {
+		return nil
+	}
+
+	err := m.rConn.DelRule(r.nftRule)
+	if err != nil {
+		log.Errorf("failed to delete rule: %v", err)
+	}
+	err = m.rConn.Flush()
+	if err != nil {
+		return err
+	}
+
+	delete(m.rules, r.GetRuleID())
+	m.ipsetStore.DeleteReferenceFromIpSet(r.nftSet.Name)
+
+	if m.ipsetStore.HasReferenceToSet(r.nftSet.Name) {
+		return nil
+	}
+
+	// we delete last IP from the set, that means we need to delete
+	// set itself and associated firewall rule too
+	m.rConn.FlushSet(r.nftSet)
+	m.rConn.DelSet(r.nftSet)
+	m.ipsetStore.deleteIpset(r.nftSet.Name)
+	return nil
+}
+
+// createDefaultAllowRules creates default allow rules for the input and output chains
+func (m *AclManager) createDefaultAllowRules() error {
+	expIn := []expr.Any{
+		&expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseNetworkHeader,
+			Offset:       12,
+			Len:          4,
+		},
+		// mask
+		&expr.Bitwise{
+			SourceRegister: 1,
+			DestRegister:   1,
+			Len:            4,
+			Mask:           []byte{0, 0, 0, 0},
+			Xor:            []byte{0, 0, 0, 0},
+		},
+		// net address
+		&expr.Cmp{
+			Register: 1,
+			Data:     []byte{0, 0, 0, 0},
+		},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+
+	_ = m.rConn.InsertRule(&nftables.Rule{
+		Table:    m.workTable,
+		Chain:    m.chainInputRules,
+		Position: 0,
+		Exprs:    expIn,
+	})
+
+	expOut := []expr.Any{
+		&expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseNetworkHeader,
+			Offset:       16,
+			Len:          4,
+		},
+		// mask
+		&expr.Bitwise{
+			SourceRegister: 1,
+			DestRegister:   1,
+			Len:            4,
+			Mask:           []byte{0, 0, 0, 0},
+			Xor:            []byte{0, 0, 0, 0},
+		},
+		// net address
+		&expr.Cmp{
+			Register: 1,
+			Data:     []byte{0, 0, 0, 0},
+		},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+
+	_ = m.rConn.InsertRule(&nftables.Rule{
+		Table:    m.workTable,
+		Chain:    m.chainOutputRules,
+		Position: 0,
+		Exprs:    expOut,
+	})
+
+	if err := m.rConn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+	return nil
+}
+
+// Flush rule/chain/set operations from the buffer
+//
+// Method also get all rules after flush and refreshes handle values in the rulesets
+func (m *AclManager) Flush() error {
+	if err := m.flushWithBackoff(); err != nil {
+		return err
+	}
+
+	if err := m.refreshRuleHandles(m.chainInputRules); err != nil {
+		log.Errorf("failed to refresh rule handles ipv4 input chain: %v", err)
+	}
+
+	if err := m.refreshRuleHandles(m.chainOutputRules); err != nil {
+		log.Errorf("failed to refresh rule handles IPv4 output chain: %v", err)
+	}
+
+	return nil
+}
+
+func (m *AclManager) addIOFiltering(ip net.IP, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, direction firewall.RuleDirection, action firewall.Action, ipset *nftables.Set, comment string) (*Rule, error) {
+	ruleId := generatePeerRuleId(ip, sPort, dPort, direction, action, ipset)
+	if r, ok := m.rules[ruleId]; ok {
+		return &Rule{
+			r.nftRule,
+			r.nftSet,
+			r.ruleID,
+			ip,
+		}, nil
+	}
+
+	var expressions []expr.Any
+
+	if proto != firewall.ProtocolALL {
+		expressions = append(expressions, &expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseNetworkHeader,
+			Offset:       uint32(9),
+			Len:          uint32(1),
+		})
+
+		protoData, err := protoToInt(proto)
+		if err != nil {
+			return nil, fmt.Errorf("convert protocol to number: %v", err)
+		}
+
+		expressions = append(expressions, &expr.Cmp{
+			Register: 1,
+			Op:       expr.CmpOpEq,
+			Data:     []byte{protoData},
+		})
+	}
+
+	rawIP := ip.To4()
+	// check if rawIP contains zeroed IPv4 0.0.0.0 value
+	// in that case not add IP match expression into the rule definition
+	if !bytes.HasPrefix(anyIP, rawIP) {
+		// source address position
+		addrOffset := uint32(12)
+		if direction == firewall.RuleDirectionOUT {
+			addrOffset += 4 // is ipv4 address length
+		}
+
+		expressions = append(expressions,
+			&expr.Payload{
+				DestRegister: 1,
+				Base:         expr.PayloadBaseNetworkHeader,
+				Offset:       addrOffset,
+				Len:          4,
+			},
+		)
+		// add individual IP for match if no ipset defined
+		if ipset == nil {
+			expressions = append(expressions,
+				&expr.Cmp{
+					Op:       expr.CmpOpEq,
+					Register: 1,
+					Data:     rawIP,
+				},
+			)
+		} else {
+			expressions = append(expressions,
+				&expr.Lookup{
+					SourceRegister: 1,
+					SetName:        ipset.Name,
+					SetID:          ipset.ID,
+				},
+			)
+		}
+	}
+
+	if sPort != nil && len(sPort.Values) != 0 {
+		expressions = append(expressions,
+			&expr.Payload{
+				DestRegister: 1,
+				Base:         expr.PayloadBaseTransportHeader,
+				Offset:       0,
+				Len:          2,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     encodePort(*sPort),
+			},
+		)
+	}
+
+	if dPort != nil && len(dPort.Values) != 0 {
+		expressions = append(expressions,
+			&expr.Payload{
+				DestRegister: 1,
+				Base:         expr.PayloadBaseTransportHeader,
+				Offset:       2,
+				Len:          2,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     encodePort(*dPort),
+			},
+		)
+	}
+
+	switch action {
+	case firewall.ActionAccept:
+		expressions = append(expressions, &expr.Verdict{Kind: expr.VerdictAccept})
+	case firewall.ActionDrop:
+		expressions = append(expressions, &expr.Verdict{Kind: expr.VerdictDrop})
+	}
+
+	userData := []byte(strings.Join([]string{ruleId, comment}, " "))
+
+	var chain *nftables.Chain
+	if direction == firewall.RuleDirectionIN {
+		chain = m.chainInputRules
+	} else {
+		chain = m.chainOutputRules
+	}
+	nftRule := m.rConn.AddRule(&nftables.Rule{
+		Table:    m.workTable,
+		Chain:    chain,
+		Exprs:    expressions,
+		UserData: userData,
+	})
+
+	rule := &Rule{
+		nftRule: nftRule,
+		nftSet:  ipset,
+		ruleID:  ruleId,
+		ip:      ip,
+	}
+	m.rules[ruleId] = rule
+	if ipset != nil {
+		m.ipsetStore.AddReferenceToIpset(ipset.Name)
+	}
+	return rule, nil
+}
+
+func (m *AclManager) createDefaultChains() (err error) {
+	// chainNameInputRules
+	chain := m.createChain(chainNameInputRules)
+	err = m.rConn.Flush()
+	if err != nil {
+		log.Debugf("failed to create chain (%s): %s", chain.Name, err)
+		return fmt.Errorf(flushError, err)
+	}
+	m.chainInputRules = chain
+
+	// chainNameOutputRules
+	chain = m.createChain(chainNameOutputRules)
+	err = m.rConn.Flush()
+	if err != nil {
+		log.Debugf("failed to create chain (%s): %s", chainNameOutputRules, err)
+		return err
+	}
+	m.chainOutputRules = chain
+
+	// netbird-acl-input-filter
+	// type filter hook input priority filter; policy accept;
+	chain = m.createFilterChainWithHook(chainNameInputFilter, nftables.ChainHookInput)
+	m.addJumpRule(chain, m.chainInputRules.Name, expr.MetaKeyIIFNAME) // to netbird-acl-input-rules
+	m.addDropExpressions(chain, expr.MetaKeyIIFNAME)
+	err = m.rConn.Flush()
+	if err != nil {
+		log.Debugf("failed to create chain (%s): %s", chain.Name, err)
+		return err
+	}
+
+	// netbird-acl-output-filter
+	// type filter hook output priority filter; policy accept;
+	chain = m.createFilterChainWithHook(chainNameOutputFilter, nftables.ChainHookOutput)
+	m.addFwdAllow(chain, expr.MetaKeyOIFNAME)
+	m.addJumpRule(chain, m.chainOutputRules.Name, expr.MetaKeyOIFNAME) // to netbird-acl-output-rules
+	m.addDropExpressions(chain, expr.MetaKeyOIFNAME)
+	err = m.rConn.Flush()
+	if err != nil {
+		log.Debugf("failed to create chain (%s): %s", chainNameOutputFilter, err)
+		return err
+	}
+
+	// netbird-acl-forward-filter
+	chainFwFilter := m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward)
+	m.addJumpRulesToRtForward(chainFwFilter) // to netbird-rt-fwd
+	m.addDropExpressions(chainFwFilter, expr.MetaKeyIIFNAME)
+
+	err = m.rConn.Flush()
+	if err != nil {
+		log.Debugf("failed to create chain (%s): %s", chainNameForwardFilter, err)
+		return fmt.Errorf(flushError, err)
+	}
+
+	if err := m.allowRedirectedTraffic(chainFwFilter); err != nil {
+		log.Errorf("failed to allow redirected traffic: %s", err)
+	}
+
+	return nil
+}
+
+// Makes redirected traffic originally destined for the host itself (now subject to the forward filter)
+// go through the input filter as well. This will enable e.g. Docker services to keep working by accessing the
+// netbird peer IP.
+func (m *AclManager) allowRedirectedTraffic(chainFwFilter *nftables.Chain) error {
+	preroutingChain := m.rConn.AddChain(&nftables.Chain{
+		Name:     chainNamePrerouting,
+		Table:    m.workTable,
+		Type:     nftables.ChainTypeFilter,
+		Hooknum:  nftables.ChainHookPrerouting,
+		Priority: nftables.ChainPriorityMangle,
+	})
+
+	m.addPreroutingRule(preroutingChain)
+
+	m.addFwmarkToForward(chainFwFilter)
+
+	if err := m.rConn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	return nil
+}
+
+func (m *AclManager) addPreroutingRule(preroutingChain *nftables.Chain) {
+	m.rConn.AddRule(&nftables.Rule{
+		Table: m.workTable,
+		Chain: preroutingChain,
+		Exprs: []expr.Any{
+			&expr.Meta{
+				Key:      expr.MetaKeyIIFNAME,
+				Register: 1,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     ifname(m.wgIface.Name()),
+			},
+			&expr.Fib{
+				Register:       1,
+				ResultADDRTYPE: true,
+				FlagDADDR:      true,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     binaryutil.NativeEndian.PutUint32(unix.RTN_LOCAL),
+			},
+			&expr.Immediate{
+				Register: 1,
+				Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected),
+			},
+			&expr.Meta{
+				Key:            expr.MetaKeyMARK,
+				Register:       1,
+				SourceRegister: true,
+			},
+		},
+	})
+}
+
+func (m *AclManager) addFwmarkToForward(chainFwFilter *nftables.Chain) {
+	m.rConn.InsertRule(&nftables.Rule{
+		Table: m.workTable,
+		Chain: chainFwFilter,
+		Exprs: []expr.Any{
+			&expr.Meta{
+				Key:      expr.MetaKeyMARK,
+				Register: 1,
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected),
+			},
+			&expr.Verdict{
+				Kind:  expr.VerdictJump,
+				Chain: m.chainInputRules.Name,
+			},
+		},
+	})
+}
+
+func (m *AclManager) addJumpRulesToRtForward(chainFwFilter *nftables.Chain) {
+	expressions := []expr.Any{
+		&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     ifname(m.wgIface.Name()),
+		},
+		&expr.Verdict{
+			Kind:  expr.VerdictJump,
+			Chain: m.routingFwChainName,
+		},
+	}
+
+	_ = m.rConn.AddRule(&nftables.Rule{
+		Table: m.workTable,
+		Chain: chainFwFilter,
+		Exprs: expressions,
+	})
+}
+
+func (m *AclManager) createChain(name string) *nftables.Chain {
+	chain := &nftables.Chain{
+		Name:  name,
+		Table: m.workTable,
+	}
+
+	chain = m.rConn.AddChain(chain)
+
+	insertReturnTrafficRule(m.rConn, m.workTable, chain)
+
+	return chain
+}
+
+func (m *AclManager) createFilterChainWithHook(name string, hookNum *nftables.ChainHook) *nftables.Chain {
+	polAccept := nftables.ChainPolicyAccept
+	chain := &nftables.Chain{
+		Name:     name,
+		Table:    m.workTable,
+		Hooknum:  hookNum,
+		Priority: nftables.ChainPriorityFilter,
+		Type:     nftables.ChainTypeFilter,
+		Policy:   &polAccept,
+	}
+
+	return m.rConn.AddChain(chain)
+}
+
+func (m *AclManager) addDropExpressions(chain *nftables.Chain, ifaceKey expr.MetaKey) []expr.Any {
+	expressions := []expr.Any{
+		&expr.Meta{Key: ifaceKey, Register: 1},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     ifname(m.wgIface.Name()),
+		},
+		&expr.Verdict{Kind: expr.VerdictDrop},
+	}
+	_ = m.rConn.AddRule(&nftables.Rule{
+		Table: m.workTable,
+		Chain: chain,
+		Exprs: expressions,
+	})
+	return nil
+}
+
+func (m *AclManager) addFwdAllow(chain *nftables.Chain, iifname expr.MetaKey) {
+	ip, _ := netip.AddrFromSlice(m.wgIface.Address().Network.IP.To4())
+	dstOp := expr.CmpOpNeq
+	expressions := []expr.Any{
+		&expr.Meta{Key: iifname, Register: 1},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     ifname(m.wgIface.Name()),
+		},
+		&expr.Payload{
+			DestRegister: 2,
+			Base:         expr.PayloadBaseNetworkHeader,
+			Offset:       16,
+			Len:          4,
+		},
+		&expr.Bitwise{
+			SourceRegister: 2,
+			DestRegister:   2,
+			Len:            4,
+			Xor:            []byte{0x0, 0x0, 0x0, 0x0},
+			Mask:           m.wgIface.Address().Network.Mask,
+		},
+		&expr.Cmp{
+			Op:       dstOp,
+			Register: 2,
+			Data:     ip.Unmap().AsSlice(),
+		},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+	_ = m.rConn.AddRule(&nftables.Rule{
+		Table: chain.Table,
+		Chain: chain,
+		Exprs: expressions,
+	})
+}
+
+func (m *AclManager) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr.MetaKey) {
+	expressions := []expr.Any{
+		&expr.Meta{Key: ifaceKey, Register: 1},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     ifname(m.wgIface.Name()),
+		},
+		&expr.Verdict{
+			Kind:  expr.VerdictJump,
+			Chain: to,
+		},
+	}
+
+	_ = m.rConn.AddRule(&nftables.Rule{
+		Table: chain.Table,
+		Chain: chain,
+		Exprs: expressions,
+	})
+}
+
+func (m *AclManager) addIpToSet(ipsetName string, ip net.IP) (*nftables.Set, error) {
+	ipset, err := m.rConn.GetSetByName(m.workTable, ipsetName)
+	rawIP := ip.To4()
+	if err != nil {
+		if ipset, err = m.createSet(m.workTable, ipsetName); err != nil {
+			return nil, fmt.Errorf("get set name: %v", err)
+		}
+
+		m.ipsetStore.newIpset(ipset.Name)
+	}
+
+	if m.ipsetStore.IsIpInSet(ipset.Name, ip) {
+		return ipset, nil
+	}
+
+	if err := m.sConn.SetAddElements(ipset, []nftables.SetElement{{Key: rawIP}}); err != nil {
+		return nil, fmt.Errorf("add set element for the first time: %v", err)
+	}
+
+	m.ipsetStore.AddIpToSet(ipset.Name, ip)
+
+	if err := m.sConn.Flush(); err != nil {
+		return nil, fmt.Errorf("flush add elements: %v", err)
+	}
+
+	return ipset, nil
+}
+
+// createSet in given table by name
+func (m *AclManager) createSet(table *nftables.Table, name string) (*nftables.Set, error) {
+	ipset := &nftables.Set{
+		Name:    name,
+		Table:   table,
+		Dynamic: true,
+		KeyType: nftables.TypeIPAddr,
+	}
+
+	if err := m.rConn.AddSet(ipset, nil); err != nil {
+		return nil, fmt.Errorf("create set: %v", err)
+	}
+
+	if err := m.rConn.Flush(); err != nil {
+		return nil, fmt.Errorf("flush created set: %v", err)
+	}
+
+	return ipset, nil
+}
+
+func (m *AclManager) flushWithBackoff() (err error) {
+	backoff := 4
+	backoffTime := 1000 * time.Millisecond
+	for i := 0; ; i++ {
+		err = m.rConn.Flush()
+		if err != nil {
+			if !strings.Contains(err.Error(), "busy") {
+				return
+			}
+			log.Error("failed to flush nftables, retrying...")
+			if i == backoff-1 {
+				return err
+			}
+			time.Sleep(backoffTime)
+			backoffTime *= 2
+			continue
+		}
+		break
+	}
+	return
+}
+
+func (m *AclManager) refreshRuleHandles(chain *nftables.Chain) error {
+	if m.workTable == nil || chain == nil {
+		return nil
+	}
+
+	list, err := m.rConn.GetRules(m.workTable, chain)
+	if err != nil {
+		return err
+	}
+
+	for _, rule := range list {
+		if len(rule.UserData) == 0 {
+			continue
+		}
+		split := bytes.Split(rule.UserData, []byte(" "))
+		r, ok := m.rules[string(split[0])]
+		if ok {
+			*r.nftRule = *rule
+		}
+	}
+
+	return nil
+}
+
+func generatePeerRuleId(
+	ip net.IP,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	direction firewall.RuleDirection,
+	action firewall.Action,
+	ipset *nftables.Set,
+) string {
+	rulesetID := ":" + strconv.Itoa(int(direction)) + ":"
+	if sPort != nil {
+		rulesetID += sPort.String()
+	}
+	rulesetID += ":"
+	if dPort != nil {
+		rulesetID += dPort.String()
+	}
+	rulesetID += ":"
+	rulesetID += strconv.Itoa(int(action))
+	if ipset == nil {
+		return "ip:" + ip.String() + rulesetID
+	}
+	return "set:" + ipset.Name + rulesetID
+}
+
+func encodePort(port firewall.Port) []byte {
+	bs := make([]byte, 2)
+	binary.BigEndian.PutUint16(bs, uint16(port.Values[0]))
+	return bs
+}
+
+func ifname(n string) []byte {
+	b := make([]byte, 16)
+	copy(b, n+"\x00")
+	return b
+}
+
+func protoToInt(protocol firewall.Protocol) (uint8, error) {
+	switch protocol {
+	case firewall.ProtocolTCP:
+		return unix.IPPROTO_TCP, nil
+	case firewall.ProtocolUDP:
+		return unix.IPPROTO_UDP, nil
+	case firewall.ProtocolICMP:
+		return unix.IPPROTO_ICMP, nil
+	}
+
+	return 0, fmt.Errorf("unsupported protocol: %s", protocol)
+}

client/firewall/nftables/ipsetstore_linux.go

@@ -0,0 +1,85 @@
+package nftables
+
+import (
+	"net"
+)
+
+type ipsetStore struct {
+	ipsetReference map[string]int
+	ipsets         map[string]map[string]struct{} // ipsetName -> list of ips
+}
+
+func newIpsetStore() *ipsetStore {
+	return &ipsetStore{
+		ipsetReference: make(map[string]int),
+		ipsets:         make(map[string]map[string]struct{}),
+	}
+}
+
+func (s *ipsetStore) ips(ipsetName string) (map[string]struct{}, bool) {
+	r, ok := s.ipsets[ipsetName]
+	return r, ok
+}
+
+func (s *ipsetStore) newIpset(ipsetName string) map[string]struct{} {
+	s.ipsetReference[ipsetName] = 0
+	ipList := make(map[string]struct{})
+	s.ipsets[ipsetName] = ipList
+	return ipList
+}
+
+func (s *ipsetStore) deleteIpset(ipsetName string) {
+	delete(s.ipsetReference, ipsetName)
+	delete(s.ipsets, ipsetName)
+}
+
+func (s *ipsetStore) DeleteIpFromSet(ipsetName string, ip net.IP) {
+	ipList, ok := s.ipsets[ipsetName]
+	if !ok {
+		return
+	}
+	delete(ipList, ip.String())
+}
+
+func (s *ipsetStore) AddIpToSet(ipsetName string, ip net.IP) {
+	ipList, ok := s.ipsets[ipsetName]
+	if !ok {
+		return
+	}
+	ipList[ip.String()] = struct{}{}
+}
+
+func (s *ipsetStore) IsIpInSet(ipsetName string, ip net.IP) bool {
+	ipList, ok := s.ipsets[ipsetName]
+	if !ok {
+		return false
+	}
+	_, ok = ipList[ip.String()]
+	return ok
+}
+
+func (s *ipsetStore) AddReferenceToIpset(ipsetName string) {
+	s.ipsetReference[ipsetName]++
+}
+
+func (s *ipsetStore) DeleteReferenceFromIpSet(ipsetName string) {
+	r, ok := s.ipsetReference[ipsetName]
+	if !ok {
+		return
+	}
+	if r == 0 {
+		return
+	}
+	s.ipsetReference[ipsetName]--
+}
+
+func (s *ipsetStore) HasReferenceToSet(ipsetName string) bool {
+	if _, ok := s.ipsetReference[ipsetName]; !ok {
+		return false
+	}
+	if s.ipsetReference[ipsetName] == 0 {
+		return false
+	}
+
+	return true
+}

client/firewall/nftables/manager_linux.go

@@ -0,0 +1,414 @@
+package nftables
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"sync"
+
+	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
+	"github.com/google/nftables/expr"
+	log "github.com/sirupsen/logrus"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+)
+
+const (
+	// tableNameNetbird is the name of the table that is used for filtering by the Netbird client
+	tableNameNetbird = "netbird"
+
+	tableNameFilter = "filter"
+	chainNameInput  = "INPUT"
+)
+
+// iFaceMapper defines subset methods of interface required for manager
+type iFaceMapper interface {
+	Name() string
+	Address() iface.WGAddress
+	IsUserspaceBind() bool
+}
+
+// Manager of iptables firewall
+type Manager struct {
+	mutex   sync.Mutex
+	rConn   *nftables.Conn
+	wgIface iFaceMapper
+
+	router     *router
+	aclManager *AclManager
+}
+
+// Create nftables firewall manager
+func Create(wgIface iFaceMapper) (*Manager, error) {
+	m := &Manager{
+		rConn:   &nftables.Conn{},
+		wgIface: wgIface,
+	}
+
+	workTable := &nftables.Table{Name: tableNameNetbird, Family: nftables.TableFamilyIPv4}
+
+	var err error
+	m.router, err = newRouter(workTable, wgIface)
+	if err != nil {
+		return nil, fmt.Errorf("create router: %w", err)
+	}
+
+	m.aclManager, err = newAclManager(workTable, wgIface, chainNameRoutingFw)
+	if err != nil {
+		return nil, fmt.Errorf("create acl manager: %w", err)
+	}
+
+	return m, nil
+}
+
+// Init nftables firewall manager
+func (m *Manager) Init(stateManager *statemanager.Manager) error {
+	workTable, err := m.createWorkTable()
+	if err != nil {
+		return fmt.Errorf("create work table: %w", err)
+	}
+
+	if err := m.router.init(workTable); err != nil {
+		return fmt.Errorf("router init: %w", err)
+	}
+
+	if err := m.aclManager.init(workTable); err != nil {
+		// TODO: cleanup router
+		return fmt.Errorf("acl manager init: %w", err)
+	}
+
+	stateManager.RegisterState(&ShutdownState{})
+
+	// We only need to record minimal interface state for potential recreation.
+	// Unlike iptables, which requires tracking individual rules, nftables maintains
+	// a known state (our netbird table plus a few static rules). This allows for easy
+	// cleanup using Reset() without needing to store specific rules.
+	if err := stateManager.UpdateState(&ShutdownState{
+		InterfaceState: &InterfaceState{
+			NameStr:       m.wgIface.Name(),
+			WGAddress:     m.wgIface.Address(),
+			UserspaceBind: m.wgIface.IsUserspaceBind(),
+		},
+	}); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+
+	// persist early
+	go func() {
+		if err := stateManager.PersistState(context.Background()); err != nil {
+			log.Errorf("failed to persist state: %v", err)
+		}
+	}()
+
+	return nil
+}
+
+// AddPeerFiltering rule to the firewall
+//
+// If comment argument is empty firewall manager should set
+// rule ID as comment for the rule
+func (m *Manager) AddPeerFiltering(
+	ip net.IP,
+	proto firewall.Protocol,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	direction firewall.RuleDirection,
+	action firewall.Action,
+	ipsetName string,
+	comment string,
+) ([]firewall.Rule, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	rawIP := ip.To4()
+	if rawIP == nil {
+		return nil, fmt.Errorf("unsupported IP version: %s", ip.String())
+	}
+
+	return m.aclManager.AddPeerFiltering(ip, proto, sPort, dPort, direction, action, ipsetName, comment)
+}
+
+func (m *Manager) AddRouteFiltering(sources []netip.Prefix, destination netip.Prefix, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action) (firewall.Rule, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	if !destination.Addr().Is4() {
+		return nil, fmt.Errorf("unsupported IP version: %s", destination.Addr().String())
+	}
+
+	return m.router.AddRouteFiltering(sources, destination, proto, sPort, dPort, action)
+}
+
+// DeletePeerRule from the firewall by rule definition
+func (m *Manager) DeletePeerRule(rule firewall.Rule) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.aclManager.DeletePeerRule(rule)
+}
+
+// DeleteRouteRule deletes a routing rule
+func (m *Manager) DeleteRouteRule(rule firewall.Rule) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.router.DeleteRouteRule(rule)
+}
+
+func (m *Manager) IsServerRouteSupported() bool {
+	return true
+}
+
+func (m *Manager) AddNatRule(pair firewall.RouterPair) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.router.AddNatRule(pair)
+}
+
+func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.router.RemoveNatRule(pair)
+}
+
+// AllowNetbird allows netbird interface traffic
+func (m *Manager) AllowNetbird() error {
+	if !m.wgIface.IsUserspaceBind() {
+		return nil
+	}
+
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	err := m.aclManager.createDefaultAllowRules()
+	if err != nil {
+		return fmt.Errorf("failed to create default allow rules: %v", err)
+	}
+
+	chains, err := m.rConn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return fmt.Errorf("list of chains: %w", err)
+	}
+
+	var chain *nftables.Chain
+	for _, c := range chains {
+		if c.Table.Name == tableNameFilter && c.Name == chainNameInput {
+			chain = c
+			break
+		}
+	}
+
+	if chain == nil {
+		log.Debugf("chain INPUT not found. Skipping add allow netbird rule")
+		return nil
+	}
+
+	rules, err := m.rConn.GetRules(chain.Table, chain)
+	if err != nil {
+		return fmt.Errorf("failed to get rules for the INPUT chain: %v", err)
+	}
+
+	if rule := m.detectAllowNetbirdRule(rules); rule != nil {
+		log.Debugf("allow netbird rule already exists: %v", rule)
+		return nil
+	}
+
+	m.applyAllowNetbirdRules(chain)
+
+	err = m.rConn.Flush()
+	if err != nil {
+		return fmt.Errorf("failed to flush allow input netbird rules: %v", err)
+	}
+
+	return nil
+}
+
+// SetLegacyManagement sets the route manager to use legacy management
+func (m *Manager) SetLegacyManagement(isLegacy bool) error {
+	return firewall.SetLegacyManagement(m.router, isLegacy)
+}
+
+// Reset firewall to the default state
+func (m *Manager) Reset(stateManager *statemanager.Manager) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	if err := m.resetNetbirdInputRules(); err != nil {
+		return fmt.Errorf("reset netbird input rules: %v", err)
+	}
+
+	if err := m.router.Reset(); err != nil {
+		return fmt.Errorf("reset router: %v", err)
+	}
+
+	if err := m.cleanupNetbirdTables(); err != nil {
+		return fmt.Errorf("cleanup netbird tables: %v", err)
+	}
+
+	if err := m.rConn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	if err := stateManager.DeleteState(&ShutdownState{}); err != nil {
+		return fmt.Errorf("delete state: %v", err)
+	}
+
+	return nil
+}
+
+func (m *Manager) resetNetbirdInputRules() error {
+	chains, err := m.rConn.ListChains()
+	if err != nil {
+		return fmt.Errorf("list chains: %w", err)
+	}
+
+	m.deleteNetbirdInputRules(chains)
+
+	return nil
+}
+
+func (m *Manager) deleteNetbirdInputRules(chains []*nftables.Chain) {
+	for _, c := range chains {
+		if c.Table.Name == tableNameFilter && c.Name == chainNameInput {
+			rules, err := m.rConn.GetRules(c.Table, c)
+			if err != nil {
+				log.Errorf("get rules for chain %q: %v", c.Name, err)
+				continue
+			}
+
+			m.deleteMatchingRules(rules)
+		}
+	}
+}
+
+func (m *Manager) deleteMatchingRules(rules []*nftables.Rule) {
+	for _, r := range rules {
+		if bytes.Equal(r.UserData, []byte(allowNetbirdInputRuleID)) {
+			if err := m.rConn.DelRule(r); err != nil {
+				log.Errorf("delete rule: %v", err)
+			}
+		}
+	}
+}
+
+func (m *Manager) cleanupNetbirdTables() error {
+	tables, err := m.rConn.ListTables()
+	if err != nil {
+		return fmt.Errorf("list tables: %w", err)
+	}
+
+	for _, t := range tables {
+		if t.Name == tableNameNetbird {
+			m.rConn.DelTable(t)
+		}
+	}
+	return nil
+}
+
+// Flush rule/chain/set operations from the buffer
+//
+// Method also get all rules after flush and refreshes handle values in the rulesets
+// todo review this method usage
+func (m *Manager) Flush() error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	return m.aclManager.Flush()
+}
+
+func (m *Manager) createWorkTable() (*nftables.Table, error) {
+	tables, err := m.rConn.ListTablesOfFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return nil, fmt.Errorf("list of tables: %w", err)
+	}
+
+	for _, t := range tables {
+		if t.Name == tableNameNetbird {
+			m.rConn.DelTable(t)
+		}
+	}
+
+	table := m.rConn.AddTable(&nftables.Table{Name: tableNameNetbird, Family: nftables.TableFamilyIPv4})
+	err = m.rConn.Flush()
+	return table, err
+}
+
+func (m *Manager) applyAllowNetbirdRules(chain *nftables.Chain) {
+	rule := &nftables.Rule{
+		Table: chain.Table,
+		Chain: chain,
+		Exprs: []expr.Any{
+			&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     ifname(m.wgIface.Name()),
+			},
+			&expr.Verdict{
+				Kind: expr.VerdictAccept,
+			},
+		},
+		UserData: []byte(allowNetbirdInputRuleID),
+	}
+	_ = m.rConn.InsertRule(rule)
+}
+
+func (m *Manager) detectAllowNetbirdRule(existedRules []*nftables.Rule) *nftables.Rule {
+	ifName := ifname(m.wgIface.Name())
+	for _, rule := range existedRules {
+		if rule.Table.Name == tableNameFilter && rule.Chain.Name == chainNameInput {
+			if len(rule.Exprs) < 4 {
+				if e, ok := rule.Exprs[0].(*expr.Meta); !ok || e.Key != expr.MetaKeyIIFNAME {
+					continue
+				}
+				if e, ok := rule.Exprs[1].(*expr.Cmp); !ok || e.Op != expr.CmpOpEq || !bytes.Equal(e.Data, ifName) {
+					continue
+				}
+				return rule
+			}
+		}
+	}
+	return nil
+}
+
+func insertReturnTrafficRule(conn *nftables.Conn, table *nftables.Table, chain *nftables.Chain) {
+	rule := &nftables.Rule{
+		Table: table,
+		Chain: chain,
+		Exprs: getEstablishedExprs(1),
+	}
+
+	conn.InsertRule(rule)
+}
+
+func getEstablishedExprs(register uint32) []expr.Any {
+	return []expr.Any{
+		&expr.Ct{
+			Key:      expr.CtKeySTATE,
+			Register: register,
+		},
+		&expr.Bitwise{
+			SourceRegister: register,
+			DestRegister:   register,
+			Len:            4,
+			Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED),
+			Xor:            binaryutil.NativeEndian.PutUint32(0),
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: register,
+			Data:     []byte{0, 0, 0, 0},
+		},
+		&expr.Counter{},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+}

client/firewall/nftables/manager_linux_test.go

@@ -0,0 +1,331 @@
+package nftables
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"net/netip"
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
+	"github.com/google/nftables/expr"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+
+	fw "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/iface"
+)
+
+var ifaceMock = &iFaceMock{
+	NameFunc: func() string {
+		return "lo"
+	},
+	AddressFunc: func() iface.WGAddress {
+		return iface.WGAddress{
+			IP: net.ParseIP("100.96.0.1"),
+			Network: &net.IPNet{
+				IP:   net.ParseIP("100.96.0.0"),
+				Mask: net.IPv4Mask(255, 255, 255, 0),
+			},
+		}
+	},
+}
+
+// iFaceMapper defines subset methods of interface required for manager
+type iFaceMock struct {
+	NameFunc    func() string
+	AddressFunc func() iface.WGAddress
+}
+
+func (i *iFaceMock) Name() string {
+	if i.NameFunc != nil {
+		return i.NameFunc()
+	}
+	panic("NameFunc is not set")
+}
+
+func (i *iFaceMock) Address() iface.WGAddress {
+	if i.AddressFunc != nil {
+		return i.AddressFunc()
+	}
+	panic("AddressFunc is not set")
+}
+
+func (i *iFaceMock) IsUserspaceBind() bool { return false }
+
+func TestNftablesManager(t *testing.T) {
+
+	// just check on the local interface
+	manager, err := Create(ifaceMock)
+	require.NoError(t, err)
+	require.NoError(t, manager.Init(nil))
+	time.Sleep(time.Second * 3)
+
+	defer func() {
+		err = manager.Reset(nil)
+		require.NoError(t, err, "failed to reset")
+		time.Sleep(time.Second)
+	}()
+
+	ip := net.ParseIP("100.96.0.1")
+
+	testClient := &nftables.Conn{}
+
+	rule, err := manager.AddPeerFiltering(
+		ip,
+		fw.ProtocolTCP,
+		nil,
+		&fw.Port{Values: []int{53}},
+		fw.RuleDirectionIN,
+		fw.ActionDrop,
+		"",
+		"",
+	)
+	require.NoError(t, err, "failed to add rule")
+
+	err = manager.Flush()
+	require.NoError(t, err, "failed to flush")
+
+	rules, err := testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules)
+	require.NoError(t, err, "failed to get rules")
+
+	require.Len(t, rules, 2, "expected 2 rules")
+
+	expectedExprs1 := []expr.Any{
+		&expr.Ct{
+			Key:      expr.CtKeySTATE,
+			Register: 1,
+		},
+		&expr.Bitwise{
+			SourceRegister: 1,
+			DestRegister:   1,
+			Len:            4,
+			Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitESTABLISHED | expr.CtStateBitRELATED),
+			Xor:            binaryutil.NativeEndian.PutUint32(0),
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: 1,
+			Data:     []byte{0, 0, 0, 0},
+		},
+		&expr.Counter{},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+	require.ElementsMatch(t, rules[0].Exprs, expectedExprs1, "expected the same expressions")
+
+	ipToAdd, _ := netip.AddrFromSlice(ip)
+	add := ipToAdd.Unmap()
+	expectedExprs2 := []expr.Any{
+		&expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseNetworkHeader,
+			Offset:       uint32(9),
+			Len:          uint32(1),
+		},
+		&expr.Cmp{
+			Register: 1,
+			Op:       expr.CmpOpEq,
+			Data:     []byte{unix.IPPROTO_TCP},
+		},
+		&expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseNetworkHeader,
+			Offset:       12,
+			Len:          4,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     add.AsSlice(),
+		},
+		&expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseTransportHeader,
+			Offset:       2,
+			Len:          2,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     []byte{0, 53},
+		},
+		&expr.Verdict{Kind: expr.VerdictDrop},
+	}
+	require.ElementsMatch(t, rules[1].Exprs, expectedExprs2, "expected the same expressions")
+
+	for _, r := range rule {
+		err = manager.DeletePeerRule(r)
+		require.NoError(t, err, "failed to delete rule")
+	}
+
+	err = manager.Flush()
+	require.NoError(t, err, "failed to flush")
+
+	rules, err = testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules)
+	require.NoError(t, err, "failed to get rules")
+	// established rule remains
+	require.Len(t, rules, 1, "expected 1 rules after deletion")
+
+	err = manager.Reset(nil)
+	require.NoError(t, err, "failed to reset")
+}
+
+func TestNFtablesCreatePerformance(t *testing.T) {
+	mock := &iFaceMock{
+		NameFunc: func() string {
+			return "lo"
+		},
+		AddressFunc: func() iface.WGAddress {
+			return iface.WGAddress{
+				IP: net.ParseIP("100.96.0.1"),
+				Network: &net.IPNet{
+					IP:   net.ParseIP("100.96.0.0"),
+					Mask: net.IPv4Mask(255, 255, 255, 0),
+				},
+			}
+		},
+	}
+
+	for _, testMax := range []int{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000} {
+		t.Run(fmt.Sprintf("Testing %d rules", testMax), func(t *testing.T) {
+			// just check on the local interface
+			manager, err := Create(mock)
+			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
+			time.Sleep(time.Second * 3)
+
+			defer func() {
+				if err := manager.Reset(nil); err != nil {
+					t.Errorf("clear the manager state: %v", err)
+				}
+				time.Sleep(time.Second)
+			}()
+
+			ip := net.ParseIP("10.20.0.100")
+			start := time.Now()
+			for i := 0; i < testMax; i++ {
+				port := &fw.Port{Values: []int{1000 + i}}
+				if i%2 == 0 {
+					_, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept HTTP traffic")
+				} else {
+					_, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionIN, fw.ActionAccept, "", "accept HTTP traffic")
+				}
+				require.NoError(t, err, "failed to add rule")
+
+				if i%100 == 0 {
+					err = manager.Flush()
+					require.NoError(t, err, "failed to flush")
+				}
+			}
+
+			t.Logf("execution avg per rule: %s", time.Since(start)/time.Duration(testMax))
+		})
+	}
+}
+
+func runIptablesSave(t *testing.T) (string, string) {
+	t.Helper()
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Command("iptables-save")
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	require.NoError(t, err, "iptables-save failed to run")
+
+	return stdout.String(), stderr.String()
+}
+
+func verifyIptablesOutput(t *testing.T, stdout, stderr string) {
+	t.Helper()
+	// Check for any incompatibility warnings
+	require.NotContains(t,
+		stderr,
+		"incompatible",
+		"iptables-save produced compatibility warning. Full stderr: %s",
+		stderr,
+	)
+
+	// Verify standard tables are present
+	expectedTables := []string{
+		"*filter",
+		"*nat",
+		"*mangle",
+	}
+
+	for _, table := range expectedTables {
+		require.Contains(t,
+			stdout,
+			table,
+			"iptables-save output missing expected table: %s\nFull stdout: %s",
+			table,
+			stdout,
+		)
+	}
+}
+
+func TestNftablesManagerCompatibilityWithIptables(t *testing.T) {
+	if check() != NFTABLES {
+		t.Skip("nftables not supported on this system")
+	}
+
+	if _, err := exec.LookPath("iptables-save"); err != nil {
+		t.Skipf("iptables-save not available on this system: %v", err)
+	}
+
+	// First ensure iptables-nft tables exist by running iptables-save
+	stdout, stderr := runIptablesSave(t)
+	verifyIptablesOutput(t, stdout, stderr)
+
+	manager, err := Create(ifaceMock)
+	require.NoError(t, err, "failed to create manager")
+	require.NoError(t, manager.Init(nil))
+
+	t.Cleanup(func() {
+		err := manager.Reset(nil)
+		require.NoError(t, err, "failed to reset manager state")
+
+		// Verify iptables output after reset
+		stdout, stderr := runIptablesSave(t)
+		verifyIptablesOutput(t, stdout, stderr)
+	})
+
+	ip := net.ParseIP("100.96.0.1")
+	_, err = manager.AddPeerFiltering(
+		ip,
+		fw.ProtocolTCP,
+		nil,
+		&fw.Port{Values: []int{80}},
+		fw.RuleDirectionIN,
+		fw.ActionAccept,
+		"",
+		"test rule",
+	)
+	require.NoError(t, err, "failed to add peer filtering rule")
+
+	_, err = manager.AddRouteFiltering(
+		[]netip.Prefix{netip.MustParsePrefix("192.168.2.0/24")},
+		netip.MustParsePrefix("10.1.0.0/24"),
+		fw.ProtocolTCP,
+		nil,
+		&fw.Port{Values: []int{443}},
+		fw.ActionAccept,
+	)
+	require.NoError(t, err, "failed to add route filtering rule")
+
+	pair := fw.RouterPair{
+		Source:      netip.MustParsePrefix("192.168.1.0/24"),
+		Destination: netip.MustParsePrefix("10.0.0.0/24"),
+		Masquerade:  true,
+	}
+	err = manager.AddNatRule(pair)
+	require.NoError(t, err, "failed to add NAT rule")
+
+	stdout, stderr = runIptablesSave(t)
+	verifyIptablesOutput(t, stdout, stderr)
+}

client/firewall/nftables/router_linux.go

@@ -0,0 +1,989 @@
+package nftables
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"strings"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
+	"github.com/google/nftables/expr"
+	"github.com/hashicorp/go-multierror"
+	log "github.com/sirupsen/logrus"
+
+	nberrors "github.com/netbirdio/netbird/client/errors"
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/internal/acl/id"
+	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
+	nbnet "github.com/netbirdio/netbird/util/net"
+)
+
+const (
+	chainNameRoutingFw  = "netbird-rt-fwd"
+	chainNameRoutingNat = "netbird-rt-postrouting"
+	chainNameForward    = "FORWARD"
+
+	userDataAcceptForwardRuleIif = "frwacceptiif"
+	userDataAcceptForwardRuleOif = "frwacceptoif"
+)
+
+const refreshRulesMapError = "refresh rules map: %w"
+
+var (
+	errFilterTableNotFound = fmt.Errorf("nftables: 'filter' table not found")
+)
+
+type router struct {
+	conn        *nftables.Conn
+	workTable   *nftables.Table
+	filterTable *nftables.Table
+	chains      map[string]*nftables.Chain
+	// rules is useful to avoid duplicates and to get missing attributes that we don't have when adding new rules
+	rules        map[string]*nftables.Rule
+	ipsetCounter *refcounter.Counter[string, []netip.Prefix, *nftables.Set]
+
+	wgIface          iFaceMapper
+	legacyManagement bool
+}
+
+func newRouter(workTable *nftables.Table, wgIface iFaceMapper) (*router, error) {
+	r := &router{
+		conn:      &nftables.Conn{},
+		workTable: workTable,
+		chains:    make(map[string]*nftables.Chain),
+		rules:     make(map[string]*nftables.Rule),
+		wgIface:   wgIface,
+	}
+
+	r.ipsetCounter = refcounter.New(
+		r.createIpSet,
+		r.deleteIpSet,
+	)
+
+	var err error
+	r.filterTable, err = r.loadFilterTable()
+	if err != nil {
+		if errors.Is(err, errFilterTableNotFound) {
+			log.Warnf("table 'filter' not found for forward rules")
+		} else {
+			return nil, fmt.Errorf("load filter table: %w", err)
+		}
+	}
+
+	return r, nil
+}
+
+func (r *router) init(workTable *nftables.Table) error {
+	r.workTable = workTable
+
+	if err := r.removeAcceptForwardRules(); err != nil {
+		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
+	}
+
+	if err := r.createContainers(); err != nil {
+		return fmt.Errorf("create containers: %w", err)
+	}
+
+	return nil
+}
+
+// Reset cleans existing nftables default forward rules from the system
+func (r *router) Reset() error {
+	// clear without deleting the ipsets, the nf table will be deleted by the caller
+	r.ipsetCounter.Clear()
+
+	return r.removeAcceptForwardRules()
+}
+
+func (r *router) loadFilterTable() (*nftables.Table, error) {
+	tables, err := r.conn.ListTablesOfFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return nil, fmt.Errorf("nftables: unable to list tables: %v", err)
+	}
+
+	for _, table := range tables {
+		if table.Name == "filter" {
+			return table, nil
+		}
+	}
+
+	return nil, errFilterTableNotFound
+}
+
+func (r *router) createContainers() error {
+	r.chains[chainNameRoutingFw] = r.conn.AddChain(&nftables.Chain{
+		Name:  chainNameRoutingFw,
+		Table: r.workTable,
+	})
+
+	insertReturnTrafficRule(r.conn, r.workTable, r.chains[chainNameRoutingFw])
+
+	prio := *nftables.ChainPriorityNATSource - 1
+	r.chains[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{
+		Name:     chainNameRoutingNat,
+		Table:    r.workTable,
+		Hooknum:  nftables.ChainHookPostrouting,
+		Priority: &prio,
+		Type:     nftables.ChainTypeNAT,
+	})
+
+	// Chain is created by acl manager
+	// TODO: move creation to a common place
+	r.chains[chainNamePrerouting] = &nftables.Chain{
+		Name:     chainNamePrerouting,
+		Table:    r.workTable,
+		Type:     nftables.ChainTypeFilter,
+		Hooknum:  nftables.ChainHookPrerouting,
+		Priority: nftables.ChainPriorityMangle,
+	}
+
+	// Add the single NAT rule that matches on mark
+	if err := r.addPostroutingRules(); err != nil {
+		return fmt.Errorf("add single nat rule: %v", err)
+	}
+
+	if err := r.acceptForwardRules(); err != nil {
+		log.Errorf("failed to add accept rules for the forward chain: %s", err)
+	}
+
+	if err := r.refreshRulesMap(); err != nil {
+		log.Errorf("failed to clean up rules from FORWARD chain: %s", err)
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf("nftables: unable to initialize table: %v", err)
+	}
+
+	return nil
+}
+
+// AddRouteFiltering appends a nftables rule to the routing chain
+func (r *router) AddRouteFiltering(
+	sources []netip.Prefix,
+	destination netip.Prefix,
+	proto firewall.Protocol,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	action firewall.Action,
+) (firewall.Rule, error) {
+
+	ruleKey := id.GenerateRouteRuleKey(sources, destination, proto, sPort, dPort, action)
+	if _, ok := r.rules[string(ruleKey)]; ok {
+		return ruleKey, nil
+	}
+
+	chain := r.chains[chainNameRoutingFw]
+	var exprs []expr.Any
+
+	switch {
+	case len(sources) == 1 && sources[0].Bits() == 0:
+		// If it's 0.0.0.0/0, we don't need to add any source matching
+	case len(sources) == 1:
+		// If there's only one source, we can use it directly
+		exprs = append(exprs, generateCIDRMatcherExpressions(true, sources[0])...)
+	default:
+		// If there are multiple sources, create or get an ipset
+		var err error
+		exprs, err = r.getIpSetExprs(sources, exprs)
+		if err != nil {
+			return nil, fmt.Errorf("get ipset expressions: %w", err)
+		}
+	}
+
+	// Handle destination
+	exprs = append(exprs, generateCIDRMatcherExpressions(false, destination)...)
+
+	// Handle protocol
+	if proto != firewall.ProtocolALL {
+		protoNum, err := protoToInt(proto)
+		if err != nil {
+			return nil, fmt.Errorf("convert protocol to number: %w", err)
+		}
+		exprs = append(exprs, &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1})
+		exprs = append(exprs, &expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     []byte{protoNum},
+		})
+
+		exprs = append(exprs, applyPort(sPort, true)...)
+		exprs = append(exprs, applyPort(dPort, false)...)
+	}
+
+	exprs = append(exprs, &expr.Counter{})
+
+	var verdict expr.VerdictKind
+	if action == firewall.ActionAccept {
+		verdict = expr.VerdictAccept
+	} else {
+		verdict = expr.VerdictDrop
+	}
+	exprs = append(exprs, &expr.Verdict{Kind: verdict})
+
+	rule := &nftables.Rule{
+		Table:    r.workTable,
+		Chain:    chain,
+		Exprs:    exprs,
+		UserData: []byte(ruleKey),
+	}
+
+	rule = r.conn.AddRule(rule)
+
+	log.Tracef("Adding route rule %s", spew.Sdump(rule))
+	if err := r.conn.Flush(); err != nil {
+		return nil, fmt.Errorf(flushError, err)
+	}
+
+	r.rules[string(ruleKey)] = rule
+
+	log.Debugf("nftables: added route rule: sources=%v, destination=%v, proto=%v, sPort=%v, dPort=%v, action=%v", sources, destination, proto, sPort, dPort, action)
+
+	return ruleKey, nil
+}
+
+func (r *router) getIpSetExprs(sources []netip.Prefix, exprs []expr.Any) ([]expr.Any, error) {
+	setName := firewall.GenerateSetName(sources)
+	ref, err := r.ipsetCounter.Increment(setName, sources)
+	if err != nil {
+		return nil, fmt.Errorf("create or get ipset for sources: %w", err)
+	}
+
+	exprs = append(exprs,
+		&expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseNetworkHeader,
+			Offset:       12,
+			Len:          4,
+		},
+		&expr.Lookup{
+			SourceRegister: 1,
+			SetName:        ref.Out.Name,
+			SetID:          ref.Out.ID,
+		},
+	)
+	return exprs, nil
+}
+
+func (r *router) DeleteRouteRule(rule firewall.Rule) error {
+	if err := r.refreshRulesMap(); err != nil {
+		return fmt.Errorf(refreshRulesMapError, err)
+	}
+
+	ruleKey := rule.GetRuleID()
+	nftRule, exists := r.rules[ruleKey]
+	if !exists {
+		log.Debugf("route rule %s not found", ruleKey)
+		return nil
+	}
+
+	if nftRule.Handle == 0 {
+		return fmt.Errorf("route rule %s has no handle", ruleKey)
+	}
+
+	setName := r.findSetNameInRule(nftRule)
+
+	if err := r.deleteNftRule(nftRule, ruleKey); err != nil {
+		return fmt.Errorf("delete: %w", err)
+	}
+
+	if setName != "" {
+		if _, err := r.ipsetCounter.Decrement(setName); err != nil {
+			return fmt.Errorf("decrement ipset reference: %w", err)
+		}
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	return nil
+}
+
+func (r *router) createIpSet(setName string, sources []netip.Prefix) (*nftables.Set, error) {
+	// overlapping prefixes will result in an error, so we need to merge them
+	sources = firewall.MergeIPRanges(sources)
+
+	set := &nftables.Set{
+		Name:  setName,
+		Table: r.workTable,
+		// required for prefixes
+		Interval: true,
+		KeyType:  nftables.TypeIPAddr,
+	}
+
+	var elements []nftables.SetElement
+	for _, prefix := range sources {
+		// TODO: Implement IPv6 support
+		if prefix.Addr().Is6() {
+			log.Printf("Skipping IPv6 prefix %s: IPv6 support not yet implemented", prefix)
+			continue
+		}
+
+		// nftables needs half-open intervals [firstIP, lastIP) for prefixes
+		// e.g. 10.0.0.0/24 becomes [10.0.0.0, 10.0.1.0), 10.1.1.1/32 becomes [10.1.1.1, 10.1.1.2) etc
+		firstIP := prefix.Addr()
+		lastIP := calculateLastIP(prefix).Next()
+
+		elements = append(elements,
+			// the nft tool also adds a line like this, see https://github.com/google/nftables/issues/247
+			// nftables.SetElement{Key: []byte{0, 0, 0, 0}, IntervalEnd: true},
+			nftables.SetElement{Key: firstIP.AsSlice()},
+			nftables.SetElement{Key: lastIP.AsSlice(), IntervalEnd: true},
+		)
+	}
+
+	if err := r.conn.AddSet(set, elements); err != nil {
+		return nil, fmt.Errorf("error adding elements to set %s: %w", setName, err)
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return nil, fmt.Errorf("flush error: %w", err)
+	}
+
+	log.Printf("Created new ipset: %s with %d elements", setName, len(elements)/2)
+
+	return set, nil
+}
+
+// calculateLastIP determines the last IP in a given prefix.
+func calculateLastIP(prefix netip.Prefix) netip.Addr {
+	hostMask := ^uint32(0) >> prefix.Masked().Bits()
+	lastIP := uint32FromNetipAddr(prefix.Addr()) | hostMask
+
+	return netip.AddrFrom4(uint32ToBytes(lastIP))
+}
+
+// Utility function to convert netip.Addr to uint32.
+func uint32FromNetipAddr(addr netip.Addr) uint32 {
+	b := addr.As4()
+	return binary.BigEndian.Uint32(b[:])
+}
+
+// Utility function to convert uint32 to a netip-compatible byte slice.
+func uint32ToBytes(ip uint32) [4]byte {
+	var b [4]byte
+	binary.BigEndian.PutUint32(b[:], ip)
+	return b
+}
+
+func (r *router) deleteIpSet(setName string, set *nftables.Set) error {
+	r.conn.DelSet(set)
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	log.Debugf("Deleted unused ipset %s", setName)
+	return nil
+}
+
+func (r *router) findSetNameInRule(rule *nftables.Rule) string {
+	for _, e := range rule.Exprs {
+		if lookup, ok := e.(*expr.Lookup); ok {
+			return lookup.SetName
+		}
+	}
+	return ""
+}
+
+func (r *router) deleteNftRule(rule *nftables.Rule, ruleKey string) error {
+	if err := r.conn.DelRule(rule); err != nil {
+		return fmt.Errorf("delete rule %s: %w", ruleKey, err)
+	}
+	delete(r.rules, ruleKey)
+
+	log.Debugf("removed route rule %s", ruleKey)
+
+	return nil
+}
+
+// AddNatRule appends a nftables rule pair to the nat chain
+func (r *router) AddNatRule(pair firewall.RouterPair) error {
+	if err := r.refreshRulesMap(); err != nil {
+		return fmt.Errorf(refreshRulesMapError, err)
+	}
+
+	if r.legacyManagement {
+		log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination)
+		if err := r.addLegacyRouteRule(pair); err != nil {
+			return fmt.Errorf("add legacy routing rule: %w", err)
+		}
+	}
+
+	if pair.Masquerade {
+		if err := r.addNatRule(pair); err != nil {
+			return fmt.Errorf("add nat rule: %w", err)
+		}
+
+		if err := r.addNatRule(firewall.GetInversePair(pair)); err != nil {
+			return fmt.Errorf("add inverse nat rule: %w", err)
+		}
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf("nftables: insert rules for %s: %v", pair.Destination, err)
+	}
+
+	return nil
+}
+
+// addNatRule inserts a nftables rule to the conn client flush queue
+func (r *router) addNatRule(pair firewall.RouterPair) error {
+	sourceExp := generateCIDRMatcherExpressions(true, pair.Source)
+	destExp := generateCIDRMatcherExpressions(false, pair.Destination)
+
+	op := expr.CmpOpEq
+	if pair.Inverse {
+		op = expr.CmpOpNeq
+	}
+
+	exprs := []expr.Any{
+		// We only care about NEW connections to mark them and later identify them in the postrouting chain for masquerading.
+		// Masquerading will take care of the conntrack state, which means we won't need to mark established connections.
+		&expr.Ct{
+			Key:      expr.CtKeySTATE,
+			Register: 1,
+		},
+		&expr.Bitwise{
+			SourceRegister: 1,
+			DestRegister:   1,
+			Len:            4,
+			Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW),
+			Xor:            binaryutil.NativeEndian.PutUint32(0),
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: 1,
+			Data:     []byte{0, 0, 0, 0},
+		},
+
+		// interface matching
+		&expr.Meta{
+			Key:      expr.MetaKeyIIFNAME,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       op,
+			Register: 1,
+			Data:     ifname(r.wgIface.Name()),
+		},
+	}
+
+	exprs = append(exprs, sourceExp...)
+	exprs = append(exprs, destExp...)
+
+	var markValue uint32 = nbnet.PreroutingFwmarkMasquerade
+	if pair.Inverse {
+		markValue = nbnet.PreroutingFwmarkMasqueradeReturn
+	}
+
+	exprs = append(exprs,
+		&expr.Immediate{
+			Register: 1,
+			Data:     binaryutil.NativeEndian.PutUint32(markValue),
+		},
+		&expr.Meta{
+			Key:            expr.MetaKeyMARK,
+			SourceRegister: true,
+			Register:       1,
+		},
+	)
+
+	ruleKey := firewall.GenKey(firewall.PreroutingFormat, pair)
+
+	if _, exists := r.rules[ruleKey]; exists {
+		if err := r.removeNatRule(pair); err != nil {
+			return fmt.Errorf("remove prerouting rule: %w", err)
+		}
+	}
+
+	r.rules[ruleKey] = r.conn.AddRule(&nftables.Rule{
+		Table:    r.workTable,
+		Chain:    r.chains[chainNamePrerouting],
+		Exprs:    exprs,
+		UserData: []byte(ruleKey),
+	})
+
+	return nil
+}
+
+// addPostroutingRules adds the masquerade rules
+func (r *router) addPostroutingRules() error {
+	// First masquerade rule for traffic coming in from WireGuard interface
+	exprs := []expr.Any{
+		// Match on the first fwmark
+		&expr.Meta{
+			Key:      expr.MetaKeyMARK,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade),
+		},
+
+		// We need to exclude the loopback interface as this changes the ebpf proxy port
+		&expr.Meta{
+			Key:      expr.MetaKeyOIFNAME,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpNeq,
+			Register: 1,
+			Data:     ifname("lo"),
+		},
+		&expr.Counter{},
+		&expr.Masq{},
+	}
+
+	r.conn.AddRule(&nftables.Rule{
+		Table: r.workTable,
+		Chain: r.chains[chainNameRoutingNat],
+		Exprs: exprs,
+	})
+
+	// Second masquerade rule for traffic going out through WireGuard interface
+	exprs2 := []expr.Any{
+		// Match on the second fwmark
+		&expr.Meta{
+			Key:      expr.MetaKeyMARK,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasqueradeReturn),
+		},
+
+		// Match WireGuard interface
+		&expr.Meta{
+			Key:      expr.MetaKeyOIFNAME,
+			Register: 1,
+		},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     ifname(r.wgIface.Name()),
+		},
+		&expr.Counter{},
+		&expr.Masq{},
+	}
+
+	r.conn.AddRule(&nftables.Rule{
+		Table: r.workTable,
+		Chain: r.chains[chainNameRoutingNat],
+		Exprs: exprs2,
+	})
+
+	return nil
+}
+
+// addLegacyRouteRule adds a legacy routing rule for mgmt servers pre route acls
+func (r *router) addLegacyRouteRule(pair firewall.RouterPair) error {
+	sourceExp := generateCIDRMatcherExpressions(true, pair.Source)
+	destExp := generateCIDRMatcherExpressions(false, pair.Destination)
+
+	exprs := []expr.Any{
+		&expr.Counter{},
+		&expr.Verdict{
+			Kind: expr.VerdictAccept,
+		},
+	}
+
+	expression := append(sourceExp, append(destExp, exprs...)...) // nolint:gocritic
+
+	ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair)
+
+	if _, exists := r.rules[ruleKey]; exists {
+		if err := r.removeLegacyRouteRule(pair); err != nil {
+			return fmt.Errorf("remove legacy routing rule: %w", err)
+		}
+	}
+
+	r.rules[ruleKey] = r.conn.AddRule(&nftables.Rule{
+		Table:    r.workTable,
+		Chain:    r.chains[chainNameRoutingFw],
+		Exprs:    expression,
+		UserData: []byte(ruleKey),
+	})
+	return nil
+}
+
+// removeLegacyRouteRule removes a legacy routing rule for mgmt servers pre route acls
+func (r *router) removeLegacyRouteRule(pair firewall.RouterPair) error {
+	ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair)
+
+	if rule, exists := r.rules[ruleKey]; exists {
+		if err := r.conn.DelRule(rule); err != nil {
+			return fmt.Errorf("remove legacy forwarding rule %s -> %s: %v", pair.Source, pair.Destination, err)
+		}
+
+		log.Debugf("nftables: removed legacy forwarding rule %s -> %s", pair.Source, pair.Destination)
+
+		delete(r.rules, ruleKey)
+	} else {
+		log.Debugf("nftables: legacy forwarding rule %s not found", ruleKey)
+	}
+
+	return nil
+}
+
+// GetLegacyManagement returns the route manager's legacy management mode
+func (r *router) GetLegacyManagement() bool {
+	return r.legacyManagement
+}
+
+// SetLegacyManagement sets the route manager to use legacy management mode
+func (r *router) SetLegacyManagement(isLegacy bool) {
+	r.legacyManagement = isLegacy
+}
+
+// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls
+func (r *router) RemoveAllLegacyRouteRules() error {
+	if err := r.refreshRulesMap(); err != nil {
+		return fmt.Errorf(refreshRulesMapError, err)
+	}
+
+	var merr *multierror.Error
+	for k, rule := range r.rules {
+		if !strings.HasPrefix(k, firewall.ForwardingFormatPrefix) {
+			continue
+		}
+		if err := r.conn.DelRule(rule); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %v", err))
+		} else {
+			delete(r.rules, k)
+		}
+
+	}
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+// acceptForwardRules adds iif/oif rules in the filter table/forward chain to make sure
+// that our traffic is not dropped by existing rules there.
+// The existing FORWARD rules/policies decide outbound traffic towards our interface.
+// In case the FORWARD policy is set to "drop", we add an established/related rule to allow return traffic for the inbound rule.
+func (r *router) acceptForwardRules() error {
+	if r.filterTable == nil {
+		log.Debugf("table 'filter' not found for forward rules, skipping accept rules")
+		return nil
+	}
+
+	fw := "iptables"
+
+	defer func() {
+		log.Debugf("Used %s to add accept forward rules", fw)
+	}()
+
+	// Try iptables first and fallback to nftables if iptables is not available
+	ipt, err := iptables.New()
+	if err != nil {
+		// filter table exists but iptables is not
+		log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err)
+
+		fw = "nftables"
+		return r.acceptForwardRulesNftables()
+	}
+
+	return r.acceptForwardRulesIptables(ipt)
+}
+
+func (r *router) acceptForwardRulesIptables(ipt *iptables.IPTables) error {
+	var merr *multierror.Error
+	for _, rule := range r.getAcceptForwardRules() {
+		if err := ipt.Insert("filter", chainNameForward, 1, rule...); err != nil {
+			merr = multierror.Append(err, fmt.Errorf("add iptables rule: %v", err))
+		} else {
+			log.Debugf("added iptables rule: %v", rule)
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+func (r *router) getAcceptForwardRules() [][]string {
+	intf := r.wgIface.Name()
+	return [][]string{
+		{"-i", intf, "-j", "ACCEPT"},
+		{"-o", intf, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"},
+	}
+}
+
+func (r *router) acceptForwardRulesNftables() error {
+	intf := ifname(r.wgIface.Name())
+
+	// Rule for incoming interface (iif) with counter
+	iifRule := &nftables.Rule{
+		Table: r.filterTable,
+		Chain: &nftables.Chain{
+			Name:     chainNameForward,
+			Table:    r.filterTable,
+			Type:     nftables.ChainTypeFilter,
+			Hooknum:  nftables.ChainHookForward,
+			Priority: nftables.ChainPriorityFilter,
+		},
+		Exprs: []expr.Any{
+			&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
+			&expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     intf,
+			},
+			&expr.Counter{},
+			&expr.Verdict{Kind: expr.VerdictAccept},
+		},
+		UserData: []byte(userDataAcceptForwardRuleIif),
+	}
+	r.conn.InsertRule(iifRule)
+
+	oifExprs := []expr.Any{
+		&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     intf,
+		},
+	}
+
+	// Rule for outgoing interface (oif) with counter
+	oifRule := &nftables.Rule{
+		Table: r.filterTable,
+		Chain: &nftables.Chain{
+			Name:     "FORWARD",
+			Table:    r.filterTable,
+			Type:     nftables.ChainTypeFilter,
+			Hooknum:  nftables.ChainHookForward,
+			Priority: nftables.ChainPriorityFilter,
+		},
+		Exprs:    append(oifExprs, getEstablishedExprs(2)...),
+		UserData: []byte(userDataAcceptForwardRuleOif),
+	}
+
+	r.conn.InsertRule(oifRule)
+
+	return nil
+}
+
+func (r *router) removeAcceptForwardRules() error {
+	if r.filterTable == nil {
+		return nil
+	}
+
+	// Try iptables first and fallback to nftables if iptables is not available
+	ipt, err := iptables.New()
+	if err != nil {
+		log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err)
+		return r.removeAcceptForwardRulesNftables()
+	}
+
+	return r.removeAcceptForwardRulesIptables(ipt)
+}
+
+func (r *router) removeAcceptForwardRulesNftables() error {
+	chains, err := r.conn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return fmt.Errorf("list chains: %v", err)
+	}
+
+	for _, chain := range chains {
+		if chain.Table.Name != r.filterTable.Name || chain.Name != chainNameForward {
+			continue
+		}
+
+		rules, err := r.conn.GetRules(r.filterTable, chain)
+		if err != nil {
+			return fmt.Errorf("get rules: %v", err)
+		}
+
+		for _, rule := range rules {
+			if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleIif)) ||
+				bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleOif)) {
+				if err := r.conn.DelRule(rule); err != nil {
+					return fmt.Errorf("delete rule: %v", err)
+				}
+			}
+		}
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf(flushError, err)
+	}
+
+	return nil
+}
+
+func (r *router) removeAcceptForwardRulesIptables(ipt *iptables.IPTables) error {
+	var merr *multierror.Error
+	for _, rule := range r.getAcceptForwardRules() {
+		if err := ipt.DeleteIfExists("filter", chainNameForward, rule...); err != nil {
+			merr = multierror.Append(err, fmt.Errorf("remove iptables rule: %v", err))
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+// RemoveNatRule removes the prerouting mark rule
+func (r *router) RemoveNatRule(pair firewall.RouterPair) error {
+	if err := r.refreshRulesMap(); err != nil {
+		return fmt.Errorf(refreshRulesMapError, err)
+	}
+
+	if err := r.removeNatRule(pair); err != nil {
+		return fmt.Errorf("remove prerouting rule: %w", err)
+	}
+
+	if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil {
+		return fmt.Errorf("remove inverse prerouting rule: %w", err)
+	}
+
+	if err := r.removeLegacyRouteRule(pair); err != nil {
+		return fmt.Errorf("remove legacy routing rule: %w", err)
+	}
+
+	if err := r.conn.Flush(); err != nil {
+		return fmt.Errorf("nftables: received error while applying rule removal for %s: %v", pair.Destination, err)
+	}
+
+	log.Debugf("nftables: removed nat rules for %s", pair.Destination)
+	return nil
+}
+
+func (r *router) removeNatRule(pair firewall.RouterPair) error {
+	ruleKey := firewall.GenKey(firewall.PreroutingFormat, pair)
+
+	if rule, exists := r.rules[ruleKey]; exists {
+		err := r.conn.DelRule(rule)
+		if err != nil {
+			return fmt.Errorf("remove prerouting rule %s -> %s: %v", pair.Source, pair.Destination, err)
+		}
+
+		log.Debugf("nftables: removed prerouting rule %s -> %s", pair.Source, pair.Destination)
+
+		delete(r.rules, ruleKey)
+	} else {
+		log.Debugf("nftables: prerouting rule %s not found", ruleKey)
+	}
+
+	return nil
+}
+
+// refreshRulesMap refreshes the rule map with the latest rules. this is useful to avoid
+// duplicates and to get missing attributes that we don't have when adding new rules
+func (r *router) refreshRulesMap() error {
+	for _, chain := range r.chains {
+		rules, err := r.conn.GetRules(chain.Table, chain)
+		if err != nil {
+			return fmt.Errorf("nftables: unable to list rules: %v", err)
+		}
+		for _, rule := range rules {
+			if len(rule.UserData) > 0 {
+				r.rules[string(rule.UserData)] = rule
+			}
+		}
+	}
+	return nil
+}
+
+// generateCIDRMatcherExpressions generates nftables expressions that matches a CIDR
+func generateCIDRMatcherExpressions(source bool, prefix netip.Prefix) []expr.Any {
+	var offset uint32
+	if source {
+		offset = 12 // src offset
+	} else {
+		offset = 16 // dst offset
+	}
+
+	ones := prefix.Bits()
+	// 0.0.0.0/0 doesn't need extra expressions
+	if ones == 0 {
+		return nil
+	}
+
+	mask := net.CIDRMask(ones, 32)
+
+	return []expr.Any{
+		&expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseNetworkHeader,
+			Offset:       offset,
+			Len:          4,
+		},
+		// netmask
+		&expr.Bitwise{
+			DestRegister:   1,
+			SourceRegister: 1,
+			Len:            4,
+			Mask:           mask,
+			Xor:            []byte{0, 0, 0, 0},
+		},
+		// net address
+		&expr.Cmp{
+			Op:       expr.CmpOpEq,
+			Register: 1,
+			Data:     prefix.Masked().Addr().AsSlice(),
+		},
+	}
+}
+
+func applyPort(port *firewall.Port, isSource bool) []expr.Any {
+	if port == nil {
+		return nil
+	}
+
+	var exprs []expr.Any
+
+	offset := uint32(2) // Default offset for destination port
+	if isSource {
+		offset = 0 // Offset for source port
+	}
+
+	exprs = append(exprs, &expr.Payload{
+		DestRegister: 1,
+		Base:         expr.PayloadBaseTransportHeader,
+		Offset:       offset,
+		Len:          2,
+	})
+
+	if port.IsRange && len(port.Values) == 2 {
+		// Handle port range
+		exprs = append(exprs,
+			&expr.Cmp{
+				Op:       expr.CmpOpGte,
+				Register: 1,
+				Data:     binaryutil.BigEndian.PutUint16(uint16(port.Values[0])),
+			},
+			&expr.Cmp{
+				Op:       expr.CmpOpLte,
+				Register: 1,
+				Data:     binaryutil.BigEndian.PutUint16(uint16(port.Values[1])),
+			},
+		)
+	} else {
+		// Handle single port or multiple ports
+		for i, p := range port.Values {
+			if i > 0 {
+				// Add a bitwise OR operation between port checks
+				exprs = append(exprs, &expr.Bitwise{
+					SourceRegister: 1,
+					DestRegister:   1,
+					Len:            4,
+					Mask:           []byte{0x00, 0x00, 0xff, 0xff},
+					Xor:            []byte{0x00, 0x00, 0x00, 0x00},
+				})
+			}
+			exprs = append(exprs, &expr.Cmp{
+				Op:       expr.CmpOpEq,
+				Register: 1,
+				Data:     binaryutil.BigEndian.PutUint16(uint16(p)),
+			})
+		}
+	}
+
+	return exprs
+}

client/firewall/nftables/router_linux_test.go

@@ -0,0 +1,716 @@
+//go:build !android
+
+package nftables
+
+import (
+	"encoding/binary"
+	"net/netip"
+	"os/exec"
+	"testing"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
+	"github.com/google/nftables/expr"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/firewall/test"
+)
+
+const (
+	// UNKNOWN is the default value for the firewall type for unknown firewall type
+	UNKNOWN = iota
+	// IPTABLES is the value for the iptables firewall type
+	IPTABLES
+	// NFTABLES is the value for the nftables firewall type
+	NFTABLES
+)
+
+func TestNftablesManager_AddNatRule(t *testing.T) {
+	if check() != NFTABLES {
+		t.Skip("nftables not supported on this OS")
+	}
+
+	for _, testCase := range test.InsertRuleTestCases {
+		t.Run(testCase.Name, func(t *testing.T) {
+			// need fw manager to init both acl mgr and router for all chains to be present
+			manager, err := Create(ifaceMock)
+			t.Cleanup(func() {
+				require.NoError(t, manager.Reset(nil))
+			})
+			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
+
+			nftablesTestingClient := &nftables.Conn{}
+
+			rtr := manager.router
+			err = rtr.AddNatRule(testCase.InputPair)
+			require.NoError(t, err, "pair should be inserted")
+
+			t.Cleanup(func() {
+				require.NoError(t, rtr.RemoveNatRule(testCase.InputPair), "failed to remove rule")
+			})
+
+			if testCase.InputPair.Masquerade {
+				// Build expected expressions for connection tracking
+				conntrackExprs := []expr.Any{
+					&expr.Ct{
+						Key:      expr.CtKeySTATE,
+						Register: 1,
+					},
+					&expr.Bitwise{
+						SourceRegister: 1,
+						DestRegister:   1,
+						Len:            4,
+						Mask:           binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW),
+						Xor:            binaryutil.NativeEndian.PutUint32(0),
+					},
+					&expr.Cmp{
+						Op:       expr.CmpOpNeq,
+						Register: 1,
+						Data:     []byte{0, 0, 0, 0},
+					},
+				}
+
+				// Build interface matching expression
+				ifaceExprs := []expr.Any{
+					&expr.Meta{
+						Key:      expr.MetaKeyIIFNAME,
+						Register: 1,
+					},
+					&expr.Cmp{
+						Op:       expr.CmpOpEq,
+						Register: 1,
+						Data:     ifname(ifaceMock.Name()),
+					},
+				}
+
+				// Build CIDR matching expressions
+				sourceExp := generateCIDRMatcherExpressions(true, testCase.InputPair.Source)
+				destExp := generateCIDRMatcherExpressions(false, testCase.InputPair.Destination)
+
+				// Combine all expressions in the correct order
+				// nolint:gocritic
+				testingExpression := append(conntrackExprs, ifaceExprs...)
+				testingExpression = append(testingExpression, sourceExp...)
+				testingExpression = append(testingExpression, destExp...)
+
+				natRuleKey := firewall.GenKey(firewall.PreroutingFormat, testCase.InputPair)
+				found := 0
+				for _, chain := range rtr.chains {
+					if chain.Name == chainNamePrerouting {
+						rules, err := nftablesTestingClient.GetRules(chain.Table, chain)
+						require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name)
+						for _, rule := range rules {
+							if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
+								// Compare expressions up to the mark setting expressions
+								require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "prerouting nat rule elements should match")
+								found = 1
+							}
+						}
+					}
+				}
+				require.Equal(t, 1, found, "should find at least 1 rule in prerouting chain")
+			}
+		})
+	}
+}
+
+func TestNftablesManager_RemoveNatRule(t *testing.T) {
+	if check() != NFTABLES {
+		t.Skip("nftables not supported on this OS")
+	}
+
+	for _, testCase := range test.RemoveRuleTestCases {
+		t.Run(testCase.Name, func(t *testing.T) {
+			manager, err := Create(ifaceMock)
+			t.Cleanup(func() {
+				require.NoError(t, manager.Reset(nil))
+			})
+			require.NoError(t, err)
+			require.NoError(t, manager.Init(nil))
+
+			rtr := manager.router
+
+			// First add the NAT rule using the router's method
+			err = rtr.AddNatRule(testCase.InputPair)
+			require.NoError(t, err, "should add NAT rule")
+
+			// Verify the rule was added
+			natRuleKey := firewall.GenKey(firewall.PreroutingFormat, testCase.InputPair)
+			found := false
+			rules, err := rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNamePrerouting])
+			require.NoError(t, err, "should list rules")
+			for _, rule := range rules {
+				if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
+					found = true
+					break
+				}
+			}
+			require.True(t, found, "NAT rule should exist before removal")
+
+			// Now remove the rule
+			err = rtr.RemoveNatRule(testCase.InputPair)
+			require.NoError(t, err, "shouldn't return error when removing rule")
+
+			// Verify the rule was removed
+			found = false
+			rules, err = rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNamePrerouting])
+			require.NoError(t, err, "should list rules after removal")
+			for _, rule := range rules {
+				if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey {
+					found = true
+					break
+				}
+			}
+			require.False(t, found, "NAT rule should not exist after removal")
+
+			// Verify the static postrouting rules still exist
+			rules, err = rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNameRoutingNat])
+			require.NoError(t, err, "should list postrouting rules")
+			foundCounter := false
+			for _, rule := range rules {
+				for _, e := range rule.Exprs {
+					if _, ok := e.(*expr.Counter); ok {
+						foundCounter = true
+						break
+					}
+				}
+				if foundCounter {
+					break
+				}
+			}
+			require.True(t, foundCounter, "static postrouting rule should remain")
+		})
+	}
+}
+
+func TestRouter_AddRouteFiltering(t *testing.T) {
+	if check() != NFTABLES {
+		t.Skip("nftables not supported on this system")
+	}
+
+	workTable, err := createWorkTable()
+	require.NoError(t, err, "Failed to create work table")
+
+	defer deleteWorkTable()
+
+	r, err := newRouter(workTable, ifaceMock)
+	require.NoError(t, err, "Failed to create router")
+	require.NoError(t, r.init(workTable))
+
+	defer func(r *router) {
+		require.NoError(t, r.Reset(), "Failed to reset rules")
+	}(r)
+
+	tests := []struct {
+		name        string
+		sources     []netip.Prefix
+		destination netip.Prefix
+		proto       firewall.Protocol
+		sPort       *firewall.Port
+		dPort       *firewall.Port
+		direction   firewall.RuleDirection
+		action      firewall.Action
+		expectSet   bool
+	}{
+		{
+			name:        "Basic TCP rule with single source",
+			sources:     []netip.Prefix{netip.MustParsePrefix("192.168.1.0/24")},
+			destination: netip.MustParsePrefix("10.0.0.0/24"),
+			proto:       firewall.ProtocolTCP,
+			sPort:       nil,
+			dPort:       &firewall.Port{Values: []int{80}},
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name: "UDP rule with multiple sources",
+			sources: []netip.Prefix{
+				netip.MustParsePrefix("172.16.0.0/16"),
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+			destination: netip.MustParsePrefix("10.0.0.0/8"),
+			proto:       firewall.ProtocolUDP,
+			sPort:       &firewall.Port{Values: []int{1024, 2048}, IsRange: true},
+			dPort:       nil,
+			direction:   firewall.RuleDirectionOUT,
+			action:      firewall.ActionDrop,
+			expectSet:   true,
+		},
+		{
+			name:        "All protocols rule",
+			sources:     []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")},
+			destination: netip.MustParsePrefix("0.0.0.0/0"),
+			proto:       firewall.ProtocolALL,
+			sPort:       nil,
+			dPort:       nil,
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "ICMP rule",
+			sources:     []netip.Prefix{netip.MustParsePrefix("192.168.0.0/16")},
+			destination: netip.MustParsePrefix("10.0.0.0/8"),
+			proto:       firewall.ProtocolICMP,
+			sPort:       nil,
+			dPort:       nil,
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "TCP rule with multiple source ports",
+			sources:     []netip.Prefix{netip.MustParsePrefix("172.16.0.0/12")},
+			destination: netip.MustParsePrefix("192.168.0.0/16"),
+			proto:       firewall.ProtocolTCP,
+			sPort:       &firewall.Port{Values: []int{80, 443, 8080}},
+			dPort:       nil,
+			direction:   firewall.RuleDirectionOUT,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "UDP rule with single IP and port range",
+			sources:     []netip.Prefix{netip.MustParsePrefix("192.168.1.1/32")},
+			destination: netip.MustParsePrefix("10.0.0.0/24"),
+			proto:       firewall.ProtocolUDP,
+			sPort:       nil,
+			dPort:       &firewall.Port{Values: []int{5000, 5100}, IsRange: true},
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionDrop,
+			expectSet:   false,
+		},
+		{
+			name:        "TCP rule with source and destination ports",
+			sources:     []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")},
+			destination: netip.MustParsePrefix("172.16.0.0/16"),
+			proto:       firewall.ProtocolTCP,
+			sPort:       &firewall.Port{Values: []int{1024, 65535}, IsRange: true},
+			dPort:       &firewall.Port{Values: []int{22}},
+			direction:   firewall.RuleDirectionOUT,
+			action:      firewall.ActionAccept,
+			expectSet:   false,
+		},
+		{
+			name:        "Drop all incoming traffic",
+			sources:     []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")},
+			destination: netip.MustParsePrefix("192.168.0.0/24"),
+			proto:       firewall.ProtocolALL,
+			sPort:       nil,
+			dPort:       nil,
+			direction:   firewall.RuleDirectionIN,
+			action:      firewall.ActionDrop,
+			expectSet:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ruleKey, err := r.AddRouteFiltering(tt.sources, tt.destination, tt.proto, tt.sPort, tt.dPort, tt.action)
+			require.NoError(t, err, "AddRouteFiltering failed")
+
+			t.Cleanup(func() {
+				require.NoError(t, r.DeleteRouteRule(ruleKey), "Failed to delete rule")
+			})
+
+			// Check if the rule is in the internal map
+			rule, ok := r.rules[ruleKey.GetRuleID()]
+			assert.True(t, ok, "Rule not found in internal map")
+
+			t.Log("Internal rule expressions:")
+			for i, expr := range rule.Exprs {
+				t.Logf("  [%d] %T: %+v", i, expr, expr)
+			}
+
+			// Verify internal rule content
+			verifyRule(t, rule, tt.sources, tt.destination, tt.proto, tt.sPort, tt.dPort, tt.direction, tt.action, tt.expectSet)
+
+			// Check if the rule exists in nftables and verify its content
+			rules, err := r.conn.GetRules(r.workTable, r.chains[chainNameRoutingFw])
+			require.NoError(t, err, "Failed to get rules from nftables")
+
+			var nftRule *nftables.Rule
+			for _, rule := range rules {
+				if string(rule.UserData) == ruleKey.GetRuleID() {
+					nftRule = rule
+					break
+				}
+			}
+
+			require.NotNil(t, nftRule, "Rule not found in nftables")
+			t.Log("Actual nftables rule expressions:")
+			for i, expr := range nftRule.Exprs {
+				t.Logf("  [%d] %T: %+v", i, expr, expr)
+			}
+
+			// Verify actual nftables rule content
+			verifyRule(t, nftRule, tt.sources, tt.destination, tt.proto, tt.sPort, tt.dPort, tt.direction, tt.action, tt.expectSet)
+		})
+	}
+}
+
+func TestNftablesCreateIpSet(t *testing.T) {
+	if check() != NFTABLES {
+		t.Skip("nftables not supported on this system")
+	}
+
+	workTable, err := createWorkTable()
+	require.NoError(t, err, "Failed to create work table")
+
+	defer deleteWorkTable()
+
+	r, err := newRouter(workTable, ifaceMock)
+	require.NoError(t, err, "Failed to create router")
+	require.NoError(t, r.init(workTable))
+
+	defer func() {
+		require.NoError(t, r.Reset(), "Failed to reset router")
+	}()
+
+	tests := []struct {
+		name     string
+		sources  []netip.Prefix
+		expected []netip.Prefix
+	}{
+		{
+			name:    "Single IP",
+			sources: []netip.Prefix{netip.MustParsePrefix("192.168.1.1/32")},
+		},
+		{
+			name: "Multiple IPs",
+			sources: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.1/32"),
+				netip.MustParsePrefix("10.0.0.1/32"),
+				netip.MustParsePrefix("172.16.0.1/32"),
+			},
+		},
+		{
+			name:    "Single Subnet",
+			sources: []netip.Prefix{netip.MustParsePrefix("192.168.0.0/24")},
+		},
+		{
+			name: "Multiple Subnets with Various Prefix Lengths",
+			sources: []netip.Prefix{
+				netip.MustParsePrefix("10.0.0.0/8"),
+				netip.MustParsePrefix("172.16.0.0/16"),
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("203.0.113.0/26"),
+			},
+		},
+		{
+			name: "Mix of Single IPs and Subnets in Different Positions",
+			sources: []netip.Prefix{
+				netip.MustParsePrefix("192.168.1.1/32"),
+				netip.MustParsePrefix("10.0.0.0/16"),
+				netip.MustParsePrefix("172.16.0.1/32"),
+				netip.MustParsePrefix("203.0.113.0/24"),
+			},
+		},
+		{
+			name: "Overlapping IPs/Subnets",
+			sources: []netip.Prefix{
+				netip.MustParsePrefix("10.0.0.0/8"),
+				netip.MustParsePrefix("10.0.0.0/16"),
+				netip.MustParsePrefix("10.0.0.1/32"),
+				netip.MustParsePrefix("192.168.0.0/16"),
+				netip.MustParsePrefix("192.168.1.0/24"),
+				netip.MustParsePrefix("192.168.1.1/32"),
+			},
+			expected: []netip.Prefix{
+				netip.MustParsePrefix("10.0.0.0/8"),
+				netip.MustParsePrefix("192.168.0.0/16"),
+			},
+		},
+	}
+
+	// Add this helper function inside TestNftablesCreateIpSet
+	printNftSets := func() {
+		cmd := exec.Command("nft", "list", "sets")
+		output, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Logf("Failed to run 'nft list sets': %v", err)
+		} else {
+			t.Logf("Current nft sets:\n%s", output)
+		}
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			setName := firewall.GenerateSetName(tt.sources)
+			set, err := r.createIpSet(setName, tt.sources)
+			if err != nil {
+				t.Logf("Failed to create IP set: %v", err)
+				printNftSets()
+				require.NoError(t, err, "Failed to create IP set")
+			}
+			require.NotNil(t, set, "Created set is nil")
+
+			// Verify set properties
+			assert.Equal(t, setName, set.Name, "Set name mismatch")
+			assert.Equal(t, r.workTable, set.Table, "Set table mismatch")
+			assert.True(t, set.Interval, "Set interval property should be true")
+			assert.Equal(t, nftables.TypeIPAddr, set.KeyType, "Set key type mismatch")
+
+			// Fetch the created set from nftables
+			fetchedSet, err := r.conn.GetSetByName(r.workTable, setName)
+			require.NoError(t, err, "Failed to fetch created set")
+			require.NotNil(t, fetchedSet, "Fetched set is nil")
+
+			// Verify set elements
+			elements, err := r.conn.GetSetElements(fetchedSet)
+			require.NoError(t, err, "Failed to get set elements")
+
+			// Count the number of unique prefixes (excluding interval end markers)
+			uniquePrefixes := make(map[string]bool)
+			for _, elem := range elements {
+				if !elem.IntervalEnd {
+					ip := netip.AddrFrom4(*(*[4]byte)(elem.Key))
+					uniquePrefixes[ip.String()] = true
+				}
+			}
+
+			// Check against expected merged prefixes
+			expectedCount := len(tt.expected)
+			if expectedCount == 0 {
+				expectedCount = len(tt.sources)
+			}
+			assert.Equal(t, expectedCount, len(uniquePrefixes), "Number of unique prefixes in set doesn't match expected")
+
+			// Verify each expected prefix is in the set
+			for _, expected := range tt.expected {
+				found := false
+				for _, elem := range elements {
+					if !elem.IntervalEnd {
+						ip := netip.AddrFrom4(*(*[4]byte)(elem.Key))
+						if expected.Contains(ip) {
+							found = true
+							break
+						}
+					}
+				}
+				assert.True(t, found, "Expected prefix %s not found in set", expected)
+			}
+
+			r.conn.DelSet(set)
+			if err := r.conn.Flush(); err != nil {
+				t.Logf("Failed to delete set: %v", err)
+				printNftSets()
+			}
+			require.NoError(t, err, "Failed to delete set")
+		})
+	}
+}
+
+func verifyRule(t *testing.T, rule *nftables.Rule, sources []netip.Prefix, destination netip.Prefix, proto firewall.Protocol, sPort, dPort *firewall.Port, direction firewall.RuleDirection, action firewall.Action, expectSet bool) {
+	t.Helper()
+
+	assert.NotNil(t, rule, "Rule should not be nil")
+
+	// Verify sources and destination
+	if expectSet {
+		assert.True(t, containsSetLookup(rule.Exprs), "Rule should contain set lookup for multiple sources")
+	} else if len(sources) == 1 && sources[0].Bits() != 0 {
+		if direction == firewall.RuleDirectionIN {
+			assert.True(t, containsCIDRMatcher(rule.Exprs, sources[0], true), "Rule should contain source CIDR matcher for %s", sources[0])
+		} else {
+			assert.True(t, containsCIDRMatcher(rule.Exprs, sources[0], false), "Rule should contain destination CIDR matcher for %s", sources[0])
+		}
+	}
+
+	if direction == firewall.RuleDirectionIN {
+		assert.True(t, containsCIDRMatcher(rule.Exprs, destination, false), "Rule should contain destination CIDR matcher for %s", destination)
+	} else {
+		assert.True(t, containsCIDRMatcher(rule.Exprs, destination, true), "Rule should contain source CIDR matcher for %s", destination)
+	}
+
+	// Verify protocol
+	if proto != firewall.ProtocolALL {
+		assert.True(t, containsProtocol(rule.Exprs, proto), "Rule should contain protocol matcher for %s", proto)
+	}
+
+	// Verify ports
+	if sPort != nil {
+		assert.True(t, containsPort(rule.Exprs, sPort, true), "Rule should contain source port matcher for %v", sPort)
+	}
+	if dPort != nil {
+		assert.True(t, containsPort(rule.Exprs, dPort, false), "Rule should contain destination port matcher for %v", dPort)
+	}
+
+	// Verify action
+	assert.True(t, containsAction(rule.Exprs, action), "Rule should contain correct action: %s", action)
+}
+
+func containsSetLookup(exprs []expr.Any) bool {
+	for _, e := range exprs {
+		if _, ok := e.(*expr.Lookup); ok {
+			return true
+		}
+	}
+	return false
+}
+
+func containsCIDRMatcher(exprs []expr.Any, prefix netip.Prefix, isSource bool) bool {
+	var offset uint32
+	if isSource {
+		offset = 12 // src offset
+	} else {
+		offset = 16 // dst offset
+	}
+
+	var payloadFound, bitwiseFound, cmpFound bool
+	for _, e := range exprs {
+		switch ex := e.(type) {
+		case *expr.Payload:
+			if ex.Base == expr.PayloadBaseNetworkHeader && ex.Offset == offset && ex.Len == 4 {
+				payloadFound = true
+			}
+		case *expr.Bitwise:
+			if ex.Len == 4 && len(ex.Mask) == 4 && len(ex.Xor) == 4 {
+				bitwiseFound = true
+			}
+		case *expr.Cmp:
+			if ex.Op == expr.CmpOpEq && len(ex.Data) == 4 {
+				cmpFound = true
+			}
+		}
+	}
+	return (payloadFound && bitwiseFound && cmpFound) || prefix.Bits() == 0
+}
+
+func containsPort(exprs []expr.Any, port *firewall.Port, isSource bool) bool {
+	var offset uint32 = 2 // Default offset for destination port
+	if isSource {
+		offset = 0 // Offset for source port
+	}
+
+	var payloadFound, portMatchFound bool
+	for _, e := range exprs {
+		switch ex := e.(type) {
+		case *expr.Payload:
+			if ex.Base == expr.PayloadBaseTransportHeader && ex.Offset == offset && ex.Len == 2 {
+				payloadFound = true
+			}
+		case *expr.Cmp:
+			if port.IsRange {
+				if ex.Op == expr.CmpOpGte || ex.Op == expr.CmpOpLte {
+					portMatchFound = true
+				}
+			} else {
+				if ex.Op == expr.CmpOpEq && len(ex.Data) == 2 {
+					portValue := binary.BigEndian.Uint16(ex.Data)
+					for _, p := range port.Values {
+						if uint16(p) == portValue {
+							portMatchFound = true
+							break
+						}
+					}
+				}
+			}
+		}
+		if payloadFound && portMatchFound {
+			return true
+		}
+	}
+	return false
+}
+
+func containsProtocol(exprs []expr.Any, proto firewall.Protocol) bool {
+	var metaFound, cmpFound bool
+	expectedProto, _ := protoToInt(proto)
+	for _, e := range exprs {
+		switch ex := e.(type) {
+		case *expr.Meta:
+			if ex.Key == expr.MetaKeyL4PROTO {
+				metaFound = true
+			}
+		case *expr.Cmp:
+			if ex.Op == expr.CmpOpEq && len(ex.Data) == 1 && ex.Data[0] == expectedProto {
+				cmpFound = true
+			}
+		}
+	}
+	return metaFound && cmpFound
+}
+
+func containsAction(exprs []expr.Any, action firewall.Action) bool {
+	for _, e := range exprs {
+		if verdict, ok := e.(*expr.Verdict); ok {
+			switch action {
+			case firewall.ActionAccept:
+				return verdict.Kind == expr.VerdictAccept
+			case firewall.ActionDrop:
+				return verdict.Kind == expr.VerdictDrop
+			}
+		}
+	}
+	return false
+}
+
+// check returns the firewall type based on common lib checks. It returns UNKNOWN if no firewall is found.
+func check() int {
+	nf := nftables.Conn{}
+	if _, err := nf.ListChains(); err == nil {
+		return NFTABLES
+	}
+
+	ip, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+	if err != nil {
+		return UNKNOWN
+	}
+	if isIptablesClientAvailable(ip) {
+		return IPTABLES
+	}
+
+	return UNKNOWN
+}
+
+func isIptablesClientAvailable(client *iptables.IPTables) bool {
+	_, err := client.ListChains("filter")
+	return err == nil
+}
+
+func createWorkTable() (*nftables.Table, error) {
+	sConn, err := nftables.New(nftables.AsLasting())
+	if err != nil {
+		return nil, err
+	}
+
+	tables, err := sConn.ListTablesOfFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, t := range tables {
+		if t.Name == tableNameNetbird {
+			sConn.DelTable(t)
+		}
+	}
+
+	table := sConn.AddTable(&nftables.Table{Name: tableNameNetbird, Family: nftables.TableFamilyIPv4})
+	err = sConn.Flush()
+
+	return table, err
+}
+
+func deleteWorkTable() {
+	sConn, err := nftables.New(nftables.AsLasting())
+	if err != nil {
+		return
+	}
+
+	tables, err := sConn.ListTablesOfFamily(nftables.TableFamilyIPv4)
+	if err != nil {
+		return
+	}
+
+	for _, t := range tables {
+		if t.Name == tableNameNetbird {
+			sConn.DelTable(t)
+		}
+	}
+}

client/firewall/nftables/rule_linux.go

@@ -0,0 +1,20 @@
+package nftables
+
+import (
+	"net"
+
+	"github.com/google/nftables"
+)
+
+// Rule to handle management of rules
+type Rule struct {
+	nftRule *nftables.Rule
+	nftSet  *nftables.Set
+	ruleID  string
+	ip      net.IP
+}
+
+// GetRuleID returns the rule id
+func (r *Rule) GetRuleID() string {
+	return r.ruleID
+}

client/firewall/nftables/state_linux.go

@@ -0,0 +1,47 @@
+package nftables
+
+import (
+	"fmt"
+
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+type InterfaceState struct {
+	NameStr       string          `json:"name"`
+	WGAddress     iface.WGAddress `json:"wg_address"`
+	UserspaceBind bool            `json:"userspace_bind"`
+}
+
+func (i *InterfaceState) Name() string {
+	return i.NameStr
+}
+
+func (i *InterfaceState) Address() device.WGAddress {
+	return i.WGAddress
+}
+
+func (i *InterfaceState) IsUserspaceBind() bool {
+	return i.UserspaceBind
+}
+
+type ShutdownState struct {
+	InterfaceState *InterfaceState `json:"interface_state,omitempty"`
+}
+
+func (s *ShutdownState) Name() string {
+	return "nftables_state"
+}
+
+func (s *ShutdownState) Cleanup() error {
+	nft, err := Create(s.InterfaceState)
+	if err != nil {
+		return fmt.Errorf("create nftables manager: %w", err)
+	}
+
+	if err := nft.Reset(nil); err != nil {
+		return fmt.Errorf("reset nftables manager: %w", err)
+	}
+
+	return nil
+}

client/firewall/port.go

@@ -1,24 +0,0 @@
-package firewall
-
-// PortProtocol is the protocol of the port
-type PortProtocol string
-
-const (
-	// PortProtocolTCP is the TCP protocol
-	PortProtocolTCP PortProtocol = "tcp"
-
-	// PortProtocolUDP is the UDP protocol
-	PortProtocolUDP PortProtocol = "udp"
-)
-
-// Port of the address for firewall rule
-type Port struct {
-	// IsRange is true Values contains two values, the first is the start port, the second is the end port
-	IsRange bool
-
-	// Values contains one value for single port, multiple values for the list of ports, or two values for the range of ports
-	Values []int
-
-	// Proto is the protocol of the port
-	Proto PortProtocol
-}

client/firewall/test/cases_linux.go

@@ -0,0 +1,49 @@
+package test
+
+import (
+	"net/netip"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+)
+
+var (
+	InsertRuleTestCases = []struct {
+		Name      string
+		InputPair firewall.RouterPair
+	}{
+		{
+			Name: "Insert Forwarding IPV4 Rule",
+			InputPair: firewall.RouterPair{
+				ID:          "zxa",
+				Source:      netip.MustParsePrefix("100.100.100.1/32"),
+				Destination: netip.MustParsePrefix("100.100.200.0/24"),
+				Masquerade:  false,
+			},
+		},
+		{
+			Name: "Insert Forwarding And Nat IPV4 Rules",
+			InputPair: firewall.RouterPair{
+				ID:          "zxa",
+				Source:      netip.MustParsePrefix("100.100.100.1/32"),
+				Destination: netip.MustParsePrefix("100.100.200.0/24"),
+				Masquerade:  true,
+			},
+		},
+	}
+
+	RemoveRuleTestCases = []struct {
+		Name      string
+		InputPair firewall.RouterPair
+		IpVersion string
+	}{
+		{
+			Name: "Remove Forwarding And Nat IPV4 Rules",
+			InputPair: firewall.RouterPair{
+				ID:          "zxa",
+				Source:      netip.MustParsePrefix("100.100.100.1/32"),
+				Destination: netip.MustParsePrefix("100.100.200.0/24"),
+				Masquerade:  true,
+			},
+		},
+	}
+)

client/firewall/uspfilter/allow_netbird.go

@@ -0,0 +1,27 @@
+//go:build !windows
+
+package uspfilter
+
+import "github.com/netbirdio/netbird/client/internal/statemanager"
+
+// Reset firewall to the default state
+func (m *Manager) Reset(stateManager *statemanager.Manager) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	m.outgoingRules = make(map[string]RuleSet)
+	m.incomingRules = make(map[string]RuleSet)
+
+	if m.nativeFirewall != nil {
+		return m.nativeFirewall.Reset(stateManager)
+	}
+	return nil
+}
+
+// AllowNetbird allows netbird interface traffic
+func (m *Manager) AllowNetbird() error {
+	if m.nativeFirewall != nil {
+		return m.nativeFirewall.AllowNetbird()
+	}
+	return nil
+}

client/firewall/uspfilter/allow_netbird_windows.go

@@ -0,0 +1,114 @@
+package uspfilter
+
+import (
+	"fmt"
+	"os/exec"
+	"syscall"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+)
+
+type action string
+
+const (
+	addRule          action = "add"
+	deleteRule       action = "delete"
+	firewallRuleName        = "Netbird"
+)
+
+// Reset firewall to the default state
+func (m *Manager) Reset(*statemanager.Manager) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	m.outgoingRules = make(map[string]RuleSet)
+	m.incomingRules = make(map[string]RuleSet)
+
+	if !isWindowsFirewallReachable() {
+		return nil
+	}
+
+	if !isFirewallRuleActive(firewallRuleName) {
+		return nil
+	}
+
+	if err := manageFirewallRule(firewallRuleName, deleteRule); err != nil {
+		return fmt.Errorf("couldn't remove windows firewall: %w", err)
+	}
+
+	return nil
+}
+
+// AllowNetbird allows netbird interface traffic
+func (m *Manager) AllowNetbird() error {
+	if !isWindowsFirewallReachable() {
+		return nil
+	}
+
+	if isFirewallRuleActive(firewallRuleName) {
+		return nil
+	}
+	return manageFirewallRule(firewallRuleName,
+		addRule,
+		"dir=in",
+		"enable=yes",
+		"action=allow",
+		"profile=any",
+		"localip="+m.wgIface.Address().IP.String(),
+	)
+}
+
+func manageFirewallRule(ruleName string, action action, extraArgs ...string) error {
+
+	args := []string{"advfirewall", "firewall", string(action), "rule", "name=" + ruleName}
+	if action == addRule {
+		args = append(args, extraArgs...)
+	}
+	netshCmd := GetSystem32Command("netsh")
+	cmd := exec.Command(netshCmd, args...)
+	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+	return cmd.Run()
+}
+
+func isWindowsFirewallReachable() bool {
+	args := []string{"advfirewall", "show", "allprofiles", "state"}
+
+	netshCmd := GetSystem32Command("netsh")
+
+	cmd := exec.Command(netshCmd, args...)
+	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+
+	_, err := cmd.Output()
+	if err != nil {
+		log.Infof("Windows firewall is not reachable, skipping default rule management. Using only user space rules. Error: %s", err)
+		return false
+	}
+
+	return true
+}
+
+func isFirewallRuleActive(ruleName string) bool {
+	args := []string{"advfirewall", "firewall", "show", "rule", "name=" + ruleName}
+
+	netshCmd := GetSystem32Command("netsh")
+
+	cmd := exec.Command(netshCmd, args...)
+	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+	_, err := cmd.Output()
+	return err == nil
+}
+
+// GetSystem32Command checks if a command can be found in the system path and returns it. In case it can't find it
+// in the path it will return the full path of a command assuming C:\windows\system32 as the base path.
+func GetSystem32Command(command string) string {
+	_, err := exec.LookPath(command)
+	if err == nil {
+		return command
+	}
+
+	log.Tracef("Command %s not found in PATH, using C:\\windows\\system32\\%s.exe path", command, command)
+
+	return "C:\\windows\\system32\\" + command + ".exe"
+}

client/firewall/uspfilter/rule.go

@@ -0,0 +1,30 @@
+package uspfilter
+
+import (
+	"net"
+
+	"github.com/google/gopacket"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+)
+
+// Rule to handle management of rules
+type Rule struct {
+	id         string
+	ip         net.IP
+	ipLayer    gopacket.LayerType
+	matchByIP  bool
+	protoLayer gopacket.LayerType
+	direction  firewall.RuleDirection
+	sPort      uint16
+	dPort      uint16
+	drop       bool
+	comment    string
+
+	udpHook func([]byte) bool
+}
+
+// GetRuleID returns the rule id
+func (r *Rule) GetRuleID() string {
+	return r.id
+}

client/firewall/uspfilter/uspfilter.go

@@ -0,0 +1,440 @@
+package uspfilter
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+	"sync"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/uuid"
+	log "github.com/sirupsen/logrus"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/device"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+)
+
+const layerTypeAll = 0
+
+var (
+	errRouteNotSupported = fmt.Errorf("route not supported with userspace firewall")
+)
+
+// IFaceMapper defines subset methods of interface required for manager
+type IFaceMapper interface {
+	SetFilter(device.PacketFilter) error
+	Address() iface.WGAddress
+}
+
+// RuleSet is a set of rules grouped by a string key
+type RuleSet map[string]Rule
+
+// Manager userspace firewall manager
+type Manager struct {
+	outgoingRules  map[string]RuleSet
+	incomingRules  map[string]RuleSet
+	wgNetwork      *net.IPNet
+	decoders       sync.Pool
+	wgIface        IFaceMapper
+	nativeFirewall firewall.Manager
+
+	mutex sync.RWMutex
+}
+
+// decoder for packages
+type decoder struct {
+	eth     layers.Ethernet
+	ip4     layers.IPv4
+	ip6     layers.IPv6
+	tcp     layers.TCP
+	udp     layers.UDP
+	icmp4   layers.ICMPv4
+	icmp6   layers.ICMPv6
+	decoded []gopacket.LayerType
+	parser  *gopacket.DecodingLayerParser
+}
+
+// Create userspace firewall manager constructor
+func Create(iface IFaceMapper) (*Manager, error) {
+	return create(iface)
+}
+
+func CreateWithNativeFirewall(iface IFaceMapper, nativeFirewall firewall.Manager) (*Manager, error) {
+	mgr, err := create(iface)
+	if err != nil {
+		return nil, err
+	}
+
+	mgr.nativeFirewall = nativeFirewall
+	return mgr, nil
+}
+
+func create(iface IFaceMapper) (*Manager, error) {
+	m := &Manager{
+		decoders: sync.Pool{
+			New: func() any {
+				d := &decoder{
+					decoded: []gopacket.LayerType{},
+				}
+				d.parser = gopacket.NewDecodingLayerParser(
+					layers.LayerTypeIPv4,
+					&d.eth, &d.ip4, &d.ip6, &d.icmp4, &d.icmp6, &d.tcp, &d.udp,
+				)
+				d.parser.IgnoreUnsupported = true
+				return d
+			},
+		},
+		outgoingRules: make(map[string]RuleSet),
+		incomingRules: make(map[string]RuleSet),
+		wgIface:       iface,
+	}
+
+	if err := iface.SetFilter(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (m *Manager) Init(*statemanager.Manager) error {
+	return nil
+}
+
+func (m *Manager) IsServerRouteSupported() bool {
+	if m.nativeFirewall == nil {
+		return false
+	} else {
+		return true
+	}
+}
+
+func (m *Manager) AddNatRule(pair firewall.RouterPair) error {
+	if m.nativeFirewall == nil {
+		return errRouteNotSupported
+	}
+	return m.nativeFirewall.AddNatRule(pair)
+}
+
+// RemoveNatRule removes a routing firewall rule
+func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error {
+	if m.nativeFirewall == nil {
+		return errRouteNotSupported
+	}
+	return m.nativeFirewall.RemoveNatRule(pair)
+}
+
+// AddPeerFiltering rule to the firewall
+//
+// If comment argument is empty firewall manager should set
+// rule ID as comment for the rule
+func (m *Manager) AddPeerFiltering(
+	ip net.IP,
+	proto firewall.Protocol,
+	sPort *firewall.Port,
+	dPort *firewall.Port,
+	direction firewall.RuleDirection,
+	action firewall.Action,
+	ipsetName string,
+	comment string,
+) ([]firewall.Rule, error) {
+	r := Rule{
+		id:        uuid.New().String(),
+		ip:        ip,
+		ipLayer:   layers.LayerTypeIPv6,
+		matchByIP: true,
+		direction: direction,
+		drop:      action == firewall.ActionDrop,
+		comment:   comment,
+	}
+	if ipNormalized := ip.To4(); ipNormalized != nil {
+		r.ipLayer = layers.LayerTypeIPv4
+		r.ip = ipNormalized
+	}
+
+	if s := r.ip.String(); s == "0.0.0.0" || s == "::" {
+		r.matchByIP = false
+	}
+
+	if sPort != nil && len(sPort.Values) == 1 {
+		r.sPort = uint16(sPort.Values[0])
+	}
+
+	if dPort != nil && len(dPort.Values) == 1 {
+		r.dPort = uint16(dPort.Values[0])
+	}
+
+	switch proto {
+	case firewall.ProtocolTCP:
+		r.protoLayer = layers.LayerTypeTCP
+	case firewall.ProtocolUDP:
+		r.protoLayer = layers.LayerTypeUDP
+	case firewall.ProtocolICMP:
+		r.protoLayer = layers.LayerTypeICMPv4
+		if r.ipLayer == layers.LayerTypeIPv6 {
+			r.protoLayer = layers.LayerTypeICMPv6
+		}
+	case firewall.ProtocolALL:
+		r.protoLayer = layerTypeAll
+	}
+
+	m.mutex.Lock()
+	if direction == firewall.RuleDirectionIN {
+		if _, ok := m.incomingRules[r.ip.String()]; !ok {
+			m.incomingRules[r.ip.String()] = make(RuleSet)
+		}
+		m.incomingRules[r.ip.String()][r.id] = r
+	} else {
+		if _, ok := m.outgoingRules[r.ip.String()]; !ok {
+			m.outgoingRules[r.ip.String()] = make(RuleSet)
+		}
+		m.outgoingRules[r.ip.String()][r.id] = r
+	}
+	m.mutex.Unlock()
+	return []firewall.Rule{&r}, nil
+}
+
+func (m *Manager) AddRouteFiltering(sources []netip.Prefix, destination netip.Prefix, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action) (firewall.Rule, error) {
+	if m.nativeFirewall == nil {
+		return nil, errRouteNotSupported
+	}
+	return m.nativeFirewall.AddRouteFiltering(sources, destination, proto, sPort, dPort, action)
+}
+
+func (m *Manager) DeleteRouteRule(rule firewall.Rule) error {
+	if m.nativeFirewall == nil {
+		return errRouteNotSupported
+	}
+	return m.nativeFirewall.DeleteRouteRule(rule)
+}
+
+// DeletePeerRule from the firewall by rule definition
+func (m *Manager) DeletePeerRule(rule firewall.Rule) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	r, ok := rule.(*Rule)
+	if !ok {
+		return fmt.Errorf("delete rule: invalid rule type: %T", rule)
+	}
+
+	if r.direction == firewall.RuleDirectionIN {
+		_, ok := m.incomingRules[r.ip.String()][r.id]
+		if !ok {
+			return fmt.Errorf("delete rule: no rule with such id: %v", r.id)
+		}
+		delete(m.incomingRules[r.ip.String()], r.id)
+	} else {
+		_, ok := m.outgoingRules[r.ip.String()][r.id]
+		if !ok {
+			return fmt.Errorf("delete rule: no rule with such id: %v", r.id)
+		}
+		delete(m.outgoingRules[r.ip.String()], r.id)
+	}
+
+	return nil
+}
+
+// SetLegacyManagement doesn't need to be implemented for this manager
+func (m *Manager) SetLegacyManagement(isLegacy bool) error {
+	if m.nativeFirewall == nil {
+		return nil
+	}
+	return m.nativeFirewall.SetLegacyManagement(isLegacy)
+}
+
+// Flush doesn't need to be implemented for this manager
+func (m *Manager) Flush() error { return nil }
+
+// DropOutgoing filter outgoing packets
+func (m *Manager) DropOutgoing(packetData []byte) bool {
+	return m.dropFilter(packetData, m.outgoingRules, false)
+}
+
+// DropIncoming filter incoming packets
+func (m *Manager) DropIncoming(packetData []byte) bool {
+	return m.dropFilter(packetData, m.incomingRules, true)
+}
+
+// dropFilter implements same logic for booth direction of the traffic
+func (m *Manager) dropFilter(packetData []byte, rules map[string]RuleSet, isIncomingPacket bool) bool {
+	m.mutex.RLock()
+	defer m.mutex.RUnlock()
+
+	d := m.decoders.Get().(*decoder)
+	defer m.decoders.Put(d)
+
+	if err := d.parser.DecodeLayers(packetData, &d.decoded); err != nil {
+		log.Tracef("couldn't decode layer, err: %s", err)
+		return true
+	}
+
+	if len(d.decoded) < 2 {
+		log.Tracef("not enough levels in network packet")
+		return true
+	}
+
+	ipLayer := d.decoded[0]
+
+	switch ipLayer {
+	case layers.LayerTypeIPv4:
+		if !m.wgNetwork.Contains(d.ip4.SrcIP) || !m.wgNetwork.Contains(d.ip4.DstIP) {
+			return false
+		}
+	case layers.LayerTypeIPv6:
+		if !m.wgNetwork.Contains(d.ip6.SrcIP) || !m.wgNetwork.Contains(d.ip6.DstIP) {
+			return false
+		}
+	default:
+		log.Errorf("unknown layer: %v", d.decoded[0])
+		return true
+	}
+
+	var ip net.IP
+	switch ipLayer {
+	case layers.LayerTypeIPv4:
+		if isIncomingPacket {
+			ip = d.ip4.SrcIP
+		} else {
+			ip = d.ip4.DstIP
+		}
+	case layers.LayerTypeIPv6:
+		if isIncomingPacket {
+			ip = d.ip6.SrcIP
+		} else {
+			ip = d.ip6.DstIP
+		}
+	}
+
+	filter, ok := validateRule(ip, packetData, rules[ip.String()], d)
+	if ok {
+		return filter
+	}
+	filter, ok = validateRule(ip, packetData, rules["0.0.0.0"], d)
+	if ok {
+		return filter
+	}
+	filter, ok = validateRule(ip, packetData, rules["::"], d)
+	if ok {
+		return filter
+	}
+
+	// default policy is DROP ALL
+	return true
+}
+
+func validateRule(ip net.IP, packetData []byte, rules map[string]Rule, d *decoder) (bool, bool) {
+	payloadLayer := d.decoded[1]
+	for _, rule := range rules {
+		if rule.matchByIP && !ip.Equal(rule.ip) {
+			continue
+		}
+
+		if rule.protoLayer == layerTypeAll {
+			return rule.drop, true
+		}
+
+		if payloadLayer != rule.protoLayer {
+			continue
+		}
+
+		switch payloadLayer {
+		case layers.LayerTypeTCP:
+			if rule.sPort == 0 && rule.dPort == 0 {
+				return rule.drop, true
+			}
+			if rule.sPort != 0 && rule.sPort == uint16(d.tcp.SrcPort) {
+				return rule.drop, true
+			}
+			if rule.dPort != 0 && rule.dPort == uint16(d.tcp.DstPort) {
+				return rule.drop, true
+			}
+		case layers.LayerTypeUDP:
+			// if rule has UDP hook (and if we are here we match this rule)
+			// we ignore rule.drop and call this hook
+			if rule.udpHook != nil {
+				return rule.udpHook(packetData), true
+			}
+
+			if rule.sPort == 0 && rule.dPort == 0 {
+				return rule.drop, true
+			}
+			if rule.sPort != 0 && rule.sPort == uint16(d.udp.SrcPort) {
+				return rule.drop, true
+			}
+			if rule.dPort != 0 && rule.dPort == uint16(d.udp.DstPort) {
+				return rule.drop, true
+			}
+		case layers.LayerTypeICMPv4, layers.LayerTypeICMPv6:
+			return rule.drop, true
+		}
+	}
+	return false, false
+}
+
+// SetNetwork of the wireguard interface to which filtering applied
+func (m *Manager) SetNetwork(network *net.IPNet) {
+	m.wgNetwork = network
+}
+
+// AddUDPPacketHook calls hook when UDP packet from given direction matched
+//
+// Hook function returns flag which indicates should be the matched package dropped or not
+func (m *Manager) AddUDPPacketHook(
+	in bool, ip net.IP, dPort uint16, hook func([]byte) bool,
+) string {
+	r := Rule{
+		id:         uuid.New().String(),
+		ip:         ip,
+		protoLayer: layers.LayerTypeUDP,
+		dPort:      dPort,
+		ipLayer:    layers.LayerTypeIPv6,
+		direction:  firewall.RuleDirectionOUT,
+		comment:    fmt.Sprintf("UDP Hook direction: %v, ip:%v, dport:%d", in, ip, dPort),
+		udpHook:    hook,
+	}
+
+	if ip.To4() != nil {
+		r.ipLayer = layers.LayerTypeIPv4
+	}
+
+	m.mutex.Lock()
+	if in {
+		r.direction = firewall.RuleDirectionIN
+		if _, ok := m.incomingRules[r.ip.String()]; !ok {
+			m.incomingRules[r.ip.String()] = make(map[string]Rule)
+		}
+		m.incomingRules[r.ip.String()][r.id] = r
+	} else {
+		if _, ok := m.outgoingRules[r.ip.String()]; !ok {
+			m.outgoingRules[r.ip.String()] = make(map[string]Rule)
+		}
+		m.outgoingRules[r.ip.String()][r.id] = r
+	}
+
+	m.mutex.Unlock()
+
+	return r.id
+}
+
+// RemovePacketHook removes packet hook by given ID
+func (m *Manager) RemovePacketHook(hookID string) error {
+	for _, arr := range m.incomingRules {
+		for _, r := range arr {
+			if r.id == hookID {
+				rule := r
+				return m.DeletePeerRule(&rule)
+			}
+		}
+	}
+	for _, arr := range m.outgoingRules {
+		for _, r := range arr {
+			if r.id == hookID {
+				rule := r
+				return m.DeletePeerRule(&rule)
+			}
+		}
+	}
+	return fmt.Errorf("hook with given id not found")
+}

client/firewall/uspfilter/uspfilter_test.go

@@ -0,0 +1,420 @@
+package uspfilter
+
+import (
+	"fmt"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/stretchr/testify/require"
+
+	fw "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/device"
+)
+
+type IFaceMock struct {
+	SetFilterFunc func(device.PacketFilter) error
+	AddressFunc   func() iface.WGAddress
+}
+
+func (i *IFaceMock) SetFilter(iface device.PacketFilter) error {
+	if i.SetFilterFunc == nil {
+		return fmt.Errorf("not implemented")
+	}
+	return i.SetFilterFunc(iface)
+}
+
+func (i *IFaceMock) Address() iface.WGAddress {
+	if i.AddressFunc == nil {
+		return iface.WGAddress{}
+	}
+	return i.AddressFunc()
+}
+
+func TestManagerCreate(t *testing.T) {
+	ifaceMock := &IFaceMock{
+		SetFilterFunc: func(device.PacketFilter) error { return nil },
+	}
+
+	m, err := Create(ifaceMock)
+	if err != nil {
+		t.Errorf("failed to create Manager: %v", err)
+		return
+	}
+
+	if m == nil {
+		t.Error("Manager is nil")
+	}
+}
+
+func TestManagerAddPeerFiltering(t *testing.T) {
+	isSetFilterCalled := false
+	ifaceMock := &IFaceMock{
+		SetFilterFunc: func(device.PacketFilter) error {
+			isSetFilterCalled = true
+			return nil
+		},
+	}
+
+	m, err := Create(ifaceMock)
+	if err != nil {
+		t.Errorf("failed to create Manager: %v", err)
+		return
+	}
+
+	ip := net.ParseIP("192.168.1.1")
+	proto := fw.ProtocolTCP
+	port := &fw.Port{Values: []int{80}}
+	direction := fw.RuleDirectionOUT
+	action := fw.ActionDrop
+	comment := "Test rule"
+
+	rule, err := m.AddPeerFiltering(ip, proto, nil, port, direction, action, "", comment)
+	if err != nil {
+		t.Errorf("failed to add filtering: %v", err)
+		return
+	}
+
+	if rule == nil {
+		t.Error("Rule is nil")
+		return
+	}
+
+	if !isSetFilterCalled {
+		t.Error("SetFilter was not called")
+		return
+	}
+}
+
+func TestManagerDeleteRule(t *testing.T) {
+	ifaceMock := &IFaceMock{
+		SetFilterFunc: func(device.PacketFilter) error { return nil },
+	}
+
+	m, err := Create(ifaceMock)
+	if err != nil {
+		t.Errorf("failed to create Manager: %v", err)
+		return
+	}
+
+	ip := net.ParseIP("192.168.1.1")
+	proto := fw.ProtocolTCP
+	port := &fw.Port{Values: []int{80}}
+	direction := fw.RuleDirectionOUT
+	action := fw.ActionDrop
+	comment := "Test rule"
+
+	rule, err := m.AddPeerFiltering(ip, proto, nil, port, direction, action, "", comment)
+	if err != nil {
+		t.Errorf("failed to add filtering: %v", err)
+		return
+	}
+
+	ip = net.ParseIP("192.168.1.1")
+	proto = fw.ProtocolTCP
+	port = &fw.Port{Values: []int{80}}
+	direction = fw.RuleDirectionIN
+	action = fw.ActionDrop
+	comment = "Test rule 2"
+
+	rule2, err := m.AddPeerFiltering(ip, proto, nil, port, direction, action, "", comment)
+	if err != nil {
+		t.Errorf("failed to add filtering: %v", err)
+		return
+	}
+
+	for _, r := range rule {
+		err = m.DeletePeerRule(r)
+		if err != nil {
+			t.Errorf("failed to delete rule: %v", err)
+			return
+		}
+	}
+
+	for _, r := range rule2 {
+		if _, ok := m.incomingRules[ip.String()][r.GetRuleID()]; !ok {
+			t.Errorf("rule2 is not in the incomingRules")
+		}
+	}
+
+	for _, r := range rule2 {
+		err = m.DeletePeerRule(r)
+		if err != nil {
+			t.Errorf("failed to delete rule: %v", err)
+			return
+		}
+	}
+
+	for _, r := range rule2 {
+		if _, ok := m.incomingRules[ip.String()][r.GetRuleID()]; ok {
+			t.Errorf("rule2 is not in the incomingRules")
+		}
+	}
+}
+
+func TestAddUDPPacketHook(t *testing.T) {
+	tests := []struct {
+		name       string
+		in         bool
+		expDir     fw.RuleDirection
+		ip         net.IP
+		dPort      uint16
+		hook       func([]byte) bool
+		expectedID string
+	}{
+		{
+			name:   "Test Outgoing UDP Packet Hook",
+			in:     false,
+			expDir: fw.RuleDirectionOUT,
+			ip:     net.IPv4(10, 168, 0, 1),
+			dPort:  8000,
+			hook:   func([]byte) bool { return true },
+		},
+		{
+			name:   "Test Incoming UDP Packet Hook",
+			in:     true,
+			expDir: fw.RuleDirectionIN,
+			ip:     net.IPv6loopback,
+			dPort:  9000,
+			hook:   func([]byte) bool { return false },
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			manager := &Manager{
+				incomingRules: map[string]RuleSet{},
+				outgoingRules: map[string]RuleSet{},
+			}
+
+			manager.AddUDPPacketHook(tt.in, tt.ip, tt.dPort, tt.hook)
+
+			var addedRule Rule
+			if tt.in {
+				if len(manager.incomingRules[tt.ip.String()]) != 1 {
+					t.Errorf("expected 1 incoming rule, got %d", len(manager.incomingRules))
+					return
+				}
+				for _, rule := range manager.incomingRules[tt.ip.String()] {
+					addedRule = rule
+				}
+			} else {
+				if len(manager.outgoingRules) != 1 {
+					t.Errorf("expected 1 outgoing rule, got %d", len(manager.outgoingRules))
+					return
+				}
+				for _, rule := range manager.outgoingRules[tt.ip.String()] {
+					addedRule = rule
+				}
+			}
+
+			if !tt.ip.Equal(addedRule.ip) {
+				t.Errorf("expected ip %s, got %s", tt.ip, addedRule.ip)
+				return
+			}
+			if tt.dPort != addedRule.dPort {
+				t.Errorf("expected dPort %d, got %d", tt.dPort, addedRule.dPort)
+				return
+			}
+			if layers.LayerTypeUDP != addedRule.protoLayer {
+				t.Errorf("expected protoLayer %s, got %s", layers.LayerTypeUDP, addedRule.protoLayer)
+				return
+			}
+			if tt.expDir != addedRule.direction {
+				t.Errorf("expected direction %d, got %d", tt.expDir, addedRule.direction)
+				return
+			}
+			if addedRule.udpHook == nil {
+				t.Errorf("expected udpHook to be set")
+				return
+			}
+		})
+	}
+}
+
+func TestManagerReset(t *testing.T) {
+	ifaceMock := &IFaceMock{
+		SetFilterFunc: func(device.PacketFilter) error { return nil },
+	}
+
+	m, err := Create(ifaceMock)
+	if err != nil {
+		t.Errorf("failed to create Manager: %v", err)
+		return
+	}
+
+	ip := net.ParseIP("192.168.1.1")
+	proto := fw.ProtocolTCP
+	port := &fw.Port{Values: []int{80}}
+	direction := fw.RuleDirectionOUT
+	action := fw.ActionDrop
+	comment := "Test rule"
+
+	_, err = m.AddPeerFiltering(ip, proto, nil, port, direction, action, "", comment)
+	if err != nil {
+		t.Errorf("failed to add filtering: %v", err)
+		return
+	}
+
+	err = m.Reset(nil)
+	if err != nil {
+		t.Errorf("failed to reset Manager: %v", err)
+		return
+	}
+
+	if len(m.outgoingRules) != 0 || len(m.incomingRules) != 0 {
+		t.Errorf("rules is not empty")
+	}
+}
+
+func TestNotMatchByIP(t *testing.T) {
+	ifaceMock := &IFaceMock{
+		SetFilterFunc: func(device.PacketFilter) error { return nil },
+	}
+
+	m, err := Create(ifaceMock)
+	if err != nil {
+		t.Errorf("failed to create Manager: %v", err)
+		return
+	}
+	m.wgNetwork = &net.IPNet{
+		IP:   net.ParseIP("100.10.0.0"),
+		Mask: net.CIDRMask(16, 32),
+	}
+
+	ip := net.ParseIP("0.0.0.0")
+	proto := fw.ProtocolUDP
+	direction := fw.RuleDirectionOUT
+	action := fw.ActionAccept
+	comment := "Test rule"
+
+	_, err = m.AddPeerFiltering(ip, proto, nil, nil, direction, action, "", comment)
+	if err != nil {
+		t.Errorf("failed to add filtering: %v", err)
+		return
+	}
+
+	ipv4 := &layers.IPv4{
+		TTL:      64,
+		Version:  4,
+		SrcIP:    net.ParseIP("100.10.0.1"),
+		DstIP:    net.ParseIP("100.10.0.100"),
+		Protocol: layers.IPProtocolUDP,
+	}
+	udp := &layers.UDP{
+		SrcPort: 51334,
+		DstPort: 53,
+	}
+
+	if err := udp.SetNetworkLayerForChecksum(ipv4); err != nil {
+		t.Errorf("failed to set network layer for checksum: %v", err)
+		return
+	}
+	payload := gopacket.Payload([]byte("test"))
+
+	buf := gopacket.NewSerializeBuffer()
+	opts := gopacket.SerializeOptions{
+		ComputeChecksums: true,
+		FixLengths:       true,
+	}
+	if err = gopacket.SerializeLayers(buf, opts, ipv4, udp, payload); err != nil {
+		t.Errorf("failed to serialize packet: %v", err)
+		return
+	}
+
+	if m.dropFilter(buf.Bytes(), m.outgoingRules, false) {
+		t.Errorf("expected packet to be accepted")
+		return
+	}
+
+	if err = m.Reset(nil); err != nil {
+		t.Errorf("failed to reset Manager: %v", err)
+		return
+	}
+}
+
+// TestRemovePacketHook tests the functionality of the RemovePacketHook method
+func TestRemovePacketHook(t *testing.T) {
+	// creating mock iface
+	iface := &IFaceMock{
+		SetFilterFunc: func(device.PacketFilter) error { return nil },
+	}
+
+	// creating manager instance
+	manager, err := Create(iface)
+	if err != nil {
+		t.Fatalf("Failed to create Manager: %s", err)
+	}
+
+	// Add a UDP packet hook
+	hookFunc := func(data []byte) bool { return true }
+	hookID := manager.AddUDPPacketHook(false, net.IPv4(192, 168, 0, 1), 8080, hookFunc)
+
+	// Assert the hook is added by finding it in the manager's outgoing rules
+	found := false
+	for _, arr := range manager.outgoingRules {
+		for _, rule := range arr {
+			if rule.id == hookID {
+				found = true
+				break
+			}
+		}
+	}
+
+	if !found {
+		t.Fatalf("The hook was not added properly.")
+	}
+
+	// Now remove the packet hook
+	err = manager.RemovePacketHook(hookID)
+	if err != nil {
+		t.Fatalf("Failed to remove hook: %s", err)
+	}
+
+	// Assert the hook is removed by checking it in the manager's outgoing rules
+	for _, arr := range manager.outgoingRules {
+		for _, rule := range arr {
+			if rule.id == hookID {
+				t.Fatalf("The hook was not removed properly.")
+			}
+		}
+	}
+}
+
+func TestUSPFilterCreatePerformance(t *testing.T) {
+	for _, testMax := range []int{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000} {
+		t.Run(fmt.Sprintf("Testing %d rules", testMax), func(t *testing.T) {
+			// just check on the local interface
+			ifaceMock := &IFaceMock{
+				SetFilterFunc: func(device.PacketFilter) error { return nil },
+			}
+			manager, err := Create(ifaceMock)
+			require.NoError(t, err)
+			time.Sleep(time.Second)
+
+			defer func() {
+				if err := manager.Reset(nil); err != nil {
+					t.Errorf("clear the manager state: %v", err)
+				}
+				time.Sleep(time.Second)
+			}()
+
+			ip := net.ParseIP("10.20.0.100")
+			start := time.Now()
+			for i := 0; i < testMax; i++ {
+				port := &fw.Port{Values: []int{1000 + i}}
+				if i%2 == 0 {
+					_, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionOUT, fw.ActionAccept, "", "accept HTTP traffic")
+				} else {
+					_, err = manager.AddPeerFiltering(ip, "tcp", nil, port, fw.RuleDirectionIN, fw.ActionAccept, "", "accept HTTP traffic")
+				}
+
+				require.NoError(t, err, "failed to add rule")
+			}
+			t.Logf("execution avg per rule: %s", time.Since(start)/time.Duration(testMax))
+		})
+	}
+}

client/iface/bind/control_android.go

@@ -0,0 +1,12 @@
+package bind
+
+import (
+	wireguard "golang.zx2c4.com/wireguard/conn"
+
+	nbnet "github.com/netbirdio/netbird/util/net"
+)
+
+func init() {
+	// ControlFns is not thread safe and should only be modified during init.
+	*wireguard.ControlFns = append(*wireguard.ControlFns, nbnet.ControlProtectSocket)
+}

client/iface/bind/endpoint.go

@@ -0,0 +1,5 @@
+package bind
+
+import wgConn "golang.zx2c4.com/wireguard/conn"
+
+type Endpoint = wgConn.StdNetEndpoint

client/iface/bind/ice_bind.go

@@ -0,0 +1,303 @@
+package bind
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/pion/stun/v2"
+	"github.com/pion/transport/v3"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+	wgConn "golang.zx2c4.com/wireguard/conn"
+)
+
+type RecvMessage struct {
+	Endpoint *Endpoint
+	Buffer   []byte
+}
+
+type receiverCreator struct {
+	iceBind *ICEBind
+}
+
+func (rc receiverCreator) CreateIPv4ReceiverFn(pc *ipv4.PacketConn, conn *net.UDPConn, rxOffload bool, msgPool *sync.Pool) wgConn.ReceiveFunc {
+	return rc.iceBind.createIPv4ReceiverFn(pc, conn, rxOffload, msgPool)
+}
+
+// ICEBind is a bind implementation with two main features:
+// 1. filter out STUN messages and handle them
+// 2. forward the received packets to the WireGuard interface from the relayed connection
+//
+// ICEBind.endpoints var is a map that stores the connection for each relayed peer. Fake address is just an IP address
+// without port, in the format of 127.1.x.x where x.x is the last two octets of the peer address. We try to avoid to
+// use the port because in the Send function the wgConn.Endpoint the port info is not exported.
+type ICEBind struct {
+	*wgConn.StdNetBind
+	RecvChan chan RecvMessage
+
+	transportNet transport.Net
+	filterFn     FilterFn
+	endpoints    map[netip.Addr]net.Conn
+	endpointsMu  sync.Mutex
+	// every time when Close() is called (i.e. BindUpdate()) we need to close exit from the receiveRelayed and create a
+	// new closed channel. With the closedChanMu we can safely close the channel and create a new one
+	closedChan   chan struct{}
+	closedChanMu sync.RWMutex // protect the closeChan recreation from reading from it.
+	closed       bool
+
+	muUDPMux sync.Mutex
+	udpMux   *UniversalUDPMuxDefault
+}
+
+func NewICEBind(transportNet transport.Net, filterFn FilterFn) *ICEBind {
+	b, _ := wgConn.NewStdNetBind().(*wgConn.StdNetBind)
+	ib := &ICEBind{
+		StdNetBind:   b,
+		RecvChan:     make(chan RecvMessage, 1),
+		transportNet: transportNet,
+		filterFn:     filterFn,
+		endpoints:    make(map[netip.Addr]net.Conn),
+		closedChan:   make(chan struct{}),
+		closed:       true,
+	}
+
+	rc := receiverCreator{
+		ib,
+	}
+	ib.StdNetBind = wgConn.NewStdNetBindWithReceiverCreator(rc)
+	return ib
+}
+
+func (s *ICEBind) Open(uport uint16) ([]wgConn.ReceiveFunc, uint16, error) {
+	s.closed = false
+	s.closedChanMu.Lock()
+	s.closedChan = make(chan struct{})
+	s.closedChanMu.Unlock()
+	fns, port, err := s.StdNetBind.Open(uport)
+	if err != nil {
+		return nil, 0, err
+	}
+	fns = append(fns, s.receiveRelayed)
+	return fns, port, nil
+}
+
+func (s *ICEBind) Close() error {
+	if s.closed {
+		return nil
+	}
+	s.closed = true
+
+	close(s.closedChan)
+
+	return s.StdNetBind.Close()
+}
+
+// GetICEMux returns the ICE UDPMux that was created and used by ICEBind
+func (s *ICEBind) GetICEMux() (*UniversalUDPMuxDefault, error) {
+	s.muUDPMux.Lock()
+	defer s.muUDPMux.Unlock()
+	if s.udpMux == nil {
+		return nil, fmt.Errorf("ICEBind has not been initialized yet")
+	}
+
+	return s.udpMux, nil
+}
+
+func (b *ICEBind) SetEndpoint(peerAddress *net.UDPAddr, conn net.Conn) (*net.UDPAddr, error) {
+	fakeUDPAddr, err := fakeAddress(peerAddress)
+	if err != nil {
+		return nil, err
+	}
+
+	// force IPv4
+	fakeAddr, ok := netip.AddrFromSlice(fakeUDPAddr.IP.To4())
+	if !ok {
+		return nil, fmt.Errorf("failed to convert IP to netip.Addr")
+	}
+
+	b.endpointsMu.Lock()
+	b.endpoints[fakeAddr] = conn
+	b.endpointsMu.Unlock()
+
+	return fakeUDPAddr, nil
+}
+
+func (b *ICEBind) RemoveEndpoint(fakeUDPAddr *net.UDPAddr) {
+	fakeAddr, ok := netip.AddrFromSlice(fakeUDPAddr.IP.To4())
+	if !ok {
+		log.Warnf("failed to convert IP to netip.Addr")
+		return
+	}
+
+	b.endpointsMu.Lock()
+	defer b.endpointsMu.Unlock()
+	delete(b.endpoints, fakeAddr)
+}
+
+func (b *ICEBind) Send(bufs [][]byte, ep wgConn.Endpoint) error {
+	b.endpointsMu.Lock()
+	conn, ok := b.endpoints[ep.DstIP()]
+	b.endpointsMu.Unlock()
+	if !ok {
+		return b.StdNetBind.Send(bufs, ep)
+	}
+
+	for _, buf := range bufs {
+		if _, err := conn.Write(buf); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *ICEBind) createIPv4ReceiverFn(pc *ipv4.PacketConn, conn *net.UDPConn, rxOffload bool, msgsPool *sync.Pool) wgConn.ReceiveFunc {
+	s.muUDPMux.Lock()
+	defer s.muUDPMux.Unlock()
+
+	s.udpMux = NewUniversalUDPMuxDefault(
+		UniversalUDPMuxParams{
+			UDPConn:  conn,
+			Net:      s.transportNet,
+			FilterFn: s.filterFn,
+		},
+	)
+	return func(bufs [][]byte, sizes []int, eps []wgConn.Endpoint) (n int, err error) {
+		msgs := getMessages(msgsPool)
+		for i := range bufs {
+			(*msgs)[i].Buffers[0] = bufs[i]
+			(*msgs)[i].OOB = (*msgs)[i].OOB[:cap((*msgs)[i].OOB)]
+		}
+		defer putMessages(msgs, msgsPool)
+		var numMsgs int
+		if runtime.GOOS == "linux" || runtime.GOOS == "android" {
+			if rxOffload {
+				readAt := len(*msgs) - (wgConn.IdealBatchSize / wgConn.UdpSegmentMaxDatagrams)
+				//nolint
+				numMsgs, err = pc.ReadBatch((*msgs)[readAt:], 0)
+				if err != nil {
+					return 0, err
+				}
+				numMsgs, err = wgConn.SplitCoalescedMessages(*msgs, readAt, wgConn.GetGSOSize)
+				if err != nil {
+					return 0, err
+				}
+			} else {
+				numMsgs, err = pc.ReadBatch(*msgs, 0)
+				if err != nil {
+					return 0, err
+				}
+			}
+		} else {
+			msg := &(*msgs)[0]
+			msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB)
+			if err != nil {
+				return 0, err
+			}
+			numMsgs = 1
+		}
+		for i := 0; i < numMsgs; i++ {
+			msg := &(*msgs)[i]
+
+			// todo: handle err
+			ok, _ := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr)
+			if ok {
+				continue
+			}
+			sizes[i] = msg.N
+			if sizes[i] == 0 {
+				continue
+			}
+			addrPort := msg.Addr.(*net.UDPAddr).AddrPort()
+			ep := &wgConn.StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation
+			wgConn.GetSrcFromControl(msg.OOB[:msg.NN], ep)
+			eps[i] = ep
+		}
+		return numMsgs, nil
+	}
+}
+
+func (s *ICEBind) filterOutStunMessages(buffers [][]byte, n int, addr net.Addr) (bool, error) {
+	for i := range buffers {
+		if !stun.IsMessage(buffers[i]) {
+			continue
+		}
+
+		msg, err := s.parseSTUNMessage(buffers[i][:n])
+		if err != nil {
+			buffers[i] = []byte{}
+			return true, err
+		}
+
+		muxErr := s.udpMux.HandleSTUNMessage(msg, addr)
+		if muxErr != nil {
+			log.Warnf("failed to handle STUN packet")
+		}
+
+		buffers[i] = []byte{}
+		return true, nil
+	}
+	return false, nil
+}
+
+func (s *ICEBind) parseSTUNMessage(raw []byte) (*stun.Message, error) {
+	msg := &stun.Message{
+		Raw: raw,
+	}
+	if err := msg.Decode(); err != nil {
+		return nil, err
+	}
+
+	return msg, nil
+}
+
+// receiveRelayed is a receive function that is used to receive packets from the relayed connection and forward to the
+// WireGuard. Critical part is do not block if the Closed() has been called.
+func (c *ICEBind) receiveRelayed(buffs [][]byte, sizes []int, eps []wgConn.Endpoint) (int, error) {
+	c.closedChanMu.RLock()
+	defer c.closedChanMu.RUnlock()
+
+	select {
+	case <-c.closedChan:
+		return 0, net.ErrClosed
+	case msg, ok := <-c.RecvChan:
+		if !ok {
+			return 0, net.ErrClosed
+		}
+		copy(buffs[0], msg.Buffer)
+		sizes[0] = len(msg.Buffer)
+		eps[0] = wgConn.Endpoint(msg.Endpoint)
+		return 1, nil
+	}
+}
+
+// fakeAddress returns a fake address that is used to as an identifier for the peer.
+// The fake address is in the format of 127.1.x.x where x.x is the last two octets of the peer address.
+func fakeAddress(peerAddress *net.UDPAddr) (*net.UDPAddr, error) {
+	octets := strings.Split(peerAddress.IP.String(), ".")
+	if len(octets) != 4 {
+		return nil, fmt.Errorf("invalid IP format")
+	}
+
+	newAddr := &net.UDPAddr{
+		IP:   net.ParseIP(fmt.Sprintf("127.1.%s.%s", octets[2], octets[3])),
+		Port: peerAddress.Port,
+	}
+	return newAddr, nil
+}
+
+func getMessages(msgsPool *sync.Pool) *[]ipv6.Message {
+	return msgsPool.Get().(*[]ipv6.Message)
+}
+
+func putMessages(msgs *[]ipv6.Message, msgsPool *sync.Pool) {
+	for i := range *msgs {
+		(*msgs)[i].OOB = (*msgs)[i].OOB[:0]
+		(*msgs)[i] = ipv6.Message{Buffers: (*msgs)[i].Buffers, OOB: (*msgs)[i].OOB}
+	}
+	msgsPool.Put(msgs)
+}

client/iface/bind/udp_mux.go

@@ -0,0 +1,441 @@
+package bind
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"sync"
+
+	"github.com/pion/ice/v3"
+	"github.com/pion/logging"
+	"github.com/pion/stun/v2"
+	"github.com/pion/transport/v3"
+	"github.com/pion/transport/v3/stdnet"
+	log "github.com/sirupsen/logrus"
+)
+
+/*
+ Most of this code was copied from https://github.com/pion/ice and modified to fulfill NetBird's requirements
+*/
+
+const receiveMTU = 8192
+
+// UDPMuxDefault is an implementation of the interface
+type UDPMuxDefault struct {
+	params UDPMuxParams
+
+	closedChan chan struct{}
+	closeOnce  sync.Once
+
+	// connsIPv4 and connsIPv6 are maps of all udpMuxedConn indexed by ufrag|network|candidateType
+	connsIPv4, connsIPv6 map[string]*udpMuxedConn
+
+	addressMapMu sync.RWMutex
+	addressMap   map[string][]*udpMuxedConn
+
+	// buffer pool to recycle buffers for net.UDPAddr encodes/decodes
+	pool *sync.Pool
+
+	mu sync.Mutex
+
+	// for UDP connection listen at unspecified address
+	localAddrsForUnspecified []net.Addr
+}
+
+const maxAddrSize = 512
+
+// UDPMuxParams are parameters for UDPMux.
+type UDPMuxParams struct {
+	Logger  logging.LeveledLogger
+	UDPConn net.PacketConn
+
+	// Required for gathering local addresses
+	// in case a un UDPConn is passed which does not
+	// bind to a specific local address.
+	Net             transport.Net
+	InterfaceFilter func(interfaceName string) bool
+}
+
+func localInterfaces(n transport.Net, interfaceFilter func(string) bool, ipFilter func(net.IP) bool, networkTypes []ice.NetworkType, includeLoopback bool) ([]net.IP, error) { //nolint:gocognit
+	ips := []net.IP{}
+	ifaces, err := n.Interfaces()
+	if err != nil {
+		return ips, err
+	}
+
+	var IPv4Requested, IPv6Requested bool
+	for _, typ := range networkTypes {
+		if typ.IsIPv4() {
+			IPv4Requested = true
+		}
+
+		if typ.IsIPv6() {
+			IPv6Requested = true
+		}
+	}
+
+	for _, iface := range ifaces {
+		if iface.Flags&net.FlagUp == 0 {
+			continue // interface down
+		}
+		if (iface.Flags&net.FlagLoopback != 0) && !includeLoopback {
+			continue // loopback interface
+		}
+
+		if interfaceFilter != nil && !interfaceFilter(iface.Name) {
+			continue
+		}
+
+		addrs, err := iface.Addrs()
+		if err != nil {
+			continue
+		}
+
+		for _, addr := range addrs {
+			var ip net.IP
+			switch addr := addr.(type) {
+			case *net.IPNet:
+				ip = addr.IP
+			case *net.IPAddr:
+				ip = addr.IP
+			}
+			if ip == nil || (ip.IsLoopback() && !includeLoopback) {
+				continue
+			}
+
+			if ipv4 := ip.To4(); ipv4 == nil {
+				if !IPv6Requested {
+					continue
+				} else if !isSupportedIPv6(ip) {
+					continue
+				}
+			} else if !IPv4Requested {
+				continue
+			}
+
+			if ipFilter != nil && !ipFilter(ip) {
+				continue
+			}
+
+			ips = append(ips, ip)
+		}
+	}
+	return ips, nil
+}
+
+// The conditions of invalidation written below are defined in
+// https://tools.ietf.org/html/rfc8445#section-5.1.1.1
+func isSupportedIPv6(ip net.IP) bool {
+	if len(ip) != net.IPv6len ||
+		isZeros(ip[0:12]) || // !(IPv4-compatible IPv6)
+		ip[0] == 0xfe && ip[1]&0xc0 == 0xc0 || // !(IPv6 site-local unicast)
+		ip.IsLinkLocalUnicast() ||
+		ip.IsLinkLocalMulticast() {
+		return false
+	}
+	return true
+}
+
+func isZeros(ip net.IP) bool {
+	for i := 0; i < len(ip); i++ {
+		if ip[i] != 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// NewUDPMuxDefault creates an implementation of UDPMux
+func NewUDPMuxDefault(params UDPMuxParams) *UDPMuxDefault {
+	if params.Logger == nil {
+		params.Logger = logging.NewDefaultLoggerFactory().NewLogger("ice")
+	}
+
+	var localAddrsForUnspecified []net.Addr
+	if addr, ok := params.UDPConn.LocalAddr().(*net.UDPAddr); !ok {
+		params.Logger.Errorf("LocalAddr is not a net.UDPAddr, got %T", params.UDPConn.LocalAddr())
+	} else if ok && addr.IP.IsUnspecified() {
+		// For unspecified addresses, the correct behavior is to return errListenUnspecified, but
+		// it will break the applications that are already using unspecified UDP connection
+		// with UDPMuxDefault, so print a warn log and create a local address list for mux.
+		params.Logger.Warn("UDPMuxDefault should not listening on unspecified address, use NewMultiUDPMuxFromPort instead")
+		var networks []ice.NetworkType
+		switch {
+
+		case addr.IP.To16() != nil:
+			networks = []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}
+
+		case addr.IP.To4() != nil:
+			networks = []ice.NetworkType{ice.NetworkTypeUDP4}
+
+		default:
+			params.Logger.Errorf("LocalAddr expected IPV4 or IPV6, got %T", params.UDPConn.LocalAddr())
+		}
+		if len(networks) > 0 {
+			if params.Net == nil {
+				var err error
+				if params.Net, err = stdnet.NewNet(); err != nil {
+					params.Logger.Errorf("failed to get create network: %v", err)
+				}
+			}
+
+			ips, err := localInterfaces(params.Net, params.InterfaceFilter, nil, networks, true)
+			if err == nil {
+				for _, ip := range ips {
+					localAddrsForUnspecified = append(localAddrsForUnspecified, &net.UDPAddr{IP: ip, Port: addr.Port})
+				}
+			} else {
+				params.Logger.Errorf("failed to get local interfaces for unspecified addr: %v", err)
+			}
+		}
+	}
+
+	return &UDPMuxDefault{
+		addressMap: map[string][]*udpMuxedConn{},
+		params:     params,
+		connsIPv4:  make(map[string]*udpMuxedConn),
+		connsIPv6:  make(map[string]*udpMuxedConn),
+		closedChan: make(chan struct{}, 1),
+		pool: &sync.Pool{
+			New: func() interface{} {
+				// big enough buffer to fit both packet and address
+				return newBufferHolder(receiveMTU + maxAddrSize)
+			},
+		},
+		localAddrsForUnspecified: localAddrsForUnspecified,
+	}
+}
+
+// LocalAddr returns the listening address of this UDPMuxDefault
+func (m *UDPMuxDefault) LocalAddr() net.Addr {
+	return m.params.UDPConn.LocalAddr()
+}
+
+// GetListenAddresses returns the list of addresses that this mux is listening on
+func (m *UDPMuxDefault) GetListenAddresses() []net.Addr {
+	if len(m.localAddrsForUnspecified) > 0 {
+		return m.localAddrsForUnspecified
+	}
+
+	return []net.Addr{m.LocalAddr()}
+}
+
+// GetConn returns a PacketConn given the connection's ufrag and network address
+// creates the connection if an existing one can't be found
+func (m *UDPMuxDefault) GetConn(ufrag string, addr net.Addr) (net.PacketConn, error) {
+	// don't check addr for mux using unspecified address
+	if len(m.localAddrsForUnspecified) == 0 && m.params.UDPConn.LocalAddr().String() != addr.String() {
+		return nil, fmt.Errorf("invalid address %s", addr.String())
+	}
+
+	var isIPv6 bool
+	if udpAddr, _ := addr.(*net.UDPAddr); udpAddr != nil && udpAddr.IP.To4() == nil {
+		isIPv6 = true
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if m.IsClosed() {
+		return nil, io.ErrClosedPipe
+	}
+
+	if conn, ok := m.getConn(ufrag, isIPv6); ok {
+		return conn, nil
+	}
+
+	c := m.createMuxedConn(ufrag)
+	go func() {
+		<-c.CloseChannel()
+		m.RemoveConnByUfrag(ufrag)
+	}()
+
+	if isIPv6 {
+		m.connsIPv6[ufrag] = c
+	} else {
+		m.connsIPv4[ufrag] = c
+	}
+
+	return c, nil
+}
+
+// RemoveConnByUfrag stops and removes the muxed packet connection
+func (m *UDPMuxDefault) RemoveConnByUfrag(ufrag string) {
+	removedConns := make([]*udpMuxedConn, 0, 2)
+
+	// Keep lock section small to avoid deadlock with conn lock
+	m.mu.Lock()
+	if c, ok := m.connsIPv4[ufrag]; ok {
+		delete(m.connsIPv4, ufrag)
+		removedConns = append(removedConns, c)
+	}
+	if c, ok := m.connsIPv6[ufrag]; ok {
+		delete(m.connsIPv6, ufrag)
+		removedConns = append(removedConns, c)
+	}
+	m.mu.Unlock()
+
+	if len(removedConns) == 0 {
+		// No need to lock if no connection was found
+		return
+	}
+
+	m.addressMapMu.Lock()
+	defer m.addressMapMu.Unlock()
+
+	for _, c := range removedConns {
+		addresses := c.getAddresses()
+		for _, addr := range addresses {
+			delete(m.addressMap, addr)
+		}
+	}
+}
+
+// IsClosed returns true if the mux had been closed
+func (m *UDPMuxDefault) IsClosed() bool {
+	select {
+	case <-m.closedChan:
+		return true
+	default:
+		return false
+	}
+}
+
+// Close the mux, no further connections could be created
+func (m *UDPMuxDefault) Close() error {
+	var err error
+	m.closeOnce.Do(func() {
+		m.mu.Lock()
+		defer m.mu.Unlock()
+
+		for _, c := range m.connsIPv4 {
+			_ = c.Close()
+		}
+		for _, c := range m.connsIPv6 {
+			_ = c.Close()
+		}
+
+		m.connsIPv4 = make(map[string]*udpMuxedConn)
+		m.connsIPv6 = make(map[string]*udpMuxedConn)
+
+		close(m.closedChan)
+
+		_ = m.params.UDPConn.Close()
+	})
+	return err
+}
+
+func (m *UDPMuxDefault) writeTo(buf []byte, rAddr net.Addr) (n int, err error) {
+	return m.params.UDPConn.WriteTo(buf, rAddr)
+}
+
+func (m *UDPMuxDefault) registerConnForAddress(conn *udpMuxedConn, addr string) {
+	if m.IsClosed() {
+		return
+	}
+
+	m.addressMapMu.Lock()
+	defer m.addressMapMu.Unlock()
+
+	existing, ok := m.addressMap[addr]
+	if !ok {
+		existing = []*udpMuxedConn{}
+	}
+	existing = append(existing, conn)
+	m.addressMap[addr] = existing
+
+	log.Debugf("ICE: registered %s for %s", addr, conn.params.Key)
+}
+
+func (m *UDPMuxDefault) createMuxedConn(key string) *udpMuxedConn {
+	c := newUDPMuxedConn(&udpMuxedConnParams{
+		Mux:       m,
+		Key:       key,
+		AddrPool:  m.pool,
+		LocalAddr: m.LocalAddr(),
+		Logger:    m.params.Logger,
+	})
+	return c
+}
+
+// HandleSTUNMessage handles STUN packets and forwards them to underlying pion/ice library
+func (m *UDPMuxDefault) HandleSTUNMessage(msg *stun.Message, addr net.Addr) error {
+
+	remoteAddr, ok := addr.(*net.UDPAddr)
+	if !ok {
+		return fmt.Errorf("underlying PacketConn did not return a UDPAddr")
+	}
+
+	// If we have already seen this address dispatch to the appropriate destination
+	// If you are using the same socket for the Host and SRFLX candidates, it might be that there are more than one
+	// muxed connection - one for the SRFLX candidate and the other one for the HOST one.
+	// We will then forward STUN packets to each of these connections.
+	m.addressMapMu.Lock()
+	var destinationConnList []*udpMuxedConn
+	if storedConns, ok := m.addressMap[addr.String()]; ok {
+		destinationConnList = append(destinationConnList, storedConns...)
+	}
+	m.addressMapMu.Unlock()
+
+	var isIPv6 bool
+	if udpAddr, _ := addr.(*net.UDPAddr); udpAddr != nil && udpAddr.IP.To4() == nil {
+		isIPv6 = true
+	}
+
+	// This block is needed to discover Peer Reflexive Candidates for which we don't know the Endpoint upfront.
+	// However, we can take a username attribute from the STUN message which contains ufrag.
+	// We can use ufrag to identify the destination conn to route packet to.
+	attr, stunAttrErr := msg.Get(stun.AttrUsername)
+	if stunAttrErr == nil {
+		ufrag := strings.Split(string(attr), ":")[0]
+
+		m.mu.Lock()
+		destinationConn := m.connsIPv4[ufrag]
+		if isIPv6 {
+			destinationConn = m.connsIPv6[ufrag]
+		}
+
+		if destinationConn != nil {
+			exists := false
+			for _, conn := range destinationConnList {
+				if conn.params.Key == destinationConn.params.Key {
+					exists = true
+					break
+				}
+			}
+			if !exists {
+				destinationConnList = append(destinationConnList, destinationConn)
+			}
+		}
+		m.mu.Unlock()
+	}
+
+	// Forward STUN packets to each destination connections even thought the STUN packet might not belong there.
+	// It will be discarded by the further ICE candidate logic if so.
+	for _, conn := range destinationConnList {
+		if err := conn.writePacket(msg.Raw, remoteAddr); err != nil {
+			log.Errorf("could not write packet: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func (m *UDPMuxDefault) getConn(ufrag string, isIPv6 bool) (val *udpMuxedConn, ok bool) {
+	if isIPv6 {
+		val, ok = m.connsIPv6[ufrag]
+	} else {
+		val, ok = m.connsIPv4[ufrag]
+	}
+	return
+}
+
+type bufferHolder struct {
+	buf []byte
+}
+
+func newBufferHolder(size int) *bufferHolder {
+	return &bufferHolder{
+		buf: make([]byte, size),
+	}
+}

client/iface/bind/udp_mux_universal.go

@@ -0,0 +1,369 @@
+package bind
+
+/*
+ Most of this code was copied from https://github.com/pion/ice and modified to fulfill NetBird's requirements.
+*/
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/pion/logging"
+	"github.com/pion/stun/v2"
+	"github.com/pion/transport/v3"
+)
+
+// FilterFn is a function that filters out candidates based on the address.
+// If it returns true, the address is to be filtered. It also returns the prefix of matching route.
+type FilterFn func(address netip.Addr) (bool, netip.Prefix, error)
+
+// UniversalUDPMuxDefault handles STUN and TURN servers packets by wrapping the original UDPConn
+// It then passes packets to the UDPMux that does the actual connection muxing.
+type UniversalUDPMuxDefault struct {
+	*UDPMuxDefault
+	params UniversalUDPMuxParams
+
+	// since we have a shared socket, for srflx candidates it makes sense to have a shared mapped address across all the agents
+	// stun.XORMappedAddress indexed by the STUN server addr
+	xorMappedMap map[string]*xorMapped
+}
+
+// UniversalUDPMuxParams are parameters for UniversalUDPMux server reflexive.
+type UniversalUDPMuxParams struct {
+	Logger                logging.LeveledLogger
+	UDPConn               net.PacketConn
+	XORMappedAddrCacheTTL time.Duration
+	Net                   transport.Net
+	FilterFn              FilterFn
+}
+
+// NewUniversalUDPMuxDefault creates an implementation of UniversalUDPMux embedding UDPMux
+func NewUniversalUDPMuxDefault(params UniversalUDPMuxParams) *UniversalUDPMuxDefault {
+	if params.Logger == nil {
+		params.Logger = logging.NewDefaultLoggerFactory().NewLogger("ice")
+	}
+	if params.XORMappedAddrCacheTTL == 0 {
+		params.XORMappedAddrCacheTTL = time.Second * 25
+	}
+
+	m := &UniversalUDPMuxDefault{
+		params:       params,
+		xorMappedMap: make(map[string]*xorMapped),
+	}
+
+	// wrap UDP connection, process server reflexive messages
+	// before they are passed to the UDPMux connection handler (connWorker)
+	m.params.UDPConn = &udpConn{
+		PacketConn: params.UDPConn,
+		mux:        m,
+		logger:     params.Logger,
+		filterFn:   params.FilterFn,
+	}
+
+	// embed UDPMux
+	udpMuxParams := UDPMuxParams{
+		Logger:  params.Logger,
+		UDPConn: m.params.UDPConn,
+		Net:     m.params.Net,
+	}
+	m.UDPMuxDefault = NewUDPMuxDefault(udpMuxParams)
+
+	return m
+}
+
+// ReadFromConn reads from the m.params.UDPConn provided upon the creation. It expects STUN packets only, however, will
+// just ignore other packets printing an warning message.
+// It is a blocking method, consider running in a go routine.
+func (m *UniversalUDPMuxDefault) ReadFromConn(ctx context.Context) {
+	buf := make([]byte, 1500)
+	for {
+		select {
+		case <-ctx.Done():
+			log.Debugf("stopped reading from the UDPConn due to finished context")
+			return
+		default:
+			n, a, err := m.params.UDPConn.ReadFrom(buf)
+			if err != nil {
+				log.Errorf("error while reading packet: %s", err)
+				continue
+			}
+			msg := &stun.Message{
+				Raw: append([]byte{}, buf[:n]...),
+			}
+			err = msg.Decode()
+			if err != nil {
+				log.Warnf("error while parsing STUN message. The packet doesn't seem to be a STUN packet: %s", err)
+				continue
+			}
+
+			err = m.HandleSTUNMessage(msg, a)
+			if err != nil {
+				log.Errorf("error while handling STUn message: %s", err)
+			}
+		}
+	}
+}
+
+// udpConn is a wrapper around UDPMux conn that overrides ReadFrom and handles STUN/TURN packets
+type udpConn struct {
+	net.PacketConn
+	mux      *UniversalUDPMuxDefault
+	logger   logging.LeveledLogger
+	filterFn FilterFn
+	// TODO: reset cache on route changes
+	addrCache sync.Map
+}
+
+func (u *udpConn) WriteTo(b []byte, addr net.Addr) (int, error) {
+	if u.filterFn == nil {
+		return u.PacketConn.WriteTo(b, addr)
+	}
+
+	if isRouted, found := u.addrCache.Load(addr.String()); found {
+		return u.handleCachedAddress(isRouted.(bool), b, addr)
+	}
+
+	return u.handleUncachedAddress(b, addr)
+}
+
+func (u *udpConn) handleCachedAddress(isRouted bool, b []byte, addr net.Addr) (int, error) {
+	if isRouted {
+		return 0, fmt.Errorf("address %s is part of a routed network, refusing to write", addr)
+	}
+	return u.PacketConn.WriteTo(b, addr)
+}
+
+func (u *udpConn) handleUncachedAddress(b []byte, addr net.Addr) (int, error) {
+	if err := u.performFilterCheck(addr); err != nil {
+		return 0, err
+	}
+	return u.PacketConn.WriteTo(b, addr)
+}
+
+func (u *udpConn) performFilterCheck(addr net.Addr) error {
+	host, err := getHostFromAddr(addr)
+	if err != nil {
+		log.Errorf("Failed to get host from address %s: %v", addr, err)
+		return nil
+	}
+
+	a, err := netip.ParseAddr(host)
+	if err != nil {
+		log.Errorf("Failed to parse address %s: %v", addr, err)
+		return nil
+	}
+
+	if isRouted, prefix, err := u.filterFn(a); err != nil {
+		log.Errorf("Failed to check if address %s is routed: %v", addr, err)
+	} else {
+		u.addrCache.Store(addr.String(), isRouted)
+		if isRouted {
+			// Extra log, as the error only shows up with ICE logging enabled
+			log.Infof("Address %s is part of routed network %s, refusing to write", addr, prefix)
+			return fmt.Errorf("address %s is part of routed network %s, refusing to write", addr, prefix)
+		}
+	}
+	return nil
+}
+
+func getHostFromAddr(addr net.Addr) (string, error) {
+	host, _, err := net.SplitHostPort(addr.String())
+	return host, err
+}
+
+// GetSharedConn returns the shared udp conn
+func (m *UniversalUDPMuxDefault) GetSharedConn() net.PacketConn {
+	return m.params.UDPConn
+}
+
+// GetListenAddresses returns the listen addr of this UDP
+func (m *UniversalUDPMuxDefault) GetListenAddresses() []net.Addr {
+	return []net.Addr{m.LocalAddr()}
+}
+
+// GetRelayedAddr creates relayed connection to the given TURN service and returns the relayed addr.
+// Not implemented yet.
+func (m *UniversalUDPMuxDefault) GetRelayedAddr(turnAddr net.Addr, deadline time.Duration) (*net.Addr, error) {
+	return nil, fmt.Errorf("not implemented yet")
+}
+
+// GetConnForURL add uniques to the muxed connection by concatenating ufrag and URL (e.g. STUN URL) to be able to support multiple STUN/TURN servers
+// and return a unique connection per server.
+func (m *UniversalUDPMuxDefault) GetConnForURL(ufrag string, url string, addr net.Addr) (net.PacketConn, error) {
+	return m.UDPMuxDefault.GetConn(fmt.Sprintf("%s%s", ufrag, url), addr)
+}
+
+// HandleSTUNMessage discovers STUN packets that carry a XOR mapped address from a STUN server.
+// All other STUN packets will be forwarded to the UDPMux
+func (m *UniversalUDPMuxDefault) HandleSTUNMessage(msg *stun.Message, addr net.Addr) error {
+
+	udpAddr, ok := addr.(*net.UDPAddr)
+	if !ok {
+		// message about this err will be logged in the UDPMux
+		return nil
+	}
+
+	if m.isXORMappedResponse(msg, udpAddr.String()) {
+		err := m.handleXORMappedResponse(udpAddr, msg)
+		if err != nil {
+			log.Debugf("%s: %v", fmt.Errorf("failed to get XOR-MAPPED-ADDRESS response"), err)
+			return nil
+		}
+		return nil
+	}
+	return m.UDPMuxDefault.HandleSTUNMessage(msg, addr)
+}
+
+// isXORMappedResponse indicates whether the message is a XORMappedAddress and is coming from the known STUN server.
+func (m *UniversalUDPMuxDefault) isXORMappedResponse(msg *stun.Message, stunAddr string) bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	// check first if it is a STUN server address because remote peer can also send similar messages but as a BindingSuccess
+	_, ok := m.xorMappedMap[stunAddr]
+	_, err := msg.Get(stun.AttrXORMappedAddress)
+	return err == nil && ok
+}
+
+// handleXORMappedResponse parses response from the STUN server, extracts XORMappedAddress attribute
+// and set the mapped address for the server
+func (m *UniversalUDPMuxDefault) handleXORMappedResponse(stunAddr *net.UDPAddr, msg *stun.Message) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	mappedAddr, ok := m.xorMappedMap[stunAddr.String()]
+	if !ok {
+		return fmt.Errorf("no XOR address mapping")
+	}
+
+	var addr stun.XORMappedAddress
+	if err := addr.GetFrom(msg); err != nil {
+		return err
+	}
+
+	m.xorMappedMap[stunAddr.String()] = mappedAddr
+	mappedAddr.SetAddr(&addr)
+
+	return nil
+}
+
+// GetXORMappedAddr returns *stun.XORMappedAddress if already present for a given STUN server.
+// Makes a STUN binding request to discover mapped address otherwise.
+// Blocks until the stun.XORMappedAddress has been discovered or deadline.
+// Method is safe for concurrent use.
+func (m *UniversalUDPMuxDefault) GetXORMappedAddr(serverAddr net.Addr, deadline time.Duration) (*stun.XORMappedAddress, error) {
+	m.mu.Lock()
+	mappedAddr, ok := m.xorMappedMap[serverAddr.String()]
+	// if we already have a mapping for this STUN server (address already received)
+	// and if it is not too old we return it without making a new request to STUN server
+	if ok {
+		if mappedAddr.expired() {
+			mappedAddr.closeWaiters()
+			delete(m.xorMappedMap, serverAddr.String())
+			ok = false
+		} else if mappedAddr.pending() {
+			ok = false
+		}
+	}
+	m.mu.Unlock()
+	if ok {
+		return mappedAddr.addr, nil
+	}
+
+	// otherwise, make a STUN request to discover the address
+	// or wait for already sent request to complete
+	waitAddrReceived, err := m.sendSTUN(serverAddr)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %s", "failed to send STUN packet", err)
+	}
+
+	// block until response was handled by the connWorker routine and XORMappedAddress was updated
+	select {
+	case <-waitAddrReceived:
+		// when channel closed, addr was obtained
+		var addr *stun.XORMappedAddress
+		m.mu.Lock()
+		// A very odd case that mappedAddr is nil.
+		// Can happen when the deadline property is larger than params.XORMappedAddrCacheTTL.
+		// Or when we don't receive a response to our m.sendSTUN request (the response is handled asynchronously) and
+		// the XORMapped expires meanwhile triggering a closure of the waitAddrReceived channel.
+		// We protect the code from panic here.
+		if mappedAddr, ok := m.xorMappedMap[serverAddr.String()]; ok {
+			addr = mappedAddr.addr
+		}
+		m.mu.Unlock()
+		if addr == nil {
+			return nil, fmt.Errorf("no XOR address mapping")
+		}
+		return addr, nil
+	case <-time.After(deadline):
+		return nil, fmt.Errorf("timeout while waiting for XORMappedAddr")
+	}
+}
+
+// sendSTUN sends a STUN request via UDP conn.
+//
+// The returned channel is closed when the STUN response has been received.
+// Method is safe for concurrent use.
+func (m *UniversalUDPMuxDefault) sendSTUN(serverAddr net.Addr) (chan struct{}, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	// if record present in the map, we already sent a STUN request,
+	// just wait when waitAddrReceived will be closed
+	addrMap, ok := m.xorMappedMap[serverAddr.String()]
+	if !ok {
+		addrMap = &xorMapped{
+			expiresAt:        time.Now().Add(m.params.XORMappedAddrCacheTTL),
+			waitAddrReceived: make(chan struct{}),
+		}
+		m.xorMappedMap[serverAddr.String()] = addrMap
+	}
+
+	req, err := stun.Build(stun.BindingRequest, stun.TransactionID)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err = m.params.UDPConn.WriteTo(req.Raw, serverAddr); err != nil {
+		return nil, err
+	}
+
+	return addrMap.waitAddrReceived, nil
+}
+
+type xorMapped struct {
+	addr             *stun.XORMappedAddress
+	waitAddrReceived chan struct{}
+	expiresAt        time.Time
+}
+
+func (a *xorMapped) closeWaiters() {
+	select {
+	case <-a.waitAddrReceived:
+		// notify was close, ok, that means we received duplicate response
+		// just exit
+		break
+	default:
+		// notify that twe have a new addr
+		close(a.waitAddrReceived)
+	}
+}
+
+func (a *xorMapped) pending() bool {
+	return a.addr == nil
+}
+
+func (a *xorMapped) expired() bool {
+	return a.expiresAt.Before(time.Now())
+}
+
+func (a *xorMapped) SetAddr(addr *stun.XORMappedAddress) {
+	a.addr = addr
+	a.closeWaiters()
+}

client/iface/bind/udp_muxed_conn.go

@@ -0,0 +1,233 @@
+package bind
+
+/*
+ Most of this code was copied from https://github.com/pion/ice and modified to fulfill NetBird's requirements
+*/
+
+import (
+	"encoding/binary"
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/pion/logging"
+	"github.com/pion/transport/v3/packetio"
+)
+
+type udpMuxedConnParams struct {
+	Mux       *UDPMuxDefault
+	AddrPool  *sync.Pool
+	Key       string
+	LocalAddr net.Addr
+	Logger    logging.LeveledLogger
+}
+
+// udpMuxedConn represents a logical packet conn for a single remote as identified by ufrag
+type udpMuxedConn struct {
+	params *udpMuxedConnParams
+	// remote addresses that we have sent to on this conn
+	addresses []string
+
+	// channel holding incoming packets
+	buf        *packetio.Buffer
+	closedChan chan struct{}
+	closeOnce  sync.Once
+	mu         sync.Mutex
+}
+
+func newUDPMuxedConn(params *udpMuxedConnParams) *udpMuxedConn {
+	p := &udpMuxedConn{
+		params:     params,
+		buf:        packetio.NewBuffer(),
+		closedChan: make(chan struct{}),
+	}
+
+	return p
+}
+
+func (c *udpMuxedConn) ReadFrom(b []byte) (n int, rAddr net.Addr, err error) {
+	buf := c.params.AddrPool.Get().(*bufferHolder) //nolint:forcetypeassert
+	defer c.params.AddrPool.Put(buf)
+
+	// read address
+	total, err := c.buf.Read(buf.buf)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	dataLen := int(binary.LittleEndian.Uint16(buf.buf[:2]))
+	if dataLen > total || dataLen > len(b) {
+		return 0, nil, io.ErrShortBuffer
+	}
+
+	// read data and then address
+	offset := 2
+	copy(b, buf.buf[offset:offset+dataLen])
+	offset += dataLen
+
+	// read address len & decode address
+	addrLen := int(binary.LittleEndian.Uint16(buf.buf[offset : offset+2]))
+	offset += 2
+
+	if rAddr, err = decodeUDPAddr(buf.buf[offset : offset+addrLen]); err != nil {
+		return 0, nil, err
+	}
+
+	return dataLen, rAddr, nil
+}
+
+func (c *udpMuxedConn) WriteTo(buf []byte, rAddr net.Addr) (n int, err error) {
+	if c.isClosed() {
+		return 0, io.ErrClosedPipe
+	}
+	// each time we write to a new address, we'll register it with the mux
+	addr := rAddr.String()
+	if !c.containsAddress(addr) {
+		c.addAddress(addr)
+	}
+
+	return c.params.Mux.writeTo(buf, rAddr)
+}
+
+func (c *udpMuxedConn) LocalAddr() net.Addr {
+	return c.params.LocalAddr
+}
+
+func (c *udpMuxedConn) SetDeadline(tm time.Time) error {
+	return nil
+}
+
+func (c *udpMuxedConn) SetReadDeadline(tm time.Time) error {
+	return nil
+}
+
+func (c *udpMuxedConn) SetWriteDeadline(tm time.Time) error {
+	return nil
+}
+
+func (c *udpMuxedConn) CloseChannel() <-chan struct{} {
+	return c.closedChan
+}
+
+func (c *udpMuxedConn) Close() error {
+	var err error
+	c.closeOnce.Do(func() {
+		err = c.buf.Close()
+		close(c.closedChan)
+	})
+	return err
+}
+
+func (c *udpMuxedConn) isClosed() bool {
+	select {
+	case <-c.closedChan:
+		return true
+	default:
+		return false
+	}
+}
+
+func (c *udpMuxedConn) getAddresses() []string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	addresses := make([]string, len(c.addresses))
+	copy(addresses, c.addresses)
+	return addresses
+}
+
+func (c *udpMuxedConn) addAddress(addr string) {
+	c.mu.Lock()
+	c.addresses = append(c.addresses, addr)
+	c.mu.Unlock()
+
+	// map it on mux
+	c.params.Mux.registerConnForAddress(c, addr)
+}
+
+func (c *udpMuxedConn) containsAddress(addr string) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for _, a := range c.addresses {
+		if addr == a {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *udpMuxedConn) writePacket(data []byte, addr *net.UDPAddr) error {
+	// write two packets, address and data
+	buf := c.params.AddrPool.Get().(*bufferHolder) //nolint:forcetypeassert
+	defer c.params.AddrPool.Put(buf)
+
+	// format of buffer | data len | data bytes | addr len | addr bytes |
+	if len(buf.buf) < len(data)+maxAddrSize {
+		return io.ErrShortBuffer
+	}
+	// data len
+	binary.LittleEndian.PutUint16(buf.buf, uint16(len(data)))
+	offset := 2
+
+	// data
+	copy(buf.buf[offset:], data)
+	offset += len(data)
+
+	// write address first, leaving room for its length
+	n, err := encodeUDPAddr(addr, buf.buf[offset+2:])
+	if err != nil {
+		return err
+	}
+	total := offset + n + 2
+
+	// address len
+	binary.LittleEndian.PutUint16(buf.buf[offset:], uint16(n))
+
+	if _, err := c.buf.Write(buf.buf[:total]); err != nil {
+		return err
+	}
+	return nil
+}
+
+func encodeUDPAddr(addr *net.UDPAddr, buf []byte) (int, error) {
+	ipData, err := addr.IP.MarshalText()
+	if err != nil {
+		return 0, err
+	}
+	total := 2 + len(ipData) + 2 + len(addr.Zone)
+	if total > len(buf) {
+		return 0, io.ErrShortBuffer
+	}
+
+	binary.LittleEndian.PutUint16(buf, uint16(len(ipData)))
+	offset := 2
+	n := copy(buf[offset:], ipData)
+	offset += n
+	binary.LittleEndian.PutUint16(buf[offset:], uint16(addr.Port))
+	offset += 2
+	copy(buf[offset:], addr.Zone)
+	return total, nil
+}
+
+func decodeUDPAddr(buf []byte) (*net.UDPAddr, error) {
+	addr := net.UDPAddr{}
+
+	offset := 0
+	ipLen := int(binary.LittleEndian.Uint16(buf[:2]))
+	offset += 2
+	// basic bounds checking
+	if ipLen+offset > len(buf) {
+		return nil, io.ErrShortBuffer
+	}
+	if err := addr.IP.UnmarshalText(buf[offset : offset+ipLen]); err != nil {
+		return nil, err
+	}
+	offset += ipLen
+	addr.Port = int(binary.LittleEndian.Uint16(buf[offset : offset+2]))
+	offset += 2
+	zone := make([]byte, len(buf[offset:]))
+	copy(zone, buf[offset:])
+	addr.Zone = string(zone)
+
+	return &addr, nil
+}

client/iface/configurer/err.go

@@ -0,0 +1,5 @@
+package configurer
+
+import "errors"
+
+var ErrPeerNotFound = errors.New("peer not found")

client/iface/configurer/kernel_unix.go

@@ -1,6 +1,6 @@
-//go:build !android
+//go:build (linux && !android) || freebsd
 
-package iface
+package configurer
 
 import (
 	"fmt"
@@ -12,23 +12,23 @@ import (
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
 
-type wGConfigurer struct {
+type KernelConfigurer struct {
 	deviceName string
 }
 
-func newWGConfigurer(deviceName string) wGConfigurer {
+func NewKernelConfigurer(deviceName string) *KernelConfigurer {
-	return wGConfigurer{
+	return &KernelConfigurer{
 		deviceName: deviceName,
 	}
 }
 
-func (c *wGConfigurer) configureInterface(privateKey string, port int) error {
+func (c *KernelConfigurer) ConfigureInterface(privateKey string, port int) error {
 	log.Debugf("adding Wireguard private key")
 	key, err := wgtypes.ParseKey(privateKey)
 	if err != nil {
 		return err
 	}
-	fwmark := 0
+	fwmark := getFwmark()
 	config := wgtypes.Config{
 		PrivateKey:   &key,
 		ReplacePeers: true,
@@ -43,8 +43,8 @@ func (c *wGConfigurer) configureInterface(privateKey string, port int) error {
 	return nil
 }
 
-func (c *wGConfigurer) updatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error {
+func (c *KernelConfigurer) UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error {
-	//parse allowed ips
+	// parse allowed ips
 	_, ipNet, err := net.ParseCIDR(allowedIps)
 	if err != nil {
 		return err
@@ -55,12 +55,13 @@ func (c *wGConfigurer) updatePeer(peerKey string, allowedIps string, keepAlive t
 		return err
 	}
 	peer := wgtypes.PeerConfig{
-		PublicKey:                   peerKeyParsed,
+		PublicKey:         peerKeyParsed,
-		ReplaceAllowedIPs:           true,
+		ReplaceAllowedIPs: false,
+		// don't replace allowed ips, wg will handle duplicated peer IP
 		AllowedIPs:                  []net.IPNet{*ipNet},
 		PersistentKeepaliveInterval: &keepAlive,
-		PresharedKey:                preSharedKey,
 		Endpoint:                    endpoint,
+		PresharedKey:                preSharedKey,
 	}
 
 	config := wgtypes.Config{
@@ -73,7 +74,7 @@ func (c *wGConfigurer) updatePeer(peerKey string, allowedIps string, keepAlive t
 	return nil
 }
 
-func (c *wGConfigurer) removePeer(peerKey string) error {
+func (c *KernelConfigurer) RemovePeer(peerKey string) error {
 	peerKeyParsed, err := wgtypes.ParseKey(peerKey)
 	if err != nil {
 		return err
@@ -94,7 +95,7 @@ func (c *wGConfigurer) removePeer(peerKey string) error {
 	return nil
 }
 
-func (c *wGConfigurer) addAllowedIP(peerKey string, allowedIP string) error {
+func (c *KernelConfigurer) AddAllowedIP(peerKey string, allowedIP string) error {
 	_, ipNet, err := net.ParseCIDR(allowedIP)
 	if err != nil {
 		return err
@@ -121,34 +122,31 @@ func (c *wGConfigurer) addAllowedIP(peerKey string, allowedIP string) error {
 	return nil
 }
 
-func (c *wGConfigurer) removeAllowedIP(peerKey string, allowedIP string) error {
+func (c *KernelConfigurer) RemoveAllowedIP(peerKey string, allowedIP string) error {
 	_, ipNet, err := net.ParseCIDR(allowedIP)
 	if err != nil {
-		return err
+		return fmt.Errorf("parse allowed IP: %w", err)
 	}
 
 	peerKeyParsed, err := wgtypes.ParseKey(peerKey)
 	if err != nil {
-		return err
+		return fmt.Errorf("parse peer key: %w", err)
 	}
 
 	existingPeer, err := c.getPeer(c.deviceName, peerKey)
 	if err != nil {
-		return err
+		return fmt.Errorf("get peer: %w", err)
 	}
 
 	newAllowedIPs := existingPeer.AllowedIPs
 
 	for i, existingAllowedIP := range existingPeer.AllowedIPs {
 		if existingAllowedIP.String() == ipNet.String() {
-			newAllowedIPs = append(existingPeer.AllowedIPs[:i], existingPeer.AllowedIPs[i+1:]...)
+			newAllowedIPs = append(existingPeer.AllowedIPs[:i], existingPeer.AllowedIPs[i+1:]...) //nolint:gocritic
 			break
 		}
 	}
 
-	if err != nil {
-		return err
-	}
 	peer := wgtypes.PeerConfig{
 		PublicKey:         peerKeyParsed,
 		UpdateOnly:        true,
@@ -161,36 +159,36 @@ func (c *wGConfigurer) removeAllowedIP(peerKey string, allowedIP string) error {
 	}
 	err = c.configure(config)
 	if err != nil {
-		return fmt.Errorf(`received error "%w" while removing allowed IP from peer on interface %s with settings: allowed ips %s`, err, c.deviceName, allowedIP)
+		return fmt.Errorf("remove allowed IP %s on interface %s: %w", allowedIP, c.deviceName, err)
 	}
 	return nil
 }
 
-func (c *wGConfigurer) getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) {
+func (c *KernelConfigurer) getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) {
 	wg, err := wgctrl.New()
 	if err != nil {
-		return wgtypes.Peer{}, err
+		return wgtypes.Peer{}, fmt.Errorf("wgctl: %w", err)
 	}
 	defer func() {
 		err = wg.Close()
 		if err != nil {
-			log.Errorf("got error while closing wgctl: %v", err)
+			log.Errorf("Got error while closing wgctl: %v", err)
 		}
 	}()
 
 	wgDevice, err := wg.Device(ifaceName)
 	if err != nil {
-		return wgtypes.Peer{}, err
+		return wgtypes.Peer{}, fmt.Errorf("get device %s: %w", ifaceName, err)
 	}
 	for _, peer := range wgDevice.Peers {
 		if peer.PublicKey.String() == peerPubKey {
 			return peer, nil
 		}
 	}
-	return wgtypes.Peer{}, fmt.Errorf("peer not found")
+	return wgtypes.Peer{}, ErrPeerNotFound
 }
 
-func (c *wGConfigurer) configure(config wgtypes.Config) error {
+func (c *KernelConfigurer) configure(config wgtypes.Config) error {
 	wg, err := wgctrl.New()
 	if err != nil {
 		return err
@@ -202,7 +200,21 @@ func (c *wGConfigurer) configure(config wgtypes.Config) error {
 	if err != nil {
 		return err
 	}
-	log.Debugf("got Wireguard device %s", c.deviceName)
 
 	return wg.ConfigureDevice(c.deviceName, config)
 }
+
+func (c *KernelConfigurer) Close() {
+}
+
+func (c *KernelConfigurer) GetStats(peerKey string) (WGStats, error) {
+	peer, err := c.getPeer(c.deviceName, peerKey)
+	if err != nil {
+		return WGStats{}, fmt.Errorf("get wireguard stats: %w", err)
+	}
+	return WGStats{
+		LastHandshake: peer.LastHandshakeTime,
+		TxBytes:       peer.TransmitBytes,
+		RxBytes:       peer.ReceiveBytes,
+	}, nil
+}