diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 000000000..a1336cdab --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +language: en-US +reviews: + profile: chill + request_changes_workflow: false + high_level_summary: true + poem: false + review_status: true + auto_review: + enabled: true + drafts: false + path_filters: + - "!**/*.tsx" + - "!**/*.ts" + - "!**/*.js" + - "!**/*.svg" + pre_merge_checks: + custom_checks: + - name: "No attribution trailers" + mode: error + instructions: >- + Fail when the PR description or any commit message carries an + attribution trailer or footer: Co-Authored-By, Claude-Session, + Generated-By, or a "Generated with"/"Generated by" tool line. + Contributors own their contributions (AGENTS.md); ask for the + lines to be removed. +chat: + auto_reply: true diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 80809e667..9b6a0edfd 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.25-bookworm +FROM golang:1.26.7-bookworm RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get -y install --no-install-recommends\ @@ -6,7 +6,6 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ iptables=1.8.9-2 \ libgl1-mesa-dev=22.3.6-1+deb12u1 \ xorg-dev=1:7.7+23 \ - libayatana-appindicator3-dev=0.5.92-1 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && go install -v golang.org/x/tools/gopls@latest diff --git a/.git-branches.toml b/.git-branches.toml index d1818090f..4c34d7928 100644 --- a/.git-branches.toml +++ b/.git-branches.toml @@ -3,7 +3,7 @@ [branches] main = "main" perennials = [] -perennial-regex = "" +perennial-regex = "^release-" [create] new-branch-type = "feature" diff --git a/.githooks/commit-msg b/.githooks/commit-msg new file mode 100755 index 000000000..e5699bd9c --- /dev/null +++ b/.githooks/commit-msg @@ -0,0 +1,26 @@ +#!/bin/bash +# Refuses commit messages that carry attribution trailers. Contributors own +# their contributions (AGENTS.md, "No Co-Authored-By or tool-attribution +# trailers"); a trailer spreads that ownership onto a tool or a bystander. + +msg_file="$1" + +# Trailer keys in any casing, with any bullet or emoji in front. +trailers='^[^[:alnum:]]*(co-authored-by|claude-session|generated-by):' +# "Generated with/by" footers, including "Generated with by". +footer='^[^[:alnum:]]*generated (with|by)( [^[:alnum:]]*by)? ' +# A footer names a product, so a capitalized word must follow the phrase +# itself. Prose such as "generated by the protobuf compiler" stays legal. +tool='[Gg][Ee][Nn][Ee][Rr][Aa][Tt][Ee][Dd] ([Ww][Ii][Tt][Hh]|[Bb][Yy])( [^[:alnum:]]*[Bb][Yy])? [^[:alnum:]]*[A-Z]' + +offending=$( { + grep -Ein "$trailers" "$msg_file" + grep -Ein "$footer" "$msg_file" | grep -E "$tool" +} | sort -un ) + +if [ -n "$offending" ]; then + echo "commit-msg: attribution trailers are not accepted in this repository:" >&2 + printf '%s\n' "$offending" | sed 's/^/ /' >&2 + echo "Remove them and commit again (see AGENTS.md)." >&2 + exit 1 +fi diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b78b1417a..ded77ec58 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,8 +3,8 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "daily" - open-pull-requests-limit: 15 + interval: "weekly" + open-pull-requests-limit: 3 groups: actions: patterns: @@ -22,9 +22,12 @@ updates: directories: - "/" schedule: - interval: "daily" + interval: "weekly" open-pull-requests-limit: 15 groups: + golang-x-packages: + patterns: + - "golang.org/x/*" aws-sdk: patterns: - "github.com/aws/aws-sdk-go-v2/*" @@ -43,3 +46,25 @@ updates: wireguard: patterns: - "golang.zx2c4.com/wireguard*" + + # Base images of the source-build Dockerfiles, pinned by digest (Chainguard + # publishes only :latest for free). Dockerfile.release files feed goreleaser + # and keep the published images as they are, so their bases are left alone. + - package-ecosystem: "docker" + directories: + - "/upload-server" + schedule: + interval: "weekly" + open-pull-requests-limit: 3 + groups: + base-images: + patterns: + - "*" + ignore: + - dependency-name: "gcr.io/distroless/base" + # Go minor and major versions move with the rest of the repository; + # patch releases and new digests of the pinned tag still come through. + - dependency-name: "golang" + update-types: + - "version-update:semver-minor" + - "version-update:semver-major" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8e68054bd..9b796f262 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -2,6 +2,12 @@ ## Issue ticket number and link + + ## Stack @@ -12,7 +18,9 @@ - [ ] Is a feature enhancement - [ ] It is a refactor - [ ] Created tests that fail without the change (if possible) -- [ ] This change does **not** modify the public API, gRPC protocols, functionality behavior, CLI / service flags, or introduce a new feature — **OR** I have discussed it with the NetBird team beforehand (link the issue / Slack thread in the description). See [CONTRIBUTING.md](https://github.com/netbirdio/netbird/blob/main/CONTRIBUTING.md#discuss-changes-with-the-netbird-team-first). +- [ ] I ran and tested this change locally — I did not rely on CI to find out whether it works +- [ ] This PR has a single purpose (not a fix + refactor + feature in one) +- [ ] This change is a trivial fix, **OR** it links an issue the NetBird team agreed on beforehand. Changes to the public API, gRPC protocols, functionality behavior, CLI / service flags, or new features always need that agreement first. See [CONTRIBUTING.md](https://github.com/netbirdio/netbird/blob/main/CONTRIBUTING.md#ticket-first-pr-second). > By submitting this pull request, you confirm that you have read and agree to the terms of the [Contributor License Agreement](https://github.com/netbirdio/netbird/blob/main/CONTRIBUTOR_LICENSE_AGREEMENT.md). diff --git a/.github/scripts/test-homebrew-cask.sh b/.github/scripts/test-homebrew-cask.sh new file mode 100755 index 000000000..9c1041bdd --- /dev/null +++ b/.github/scripts/test-homebrew-cask.sh @@ -0,0 +1,338 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { + echo "::error::$*" >&2 + exit 1 +} + +if [[ ${RUNNER_ENVIRONMENT:-} != github-hosted || ${RUNNER_OS:-} != macOS || $(uname -s) != Darwin ]]; then + fail "This test installs a system daemon and must run on a disposable GitHub macOS runner." +fi +if [[ $EUID == 0 ]]; then + fail "Run this script as the Homebrew user, not root." +fi + +readonly test_dir="${RUNNER_TEMP:?}/homebrew-cask" +readonly results_dir="$test_dir/results" +readonly app='/Applications/Netbird UI.app' +readonly plist='/Library/LaunchDaemons/netbird.plist' +readonly cask='netbirdio/tap/netbird-ui' +readonly formula='netbirdio/tap/netbird' +readonly published_cask="$test_dir/published-netbird-ui.rb" +readonly legacy_cask="$test_dir/legacy-netbird-ui.rb" +readonly rendered_cask="$test_dir/rendered-netbird-ui.rb" +readonly fixture_dir="$test_dir/fixture" +readonly serve_dir="$test_dir/serve" +readonly fixture_zip="$serve_dir/netbird-ui.zip" +readonly fixture_port=18080 +readonly fixture_url="http://127.0.0.1:$fixture_port/netbird-ui.zip" +readonly marker="$test_dir/installer.marker" + +mkdir -p "$results_dir" "$fixture_dir/netbird_ui_darwin" "$serve_dir" "$test_dir/downloads" +exec > >(tee "$results_dir/test.log") 2>&1 + +sudo -n true +if command -v netbird || [[ -e "$app" || -e "$plist" ]] || pgrep -x netbird-ui; then + fail "The runner already has NetBird installed or running." +fi +if sudo launchctl print system/netbird > "$results_dir/initial-service.log" 2>&1; then + fail "The runner already has a NetBird service loaded." +fi + +install_attempted=false +server_pid='' +daemon_pid='' +version='' + +stop_ui() { + local status=0 + sudo pkill -x netbird-ui || status=$? + # pkill returns 1 when the UI is already closed. + [[ $status == 0 || $status == 1 ]] +} + +cleanup() { + local status=$? + trap - EXIT + set +e + + if [[ $install_attempted == true ]]; then + stop_ui || status=1 + if [[ -S /var/run/netbird.sock ]]; then + sudo netbird down || status=1 + fi + if brew list --cask "$cask" >/dev/null 2>&1 || [[ -e "$app" ]]; then + brew uninstall --cask --force "$cask" || status=1 + fi + # A failed cask install can leave a daemon even after Homebrew rolls back the app. + if sudo launchctl print system/netbird > "$results_dir/cleanup-service.log" 2>&1; then + sudo netbird service stop || status=1 + fi + if [[ -e "$plist" ]]; then + sudo netbird service uninstall || status=1 + fi + fi + if [[ -f /var/log/netbird/client.log ]]; then + sudo cat /var/log/netbird/client.log > "$results_dir/client.log" || status=1 + fi + if command -v netbird >/dev/null; then + brew uninstall --formula "$formula" || status=1 + fi + if [[ -n $server_pid ]]; then + kill "$server_pid" 2>/dev/null || true + fi + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +run_logged() { + local name=$1 + shift + "$@" 2>&1 | tee "$results_dir/$name.log" +} + +cask_field() { + local stanza=$1 file=$2 + sed -nE "s/^[[:space:]]*$stanza \"([^\"]+)\".*/\\1/p" "$file" +} + +release_fields() { + local file=$1 + grep -E '^[[:space:]]*(version|url|sha256|app) ' "$file" +} + +use_cask() { + local file=$1 + cp "$file" "$tap_dir/Casks/netbird-ui.rb" +} + +# The released installer opens the UI as root, which never returns on a headless +# runner. The cask only needs two script paths and a version argument, so the test +# ships a stub bundle that records what it received and starts the daemon. +build_fixture() { + local bundle="$fixture_dir/netbird_ui_darwin" + printf '#!/bin/sh\nexit 0\n' > "$bundle/netbird-ui" + chmod 755 "$bundle/netbird-ui" + # After a bootout launchd keeps tearing the previous daemon down for a couple of + # seconds, and loading the same label again fails until that finishes. + cat > "$bundle/installer.sh" < '$marker' +netbird service install +attempt=0 +until netbird service start; do + attempt=\$((attempt + 1)) + [ "\$attempt" -lt 15 ] || exit 1 + sleep 1 +done +EOF + printf '#!/bin/sh\nexit 0\n' > "$bundle/uninstaller.sh" + # Shipped without the executable bit so the 0755 seen after install can only come from the cask. + chmod 644 "$bundle/installer.sh" "$bundle/uninstaller.sh" + rm -f "$fixture_zip" + (cd "$fixture_dir" && zip -qr "$fixture_zip" netbird_ui_darwin) +} + +start_fixture_server() { + python3 -m http.server "$fixture_port" --bind 127.0.0.1 --directory "$serve_dir" \ + > "$results_dir/fixture-server.log" 2>&1 & + server_pid=$! + local attempt + for attempt in {1..20}; do + if curl --silent --fail --output /dev/null "$fixture_url"; then + return + fi + sleep 0.5 + done + fail "The fixture HTTP server did not come up on port $fixture_port." +} + +assert_published_layout() { + local url archive script + while read -r url; do + archive="$test_dir/downloads/${url##*/}" + curl --fail --location --silent --retry 3 --output "$archive" "$url" + for script in installer.sh uninstaller.sh; do + unzip -l "$archive" | grep -q " netbird_ui_darwin/$script\$" || + fail "The published archive ${url##*/} has no netbird_ui_darwin/$script." + done + done < <(cask_field url "$published_cask") +} + +assert_no_deprecations() { + if grep -Ei '(postflight|uninstall_preflight).*deprecated|deprecated.*(postflight|uninstall_preflight)' "$@"; then + fail "Homebrew reported a deprecated cask lifecycle hook." + fi +} + +wait_for_daemon() { + local attempt + for attempt in {1..30}; do + if sudo launchctl print system/netbird > "$results_dir/service.log" 2>&1 && + grep -Eq '^[[:space:]]*state = running$' "$results_dir/service.log"; then + return + fi + sleep 1 + done + cat "$results_dir/service.log" + fail "The installed daemon did not reach the running state." +} + +wait_for_exit() { + local pid=$1 attempt + for attempt in {1..30}; do + if ! sudo kill -0 "$pid" 2>/dev/null; then + return + fi + sleep 1 + done + fail "Daemon process $pid is still running after removal." +} + +assert_service_absent() { + if sudo launchctl print system/netbird > "$results_dir/removed-service.log" 2>&1; then + fail "The NetBird service is still loaded after removal." + fi +} + +assert_installed() { + local script + [[ -f $marker ]] || fail "The cask did not run installer.sh." + grep -qx "version=$version" "$marker" || fail "installer.sh did not receive the cask version: $(cat "$marker")" + grep -qx 'uid=0' "$marker" || fail "installer.sh did not run as root: $(cat "$marker")" + [[ -d "$app" && -x "$app/netbird-ui" ]] || fail "The UI was not installed." + for script in installer.sh uninstaller.sh; do + [[ $(stat -f '%Lp' "$app/$script") == 755 ]] || fail "Incorrect permissions on $script." + done + [[ -f "$plist" ]] || fail "The installer did not create the daemon plist." + wait_for_daemon + daemon_pid=$(awk '/^[[:space:]]*pid = / { print $3; exit }' "$results_dir/service.log") + [[ $daemon_pid =~ ^[0-9]+$ ]] || fail "The running daemon has no PID." + sudo kill -0 "$daemon_pid" +} + +assert_uninstalled() { + local log=$1 + assert_no_deprecations "$log" + [[ ! -e "$app" ]] || fail "The UI app remains after uninstall." + [[ ! -e "$plist" ]] || fail "The daemon plist remains after uninstall." + assert_service_absent + wait_for_exit "$daemon_pid" + [[ $(netbird version) == "$version" ]] || fail "Cask uninstall removed the CLI dependency." +} + +installed_caskfiles() { + local extension=$1 + find "$(brew --caskroom)/netbird-ui/.metadata" -name "netbird-ui.$extension" 2>/dev/null +} + +assert_legacy_metadata() { + installed_caskfiles rb | grep -q . || fail "The legacy cask did not leave a Ruby caskfile behind." +} + +assert_steps_metadata() { + if installed_caskfiles rb | grep -q .; then + fail "Homebrew still keeps the legacy Ruby caskfile after reinstall." + fi + installed_caskfiles json | grep -q . || fail "Homebrew did not save the reinstalled cask as JSON." +} + +brew --version +sw_vers +brew tap netbirdio/tap "${GITHUB_WORKSPACE:?}/.homebrew-cask-tap" +tap_dir=$(brew --repository netbirdio/tap) +readonly tap_dir + +[[ -f "$tap_dir/Casks/netbird-ui.rb" ]] || fail "The tap has no Casks/netbird-ui.rb." +cp "$tap_dir/Casks/netbird-ui.rb" "$published_cask" +cp "$published_cask" "$results_dir/published-netbird-ui.rb" + +version=$(brew info --json=v2 --formula "$formula" | jq -r '.formulae[0].versions.stable') +readonly version +[[ -n $version && $version != null ]] || fail "Could not read the formula version from the tap." + +assert_published_layout + +build_fixture +fixture_sha=$(shasum -a 256 "$fixture_zip" | cut -d' ' -f1) +readonly fixture_sha +start_fixture_server + +export PROJECT=netbird-ui VERSION="$version" +export AMD="$fixture_zip" ARM="$fixture_zip" AMD_URL="$fixture_url" ARM_URL="$fixture_url" +gomplate -f "$GITHUB_WORKSPACE/client/ui/netbird-ui.rb.tmpl" -o "$rendered_cask" +cp "$rendered_cask" "$results_dir/rendered-netbird-ui.rb" + +sed -E "s|^([[:space:]]*version) \"[^\"]+\"|\\1 \"$version\"|; s|^([[:space:]]*url) \"[^\"]+\"|\\1 \"$fixture_url\"|; s|^([[:space:]]*sha256) \"[^\"]+\"|\\1 \"$fixture_sha\"|" \ + "$published_cask" > "$legacy_cask" +cp "$legacy_cask" "$results_dir/legacy-netbird-ui.rb" +if ! diff <(release_fields "$legacy_cask") <(release_fields "$rendered_cask"); then + fail "The rendered cask changes release data, not only lifecycle stanzas." +fi + +use_cask "$rendered_cask" +brew info --json=v2 --cask "$cask" > "$results_dir/cask.json" 2> "$results_dir/load.log" +cat "$results_dir/load.log" +assert_no_deprecations "$results_dir/load.log" +run_logged style brew style --cask --only-cops=Cask/InstallSteps "$cask" + +run_logged install-cli brew install --formula "$formula" +[[ $(netbird version) == "$version" ]] || fail "The installed CLI does not report the formula version." + +for scenario in running stopped missing; do + echo "::group::Uninstall with $scenario service" + install_attempted=true + sudo rm -f "$marker" + run_logged "install-$scenario" brew install --cask "$cask" + assert_no_deprecations "$results_dir/install-$scenario.log" + assert_installed + stop_ui + + case "$scenario" in + running) ;; + stopped) + run_logged stop-daemon sudo netbird service stop + wait_for_exit "$daemon_pid" + [[ -f "$plist" ]] || fail "Stopping the daemon unexpectedly removed its plist." + ;; + missing) + run_logged stop-missing-daemon sudo netbird service stop + run_logged remove-daemon sudo netbird service uninstall + wait_for_exit "$daemon_pid" + [[ ! -e "$plist" ]] || fail "The missing-service scenario still has a plist." + assert_service_absent + ;; + *) fail "Unknown uninstall scenario: $scenario" ;; + esac + + run_logged "uninstall-$scenario" brew uninstall --cask "$cask" + assert_uninstalled "$results_dir/uninstall-$scenario.log" + echo "::endgroup::" +done + +# Every existing user first meets the new cask through an upgrade of the published +# one, whose legacy flight blocks Homebrew replays from the saved Ruby caskfile. +echo "::group::Reinstall over the published legacy cask" +install_attempted=true +use_cask "$legacy_cask" +sudo rm -f "$marker" +run_logged install-legacy brew install --cask "$cask" +assert_installed +assert_legacy_metadata +stop_ui + +use_cask "$rendered_cask" +sudo rm -f "$marker" +run_logged reinstall-legacy brew reinstall --cask "$cask" +assert_installed +assert_steps_metadata +stop_ui + +run_logged uninstall-legacy brew uninstall --cask "$cask" +assert_uninstalled "$results_dir/uninstall-legacy.log" +echo "::endgroup::" diff --git a/.github/workflows/agent-network-e2e.yml b/.github/workflows/agent-network-e2e.yml index d78e3bbd3..41bfe153d 100644 --- a/.github/workflows/agent-network-e2e.yml +++ b/.github/workflows/agent-network-e2e.yml @@ -5,6 +5,20 @@ on: schedule: - cron: "0 3 * * *" workflow_dispatch: + inputs: + bedrock_model: + description: >- + Bedrock inference-profile id to drive the matrix with, exactly as + AWS issues it. Leave empty for the Sonnet 4.6 default. + required: false + default: "" + test_pattern: + description: >- + Package pattern to run. Defaults to the whole suite; narrow it to one + package (e.g. ./e2e/agentnetwork/...) when a run only needs that + package's answer and not the sixteen minutes the container suite costs. + required: false + default: "./e2e/..." concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -38,7 +52,7 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-anet-e2e-buildx-${{ hashFiles('go.sum', 'combined/Dockerfile.multistage', 'proxy/Dockerfile.multistage', 'e2e/harness/Dockerfile.client') }} + key: ${{ runner.os }}-anet-e2e-buildx-${{ hashFiles('go.sum', 'combined/Dockerfile.multistage', 'proxy/Dockerfile.multistage', 'e2e/harness/Dockerfile.client', 'e2e/harness/Dockerfile.nblink') }} restore-keys: | ${{ runner.os }}-anet-e2e-buildx- @@ -51,6 +65,9 @@ jobs: # token (and URL, for gateways) is unset, so partial coverage is fine. OPENAI_TOKEN: ${{ secrets.E2E_OPENAI_TOKEN }} ANTHROPIC_TOKEN: ${{ secrets.E2E_ANTHROPIC_TOKEN }} + # Moonshot AI platform key (platform.kimi.ai); drives both Kimi wire + # shapes (OpenAI /v1 and Anthropic /anthropic) through kimi_api. + KIMI_TOKEN: ${{ secrets.E2E_KIMI_TOKEN }} VERCEL_URL: ${{ secrets.E2E_VERCEL_URL }} VERCEL_TOKEN: ${{ secrets.E2E_VERCEL_TOKEN }} OPENROUTER_URL: ${{ secrets.E2E_OPENROUTER_URL }} @@ -59,10 +76,16 @@ jobs: CLOUDFLARE_TOKEN: ${{ secrets.E2E_CLOUDFLARE_TOKEN }} AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.E2E_AWS_BEARER_TOKEN_BEDROCK }} AWS_REGION: ${{ secrets.E2E_AWS_REGION }} + # Bedrock model override: dispatch input wins, then the repo variable, else the test default. + AWS_BEDROCK_MODEL: ${{ inputs.bedrock_model || vars.E2E_AWS_BEDROCK_MODEL }} # Vertex (Anthropic-on-Vertex): SA + project required; region defaults # to "global", model to a pinned claude snapshot. GOOGLE_VERTEX_SA_BASE64: ${{ secrets.E2E_GOOGLE_VERTEX_SA_BASE64 }} GOOGLE_VERTEX_PROJECT: ${{ secrets.E2E_GOOGLE_VERTEX_PROJECT }} GOOGLE_VERTEX_REGION: ${{ secrets.E2E_GOOGLE_VERTEX_REGION }} GOOGLE_VERTEX_MODEL: ${{ secrets.E2E_GOOGLE_VERTEX_MODEL }} - run: go test -tags e2e -timeout 40m -v ./e2e/... + # Read through an env var rather than interpolated into the run + # script: a dispatch input reaching a shell command directly is a + # script-injection seam, however trusted the dispatcher. + TEST_PATTERN: ${{ inputs.test_pattern || './e2e/...' }} + run: go test -tags e2e -timeout 40m -v "$TEST_PATTERN" diff --git a/.github/workflows/buf.yml b/.github/workflows/buf.yml new file mode 100644 index 000000000..cc7629534 --- /dev/null +++ b/.github/workflows/buf.yml @@ -0,0 +1,48 @@ +name: protobuf checks +on: + push: + branches: + - main + - "release-*" + pull_request: + paths: + - ".github/workflows/buf.yml" + - "**/buf.yaml" + - "**/buf.lock" + - "**/buf.gen.yaml" + - "**.proto" +permissions: + contents: read + pull-requests: read +jobs: + buf: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: bufbuild/buf-action@8c6a16e16f12ba20b6470afa9c2ba9b5ba8c97c3 # v1.5.0 + with: + push: false + archive: false + pr_comment: false + lint: false + format: false + # A push that creates a branch carries no `before` commit, so the + # action's default baseline is the all-zero SHA and `buf breaking` + # dies cloning it. Skipping costs nothing: every commit on a freshly + # cut release branch should have already passed this check on main. + breaking: ${{ !github.event.created }} + # The alternative is to compare against the default branch instead of + # skipping. Not used: buf clones the baseline when the job runs, so a + # main that has moved on since the branch was cut reads as protos + # deleted on the release branch. Resolving to an empty string on every + # other event is what keeps the action's own default in place, which + # stacked pull requests need. + # breaking_against: >- + # ${{ github.event.created + # && format('{0}#format=git,branch={1}', + # github.event.repository.clone_url, + # github.event.repository.default_branch) + # || '' }} diff --git a/.github/workflows/check-license-dependencies.yml b/.github/workflows/check-license-dependencies.yml index 17c9fdc8d..27c59f8d6 100644 --- a/.github/workflows/check-license-dependencies.yml +++ b/.github/workflows/check-license-dependencies.yml @@ -2,7 +2,7 @@ name: Check License Dependencies on: push: - branches: [main] + branches: [main, "release-*"] paths: - "go.mod" - "go.sum" @@ -34,7 +34,7 @@ jobs: while IFS= read -r dir; do echo "=== Checking $dir ===" # Search for problematic imports, excluding test files - RESULTS=$(grep -r "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\)" "$dir" --include="*.go" 2>/dev/null | grep -v "_test.go" | grep -v "test_" | grep -v "/test/" | grep -v "tools/idp-migrate/" || true) + RESULTS=$(grep -r "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\)" "$dir" --include="*.go" 2>/dev/null | grep -v "_test.go" | grep -v "test_" | grep -v "/test/" | grep -v "tools/idp-migrate/" | grep -v "tools/mysql-migrate/" || true) if [ -n "$RESULTS" ]; then echo "❌ Found problematic dependencies:" echo "$RESULTS" @@ -93,7 +93,7 @@ jobs: IMPORTERS=$(go list -json -deps ./... 2>/dev/null | jq -r "select(.Imports[]? == \"$package\") | .ImportPath") # Check if any importer is NOT in management/signal/relay - BSD_IMPORTER=$(echo "$IMPORTERS" | grep -v "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\|combined\|tools/idp-migrate\)" | head -1) + BSD_IMPORTER=$(echo "$IMPORTERS" | grep -v "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\|combined\|tools/idp-migrate\|tools/mysql-migrate\)" | head -1) if [ -n "$BSD_IMPORTER" ]; then echo "❌ $package ($license) is imported by BSD-licensed code: $BSD_IMPORTER" diff --git a/.github/workflows/docs-ack.yml b/.github/workflows/docs-ack.yml index 7e34e2f8a..edd1eebff 100644 --- a/.github/workflows/docs-ack.yml +++ b/.github/workflows/docs-ack.yml @@ -12,6 +12,8 @@ jobs: docs-ack: name: Require docs PR URL or explicit "not needed" runs-on: ubuntu-latest + # Crowdin's translation-sync service PRs are auto-generated without the PR template. + if: github.event.pull_request.user.login != 'netbirddev' steps: - name: Read PR body diff --git a/.github/workflows/frontend-ui.yml b/.github/workflows/frontend-ui.yml new file mode 100644 index 000000000..2ad43c581 --- /dev/null +++ b/.github/workflows/frontend-ui.yml @@ -0,0 +1,99 @@ +name: UI Frontend + +on: + pull_request: + paths: + - "client/ui/frontend/**" + - "client/ui/i18n/**" + - "client/ui/**/*.go" + - ".github/workflows/frontend-ui.yml" + push: + branches: + - main + - "release-*" + paths: + - "client/ui/frontend/**" + - "client/ui/i18n/**" + - "client/ui/**/*.go" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }} + cancel-in-progress: true + +jobs: + lint-and-build: + name: Lint & Build + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: client/ui/frontend + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@v7 + with: + node-version: "22" + + - name: Set up pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + with: + version: 11 + + # Bindings are generated by wails3 from the Go service definitions and + # are not checked in (see client/ui/frontend/bindings/). Without them, + # typecheck/build fail on missing module imports. + - name: Set up Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: "go.mod" + cache: false + + # wails3 CLI links against GTK4 / WebKitGTK 6.0 via its internal/operatingsystem + # package, so the dev libraries must be present before `go install`. + - name: Install Wails Linux system dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + pkg-config \ + libgtk-4-dev \ + libwebkitgtk-6.0-dev + + - name: Install wails3 CLI + # Version derived from go.mod so the binding generator always matches + # the wails runtime the daemon links against. + working-directory: ${{ github.workspace }} + run: | + WAILS_VERSION=$(go list -m -f '{{.Version}}' github.com/wailsapp/wails/v3) + go install github.com/wailsapp/wails/v3/cmd/wails3@$WAILS_VERSION + + - name: Get pnpm store directory + id: pnpm-store + run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + + - name: Cache pnpm store + uses: actions/cache@v6 + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-pnpm-${{ hashFiles('client/ui/frontend/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm- + + - name: Install dependencies + run: pnpm install --frozen-lockfile --ignore-scripts + + - name: Generate Wails bindings + run: pnpm run bindings + + - name: Lint, typecheck, format + run: pnpm check + + - name: Build + run: pnpm build diff --git a/.github/workflows/golang-test-darwin.yml b/.github/workflows/golang-test-darwin.yml index 748e3f996..dec57dc80 100644 --- a/.github/workflows/golang-test-darwin.yml +++ b/.github/workflows/golang-test-darwin.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: concurrency: @@ -45,7 +46,17 @@ jobs: run: git --no-pager diff --exit-code - name: Test - run: NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -coverprofile=coverage.txt -tags 'devcert privileged' -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 $(go list ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/testutil/privileged) + # Exclude the client/ui package itself: its main.go uses //go:embed + # all:frontend/dist, which fails to compile until the frontend has been + # built, and its release pipeline runs `pnpm build` before goreleaser. + # The pattern is anchored so the subpackages (services, preferences, + # i18n, authsession) still run: they hold Go-side unit tests and need no + # frontend bundle. + # `go list -e` lets the listing succeed even though the embed fails to + # resolve; the grep then drops the broken package by path. Without -e, + # go list aborts with empty stdout and `go test` falls back to the repo + # root, which has no Go files. + run: NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -coverprofile=coverage.txt -tags 'devcert privileged' -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 $(go list -e ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e '/client/ui$' -e /client/testutil/privileged) - name: Upload coverage reports to Codecov uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f #v7.0.0 diff --git a/.github/workflows/golang-test-freebsd.yml b/.github/workflows/golang-test-freebsd.yml index 9c795e783..7bd48e3d0 100644 --- a/.github/workflows/golang-test-freebsd.yml +++ b/.github/workflows/golang-test-freebsd.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: concurrency: @@ -32,7 +33,7 @@ jobs: with: usesh: true copyback: false - release: "15.0" + release: "15.1" envs: "GO_VERSION" prepare: | pkg install -y curl pkgconf xorg diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml index ce53261a4..880370ede 100644 --- a/.github/workflows/golang-test-linux.yml +++ b/.github/workflows/golang-test-linux.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: concurrency: @@ -53,7 +54,7 @@ jobs: - name: Install dependencies if: steps.cache.outputs.cache-hit != 'true' - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev + run: sudo apt update && sudo apt install -y -q libgtk-4-dev libwebkitgtk-6.0-dev libsoup-3.0-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev - name: Install 32-bit libpcap if: steps.cache.outputs.cache-hit != 'true' @@ -145,7 +146,7 @@ jobs: ${{ runner.os }}-gotest-cache- - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev + run: sudo apt update && sudo apt install -y -q libgtk-4-dev libwebkitgtk-6.0-dev libsoup-3.0-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev - name: Install 32-bit libpcap if: matrix.arch == '386' @@ -158,7 +159,16 @@ jobs: run: git --no-pager diff --exit-code - name: Test - run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} CI=true go test -coverprofile=coverage.txt -tags 'devcert privileged' -exec 'sudo --preserve-env=CI,CGO_ENABLED' -timeout 10m -p 1 $(go list ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/testutil/privileged) + # Exclude client/ui: its main.go uses //go:embed all:frontend/dist, + # which fails to compile until the frontend has been built, and its + # release pipeline runs `pnpm build` before goreleaser. The subpackages + # go with it because this runner's gtk4 is older than the wails runtime + # needs; the Client UI / Unit job below covers them instead. + # `go list -e` lets the listing succeed even though the embed fails to + # resolve; the grep then drops the broken package by path. Without -e, + # go list aborts with empty stdout and `go test` falls back to the repo + # root, which has no Go files. + run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} CI=true go test -coverprofile=coverage.txt -tags 'devcert privileged' -exec 'sudo --preserve-env=CI,CGO_ENABLED' -timeout 10m -p 1 $(go list -e ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui -e /client/testutil/privileged) - name: Upload coverage reports to Codecov if: matrix.arch == 'amd64' @@ -168,6 +178,71 @@ jobs: slug: netbirdio/netbird flags: unit,client + test_client_ui: + name: "Client UI / Unit" + # Pinned to 24.04 rather than the 22.04 the other client jobs use: the wails + # runtime's linux cgo layer needs GtkFileDialog, which arrived in gtk4 4.10, + # and jammy ships 4.6. Not ubuntu-latest, so a runner image rollover cannot + # move this out from under us. + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Install Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: "go.mod" + cache: false + + - name: Install dependencies + run: sudo apt update && sudo apt install -y -q libgtk-4-dev libwebkitgtk-6.0-dev libsoup-3.0-dev + + - name: Test + # client/ui itself stays out: its main.go embeds all:frontend/dist, + # which only exists after `pnpm build`. The subpackages carry the + # Go-side unit tests, including the window manager re-entrancy + # regression test, and need no frontend bundle. + run: CGO_ENABLED=1 go test -timeout 5m ./client/ui/authsession/... ./client/ui/i18n/... ./client/ui/preferences/... ./client/ui/services/... + + test_client_pkcs11: + name: "Client PKCS#11 / Unit" + # The deb and rpm packages ship the client built with the pkcs11 tag, which loads + # PKCS#11 modules through purego. The other jobs build without the tag, so this one + # compiles that driver and signs with a real SoftHSM token, the build release ships. + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Install Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: "go.mod" + cache: false + + - name: Install SoftHSM + run: sudo apt update && sudo apt install -y -q softhsm2 + + - name: Initialise token + run: | + mkdir -p "$RUNNER_TEMP/softhsm/tokens" + printf 'directories.tokendir = %s\nobjectstore.backend = file\n' "$RUNNER_TEMP/softhsm/tokens" > "$RUNNER_TEMP/softhsm/softhsm2.conf" + echo "SOFTHSM2_CONF=$RUNNER_TEMP/softhsm/softhsm2.conf" >> "$GITHUB_ENV" + SOFTHSM2_CONF="$RUNNER_TEMP/softhsm/softhsm2.conf" softhsm2-util --init-token --free --label netbird --pin 1234 --so-pin 1234 + + - name: Vet + run: CGO_ENABLED=0 go vet -tags pkcs11 ./client/internal/pkcs11/... ./client/internal/certproof/... + + - name: Test + env: + NB_TEST_PKCS11_URI: "pkcs11:token=netbird?module-path=/usr/lib/softhsm/libsofthsm2.so&pin-value=1234" + NB_TEST_PKCS11_DISPOSABLE: "1" + run: CGO_ENABLED=0 go test -tags pkcs11 -timeout 5m ./client/internal/pkcs11/... ./client/internal/certproof/... test_client_on_docker: name: "Client (Docker) / Unit" @@ -203,6 +278,9 @@ jobs: ${{ runner.os }}-gotest-cache- - name: Run tests in container + # Unlike the native job above, this one drops all of client/ui including + # the subpackages: the alpine container has no gtk4/webkitgtk, so the + # Wails application package they import would fail to link. env: HOST_GOCACHE: ${{ steps.go-env.outputs.cache_dir }} HOST_GOMODCACHE: ${{ steps.go-env.outputs.modcache_dir }} @@ -225,11 +303,11 @@ jobs: -e GOCACHE=${CONTAINER_GOCACHE} \ -e GOMODCACHE=${CONTAINER_GOMODCACHE} \ -e CONTAINER=${CONTAINER} \ - golang:1.25-alpine \ + golang:1.26.7-alpine \ sh -c ' \ apk update; apk add --no-cache \ ca-certificates iptables ip6tables dbus dbus-dev libpcap-dev build-base; \ - go test -buildvcs=false -tags "devcert privileged" -v -timeout 10m -p 1 $(go list -buildvcs=false ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui -e /upload-server -e /client/testutil/privileged) + go test -buildvcs=false -tags "devcert privileged" -v -timeout 10m -p 1 $(go list -e -buildvcs=false ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui -e /upload-server -e /client/testutil/privileged -e tools/mysql-migrate) ' test_relay: @@ -473,14 +551,32 @@ jobs: if: matrix.store == 'mysql' run: docker pull mlsmaycon/warmed-mysql:8 + # The -json stream goes through tools/gotestsummary so the log shows one + # line per test, the output of failed tests, the head of a timeout panic + # with the still-running tests, and the slowest tests per package. - name: Test + shell: bash run: | + set -o pipefail CGO_ENABLED=1 GOARCH=${{ matrix.arch }} \ NETBIRD_STORE_ENGINE=${{ matrix.store }} \ CI=true \ - go test -tags=devcert -coverprofile=coverage.txt \ + go test -json -tags=devcert -coverprofile=coverage.txt \ -exec "sudo --preserve-env=CI,NETBIRD_STORE_ENGINE" \ - -timeout 20m ./management/... ./shared/management/... + -timeout 20m ./management/... ./shared/management/... \ + | tee management-test-events.jsonl \ + | go run ./tools/gotestsummary + + # The summary trims long outputs; the raw stream keeps every line for + # the failures that need it. A green run has no use for it. + - name: Upload raw test events + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: management-unit-test-events-${{ matrix.store }} + path: management-test-events.jsonl + if-no-files-found: ignore + retention-days: 14 - name: Upload coverage reports to Codecov if: matrix.arch == 'amd64' @@ -722,17 +818,35 @@ jobs: - name: Install modules run: go mod tidy + - name: Run Mage + uses: magefile/mage-action@a662bd8c29d8106879588cfff83b2faf6e6f59db # v4.0.0 + with: + install-only: true + - name: check git status run: git --no-pager diff --exit-code + # Same summary as the unit job: a timeout here names the tests still + # running instead of ending in a goroutine dump. - name: Test + shell: bash run: | + set -o pipefail CGO_ENABLED=1 GOARCH=${{ matrix.arch }} \ NETBIRD_STORE_ENGINE=${{ matrix.store }} \ CI=true \ - go test -tags=integration -coverprofile=coverage.txt \ - -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' \ - -timeout 20m ./management/server/http/... + mage integrationtest:all -gotestflags="-json -coverprofile=coverage.txt" \ + | tee management-integration-test-events.jsonl \ + | go run ./tools/gotestsummary + + - name: Upload raw test events + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: management-integration-test-events-${{ matrix.store }} + path: management-integration-test-events.jsonl + if-no-files-found: ignore + retention-days: 14 - name: Upload coverage reports to Codecov if: matrix.arch == 'amd64' diff --git a/.github/workflows/golang-test-windows.yml b/.github/workflows/golang-test-windows.yml index b61c87cf6..49be3af58 100644 --- a/.github/workflows/golang-test-windows.yml +++ b/.github/workflows/golang-test-windows.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: env: @@ -65,8 +66,17 @@ jobs: - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOCACHE=${{ env.modcache }} - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe mod tidy - name: Generate test script + # Exclude the client/ui package itself: its main.go uses //go:embed + # all:frontend/dist, which fails to compile until the frontend has been + # built, and its release pipeline runs `pnpm build` before goreleaser. + # The pattern is anchored so the subpackages (services, preferences, + # i18n, authsession) still run: they hold Go-side unit tests and need no + # frontend bundle. + # `go list -e` lets the listing succeed even though the embed fails to + # resolve; the Where-Object pipeline then drops the broken package by + # path. Without -e, go list aborts with empty stdout. run: | - $packages = go list ./... | Where-Object { $_ -notmatch '/management' } | Where-Object { $_ -notmatch '/relay' } | Where-Object { $_ -notmatch '/signal' } | Where-Object { $_ -notmatch '/proxy' } | Where-Object { $_ -notmatch '/combined' } + $packages = go list -e ./... | Where-Object { $_ -notmatch '/management' } | Where-Object { $_ -notmatch '/relay' } | Where-Object { $_ -notmatch '/signal' } | Where-Object { $_ -notmatch '/proxy' } | Where-Object { $_ -notmatch '/combined' } | Where-Object { $_ -notmatch '/client/ui$' } | Where-Object { $_ -notmatch '/tools/mysql-migrate' } $goExe = "C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe" $cmd = "$goExe test -tags `"devcert privileged`" -timeout 10m -p 1 $($packages -join ' ') > test-out.txt 2>&1" Set-Content -Path "${{ github.workspace }}\run-tests.cmd" -Value $cmd diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 511e54b62..843177b88 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -22,7 +22,15 @@ jobs: uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2 with: ignore_words_list: erro,clienta,hastable,iif,groupd,testin,groupe,cros,ans,deriver,te,userA,ede,additionals,flate,recordin,unparseable - skip: go.mod,go.sum,**/proxy/web/** + # Non-English UI translations trip codespell on real foreign words + # (de: "Sie", "oder", "ist"). Only en/common.json is the source of + # truth that should be spell-checked. List each translated locale + # dir below and add new ones as languages are added under + # client/ui/i18n/locales/. Single-star globs are matched per path + # segment by codespell and behave the same across versions; the + # recursive "**" form did not take effect with the codespell shipped + # by this action. + skip: go.mod,go.sum,*/proxy/web/*,*pnpm-lock.yaml,*package-lock.json,*/locales/de/*,*/locales/es/*,*/locales/fr/*,*/locales/gl/*,*/locales/hu/*,*/locales/it/*,*/locales/pt/*,*/locales/ru/*,*/locales/zh-CN/*,*/i18n/TRANSLATING.md golangci: strategy: fail-fast: false @@ -37,7 +45,7 @@ jobs: display_name: Linux name: ${{ matrix.display_name }} runs-on: ${{ matrix.os }} - timeout-minutes: 15 + timeout-minutes: 25 steps: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -54,7 +62,16 @@ jobs: cache: false - name: Install dependencies if: matrix.os == 'ubuntu-latest' - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev libpcap-dev + run: sudo apt update && sudo apt install -y -q libgtk-4-dev libwebkitgtk-6.0-dev libsoup-3.0-dev libgl1-mesa-dev xorg-dev libpcap-dev + - name: Stub Wails frontend bundle + # client/ui/main.go has //go:embed all:frontend/dist. The + # directory is produced by `pnpm run build` and is gitignored, so + # lint-only runs (no frontend toolchain) need a placeholder file + # for the embed pattern to match. + shell: bash + run: | + mkdir -p client/ui/frontend/dist + touch client/ui/frontend/dist/.embed-placeholder - name: golangci-lint uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee #v9.2.1 with: @@ -62,4 +79,50 @@ jobs: skip-cache: true skip-save-cache: true cache-invalidation-interval: 0 - args: --timeout=12m + args: --timeout=20m + + # Separate job rather than extra rows in the matrix above: those rows pick a + # GOOS by picking a runner OS, while android/ios are cross-compiled from + # ubuntu — an `include` entry with os: ubuntu-latest would merge into the + # Linux row instead of adding one. The package path is restricted because a + # whole-repo run under GOOS=android pulls *_linux.go files into packages that + # have no android counterpart. + golangci-mobile: + strategy: + fail-fast: false + matrix: + include: + - goos: android + goarch: arm64 + packages: ./client/android/... + display_name: Android + - goos: ios + goarch: arm64 + packages: ./client/ios/... + display_name: iOS + name: ${{ matrix.display_name }} + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + CGO_ENABLED: 0 + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Install Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: "go.mod" + cache: false + - name: golangci-lint + uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee #v9.2.1 + with: + version: latest + install-mode: binary + skip-cache: true + skip-save-cache: true + cache-invalidation-interval: 0 + args: --timeout=20m ${{ matrix.packages }} diff --git a/.github/workflows/install-script-test.yml b/.github/workflows/install-script-test.yml index 1514caedc..61709501c 100644 --- a/.github/workflows/install-script-test.yml +++ b/.github/workflows/install-script-test.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: paths: - "release_files/install.sh" diff --git a/.github/workflows/mobile-build-validation.yml b/.github/workflows/mobile-build-validation.yml index 44e912c73..613a39b3d 100644 --- a/.github/workflows/mobile-build-validation.yml +++ b/.github/workflows/mobile-build-validation.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: concurrency: @@ -14,6 +15,14 @@ jobs: android_build: name: "Android / Build" runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + goarch: [arm64, arm, amd64, "386"] + env: + CGO_ENABLED: 0 + GOOS: android + GOARCH: ${{ matrix.goarch }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -23,35 +32,23 @@ jobs: uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: "go.mod" - - name: Setup Android SDK - uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1 - with: - cmdline-tools-version: 8512546 - - name: Setup Java - uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 - with: - java-version: "11" - distribution: "adopt" - - name: NDK Cache - id: ndk-cache - uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 - with: - path: /usr/local/lib/android/sdk/ndk - key: ndk-cache-23.1.7779620 - - name: Setup NDK - run: /usr/local/lib/android/sdk/cmdline-tools/7.0/bin/sdkmanager --install "ndk;23.1.7779620" - - name: install gomobile - run: go install golang.org/x/mobile/cmd/gomobile@v0.0.0-20251113184115-a159579294ab - - name: gomobile init - run: gomobile init - - name: build android netbird lib - run: PATH=$PATH:$(go env GOPATH) gomobile bind -o $GITHUB_WORKSPACE/netbird.aar -javapkg=io.netbird.gomobile -ldflags="-checklinkname=0 -X golang.zx2c4.com/wireguard/ipc.socketDirectory=/data/data/io.netbird.client/cache/wireguard -X github.com/netbirdio/netbird/version.version=buildtest" $GITHUB_WORKSPACE/client/android - env: - CGO_ENABLED: 0 - ANDROID_NDK_HOME: /usr/local/lib/android/sdk/ndk/23.1.7779620 + - name: Build Android bridge + run: go build ./client/android/... + - name: Vet Android bridge + if: matrix.goarch == 'arm64' + run: go vet ./client/android/... + ios_build: name: "iOS / Build" - runs-on: macos-latest + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + goarch: [arm64, amd64] + env: + CGO_ENABLED: 0 + GOOS: ios + GOARCH: ${{ matrix.goarch }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -61,11 +58,7 @@ jobs: uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: "go.mod" - - name: install gomobile - run: go install golang.org/x/mobile/cmd/gomobile@v0.0.0-20251113184115-a159579294ab - - name: gomobile init - run: gomobile init - - name: build iOS netbird lib - run: PATH=$PATH:$(go env GOPATH) gomobile bind -target=ios -bundleid=io.netbird.framework -ldflags="-X github.com/netbirdio/netbird/version.version=buildtest" -o ./NetBirdSDK.xcframework ./client/ios/NetBirdSDK - env: - CGO_ENABLED: 0 + # No `go vet` counterpart: every ios target requires external (cgo) + # linking, which needs an Xcode toolchain the runner does not have. + - name: Build iOS SDK + run: go build ./client/ios/... diff --git a/.github/workflows/no-new-replace.yml b/.github/workflows/no-new-replace.yml new file mode 100644 index 000000000..b906ce450 --- /dev/null +++ b/.github/workflows/no-new-replace.yml @@ -0,0 +1,78 @@ +name: No New Replace Directives + +on: + pull_request: + paths: + - "go.mod" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }} + cancel-in-progress: true + +jobs: + check-replace-directives: + name: check-replace-directives + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: go.mod + + - name: Compare replace directives against the base branch + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + + # A replace directive only applies when this module is the main + # module. Anything importing netbird as a library, the embedded + # clients among them, resolves the replaced path upstream instead and + # fails to build against whatever the replacement provides. Requiring + # a fork under its own module path avoids that; a replace does not. + # + # go.mod is parsed rather than diffed so that reordering, comments and + # single-line versus block syntax do not register as changes. + # + # Versions are part of the key because a replace can be scoped to one + # version of a module. Keyed on paths alone, retargeting such a + # directive at a different version would read as unchanged. + list_replaces() { + go mod edit -json "$1" \ + | jq -r ' + def ref: .Path + (if (.Version // "") == "" then "" else " " + .Version end); + (.Replace // [])[] | "\(.Old | ref) => \(.New | ref)" + ' \ + | sort + } + + git show "${BASE_SHA}:go.mod" > /tmp/base-go.mod + list_replaces /tmp/base-go.mod > /tmp/base-replaces + list_replaces go.mod > /tmp/head-replaces + + added=$(comm -13 /tmp/base-replaces /tmp/head-replaces) + if [ -n "$added" ]; then + echo "::error::This PR adds a replace directive to go.mod:" + echo "$added" | sed 's/^/ /' + echo "" + echo "A replace directive applies only to the main module, so it does not" + echo "reach anything that imports netbird as a library. Require the module" + echo "under a path you control instead, as done for github.com/netbirdio/go-nat." + exit 1 + fi + + removed=$(comm -23 /tmp/base-replaces /tmp/head-replaces) + if [ -n "$removed" ]; then + echo "This PR removes replace directives:" + echo "$removed" | sed 's/^/ /' + fi + echo "No new replace directives." diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml index 67d65356c..5b769e2f7 100644 --- a/.github/workflows/pr-title-check.yml +++ b/.github/workflows/pr-title-check.yml @@ -7,6 +7,8 @@ on: jobs: check-title: runs-on: ubuntu-latest + # Crowdin's translation-sync service PRs are auto-generated with a fixed title. + if: github.event.pull_request.user.login != 'netbirddev' steps: - name: Validate PR title prefix uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -16,6 +18,8 @@ jobs: const allowedTags = [ 'management', 'client', + 'android', + 'ios', 'signal', 'proxy', 'relay', diff --git a/.github/workflows/redhat-certify.yml b/.github/workflows/redhat-certify.yml new file mode 100644 index 000000000..96d50de9f --- /dev/null +++ b/.github/workflows/redhat-certify.yml @@ -0,0 +1,207 @@ +name: Red Hat Certification + +# Certify published UBI images in the Red Hat Ecosystem Catalog. Called by +# release.yml on stable tags, or run by hand to (re)certify any released +# version. preflight submits every architecture of an image's manifest list +# to Pyxis; auto-publish on the component makes it public once certified. +# +# Each component's Partner Connect ID comes from the REDHAT_CERT_ID_ +# repository variable, e.g. REDHAT_CERT_ID_CLIENT_ROOTLESS. The run fails +# before certifying anything if a selected component's variable is not set. + +on: + workflow_call: + inputs: + component: + type: string + required: true + version: + type: string + required: true + secrets: + PYXIS_API_TOKEN: + required: true + workflow_dispatch: + inputs: + component: + description: "Component to certify" + type: choice + required: true + default: all + options: + - all + - client-rootless + - reverse-proxy + - netbird-server + - management + - signal + - relay + version: + description: "Released version, e.g. v0.80.0" + type: string + required: true + +permissions: + contents: read + +jobs: + resolve: + name: Resolve components + runs-on: ubuntu-24.04 + outputs: + version: ${{ steps.resolve.outputs.version }} + matrix: ${{ steps.resolve.outputs.matrix }} + steps: + - name: Resolve components and images + id: resolve + env: + COMPONENT: ${{ inputs.component }} + INPUT_VERSION: ${{ inputs.version }} + REPO_VARS: ${{ toJSON(vars) }} + run: | + set -euo pipefail + version="${INPUT_VERSION#v}" + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Only stable x.y.z versions are certified, got '${INPUT_VERSION}'" + exit 1 + fi + # name, image repository, tag suffix (must match .goreleaser.yaml). + # Keep the names in sync with the workflow_dispatch options above. + components=( + "client-rootless ghcr.io/netbirdio/netbird -rootless-ubi" + "reverse-proxy ghcr.io/netbirdio/reverse-proxy -ubi" + "netbird-server ghcr.io/netbirdio/netbird-server -ubi" + "management ghcr.io/netbirdio/management -ubi" + "signal ghcr.io/netbirdio/signal -ubi" + "relay ghcr.io/netbirdio/relay -ubi" + ) + matrix="[]" + missing=() + for c in "${components[@]}"; do + read -r name repo suffix <<< "$c" + [[ "$COMPONENT" == all || "$COMPONENT" == "$name" ]] || continue + var="REDHAT_CERT_ID_${name^^}"; var="${var//-/_}" + id="$(jq -r --arg v "$var" '.[$v] // empty' <<< "$REPO_VARS")" + if [[ -z "$id" ]]; then + missing+=("$var") + continue + fi + matrix="$(jq -c --arg n "$name" --arg t "${version}${suffix}" --arg r "${repo}:${version}${suffix}" --arg i "$id" \ + '. + [{component: $n, tag: $t, ref: $r, component_id: $i}]' <<< "$matrix")" + done + if (( ${#missing[@]} )); then + echo "::error::Set these repository variables to the Partner Connect component IDs: ${missing[*]}" + exit 1 + fi + if [[ "$matrix" == "[]" ]]; then + echo "::error::No component to certify for '${COMPONENT}'" + exit 1 + fi + echo "Components to certify: ${matrix}" + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" + + certify: + name: "Certify ${{ matrix.component }} UBI image" + needs: resolve + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.resolve.outputs.matrix) }} + env: + PREFLIGHT_VERSION: "1.21.0" + # sha256 of preflight-linux-amd64 from the 1.21.0 GitHub release. + # Red Hat publishes no checksum file, so the value is pinned here. + PREFLIGHT_SHA256: "5e653135503c72f8702bbe31d7643197d12937c68086879133dd6b9650a9a449" + steps: + - name: Verify the multi-arch image is on ghcr.io + env: + IMAGE_REF: ${{ matrix.ref }} + run: | + set -euo pipefail + docker buildx imagetools inspect "$IMAGE_REF" --raw > manifest.json + for arch in amd64 arm64; do + if ! jq -e --arg a "$arch" '.manifests[] | select(.platform.architecture == $a)' manifest.json > /dev/null; then + echo "::error::${IMAGE_REF} has no ${arch} manifest" + exit 1 + fi + done + echo "Manifest list for ${IMAGE_REF}:" + jq -r '.manifests[] | "\(.platform.os)/\(.platform.architecture) \(.digest)"' manifest.json + + - name: Install preflight + run: | + set -euo pipefail + curl -fsSL --proto '=https' --proto-redir '=https' -o preflight \ + "https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/download/${PREFLIGHT_VERSION}/preflight-linux-amd64" + echo "${PREFLIGHT_SHA256} preflight" | sha256sum -c - + chmod +x preflight + ./preflight --version + + - name: Run preflight checks and submit to Red Hat + env: + IMAGE_REF: ${{ matrix.ref }} + PFLT_PYXIS_API_TOKEN: ${{ secrets.PYXIS_API_TOKEN }} + PFLT_CERTIFICATION_COMPONENT_ID: ${{ matrix.component_id }} + PFLT_ARTIFACTS: artifacts + PFLT_LOGFILE: artifacts/preflight.log + PFLT_LOGLEVEL: info + PFLT_JUNIT: "true" + run: | + set -euo pipefail + # No --platform: preflight walks the manifest list and submits every + # architecture in one run, grouped under one manifest-list digest. + # preflight does not create the PFLT_LOGFILE directory, and --submit + # fails if the log file is missing. + mkdir -p artifacts + ./preflight check container "$IMAGE_REF" --submit + + - name: Fail if any check did not pass + run: | + set -euo pipefail + shopt -s nullglob globstar + results=(artifacts/**/results.json) + if [[ ${#results[@]} -eq 0 ]]; then + echo "::error::preflight produced no results.json" + exit 1 + fi + status=0 + for f in "${results[@]}"; do + arch="$(basename "$(dirname "$f")")" + passed="$(jq -r '.passed' "$f")" + failed="$(jq -r '[.results.failed[]?.name] | join(", ")' "$f")" + echo "${arch}: passed=${passed} ${failed:+failed checks: ${failed}}" + [[ "$passed" == "true" ]] || status=1 + done + exit $status + + - name: Upload preflight artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: redhat-preflight-${{ matrix.component }}-${{ needs.resolve.outputs.version }} + path: artifacts/ + retention-days: 30 + + - name: Wait for Pyxis to mark both architectures certified + env: + TAG: ${{ matrix.tag }} + COMPONENT_ID: ${{ matrix.component_id }} + PFLT_PYXIS_API_TOKEN: ${{ secrets.PYXIS_API_TOKEN }} + run: | + set -euo pipefail + # Filter on the tag server-side so older versions are found past the first page. + url="https://catalog.redhat.com/api/containers/v1/projects/certification/id/${COMPONENT_ID}/images?filter=repositories.tags.name==${TAG}&page_size=100" + for attempt in $(seq 1 20); do + certified="$(curl -fsS --proto '=https' --proto-redir '=https' -H "X-API-KEY: ${PFLT_PYXIS_API_TOKEN}" "$url" \ + | jq -r --arg t "$TAG" '[.data[] | select(.repositories[]?.tags[]?.name == $t) | select(.certified == true) | .architecture] | unique | join(",")')" + echo "attempt ${attempt}: certified architectures for ${TAG}: ${certified:-none}" + if [[ "$certified" == "amd64,arm64" ]]; then + echo "Both architectures certified. Auto-publish is enabled on the component, so the catalog updates on its own." + exit 0 + fi + sleep 30 + done + echo "::error::Pyxis has not marked both architectures certified after 10 minutes. Check https://connect.redhat.com/component/view/${COMPONENT_ID}/images" + exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cd431a389..22ab7f566 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,10 +6,11 @@ on: - "v*" branches: - main + - "release-*" pull_request: env: - SIGN_PIPE_VER: "v0.1.6" + SIGN_PIPE_VER: "v0.1.8" GORELEASER_VER: "v2.16.0" PRODUCT_NAME: "NetBird" COPYRIGHT: "NetBird GmbH" @@ -22,8 +23,50 @@ concurrency: cancel-in-progress: true jobs: + freebsd_port_trigger: + name: "FreeBSD Port / Trigger" + runs-on: ubuntu-22.04 + outputs: + run: ${{ steps.decide.outputs.run }} + version: ${{ steps.decide.outputs.version }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Decide whether the port job runs + id: decide + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + run=false + case "$GITHUB_EVENT_NAME" in + push) + if [[ "$GITHUB_REF" == "refs/heads/main" ]]; then + run=true + elif [[ "$GITHUB_REF" == refs/tags/* && "$GITHUB_REF_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + # The port tracks stable releases only, so a canary or rc tag has nothing to submit. + run=true + echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + fi + ;; + pull_request) + git fetch --depth=1 origin "$BASE_SHA" + if git diff --name-only "$BASE_SHA" HEAD | \ + grep -qE '^(release_files/freebsd-port-.*\.sh|\.github/workflows/release\.yml)$'; then + run=true + fi + ;; + esac + echo "run=$run" >> "$GITHUB_OUTPUT" + echo "FreeBSD port job runs: $run" + release_freebsd_port: name: "FreeBSD Port / Build & Test" + needs: freebsd_port_trigger + if: needs.freebsd_port_trigger.outputs.run == 'true' runs-on: ubuntu-22.04 steps: - name: Checkout @@ -32,18 +75,19 @@ jobs: persist-credentials: false - name: Generate FreeBSD port diff - run: bash -x release_files/freebsd-port-diff.sh - - - name: Generate FreeBSD port issue body - run: bash -x release_files/freebsd-port-issue-body.sh + env: + TARGET_VERSION: ${{ needs.freebsd_port_trigger.outputs.version }} + run: bash -x release_files/freebsd-port-diff.sh "$TARGET_VERSION" - name: Check if diff was generated id: check_diff run: | - if ls netbird-*.diff 1> /dev/null 2>&1; then - echo "diff_exists=true" >> $GITHUB_OUTPUT + shopt -s nullglob + diffs=(netbird-*.diff) + if [[ ${#diffs[@]} -gt 0 ]]; then + echo "diff_exists=true" >> "$GITHUB_OUTPUT" else - echo "diff_exists=false" >> $GITHUB_OUTPUT + echo "diff_exists=false" >> "$GITHUB_OUTPUT" echo "No diff file generated (port may already be up to date)" fi @@ -51,10 +95,18 @@ jobs: if: steps.check_diff.outputs.diff_exists == 'true' id: version run: | - VERSION=$(ls netbird-*.diff | sed 's/netbird-\(.*\)\.diff/\1/') - echo "version=$VERSION" >> $GITHUB_OUTPUT + diffs=(netbird-*.diff) + VERSION="${diffs[0]#netbird-}" + VERSION="${VERSION%.diff}" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "Generated files for version: $VERSION" - cat netbird-*.diff + cat "${diffs[0]}" + + - name: Generate FreeBSD port issue body + if: steps.check_diff.outputs.diff_exists == 'true' + env: + TARGET_VERSION: ${{ steps.version.outputs.version }} + run: bash -x release_files/freebsd-port-issue-body.sh "" "$TARGET_VERSION" - name: Read Go version from go.mod id: goversion @@ -68,7 +120,7 @@ jobs: with: usesh: true copyback: false - release: "15.0" + release: "15.1" envs: "GO_VERSION" prepare: | # Install required packages @@ -185,6 +237,22 @@ jobs: run: bash shared/management/http/api/generate.sh - name: check git status run: git --no-pager diff --exit-code + - name: Generate RPM changelog from git tags + # nfpm embeds changelog.yml into the RPM; Red Hat software certification + # requires a changelog. Generated, not committed (see .gitignore). + # chglog is a go.mod tool directive, so go.sum pins it and its deps. + run: bash release_files/rpm-changelog.sh + - name: Fill the RPM ISA provide version + # nfpm cannot emit rpmbuild's ISA provide and GoReleaser cannot template it. + run: bash release_files/rpm-provides.sh + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + - name: Install proxy web dependencies for license collection + # release_files/collect-licenses.sh -w reads the proxy UI's license terms from node_modules. + working-directory: proxy/web + run: npm ci --ignore-scripts - name: Set up QEMU uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 #v4.1.0 - name: Set up Docker Buildx @@ -214,24 +282,28 @@ jobs: echo "GPG_RPM_KEY_FILE=/tmp/gpg-rpm-signing-key.asc" >> $GITHUB_ENV - name: Install goversioninfo - run: go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@233067e + run: go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@b66839b # v1.7.0 - name: Generate windows syso amd64 - run: goversioninfo -icon client/ui/assets/netbird.ico -manifest client/manifest.xml -product-name ${{ env.PRODUCT_NAME }} -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/resources_windows_amd64.syso + run: goversioninfo -icon client/ui/build/windows/icon.ico -manifest client/manifest.xml -product-name ${{ env.PRODUCT_NAME }} -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/resources_windows_amd64.syso - name: Generate windows syso arm64 - run: goversioninfo -arm -64 -icon client/ui/assets/netbird.ico -manifest client/manifest.xml -product-name ${{ env.PRODUCT_NAME }} -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/resources_windows_arm64.syso + run: goversioninfo -arm -64 -icon client/ui/build/windows/icon.ico -manifest client/manifest.xml -product-name ${{ env.PRODUCT_NAME }} -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/resources_windows_arm64.syso - name: Run GoReleaser id: goreleaser uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2 with: version: ${{ env.GORELEASER_VER }} - args: release --clean ${{ env.flags }} + args: release --config .goreleaser.generated.yaml --clean ${{ env.flags }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }} UPLOAD_DEBIAN_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }} UPLOAD_YUM_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }} GPG_RPM_KEY_FILE: ${{ env.GPG_RPM_KEY_FILE }} - NFPM_NETBIRD_RPM_PASSPHRASE: ${{ secrets.GPG_RPM_PASSPHRASE }} + # One per nfpm id: GoReleaser looks the passphrase up as NFPM__PASSPHRASE. + NFPM_NETBIRD_RPM_AMD64_PASSPHRASE: ${{ secrets.GPG_RPM_PASSPHRASE }} + NFPM_NETBIRD_RPM_ARM64_PASSPHRASE: ${{ secrets.GPG_RPM_PASSPHRASE }} + NFPM_NETBIRD_RPM_ARM_PASSPHRASE: ${{ secrets.GPG_RPM_PASSPHRASE }} + NFPM_NETBIRD_RPM_386_PASSPHRASE: ${{ secrets.GPG_RPM_PASSPHRASE }} SKIP_PUBLISH: ${{ env.SKIP_PUBLISH }} SKIP_DOCKER_PUSH: ${{ env.SKIP_DOCKER_PUSH }} - name: Verify RPM signatures @@ -254,15 +326,23 @@ jobs: id: tag_and_push_images if: | (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || - (github.event_name == 'push' && github.ref == 'refs/heads/main') + (github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release-'))) run: | set -euo pipefail + # $GITHUB_REF / $GITHUB_EVENT_NAME are read from the runner + # environment rather than substituted into this script with the + # workflow expression syntax: branch names may legally contain + # $(…), and interpolating github.ref would execute it. resolve_tags() { - if [[ "${{ github.event_name }}" == "pull_request" ]]; then + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then echo "pr-${{ github.event.pull_request.number }}" - else + elif [[ "$GITHUB_REF" == "refs/heads/main" ]]; then echo "main sha-$(git rev-parse --short HEAD)" + else + # Release branches get an immutable sha-* tag only — the floating + # "main" tag must never move from a release branch. + echo "sha-$(git rev-parse --short HEAD)" fi } @@ -278,10 +358,17 @@ jobs: image_refs=() tag_and_push() { - local src="$1" img_name tag dst + local src="$1" img_name tag dst variant="" img_name="${src%%:*}" + # Variants share a repository with their default image, so keep + # their tag suffixes. Order matters: the first matching pattern wins. + case "$src" in + *-rootless-ubi-amd64) variant="-rootless-ubi" ;; + *-rootless-amd64) variant="-rootless" ;; + *-ubi-amd64) variant="-ubi" ;; + esac for tag in $(resolve_tags); do - dst="${img_name}:${tag}" + dst="${img_name}:${tag}${variant}" echo "Tagging ${src} -> ${dst}" docker tag "$src" "$dst" docker push "$dst" @@ -327,7 +414,11 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 with: name: linux-packages - path: dist/netbird_linux** + path: | + dist/netbird_linux** + dist/netbird-pkcs11_linux** + dist/netbird-deb-32bit_linux** + dist/netbird-rpm-*_linux** retention-days: 7 - name: upload windows packages id: upload_windows_packages @@ -344,6 +435,24 @@ jobs: path: dist/netbird_darwin** retention-days: 7 + # Certify the UBI images in the Red Hat Ecosystem Catalog on stable tags. + # See redhat-certify.yml, which can also be run by hand for any released version. + redhat_certification: + name: "Red Hat" + needs: release + if: | + github.repository == 'netbirdio/netbird' && + startsWith(github.ref, 'refs/tags/v') && + !contains(github.ref_name, '-') + permissions: + contents: read + uses: ./.github/workflows/redhat-certify.yml + with: + component: all + version: ${{ github.ref_name }} + secrets: + PYXIS_API_TOKEN: ${{ secrets.PYXIS_API_TOKEN }} + release_ui: runs-on: ubuntu-latest outputs: @@ -397,8 +506,18 @@ jobs: - name: check git status run: git --no-pager diff --exit-code + - name: Set up Node.js + uses: actions/setup-node@v7 + with: + node-version: '22' + + - name: Set up pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + with: + version: 11 + - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libappindicator3-dev gir1.2-appindicator3-0.1 libxxf86vm-dev gcc-mingw-w64-x86-64 + run: sudo apt update && sudo apt install -y -q libgtk-4-dev libwebkitgtk-6.0-dev libsoup-3.0-dev gcc-mingw-w64-x86-64 - name: Decode GPG signing key if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository @@ -416,11 +535,17 @@ jobs: tar -xf llvm-mingw-20250709-ucrt-ubuntu-22.04-x86_64.tar.xz echo "/tmp/llvm-mingw-20250709-ucrt-ubuntu-22.04-x86_64/bin" >> $GITHUB_PATH - name: Install goversioninfo - run: go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@233067e + run: go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@b66839b # v1.7.0 + - name: Install wails3 CLI + # Version derived from go.mod so the binding generator always matches + # the wails runtime the binary links against. + run: | + WAILS_VERSION=$(go list -m -f '{{.Version}}' github.com/wailsapp/wails/v3) + go install github.com/wailsapp/wails/v3/cmd/wails3@$WAILS_VERSION - name: Generate windows syso amd64 - run: goversioninfo -64 -icon client/ui/assets/netbird.ico -manifest client/ui/manifest.xml -product-name ${{ env.PRODUCT_NAME }}-"UI" -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/ui/resources_windows_amd64.syso + run: goversioninfo -64 -icon client/ui/build/windows/icon.ico -manifest client/ui/build/windows/wails.exe.manifest -product-name ${{ env.PRODUCT_NAME }}-"UI" -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/ui/resources_windows_amd64.syso - name: Generate windows syso arm64 - run: goversioninfo -arm -64 -icon client/ui/assets/netbird.ico -manifest client/ui/manifest.xml -product-name ${{ env.PRODUCT_NAME }}-"UI" -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/ui/resources_windows_arm64.syso + run: goversioninfo -arm -64 -icon client/ui/build/windows/icon.ico -manifest client/ui/build/windows/wails.exe.manifest -product-name ${{ env.PRODUCT_NAME }}-"UI" -copyright "${{ env.COPYRIGHT }}" -ver-major ${{ steps.semver_parser.outputs.major }} -ver-minor ${{ steps.semver_parser.outputs.minor }} -ver-patch ${{ steps.semver_parser.outputs.patch }} -ver-build 0 -file-version ${{ steps.semver_parser.outputs.fullversion }}.0 -product-version ${{ steps.semver_parser.outputs.fullversion }}.0 -o client/ui/resources_windows_arm64.syso - name: Run GoReleaser uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2 @@ -459,6 +584,132 @@ jobs: path: dist/ retention-days: 3 + release_ui_gtk3: + # Legacy GTK3/WebKit2GTK 4.1 UI build for distros without WebKitGTK 6.0 + # (Ubuntu 22.04, Debian 12, RHEL 9, Fedora <=39). Runs on ubuntu-22.04 so + # the binary links against the oldest supported glibc. + runs-on: ubuntu-22.04 + outputs: + release_ui_gtk3_artifact_url: ${{ steps.upload_release_ui_gtk3.outputs.artifact-url }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 # It is required for GoReleaser to work properly + persist-credentials: false + + - name: Parse semver string + id: semver_parser + uses: netbirdio/shared-actions/actions/parse-semver@be5df6047383da2236e02243cceb857d8567c27e # v0.0.2 + + - name: Set snapshot flag + if: ${{ !startsWith(github.ref, 'refs/tags/v') }} + run: | + echo "flags=--snapshot" >> $GITHUB_ENV + + - name: Set build vars + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + run: | + if [[ "x-${{ steps.semver_parser.outputs.prerelease }}" == "x-" && "x-${{ github.repository }}" == "x-netbirdio/netbird" ]]; then + echo "x-${{ github.repository }}" + echo "x-${{ steps.semver_parser.outputs.prerelease }}" + echo "SKIP_PUBLISH=false" >> $GITHUB_ENV + else + echo "x-${{ github.repository }}" + echo "x-${{ steps.semver_parser.outputs.prerelease }}" + fi + + - name: Set up Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: "go.mod" + cache: false + - name: Cache Go modules + # Restore-only from the release_ui cache written by trusted runs; the + # module cache is identical (same go.sum) and stale build-cache + # entries just miss. + uses: actions/cache/restore@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-ui-go-releaser-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-ui-go-releaser- + + - name: Install modules + run: go mod tidy + + - name: check git status + run: git --no-pager diff --exit-code + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + + - name: Set up pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + with: + version: 11 + + - name: Install dependencies + run: sudo apt update && sudo apt install -y -q libgtk-3-dev libwebkit2gtk-4.1-dev + + - name: Decode GPG signing key + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + env: + GPG_RPM_PRIVATE_KEY: ${{ secrets.GPG_RPM_PRIVATE_KEY }} + run: | + echo "$GPG_RPM_PRIVATE_KEY" | base64 -d > /tmp/gpg-rpm-signing-key.asc + echo "GPG_RPM_KEY_FILE=/tmp/gpg-rpm-signing-key.asc" >> $GITHUB_ENV + + - name: Install wails3 CLI + # Version derived from go.mod so the binding generator always matches + # the wails runtime the binary links against. + # -tags gtk3: the CLI links the wails runtime's cgo packages, and the + # default tags request gtk4/webkitgtk-6.0 pkg-config entries that do + # not exist on ubuntu-22.04. + run: | + WAILS_VERSION=$(go list -m -f '{{.Version}}' github.com/wailsapp/wails/v3) + go install -tags gtk3 github.com/wailsapp/wails/v3/cmd/wails3@$WAILS_VERSION + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2 + with: + version: ${{ env.GORELEASER_VER }} + args: release --config .goreleaser_ui_gtk3.yaml --clean ${{ env.flags }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + UPLOAD_DEBIAN_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }} + UPLOAD_YUM_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }} + GPG_RPM_KEY_FILE: ${{ env.GPG_RPM_KEY_FILE }} + NFPM_NETBIRD_UI_RPM_GTK3_PASSPHRASE: ${{ secrets.GPG_RPM_PASSPHRASE }} + SKIP_PUBLISH: ${{ env.SKIP_PUBLISH }} + - name: Verify RPM signatures + run: | + docker run --rm -v $(pwd)/dist:/dist fedora:41 bash -c ' + dnf install -y -q rpm-sign curl >/dev/null 2>&1 + curl -sSL https://pkgs.netbird.io/yum/repodata/repomd.xml.key -o /tmp/rpm-pub.key + rpm --import /tmp/rpm-pub.key + echo "=== Verifying RPM signatures ===" + for rpm_file in /dist/*.rpm; do + [ -f "$rpm_file" ] || continue + echo "--- $(basename $rpm_file) ---" + rpm -K "$rpm_file" + done + ' + - name: Clean up GPG key + if: always() + run: rm -f /tmp/gpg-rpm-signing-key.asc + - name: upload non tags for debug purposes + id: upload_release_ui_gtk3 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: release-ui-gtk3 + path: dist/ + retention-days: 3 + release_ui_darwin: runs-on: macos-latest outputs: @@ -489,6 +740,20 @@ jobs: run: go mod tidy - name: check git status run: git --no-pager diff --exit-code + - name: Set up Node.js + uses: actions/setup-node@v7 + with: + node-version: '22' + - name: Set up pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + with: + version: 11 + - name: Install wails3 CLI + # Version derived from go.mod so the binding generator always matches + # the wails runtime the binary links against. + run: | + WAILS_VERSION=$(go list -m -f '{{.Version}}' github.com/wailsapp/wails/v3) + go install github.com/wailsapp/wails/v3/cmd/wails3@$WAILS_VERSION - name: Run GoReleaser id: goreleaser uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2 @@ -576,23 +841,6 @@ jobs: - name: Move wintun.dll into dist run: mv ${{ env.downloadPath }}\wintun\bin\${{ matrix.wintun_arch }}\wintun.dll ${{ github.workspace }}\dist\${{ env.PackageWorkdir }}\ - - name: Download Mesa3D (amd64 only) - id: download-mesa3d - if: matrix.arch == 'amd64' - uses: netbirdio/shared-actions/actions/win-download-and-verify@be5df6047383da2236e02243cceb857d8567c27e # v0.0.2 - with: - url: https://pkgs.netbird.io/mesa3d/MesaForWindows-x64-20.1.8.7z - destination: ${{ env.downloadPath }}\mesa3d.7z - sha256: 71c7cb64ec229a1d6b8d62fa08e1889ed2bd17c0eeede8689daf0f25cb31d6b9 - - - name: Extract Mesa3D driver (amd64 only) - if: matrix.arch == 'amd64' - run: 7z x -o"${{ env.downloadPath }}" "${{ env.downloadPath }}/mesa3d.7z" - - - name: Move opengl32.dll into dist (amd64 only) - if: matrix.arch == 'amd64' - run: mv ${{ env.downloadPath }}\opengl32.dll ${{ github.workspace }}\dist\${{ env.PackageWorkdir }}\ - - name: Download EnVar plugin for NSIS uses: netbirdio/shared-actions/actions/win-download-and-verify@be5df6047383da2236e02243cceb857d8567c27e # v0.0.2 with: @@ -615,6 +863,28 @@ jobs: if: matrix.arch == 'amd64' run: 7z x -o"${{ github.workspace }}/NSIS_Plugins" "${{ github.workspace }}/ShellExecAsUser_amd64-Unicode.7z" + - name: Set up Go for wails3 CLI + uses: actions/setup-go@v6 + with: + go-version-file: "go.mod" + cache: false + + - name: Install wails3 CLI + # Version derived from go.mod so the bootstrapper payload always + # matches the wails runtime the binary links against. + shell: bash + run: | + WAILS_VERSION=$(go list -m -f '{{.Version}}' github.com/wailsapp/wails/v3) + go install github.com/wailsapp/wails/v3/cmd/wails3@$WAILS_VERSION + + - name: Stage WebView2 bootstrapper for installers + # Both client/installer.nsis and client/netbird.wxs reference + # client/MicrosoftEdgeWebview2Setup.exe. wails3 writes it there. + # The signing pipeline (netbirdio/sign-pipelines) does the same + # step for release builds; this mirrors it for PR sanity testing. + shell: bash + run: wails3 generate webview2bootstrapper -dir client + - name: Build NSIS installer shell: pwsh env: @@ -653,7 +923,7 @@ jobs: comment_release_artifacts: name: Comment release artifacts runs-on: ubuntu-latest - needs: [release, release_ui, release_ui_darwin] + needs: [release, release_ui, release_ui_gtk3, release_ui_darwin] if: ${{ always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} permissions: contents: read @@ -665,12 +935,14 @@ jobs: env: RELEASE_RESULT: ${{ needs.release.result }} RELEASE_UI_RESULT: ${{ needs.release_ui.result }} + RELEASE_UI_GTK3_RESULT: ${{ needs.release_ui_gtk3.result }} RELEASE_UI_DARWIN_RESULT: ${{ needs.release_ui_darwin.result }} RELEASE_ARTIFACT_URL: ${{ needs.release.outputs.release_artifact_url }} LINUX_PACKAGES_ARTIFACT_URL: ${{ needs.release.outputs.linux_packages_artifact_url }} WINDOWS_PACKAGES_ARTIFACT_URL: ${{ needs.release.outputs.windows_packages_artifact_url }} MACOS_PACKAGES_ARTIFACT_URL: ${{ needs.release.outputs.macos_packages_artifact_url }} RELEASE_UI_ARTIFACT_URL: ${{ needs.release_ui.outputs.release_ui_artifact_url }} + RELEASE_UI_GTK3_ARTIFACT_URL: ${{ needs.release_ui_gtk3.outputs.release_ui_gtk3_artifact_url }} RELEASE_UI_DARWIN_ARTIFACT_URL: ${{ needs.release_ui_darwin.outputs.release_ui_darwin_artifact_url }} GHCR_IMAGES_MARKDOWN: ${{ needs.release.outputs.ghcr_images }} with: @@ -693,6 +965,7 @@ jobs: ['Windows packages', process.env.WINDOWS_PACKAGES_ARTIFACT_URL, process.env.RELEASE_RESULT], ['macOS packages', process.env.MACOS_PACKAGES_ARTIFACT_URL, process.env.RELEASE_RESULT], ['UI artifacts', process.env.RELEASE_UI_ARTIFACT_URL, process.env.RELEASE_UI_RESULT], + ['UI GTK3 artifacts', process.env.RELEASE_UI_GTK3_ARTIFACT_URL, process.env.RELEASE_UI_GTK3_RESULT], ['UI macOS artifacts', process.env.RELEASE_UI_DARWIN_ARTIFACT_URL, process.env.RELEASE_UI_DARWIN_RESULT], ]; @@ -749,7 +1022,7 @@ jobs: trigger_signer: runs-on: ubuntu-latest - needs: [release, release_ui, release_ui_darwin, test_windows_installer] + needs: [release, release_ui, release_ui_gtk3, release_ui_darwin, test_windows_installer] if: startsWith(github.ref, 'refs/tags/') steps: - name: Trigger binaries sign pipelines diff --git a/.github/workflows/reverse-proxy-e2e.yml b/.github/workflows/reverse-proxy-e2e.yml new file mode 100644 index 000000000..934a6d4c9 --- /dev/null +++ b/.github/workflows/reverse-proxy-e2e.yml @@ -0,0 +1,53 @@ +name: Reverse Proxy E2E + +on: + pull_request: + paths: + - '.github/workflows/reverse-proxy-e2e.yml' + - '**/*.go' + - 'e2e/**' + - 'shared/management/**' + - 'combined/Dockerfile*' + - 'proxy/Dockerfile*' + - '.dockerignore' + - 'go.mod' + - 'go.sum' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + reverse-proxy: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Install Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version-file: go.mod + + - name: Set up Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + - name: Cache Docker layers + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: /tmp/.reverse-proxy-buildx-cache + key: ${{ runner.os }}-reverse-proxy-e2e-${{ hashFiles('go.sum', 'combined/Dockerfile.multistage', 'proxy/Dockerfile.multistage') }} + restore-keys: | + ${{ runner.os }}-reverse-proxy-e2e- + + - name: Run reverse-proxy E2E + env: + NB_E2E_BUILDX_CACHE: /tmp/.reverse-proxy-buildx-cache + run: go test -tags e2e -timeout 25m -v ./e2e/reverseproxy/... diff --git a/.github/workflows/sync-tag.yml b/.github/workflows/sync-tag.yml index d99f88b54..608f3c6d7 100644 --- a/.github/workflows/sync-tag.yml +++ b/.github/workflows/sync-tag.yml @@ -9,21 +9,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }} cancel-in-progress: true -# Receiving workflows (cloud sync-tag, mobile bump-netbird) expect the short -# tag form (e.g. v0.30.0), not refs/tags/v0.30.0 — github.ref_name, not github.ref. +# The receiving bump-netbird workflows expect the short tag form +# (e.g. v0.30.0), not refs/tags/v0.30.0 — github.ref_name, not github.ref. jobs: - trigger_sync_tag: - runs-on: ubuntu-latest - steps: - - name: Trigger release tag sync - uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2 - with: - workflow: sync-tag.yml - ref: main - repo: ${{ secrets.UPSTREAM_REPO }} - token: ${{ secrets.NC_GITHUB_TOKEN }} - inputs: '{ "tag": "${{ github.ref_name }}" }' - trigger_android_bump: runs-on: ubuntu-latest if: github.event.created && !github.event.deleted && startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-') @@ -49,3 +37,16 @@ jobs: repo: netbirdio/ios-client token: ${{ secrets.NC_GITHUB_TOKEN }} inputs: '{ "tag": "${{ github.ref_name }}" }' + + trigger_dashboard_bump: + runs-on: ubuntu-latest + if: github.event.created && !github.event.deleted && startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-') + steps: + - name: Trigger dashboard wasm client bump + uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2 + with: + workflow: bump-netbird.yml + ref: main + repo: netbirdio/dashboard + token: ${{ secrets.NC_GITHUB_TOKEN }} + inputs: '{ "tag": "${{ github.ref_name }}" }' diff --git a/.github/workflows/test-homebrew-cask.yml b/.github/workflows/test-homebrew-cask.yml new file mode 100644 index 000000000..a75951425 --- /dev/null +++ b/.github/workflows/test-homebrew-cask.yml @@ -0,0 +1,46 @@ +name: Test Homebrew cask + +on: + pull_request: + paths: + - "client/ui/netbird-ui.rb.tmpl" + - ".github/scripts/test-homebrew-cask.sh" + - ".github/workflows/test-homebrew-cask.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }} + cancel-in-progress: true + +jobs: + install-uninstall: + runs-on: macos-latest + timeout-minutes: 20 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Clone the Homebrew tap + run: git clone https://github.com/netbirdio/homebrew-tap.git .homebrew-cask-tap + + - name: Update Homebrew and install gomplate + # The runner image disables auto-update; the cask steps DSL needs Homebrew 6.0.20 or newer. + run: | + brew update + brew install gomplate + + - name: Install and uninstall the cask + run: .github/scripts/test-homebrew-cask.sh + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: homebrew-cask-results + path: ${{ runner.temp }}/homebrew-cask/results + if-no-files-found: ignore diff --git a/.github/workflows/test-infrastructure-files.yml b/.github/workflows/test-infrastructure-files.yml index 0a4f2e371..4875adc31 100644 --- a/.github/workflows/test-infrastructure-files.yml +++ b/.github/workflows/test-infrastructure-files.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: paths: - "infrastructure_files/**" @@ -249,78 +250,56 @@ jobs: docker compose exec management ls -l /var/lib/netbird/ | grep -i GeoLite2-City_[0-9]*.mmdb docker compose exec management ls -l /var/lib/netbird/ | grep -i geonames_[0-9]*.db - test-getting-started-script: + test-legacy-getting-started-scripts: runs-on: ubuntu-latest steps: - - name: Install jq - run: sudo apt-get install -y jq - - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - name: run script with Zitadel PostgreSQL - run: NETBIRD_DOMAIN=use-ip bash -x infrastructure_files/getting-started-with-zitadel.sh + - name: Verify enterprise Traefik tag validation + run: bash infrastructure_files/tests/test-getting-started-enterprise.sh - - name: test Caddy file gen postgres - run: test -f Caddyfile + - name: Verify community Traefik image validation + run: bash infrastructure_files/tests/test-getting-started.sh - - name: test docker-compose file gen postgres - run: test -f docker-compose.yml + - name: Verify community Compose network scoping + run: bash infrastructure_files/tests/test-getting-started-networks.sh - - name: test management.json file gen postgres - run: test -f management.json + - name: Verify migration Compose network scoping + run: bash infrastructure_files/tests/test-getting-started-networks.sh --migration - - name: test turnserver.conf file gen postgres + - name: Verify fresh-install session cookie key hardening run: | - set -x - test -f turnserver.conf - grep external-ip turnserver.conf + grep -Fxq ' SESSION_COOKIE_ENCRYPTION_KEY=$(openssl rand -base64 32)' infrastructure_files/getting-started.sh + grep -Fxq ' sessionCookieEncryptionKey: "$SESSION_COOKIE_ENCRYPTION_KEY"' infrastructure_files/getting-started.sh + grep -Fxq ' install -m 600 /dev/null config.yaml' infrastructure_files/getting-started.sh + grep -Fxq ' openssl rand -base64 32' infrastructure_files/getting-started-enterprise.sh + grep -Fxq ' NETBIRD_SESSION_COOKIE_ENCRYPTION_KEY=$(rand_b64_key)' infrastructure_files/getting-started-enterprise.sh + grep -Fxq ' sessionCookieEncryptionKey: "${NETBIRD_SESSION_COOKIE_ENCRYPTION_KEY}"' infrastructure_files/getting-started-enterprise.sh - - name: test zitadel.env file gen postgres - run: test -f zitadel.env - - - name: test dashboard.env file gen postgres - run: test -f dashboard.env - - - name: test relay.env file gen postgres - run: test -f relay.env - - - name: test zdb.env file gen postgres - run: test -f zdb.env - - - name: Postgres run cleanup + - name: Verify Dex retirement notice run: | - docker compose down --volumes --rmi all - rm -rf docker-compose.yml Caddyfile zitadel.env dashboard.env machinekey/zitadel-admin-sa.token turnserver.conf management.json zdb.env + if infrastructure_files/getting-started-with-dex.sh >stdout.txt 2>stderr.txt; then + echo "Expected the retired Dex installer to fail" + exit 1 + fi + test ! -s stdout.txt + grep -Fq "Dex support is not deprecated." stderr.txt + grep -Fq "https://docs.netbird.io/selfhosted/selfhosted-quickstart" stderr.txt + grep -Fq "https://docs.netbird.io/selfhosted/identity-providers/local" stderr.txt + grep -Fq "removed in NetBird v0.80" stderr.txt - - name: run script with Zitadel CockroachDB - run: bash -x infrastructure_files/getting-started-with-zitadel.sh - env: - NETBIRD_DOMAIN: use-ip - ZITADEL_DATABASE: cockroach - - - name: test Caddy file gen CockroachDB - run: test -f Caddyfile - - - name: test docker-compose file gen CockroachDB - run: test -f docker-compose.yml - - - name: test management.json file gen CockroachDB - run: test -f management.json - - - name: test turnserver.conf file gen CockroachDB + - name: Verify Zitadel retirement notice run: | - set -x - test -f turnserver.conf - grep external-ip turnserver.conf - - - name: test zitadel.env file gen CockroachDB - run: test -f zitadel.env - - - name: test dashboard.env file gen CockroachDB - run: test -f dashboard.env - - - name: test relay.env file gen CockroachDB - run: test -f relay.env + if bash infrastructure_files/getting-started-with-zitadel.sh >stdout.txt 2>stderr.txt; then + echo "Expected the retired Zitadel installer to fail" + exit 1 + fi + test ! -s stdout.txt + grep -Fq "Zitadel support and existing Zitadel deployments are not deprecated." stderr.txt + grep -Fq "https://docs.netbird.io/selfhosted/selfhosted-quickstart" stderr.txt + grep -Fq "https://docs.netbird.io/selfhosted/identity-providers/zitadel" stderr.txt + grep -Fq "https://docs.netbird.io/selfhosted/selfhosted-guide" stderr.txt + grep -Fq "removed in NetBird v0.80" stderr.txt diff --git a/.github/workflows/ui-translations.yml b/.github/workflows/ui-translations.yml new file mode 100644 index 000000000..9ac524495 --- /dev/null +++ b/.github/workflows/ui-translations.yml @@ -0,0 +1,43 @@ +name: UI Translations + +on: + pull_request: + paths: + - "client/ui/i18n/locales/**" + - "client/ui/i18n/check-translations.mjs" + - ".github/workflows/ui-translations.yml" + push: + branches: + - main + paths: + - "client/ui/i18n/locales/**" + - "client/ui/i18n/check-translations.mjs" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }} + cancel-in-progress: true + +jobs: + check-translations: + name: Check translation key parity + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@v7 + with: + node-version: "22" + + # English (en) is the source of truth for translation keys. Locales declared + # in _index.json fail on orphaned keys or placeholder mismatches; missing + # keys only warn, since they fall back to English at runtime. + - name: Check translation key parity + run: node client/ui/i18n/check-translations.mjs diff --git a/.github/workflows/wasm-build-validation.yml b/.github/workflows/wasm-build-validation.yml index 35855918d..5f21472e5 100644 --- a/.github/workflows/wasm-build-validation.yml +++ b/.github/workflows/wasm-build-validation.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: concurrency: @@ -27,7 +28,7 @@ jobs: with: go-version-file: "go.mod" - name: Install dependencies - run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev libpcap-dev + run: sudo apt update && sudo apt install -y -q libgtk-4-dev libwebkitgtk-6.0-dev libsoup-3.0-dev libgl1-mesa-dev xorg-dev libpcap-dev - name: Install golangci-lint uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee #v9.2.1 with: diff --git a/.gitignore b/.gitignore index 305f3cb50..5c01f6e60 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,10 @@ vendor/ /netbird client/netbird-electron/ management/server/types/testdata/ + +# generated by chglog in the release workflow, embedded into the RPM +changelog.yml + +# generated by rpm-provides.sh, the config GoReleaser actually runs +.goreleaser.generated.yaml +.chglog.yml diff --git a/.golangci.yaml b/.golangci.yaml index 900af4ac0..e350b9de7 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -114,6 +114,16 @@ linters: - linters: - staticcheck text: "QF1012" + # client/ui/main.go uses //go:embed all:frontend/dist; the + # directory is populated by `pnpm build` in the release pipeline + # and missing at lint time, so the embed parses to "no matching + # files found" — surfaced by golangci-lint's typecheck pre-pass. + # Suppress just that one diagnostic; the rest of the package + # (services/, tray.go, grpc.go, ...) still gets linted normally. + - linters: + - typecheck + path: client/ui/main\.go + text: "pattern all:frontend/dist" paths: - third_party$ - builtin$ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index a2640dc8e..41c330a64 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -40,6 +40,73 @@ builds: tags: - load_wgnt_from_rsrc + # deb and rpm packages target glibc distributions, so on amd64 and arm64, the only + # architectures with a PKCS#11 driver, they carry the PKCS#11 store, which links libc. + # Tarballs, the Alpine-based images and the other architectures keep the static build. + - id: netbird-pkcs11 + dir: client + binary: netbird + env: [CGO_ENABLED=0] + goos: + - linux + goarch: + - amd64 + - arm64 + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + tags: + - load_wgnt_from_rsrc + - pkcs11 + + # The 32-bit deb architectures, which have no PKCS#11 driver. + - id: netbird-deb-32bit + dir: client + binary: netbird + env: [CGO_ENABLED=0] + goos: + - linux + goarch: + - arm + - 386 + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + tags: + - load_wgnt_from_rsrc + + # Single-arch builds: nfpm provides is not templated, so the RPM splits per arch. The + # ones with a PKCS#11 driver carry the store like the deb build above. + - &netbird_rpm_build + id: netbird-rpm-amd64 + dir: client + binary: netbird + env: [CGO_ENABLED=0] + goos: [linux] + goarch: [amd64] + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + tags: + - load_wgnt_from_rsrc + - pkcs11 + + - <<: *netbird_rpm_build + id: netbird-rpm-arm64 + goarch: [arm64] + + - <<: *netbird_rpm_build + id: netbird-rpm-arm + goarch: [arm] + tags: + - load_wgnt_from_rsrc + + - <<: *netbird_rpm_build + id: netbird-rpm-386 + goarch: [386] + tags: + - load_wgnt_from_rsrc + - id: netbird-static dir: client binary: netbird @@ -168,6 +235,29 @@ builds: - -s -w -X main.Version={{.Version}} -X main.Commit={{.Commit}} -X main.BuildDate={{.CommitDate}} mod_timestamp: "{{ .CommitTimestamp }}" + - id: nblink + dir: client/link/cmd/nblink + env: [CGO_ENABLED=0] + binary: nblink + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + - arm + goarm: + - 7 + ignore: + - goos: windows + goarch: arm + - goos: darwin + goarch: arm + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} + mod_timestamp: "{{ .CommitTimestamp }}" + - id: netbird-idp-migrate dir: tools/idp-migrate env: @@ -190,6 +280,28 @@ builds: - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser mod_timestamp: "{{ .CommitTimestamp }}" + - id: netbird-mysql-migrate + dir: tools/mysql-migrate + env: + - CGO_ENABLED=1 + - >- + {{- if eq .Runtime.Goos "linux" }} + {{- if eq .Arch "arm64"}}CC=aarch64-linux-gnu-gcc{{- end }} + {{- if eq .Arch "arm"}}CC=arm-linux-gnueabihf-gcc{{- end }} + {{- end }} + binary: netbird-mysql-migrate + goos: + - linux + goarch: + - amd64 + - arm64 + - arm + goarm: + - 7 + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + universal_binaries: - id: netbird @@ -206,38 +318,106 @@ archives: builds: - netbird-idp-migrate name_template: "netbird-idp-migrate_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + - id: netbird-mysql-migrate + builds: + - netbird-mysql-migrate + name_template: "netbird-mysql-migrate_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + - id: nblink + builds: + - nblink + name_template: "nblink_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + # goreleaser archives as tar.gz everywhere unless told otherwise, and + # nblink is downloaded and extracted by hand on Windows too. + format_overrides: + - goos: windows + format: zip nfpms: - maintainer: Netbird description: Netbird client. homepage: https://netbird.io/ license: BSD-3-Clause + vendor: NetBird id: netbird_deb bindir: /usr/bin builds: - - netbird + - netbird-pkcs11 + - netbird-deb-32bit formats: - deb scripts: postinstall: "release_files/post_install.sh" preremove: "release_files/pre_remove.sh" - - maintainer: Netbird + - &netbird_rpm + maintainer: Netbird description: Netbird client. homepage: https://netbird.io/ license: BSD-3-Clause - id: netbird_rpm + vendor: NetBird + id: netbird_rpm_amd64 bindir: /usr/bin - builds: - - netbird + ids: + - netbird-rpm-amd64 formats: - rpm + # Red Hat certification (RPM Version Handling) requires rpmbuild's ISA + # provide, which nfpm does not emit. The version is filled in by the release job. + provides: + - "netbird(x86-64) = @RPM_EVR@" + # The client verifies TLS to management and signal against the system trust + # store. Red Hat software certification (RPM Dependency Tracking) also + # rejects packages that declare no dependencies at all. + dependencies: + - ca-certificates + # Generated in CI by chglog from git tags; Red Hat certification requires an + # RPM changelog (RPM Version Handling subtest). + changelog: changelog.yml + # License, documentation and a config file so the RPM Provenance subtest sees + # %license, %doc and %config entries instead of a bare binary. + contents: + - src: LICENSE + dst: /usr/share/licenses/netbird/LICENSE + type: license + - src: README.md + dst: /usr/share/doc/netbird/README.md + type: doc + - src: release_files/netbird.sysconfig + dst: /etc/sysconfig/netbird + type: config|noreplace + # May hold the PKCS#11 token PIN (NB_TPM_PIN), so readable by root alone. + file_info: + mode: 0600 scripts: postinstall: "release_files/post_install.sh" preremove: "release_files/pre_remove.sh" rpm: + summary: NetBird client + group: Applications/Internet + packager: NetBird signature: key_file: '{{ if index .Env "GPG_RPM_KEY_FILE" }}{{ .Env.GPG_RPM_KEY_FILE }}{{ end }}' + + - <<: *netbird_rpm + id: netbird_rpm_arm64 + ids: + - netbird-rpm-arm64 + provides: + - "netbird(aarch-64) = @RPM_EVR@" + + - <<: *netbird_rpm + id: netbird_rpm_arm + ids: + - netbird-rpm-arm + provides: + - "netbird(armv6hl-32) = @RPM_EVR@" + + - <<: *netbird_rpm + id: netbird_rpm_386 + ids: + - netbird-rpm-386 + provides: + - "netbird(x86-32) = @RPM_EVR@" dockers_v2: - id: netbird disable: "{{ .Env.SKIP_DOCKER_PUSH }}" @@ -271,8 +451,8 @@ dockers_v2: - netbirdio/netbird - ghcr.io/netbirdio/netbird tags: - - "v{{ .Version }}-rootless" - - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}latest{{ end }}" + - "{{ .Version }}-rootless" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}rootless-latest{{ end }}" dockerfile: client/Dockerfile-rootless extra_files: - client/netbird-entrypoint.sh @@ -287,6 +467,43 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: netbird-rootless-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird + images: + - netbirdio/netbird + - ghcr.io/netbirdio/netbird + tags: + - "{{ .Version }}-rootless-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}rootless-ubi-latest{{ end }}" + dockerfile: client/Dockerfile-rootless.ubi + extra_files: + - client/netbird-entrypoint.sh + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -t load_wgnt_from_rsrc "{{ .ContextDir }}/licenses" ./client amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=0 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: relay disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -309,6 +526,41 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: relay-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-relay + images: + - netbirdio/relay + - ghcr.io/netbirdio/relay + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: relay/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l relay/LICENSE "{{ .ContextDir }}/licenses" ./relay amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=0 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: signal disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -331,6 +583,41 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: signal-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-signal + images: + - netbirdio/signal + - ghcr.io/netbirdio/signal + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: signal/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l signal/LICENSE "{{ .ContextDir }}/licenses" ./signal amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=0 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: management disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -353,6 +640,41 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: management-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-mgmt + images: + - netbirdio/management + - ghcr.io/netbirdio/management + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: management/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l management/LICENSE "{{ .ContextDir }}/licenses" ./management amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=1 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: upload disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -363,7 +685,7 @@ dockers_v2: tags: - "{{ .Version }}" - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}latest{{ end }}" - dockerfile: upload-server/Dockerfile + dockerfile: upload-server/Dockerfile.release platforms: - linux/amd64 - linux/arm64 @@ -397,6 +719,41 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: netbird-server-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-server + images: + - netbirdio/netbird-server + - ghcr.io/netbirdio/netbird-server + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: combined/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l combined/LICENSE "{{ .ContextDir }}/licenses" ./combined amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=1 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: netbird-proxy disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -419,6 +776,63 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: nblink + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - nblink + images: + - netbirdio/nblink + - ghcr.io/netbirdio/nblink + tags: + - "{{ .Version }}" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}latest{{ end }}" + dockerfile: client/Dockerfile-nblink + platforms: + - linux/amd64 + - linux/arm64 + - linux/arm + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "nblink" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" + - id: proxy-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-proxy + images: + - netbirdio/reverse-proxy + - ghcr.io/netbirdio/reverse-proxy + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: proxy/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l proxy/LICENSE -w "{{ .ContextDir }}/licenses" ./proxy/cmd/proxy amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=0 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" brews: - ids: @@ -451,7 +865,10 @@ uploads: - name: yum skip: "{{ .Env.SKIP_PUBLISH }}" ids: - - netbird_rpm + - netbird_rpm_amd64 + - netbird_rpm_arm64 + - netbird_rpm_arm + - netbird_rpm_386 mode: archive target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }} username: dev@wiretrustee.com @@ -466,6 +883,13 @@ checksum: - glob: ./infrastructure_files/migrate-to-enterprise.sh release: + # The signing pipeline (netbirdio/sign-pipelines, dispatched by + # trigger_signer) marks the release latest once the Windows and macOS + # artifacts are signed. Without this override goreleaser marks it latest + # at publish time, while those artifacts are still unsigned. + make_latest: false + # Mark x.y.z-rc.* and other prerelease tags as prereleases on GitHub. + prerelease: auto extra_files: - glob: ./infrastructure_files/getting-started-with-zitadel.sh - glob: ./release_files/install.sh diff --git a/.goreleaser_ui.yaml b/.goreleaser_ui.yaml index 6f9b7c059..24903188f 100644 --- a/.goreleaser_ui.yaml +++ b/.goreleaser_ui.yaml @@ -2,6 +2,15 @@ version: 2 env: - SKIP_PUBLISH={{ if index .Env "SKIP_PUBLISH" }}{{ .Env.SKIP_PUBLISH }}{{ else }}true{{ end }} project_name: netbird-ui + +before: + hooks: + # Bindings are gitignored; regenerate before the frontend build so + # the @wailsio/runtime Vite plugin can resolve them (vite refuses to + # build without them). + - sh -c 'cd client/ui && wails3 generate bindings -clean=true -ts' + - sh -c 'cd client/ui/frontend && pnpm install --frozen-lockfile && pnpm build' + builds: - id: netbird-ui dir: client/ui @@ -15,6 +24,8 @@ builds: ldflags: - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser mod_timestamp: "{{ .CommitTimestamp }}" + tags: + - production - id: netbird-ui-windows-amd64 dir: client/ui @@ -30,6 +41,8 @@ builds: - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser - -H windowsgui mod_timestamp: "{{ .CommitTimestamp }}" + tags: + - production - id: netbird-ui-windows-arm64 dir: client/ui @@ -46,6 +59,8 @@ builds: - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser - -H windowsgui mod_timestamp: "{{ .CommitTimestamp }}" + tags: + - production archives: - id: linux-arch @@ -62,6 +77,8 @@ nfpms: - maintainer: Netbird description: Netbird client UI. homepage: https://netbird.io/ + license: BSD-3-Clause + vendor: NetBird id: netbird_ui_deb package_name: netbird-ui builds: @@ -71,16 +88,26 @@ nfpms: scripts: postinstall: "release_files/ui-post-install.sh" contents: - - src: client/ui/build/netbird.desktop - dst: /usr/share/applications/netbird.desktop - - src: client/ui/assets/netbird.png + - src: client/ui/build/linux/netbird.desktop + dst: /usr/share/applications/org.wails.netbird.desktop + - src: client/ui/build/appicon.png dst: /usr/share/pixmaps/netbird.png + # Names the polkit action for the elevation prompt the app raises when an + # unprivileged user changes a privileged setting; without it the dialog + # shows a raw command line. + - src: client/ui/build/linux/polkit/io.netbird.settings.policy + dst: /usr/share/polkit-1/actions/io.netbird.settings.policy dependencies: - - netbird + - netbird (>= 0.75.0) + - libgtk-4-1 (>= 4.14) + - libwebkitgtk-6.0-4 + - xdg-utils - maintainer: Netbird description: Netbird client UI. homepage: https://netbird.io/ + license: BSD-3-Clause + vendor: NetBird id: netbird_ui_rpm package_name: netbird-ui builds: @@ -90,12 +117,21 @@ nfpms: scripts: postinstall: "release_files/ui-post-install.sh" contents: - - src: client/ui/build/netbird.desktop - dst: /usr/share/applications/netbird.desktop - - src: client/ui/assets/netbird.png + - src: client/ui/build/linux/netbird.desktop + dst: /usr/share/applications/org.wails.netbird.desktop + - src: client/ui/build/appicon.png dst: /usr/share/pixmaps/netbird.png + # Names the polkit action for the elevation prompt the app raises when an + # unprivileged user changes a privileged setting; without it the dialog + # shows a raw command line. + - src: client/ui/build/linux/polkit/io.netbird.settings.policy + dst: /usr/share/polkit-1/actions/io.netbird.settings.policy dependencies: - - netbird + - netbird >= 0.75.0 + - (gtk4 >= 4.14 or libgtk-4-1 >= 4.14) + - (webkitgtk6.0 or libwebkitgtk-6_0-4) + - xdg-utils + rpm: signature: key_file: '{{ if index .Env "GPG_RPM_KEY_FILE" }}{{ .Env.GPG_RPM_KEY_FILE }}{{ end }}' @@ -118,3 +154,11 @@ uploads: target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }} username: dev@wiretrustee.com method: PUT + +release: + # Uploads into the release created by the main .goreleaser.yaml run. + # make_latest stays false everywhere: the signing pipeline + # (netbirdio/sign-pipelines) marks the release latest after the Windows + # and macOS artifacts are signed. + make_latest: false + prerelease: auto diff --git a/.goreleaser_ui_darwin.yaml b/.goreleaser_ui_darwin.yaml index 0a0082075..8ca0e8da6 100644 --- a/.goreleaser_ui_darwin.yaml +++ b/.goreleaser_ui_darwin.yaml @@ -1,6 +1,15 @@ version: 2 project_name: netbird-ui + +before: + hooks: + # Bindings are gitignored; regenerate before the frontend build so + # the @wailsio/runtime Vite plugin can resolve them (vite refuses to + # build without them). + - sh -c 'cd client/ui && wails3 generate bindings -clean=true -ts' + - sh -c 'cd client/ui/frontend && pnpm install --frozen-lockfile && pnpm build' + builds: - id: netbird-ui-darwin dir: client/ui @@ -21,7 +30,7 @@ builds: - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser mod_timestamp: "{{ .CommitTimestamp }}" tags: - - load_wgnt_from_rsrc + - production universal_binaries: - id: netbird-ui-darwin @@ -34,3 +43,11 @@ checksum: name_template: "{{ .ProjectName }}_darwin_checksums.txt" changelog: disable: true + +release: + # Uploads into the release created by the main .goreleaser.yaml run. + # make_latest stays false everywhere: the signing pipeline + # (netbirdio/sign-pipelines) marks the release latest after the Windows + # and macOS artifacts are signed. + make_latest: false + prerelease: auto diff --git a/.goreleaser_ui_gtk3.yaml b/.goreleaser_ui_gtk3.yaml new file mode 100644 index 000000000..a9b2ca650 --- /dev/null +++ b/.goreleaser_ui_gtk3.yaml @@ -0,0 +1,144 @@ +version: 2 +env: + - SKIP_PUBLISH={{ if index .Env "SKIP_PUBLISH" }}{{ .Env.SKIP_PUBLISH }}{{ else }}true{{ end }} +project_name: netbird-ui + +before: + hooks: + # Bindings are gitignored; regenerate before the frontend build so + # the @wailsio/runtime Vite plugin can resolve them (vite refuses to + # build without them). + # -f '-tags gtk3': the generator type-checks client/ui, whose cgo imports + # would otherwise resolve gtk4/webkitgtk-6.0 pkg-config entries that do + # not exist on ubuntu-22.04. + - sh -c 'cd client/ui && wails3 generate bindings -clean=true -ts -f "-tags gtk3"' + - sh -c 'cd client/ui/frontend && pnpm install --frozen-lockfile && pnpm build' + +builds: + # Legacy GTK3 / WebKit2GTK 4.1 build for distros without WebKitGTK 6.0 + # (Ubuntu 22.04, Debian 12, RHEL 9, Fedora <=39). The gtk3 tag flips the + # Wails Linux backend to the GTK3 stack and swaps our GTK4-only XEmbed + # tray host for the pure-Go stub (client/ui/xembed_host_gtk3_linux.go). + # Must be built on the oldest supported glibc (ubuntu-22.04 runner). + - id: netbird-ui-gtk3 + dir: client/ui + binary: netbird-ui + env: + - CGO_ENABLED=1 + goos: + - linux + goarch: + - amd64 + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + tags: + - production + - gtk3 + +archives: + - id: linux-gtk3-arch + name_template: "{{ .ProjectName }}-linux-gtk3_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + builds: + - netbird-ui-gtk3 + +nfpms: + # Mutually-exclusive alternative to the GTK4 netbird-ui package -- both + # ship the same /usr/bin/netbird-ui from the shared stable/yum repos, so + # this one carries its own name and conflicts with the GTK4 package. + - maintainer: Netbird + description: Netbird client UI. + homepage: https://netbird.io/ + license: BSD-3-Clause + vendor: NetBird + id: netbird_ui_deb_gtk3 + package_name: netbird-ui-gtk3 + file_name_template: "{{ .PackageName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + builds: + - netbird-ui-gtk3 + formats: + - deb + scripts: + postinstall: "release_files/ui-post-install.sh" + contents: + - src: client/ui/build/linux/netbird.desktop + dst: /usr/share/applications/org.wails.netbird.desktop + - src: client/ui/build/appicon.png + dst: /usr/share/pixmaps/netbird.png + conflicts: + - netbird-ui + replaces: + - netbird-ui + dependencies: + - netbird (>= 0.75.0) + - libgtk-3-0 + - libwebkit2gtk-4.1-0 + - xdg-utils + + - maintainer: Netbird + description: Netbird client UI. + homepage: https://netbird.io/ + license: BSD-3-Clause + vendor: NetBird + id: netbird_ui_rpm_gtk3 + package_name: netbird-ui-gtk3 + file_name_template: "{{ .PackageName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + builds: + - netbird-ui-gtk3 + formats: + - rpm + scripts: + postinstall: "release_files/ui-post-install.sh" + contents: + - src: client/ui/build/linux/netbird.desktop + dst: /usr/share/applications/org.wails.netbird.desktop + - src: client/ui/build/appicon.png + dst: /usr/share/pixmaps/netbird.png + # No `replaces` here: nfpm maps it to rpm Obsoletes, which would make + # dnf swap installed GTK4 netbird-ui packages for this one on upgrade. + conflicts: + - netbird-ui + dependencies: + - netbird >= 0.75.0 + - (gtk3 or libgtk-3-0) + - (webkit2gtk4.1 or libwebkit2gtk-4_1-0) + - xdg-utils + + rpm: + signature: + key_file: '{{ if index .Env "GPG_RPM_KEY_FILE" }}{{ .Env.GPG_RPM_KEY_FILE }}{{ end }}' + +# The GTK4 UI job shares project_name, so the default checksum file name would +# collide with it on the shared GitHub release. +checksum: + name_template: "{{ .ProjectName }}_gtk3_checksums.txt" + +changelog: + disable: true + +uploads: + - name: debian + skip: "{{ .Env.SKIP_PUBLISH }}" + ids: + - netbird_ui_deb_gtk3 + mode: archive + target: https://pkgs.wiretrustee.com/debian/pool/{{ .ArtifactName }};deb.distribution=stable;deb.component=main;deb.architecture={{ if .Arm }}armhf{{ else }}{{ .Arch }}{{ end }};deb.package= + username: dev@wiretrustee.com + method: PUT + + - name: yum + skip: "{{ .Env.SKIP_PUBLISH }}" + ids: + - netbird_ui_rpm_gtk3 + mode: archive + target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }} + username: dev@wiretrustee.com + method: PUT + +release: + # Uploads into the release created by the main .goreleaser.yaml run. + # make_latest stays false everywhere: the signing pipeline + # (netbirdio/sign-pipelines) marks the release latest after the Windows + # and macOS artifacts are signed. + make_latest: false + prerelease: auto diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..3838a7913 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,783 @@ +# NetBird Agent Guidelines + +**NetBird** is an open source connectivity platform: a WireGuard®-based overlay +network with a control plane. The **agent** (`client/`) runs on user machines as +a privileged daemon and manages the WireGuard interface, routing, firewall, and +DNS. **Management** (`management/`) is the control plane and REST/gRPC API, +**Signal** (`signal/`) brokers peer handshakes, **Relay** (`relay/`) carries +traffic when a direct tunnel is impossible, and **Proxy** (`proxy/`) is the +identity-aware proxy behind Agent Network. + +This file applies to the whole repository, and is the single source of truth for +agent guidance here. `CLAUDE.md` is a one-line pointer to it — keep the guidance +in this file, not duplicated there. + +## Contents + +- [STOP and ask the user before](#stop-and-ask-the-user-before) +- [Quick reference](#quick-reference) +- [Structure](#structure) +- [Where to look](#where-to-look) +- [Security](#security) +- [Agent conventions](#agent-conventions) +- [Repo-wide principles](#repo-wide-principles) +- [Type safety](#type-safety) +- [Concurrency and lifecycle](#concurrency-and-lifecycle) +- [Error handling](#error-handling) +- [Comments](#comments) +- [Testing](#testing) +- [Pitfalls](#pitfalls) +- [Commits, PRs, releases](#commits-prs-releases) +- [After you push: CI and review bots](#after-you-push-ci-and-review-bots) +- [Discussion and support](#discussion-and-support) + +## STOP and ask the user before + +- **Opening a pull request for anything beyond a trivial fix, without an agreed + ticket.** Ask the user directly: *"Is there a discussion or issue for this + change?"* NetBird is discussion-first — community reports start in + [Discussions](https://github.com/netbirdio/netbird/discussions), DevRel + validates them, and only validated discussions become issues. A PR that + changes behavior with no linked issue may be closed on arrival. If there is no + ticket, offer to draft the discussion post **instead of** the PR, and wait for + the user's call. Only typos, broken links, documentation corrections, and + one-line fixes that already have an issue can skip this. +- **Designing in any high-risk area** (see + [CONTRIBUTING.md](CONTRIBUTING.md#high-risk-areas)): public API and OpenAPI + schema, gRPC protos, behavior existing deployments would notice after an + upgrade, peer connectivity (ICE, NAT traversal, relay selection, WireGuard® or + Rosenpass key handling), client system integration (routing, firewall, DNS, + interface), authentication and authorization, CLI or service flags, config + file format, daemon IPC, store schema and migrations, or a new feature. The + design gets agreed in the ticket before code is written. +- **Writing a store migration or changing a persisted model.** Migrations are + one-way in the field and both the GORM and pgx paths may need the change. +- **Hand-editing generated code.** `*.pb.go`, `*.gen.go`, and mocks are outputs. + Edit the source (`.proto`, `openapi.yml`) and rerun the matching + `generate.sh`. +- **Adding, removing, or bumping a dependency**, and never vendor a fork. +- **Weakening a security control** — authentication, authorization, certificate + verification, privilege dropping, or peer identity checks — even when it is + the fastest way to make a test pass. +- **Force-pushing to `main`**, force-pushing any branch that is already under + review, amending pushed commits, or bypassing hooks with `--no-verify`. + +## Quick reference + +```bash +# Build +go build ./... +cd client && CGO_ENABLED=0 go build . # agent +cd management && go build . # management service +cd signal && go build . # signal service + +# Verify (run before every push) +go fmt ./... +make lint # golangci-lint on files changed vs origin/main (also the pre-push hook) +make lint-all # full-repository lint, matches CI +make test-unit # host-safe unit tests, -tags devcert, no sudo +make test-privileged # privileged-tagged suite in a Docker container with NET_ADMIN +make setup-hooks # wire .githooks: pre-push runs make lint, commit-msg refuses attribution trailers + +# Narrow runs +go test ./client/internal/dns/... +go test -race -run TestPeerConn ./client/internal/peer/... +PRIV_RUN=TestNftablesManager PRIV_PKGS=./client/firewall/nftables/... make test-privileged + +# Code generation (never hand-edit the output) +./shared/management/http/api/generate.sh # REST types from openapi.yml +./shared/management/proto/generate.sh +./shared/signal/proto/generate.sh +./client/proto/generate.sh +./flow/proto/generate.sh + +# Run locally (lab only, never on a machine you rely on) +sudo ./client/netbird up --log-level debug --log-file console +sudo ./client/netbird down # teardown: restores routing, firewall, DNS +./signal/signal run --log-level debug --log-file console +./management/management management --log-level debug --log-file console --config ./management.json +``` + +`netbird up` needs root and rewrites the host's routing table, firewall rules, +DNS configuration, and WireGuard® interface. Run it only in a disposable test +environment (a VM, container, or throwaway host) that you can rebuild, never on +a workstation or server whose connectivity matters. Run `sudo netbird down` +before you stop working, before rebuilding the binary, and on every failure +path, so the host's networking state is restored instead of left half-applied. +See [Pitfalls](#pitfalls) for why cleanup on every exit path matters. + +## Structure + +```text +netbird/ +├── client/ NetBird agent +│ ├── cmd/ agent CLI +│ ├── internal/ agent business logic (engine, peer, dns, routemanager, ...) +│ ├── server/ daemon for background execution +│ ├── proto/ daemon gRPC protos +│ ├── iface/ WireGuard® interface management +│ ├── firewall/ nftables, iptables, pf, WFP, userspace backends +│ ├── ssh/ built-in SSH server and client +│ ├── ui/ desktop UI (Wails v3 + React) +│ ├── android/, ios/ mobile bindings +│ ├── wasm/ WebAssembly build +│ └── mdm/, system/ MDM policy, host information +├── management/ control plane +│ └── server/ account, peer, groups, networks, posture, permissions, +│ settings, store, http (REST), idp, integrations, migration +├── signal/ handshake broker (peer/, server/) +├── relay/ relay service (protocol/, server/, healthcheck/) +├── proxy/ identity-aware proxy (llm/, acme/, accesslog/, middleware/, tcp/, udp/) +├── agent-network/ Agent Network overview +├── shared/ imported by both agent and services +│ ├── management/ proto/, client/, http/api (OpenAPI + generated types) +│ ├── signal/ proto/, client/ +│ └── relay/, auth/, sshauth/, metrics/ +├── e2e/ end-to-end suites and harness +├── encryption/, dns/, route/, stun/, sharedsock/, util/, flow/ +├── infrastructure_files/ docker compose and getting-started templates +└── release_files/ files packaged into releases +``` + +## Where to look + +| Task | Location | +| --------------------------- | ------------------------------------------------------------ | +| REST API / OpenAPI | `shared/management/http/api/` + `management/server/http/` | +| Management gRPC protocol | `shared/management/proto/` | +| Signal protocol | `shared/signal/proto/` | +| Daemon IPC protocol | `client/proto/` | +| Peer connection and NAT | `client/internal/peer/` | +| Network map handling | `client/internal/engine.go`, `shared/management/networkmap/` | +| Routing | `client/internal/routemanager/`, `route/` | +| Firewall backends | `client/firewall/` | +| DNS | `client/internal/dns/`, `dns/` | +| WireGuard® interface | `client/iface/` | +| Persistence and migrations | `management/server/store/`, `management/server/migration/` | +| IdP integrations | `management/server/idp/` | +| Permissions model | `management/server/permissions/` | +| LLM routing / Agent Network | `proxy/internal/llm/`, `agent-network/` | +| End-to-end tests | `e2e/` | + +## Security + +### Never fail open + +When a security check — access control, an IP restriction, an auth decision — +hits an error such as an unparseable value, an unavailable lookup, or a state it +does not recognize, it must **deny**. Never skip the check or allow the request +through because the check itself failed, and make the `default` and unknown cases +of a security-related `switch` deny rather than fall through. + +### Daemon RPC input is untrusted + +The agent runs as root (LocalSystem on Windows), so a daemon RPC crosses a +privilege boundary: treat every field as untrusted input rather than as something +the UI or CLI validated on the way in. + +When you add or change an RPC, ask what the handler does with caller input while +running as root. If the answer touches a filesystem path, a URL or host, or a +privileged state change, it needs a gate **in the handler** — a check in the client +that normally calls it is not a check at all. + +- **A caller-supplied path the daemon opens.** Never `os.Open` it as root. + Constrain it, then open it *as the caller* with `ipcauth.OpenOwnedFile`, which + opens `O_NOFOLLOW`, requires a regular file, and refuses a file the caller does + not own — so a symlink or hardlink aimed at a root-only file is rejected. +- **A caller-supplied URL or host the daemon fetches.** Restrict the scheme and + allow only known hosts for unprivileged callers. Prefer a lexical host + allowlist plus TLS verification over "resolve the host, then reject private + IPs": the resolve-then-trust pattern has a DNS-rebinding race (public IP at + check time, attacker IP at connect time), while a name allowlist has no IP + check to race. Never accept `http://` where `https://` is expected. +- **A privileged state change** (SSH root login, management URL, deregistration) + gates on the caller identity from `ipcauth.CallerIdentity(ctx)`. + +Caller identity comes from the kernel — `SO_PEERCRED`, `LOCAL_PEERCRED`, or the +named-pipe client token — and never from an RPC field. When +`ipcauth.CallerIdentity` reports that it could not determine an identity, **deny**; +do not fall back to treating the caller as the transport peer. + +## Agent conventions + +### Three networking modes + +Where packets actually flow depends on the mode the agent is running in. The +three are not interchangeable, so establish which one a change applies to — and +what it should do in the other two — before you write it. + +- **kernel mode** (Linux only): in-kernel WireGuard®. The kernel handles both + peer-to-peer and routed traffic, and ACLs are iptables or nftables rules. The + client programs kernel facilities but never sees the traffic itself. +- **userspace mode** (wireguard-go with a TUN): wireguard-go runs in-process. The + kernel handles peer-to-peer traffic once it leaves the TUN, while routed traffic + — exit nodes and network routes — goes through the userspace forwarder, which + terminates the connection and re-establishes it over OS sockets. Used on + platforms without kernel WireGuard® or when the user opts out. +- **netstack mode**: wireguard-go in-process with no TUN and no kernel + networking. The forwarder does all routing by stitching userspace sockets, and + listeners such as the embedded SSH and DNS servers bind on a gVisor netstack. + Used where the process cannot create a TUN device, such as the embedded client + (`client/embed/`) and the WASM build. + +### The overlay interface is not "WireGuard" + +Do not put "WireGuard" in identifiers or comments unless the code is genuinely +coupled to WireGuard® specifically — a wireguard-go call, a handshake field, a +kernel WireGuard® netlink attribute. For the interface, the host, peers, or +traffic in general, say "the NetBird interface", "the interface", or "the overlay". +Most firewall, routing, and DNS code is transport-agnostic, so a WireGuard® +reference there is simply inaccurate and rots as the transports change. + +### IPv6 is a soft feature + +The IPv6 overlay is opt-in dual-stack, and capability can change at runtime. Treat +it as soft rather than a requirement: + +- Gate local v6 paths on the interface accessor (`wgIface.Address().HasIPv6()`), + not on raw state fields, and skip the v6 path when the host has no v6 rather + than returning an error. +- Treat an empty or unparseable peer v6 address as "no v6 for that peer" and skip + it, keeping the v4 path working. +- Never let a missing v6 break v4. Fail-closed is for security checks; a + capability mismatch skips the v6 work and carries on. + +### Environment variables + +Name the variable in a constant and parse booleans with `strconv.ParseBool` rather +than comparing strings inline, so an unexpected value is logged instead of +silently meaning false: + +```go +const EnvDisableFeature = "NB_DISABLE_FEATURE" + +func isDisabledByEnv() bool { + val := os.Getenv(EnvDisableFeature) + if val == "" { + return false + } + disabled, err := strconv.ParseBool(val) + if err != nil { + log.Warnf("failed to parse %s: %v", EnvDisableFeature, err) + return false + } + return disabled +} +``` + +### Validating against protocol specs + +When a change depends on what a protocol actually mandates, read the specification +text from the [IETF datatracker](https://datatracker.ietf.org/) rather than a +summary, and check that you have the current RFC — the widely cited one for a +protocol is often superseded. Cite the section, not just the document, so a +reviewer can jump straight to the rule. + +## Repo-wide principles + +1. **Run `go fmt` on every modified Go file.** Formatting is not optional. +2. **Zero unaddressed linter warnings.** Fix what `golangci-lint` reports on code + you touch, and delete imports, helpers, and parameters your refactor orphaned. + Exception: unused parameters in shared code may be consumed by builds outside + this repository — do not remove them, ask instead. +3. **Function comments are mandatory for exported functions**, written as full + sentences with a period, starting with the identifier name. +4. **Prefer private functions and constants.** Export only what a caller outside + the package genuinely needs. +5. **Early returns and guard clauses.** Handle errors and edge cases first + instead of nesting `if`/`else` chains. +6. **Split complex functions.** If a function trips a complexity warning, break + it into named helpers rather than silencing the warning. +7. **Avoid LLM-slop tells:** em dashes, hedging narration, restating the diff in + prose, trailing summaries. Defaults, not absolute bans. Applies to code, + comments, commit messages, and PR descriptions alike. +8. **Concurrency: do a two-pass race analysis after every change** that touches + shared state, including reads of existing maps and slices. Guard them with a + mutex (or an atomic or channel where that fits better), keep critical + sections short, and run `go test -race` on the touched packages. See + [Concurrency and lifecycle](#concurrency-and-lifecycle) for the failure modes + to check for. +9. **Cross-platform builds must keep working.** The agent targets Linux, macOS, + Windows, FreeBSD, Android, and iOS. When you add a platform-specific file, + add the counterpart or a build-tagged fallback for the others. +10. **Never hand-edit generated files.** Change the source and regenerate. +11. **Never log secrets** — private keys, setup keys, tokens, PAT values — and + keep peer IPs and hostnames out of logs above debug level. + +## Type safety + +**No bare primitives for domain concepts.** A `string` parameter for an account +ID next to a `string` parameter for a peer ID is two bugs waiting to happen, +because the compiler cannot catch the swap. Declare the type once and use it +throughout, converting only at the boundaries where data enters or leaves — +protobuf, gRPC, HTTP, an external library. + +```go +type ServiceID string +type AccountID string + +// Internal: typed all the way through +func (r *Router) RemoveRoute(host SNIHost, svcID ServiceID) { ... } + +// Proto boundary: convert once, on the way in and on the way out +svcID := ServiceID(mapping.GetId()) +req.ServiceId = string(svcID) +``` + +- **IP addresses are `netip.Addr`**, not `string` and not `net.IP`. Parse at the + boundary and pass the typed value inward. +- **Always `Unmap()`** after parsing an address, after converting from `net.IP`, + and after extracting one from `RemoteAddr()`. This normalizes a v4-mapped v6 + address (`::ffff:10.1.2.3`) to plain v4 so IPv4 rules match it. A stored or + compared mapped address silently fails to match those rules. +- **Ports are `uint16`** internally; use `int` only where a library forces it and + convert immediately. +- **Enums are a typed string with constants**, so the valid set is discoverable + and a typo fails to compile. +- **Map keys follow the same rule**, and must be a real type (`type ServiceID + string`) rather than an alias (`type serviceID = string`) — an alias silently + accepts bare strings. + +## Concurrency and lifecycle + +Beyond the mutex hygiene in the principles above, check for these failure +modes. + +- **Never read a struct field inside a goroutine** when another goroutine may nil + or reassign it. Pass the value as a parameter, or capture it into a local before + launching. This matters most when `Stop()` nils a field without waiting for the + goroutine to finish. + + ```go + go func(ifaceName string) { // good: passed in, cannot be nilled underneath + m.Start(ctx, ifaceName) + }(iface.Name()) + ``` + +- **Never wait on a channel while holding a lock the sender needs.** Copy what you + need out from under the lock, release it, then wait. + + ```go + func (m *Manager) Stop() { + m.mu.Lock() + cancel, done := m.cancel, m.done + m.mu.Unlock() + if cancel != nil { + cancel() + <-done + } + } + ``` + +- **`Stop`/`Close` must be idempotent** — guard on an already-stopped flag or a + nil cancel — and must release the state they guarded. Clear maps and caches; + a cancelled goroutine holding a live map still pins that memory. Note that a + nil map only panics on writes; reads and iteration behave like an empty map, + so where post-close use must be rejected, check the stopped flag explicitly. +- **Publish coupled state only after every fallible step succeeds.** When several + fields form an invariant, build them into locals and assign them to the receiver + at the end. Assigning as you go leaves the object half-initialized when a later + step fails, so a readiness predicate reports ready while a coupled field is nil. + If an earlier step already had an external side effect — a created chain, an + opened handle, an inserted rule — roll it back before returning the error. +- **Clean up what you own on constructor error paths.** Once a constructor has + started something, every later error path must undo it: cancel a goroutine and + wait for it to exit, stop a ticker, close a watcher. The object is never + returned, so its `Close` will never run. +- **A failed `Start` must undo everything it started.** When a component brings up + several subsystems in sequence — connection manager, watchers, routing, DNS, + flow, persisted state — a failure partway through has to tear down the ones + already running, not just close the handle the error came from. Put the + already-started guard *before* that teardown path, so a rejected second `Start` + cannot dismantle the one that is running. + +## Error handling + +Use single-assignment form when the error is only needed inside the `if`: + +```go +// Good +if err := someCall(); err != nil { + return fmt.Errorf("context: %w", err) +} + +// Bad - unnecessary split +err := someCall() +if err != nil { + return fmt.Errorf("context: %w", err) +} +``` + +Use multiple assignment when the value is needed after the block: + +```go +result, err := someCall() +if err != nil { + return fmt.Errorf("context: %w", err) +} +``` + +Add short, meaningful context, and **do not** start `fmt.Errorf` messages with +obvious words like "failed to" or "error": + +```go +// Good +return fmt.Errorf("parse remote address: %w", err) +return fmt.Errorf("listen on %s: %w", addr, err) + +// Bad +return fmt.Errorf("failed to parse remote address: %w", err) +return fmt.Errorf("error listening on %s: %w", addr, err) + +// "failed" is fine in log messages +log.Debugf("failed to parse remote address: %v", err) +``` + +Skip the wrapping when a function only extracts or delegates and the wrap would +add nothing: + +```go +func parseAddr(addr string) (string, int, error) { + host, portStr, err := net.SplitHostPort(addr) + if err != nil { + return "", 0, err + } + // ... +} +``` + +Log the errors you choose not to act on: + +- `log.Debugf()` for errors that do not affect program flow but help debugging. +- `log.Tracef()` for very verbose errors that would otherwise spam logs. +- **Never ignore** errors from writes, network sends, or critical cleanup. +- Close errors may be ignored for read-only operations; log them at debug for + writes. + +**Do not log and return the same error.** It gets reported twice, from two places, +and the second reader cannot tell whether it happened once or twice. Return it and +let the caller decide. The exception is an API handler that has already written a +response. Internal helpers return errors rather than logging and swallowing them. + +**Never return a typed nil as an error.** A nil `*MyError` stored in an `error` +interface is not nil, so `err != nil` is true and callers take the failure path on +success. Return the error only where it is actually set: + +```go +if _, err := conn.Write(buf); err != nil { // good + return err +} +return nil +``` + +**Accumulate with `multierror` when an operation should continue past individual +failures** — teardown, cleanup, or setup where partial success is acceptable. +`client/errors.FormatErrorOrNil` returns nil for an empty accumulator, so callers +still see a plain nil on full success: + +```go +func (m *Manager) Cleanup() error { + var merr *multierror.Error + for _, r := range m.resources { + if err := r.Close(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("close %s: %w", r.Name, err)) + } + } + return nberrors.FormatErrorOrNil(merr) +} +``` + +| Scenario | Approach | Why | +| --------------------- | --------------------- | ----------------------------------------- | +| Cleanup / teardown | Accumulate | Clean up as much as possible | +| Setup with rollback | Abort on first error | Partial state is invalid; undo what stuck | +| Setup with partial OK | Accumulate | Degraded operation is still useful | + +## Comments + +Comment the **why**, never the **what**. Default to no comment, and add one only +when a hidden constraint or workaround would surprise a future reader. Never +reference the current task, PR, or your own changes in a comment. + +```go +// Bad - trailing comments explaining the obvious +defer localConn.Close() // Close the connection +if err != nil { // Check if error occurred + +// Good +defer localConn.Close() + +// Good - explains a non-obvious constraint +// Use incremental checksum update per RFC 1624 for performance. +checksum = updateChecksum(checksum, oldPort, newPort) +``` + +### Length budget + +Neither of these is linter-enforced, so they are conventions the surrounding code +mostly follows rather than hard limits: + +- **Around 90 characters per line.** Wrap the comment rather than running well past + it. +- **Roughly 250 characters per comment**, about three wrapped lines. Doc comments + on exported identifiers may exceed it when the API genuinely needs the + explanation; inline comments inside a function body rarely should. + +The budget is a smell detector, not a rule to game. Do not compress a needed +explanation into cryptic shorthand to fit — if a block of code needs more than +250 characters of prose, the code is doing too much. Fix the code: + +- **Extract a named function.** A well-named function replaces the comment: the + name says *what*, the body shows *how*, and the comment you no longer write + was the *what* anyway. Clean Code calls this "explain yourself in code". +- **Extract a named constant or predicate.** `if isExpiredSetupKey(key)` needs + no comment; `if key.ExpiresAt.Before(now) && !key.Revoked && key.UsageLimit > 0` + does. +- **Keep the surviving comment for the why** — the RFC, the kernel quirk, the + ordering constraint. That part is usually one or two lines. + +### Long switch and if/else chains + +A `switch` whose cases carry multi-line explanations is the usual place this +budget is breached, and the comment is a symptom. In order of preference: + +1. **Extract each case body into a named function.** The case becomes one line, + the name carries the meaning, and the switch reads as a table of contents. +2. **Replace the switch with a lookup table** — `map[Kind]handlerFunc` — when the + branches are uniform. Adding a case stops meaning editing a growing function. +3. **Replace conditional with polymorphism** when branches vary by type and the + same switch shape starts appearing in more than one place. Clean Code's rule + of thumb: tolerate a switch statement if it appears **once**, is buried in a + factory that returns an interface, and no other switch dispatches on the same + type. A second switch over the same enum is the signal to introduce the + interface. + +Do not restructure a switch purely to satisfy the budget when the cases are one +line each and self-evident — a flat, boring `switch` over an enum is fine and +needs no comments at all. + +Explanatory comments in tests are welcome — they document the scenario being set +up, and the 250-character budget does not apply to them. + +## Testing + +- **Unit tests** live beside the code as `_test.go`. `make test-unit` runs the + host-safe set with `-tags devcert` and no sudo. +- **Privileged tests** carry the `privileged` build tag and mutate host + networking. They run through `make test-privileged`, inside a Docker container + with `NET_ADMIN`. Never bypass that harness by running them directly on the + host. +- **End-to-end suites** live in `e2e/` with a shared harness. +- **Test real behavior, not API existence.** Assert on the observable end state + a consumer would see — bytes that arrived, the packet after translation, the + row after the write — not merely that a method exists or returns an error. +- **Avoid mocks for code we own.** Exercise the real store, manager, or + controller and assert what the caller actually receives. +- **`require` for setup and preconditions, `assert` for the conditions under + test.** Use `require` whenever a later line would panic or be meaningless + otherwise. +- **Message guidance:** optional for `NoError`/`Error`; always give context for + comparison, boolean, and collection assertions. +- **Reproduce a bug before fixing it.** Write the test, watch it fail *for the + reason you expect* — a test that fails for an unrelated reason proves nothing — + then apply the fix and confirm it passes. Add the thin surrounding cases while + you are there. +- **Use `t.Setenv`** rather than `os.Setenv` so the previous value is restored on + cleanup. To test the unset case, call `t.Setenv` first to register the restore, + then `os.Unsetenv`. +- **Prefer `t.Cleanup` over `defer`** in any test with parallel subtests: the + parent function returns, running its `defer`s, while parallel subtests are + still suspended. Sequential subtests finish inside `t.Run`, so `defer` is safe + there, but `t.Cleanup` works in both cases. +- **Explanatory comments in tests are welcome.** Describe the scenario being set + up; the comment budget below does not apply to them. + +```go +server, err := StartTestServer() +require.NoError(t, err, "Test server setup must succeed") +defer server.Close() + +result, err := client.DoOperation() +assert.NoError(t, err) +assert.Equal(t, expectedResult, result, "Result should match expected") +``` + +## Pitfalls + +- **The agent runs as root.** Anything touching routing, firewall, DNS, or the + interface can take a user's machine off the network. Prefer a reversible + change and make sure cleanup runs on every exit path. +- **Management has two account loaders** (GORM and pgx). Adding a relation to an + account often means updating both, or it silently comes back empty in + production. +- **`go test ./...` without `-tags devcert` skips tests** that need the + development certificate. Use `make test-unit`. +- **`make lint` only checks the diff against `origin/main`.** CI runs + `make lint-all`; run it too before pushing a large change. +- **Protos are consumed by released clients.** An old agent must keep working + against a new Management, so fields are added, never renumbered or removed. +- **Windows requires the wintun driver**, and the daemon serves a named pipe + (`npipe://netbird`) rather than loopback TCP. Loopback TCP carries no caller + identity, so privileged operations are refused over it. + +## Commits, PRs, releases + +- **PR titles must start with a bracketed tag.** Before you propose a title, + **read [`.github/workflows/pr-title-check.yml`](.github/workflows/pr-title-check.yml) + and take the allowed tags from the `allowedTags` array in that file.** It is + the only source of truth, it changes as components are added, and the check + runs on every title edit — a tag that is not in that array is a red build. Do + not rely on a list memorized from anywhere else, including this file. + + ```text + [client] Authorize daemon IPC callers by their local identity + [management,client] Add MDM policy support + ``` + + Multiple tags are comma-separated inside one pair of brackets. Match the tag + to the component you actually changed, not to the one you read the most. + +- **Use the repository's PR template.** Fill in + [`.github/pull_request_template.md`](.github/pull_request_template.md) rather + than replacing it with your own summary: describe the change, link the issue, + tick the checklist honestly (including "ran locally" and "single purpose"), + and complete the documentation section. Do not tick a box you have not + verified, and do not delete rows that do not apply — the docs gate in CI reads + that section and fails when it is missing. + +- **Keep the PR description short.** Under 1000 words on top of the template's + own text, and usually far less — a few paragraphs. Reviewers read the diff; + the description exists to explain what the diff cannot say for itself. This is + well below what an agent will produce by default, so cut before you post. + +- **Body: why before what.** Lead with the problem and the reason for this + approach, then the shape of the change. No bullet list of files changed, no + per-function walkthrough, no restating the diff in prose, no trailing summary + section, no self-congratulatory closing line. + +- **No `Co-Authored-By` or tool-attribution trailers in the PR description**, + and none in commits either. Contributors own their contributions. Whatever + tooling produced the diff, the person opening the PR is its author: they have + read every line, they can explain why it works, they can answer review + questions without going back to a model, and they are accountable for the + consequences of merging it. Do not add a trailer, footer, or description line + that spreads that ownership onto a tool. + +- **Commit subjects follow the same `[scope] Subject` convention.** Keep the + subject short, and use the body for why before what. No bullet lists of files + changed. + +- **Push review fixes as separate commits.** The PR is squashed on merge, so + there is no reason to rewrite history mid-review; many small commits make the + re-review readable. + +- **Do not force-push a branch that is under review.** A force-push detaches + existing review comments from the lines they were written against, destroys + the "changes since your last review" diff a reviewer relies on, and discards + the CI history that showed which commit broke what. Add commits instead — + including for fixups and reverts. Force-push only when there is no + alternative: a rebase to clear a genuine conflict, or removing a secret or a + large binary that was committed by mistake. When you must, ask the user first, + then say so in a PR comment so reviewers know their anchors moved. Never + force-push `main`, and never force-push a branch you do not own. + +- **One PR, one purpose.** Split refactors out of fixes and fixes out of + features. + +- **Keep the PR small.** Size is the single strongest predictor of how long a PR + waits. Aim for **under ~400 changed lines across under ~20 files**; past + roughly **1000 lines or 50 files** a community PR is likely to be sent back to + be split, or left unreviewed until it is. Large PRs from outside the core team + may be blocked outright when the size was never agreed in the ticket — + reviewing a sprawling change against a privileged networking daemon is a + security risk in itself, not just a time cost. + + Judge the size by hand-written code: exclude generated output, `go.sum`, + vendored files, and test fixtures from the estimate, but do not use their + presence to argue a 3000-line PR is small. + + When a change genuinely cannot be small — a protocol migration, a + cross-component rename — agree the split in the ticket **before** writing + code, and land it as a sequence of PRs that each build, test, and make sense + on their own. Propose that split to the user rather than opening one large PR + and hoping. + + Prefer GitHub's stacked pull requests for such a sequence, rather than + hand-managing base branches: open each PR against the branch below it instead of + `main`, so every PR's diff shows only its own change. Merging a layer retargets + the PRs above it, and branch protections and required checks on the base branch + still apply to each one. + +- **User-facing changes need a docs PR** in + [netbirdio/docs](https://github.com/netbirdio/docs), linked from the PR + description. + +## After you push: CI and review bots + +Opening the PR is not the end of the task. Watch the run, read what the bots +say, and drive the PR to green before you report the work as done. + +```bash +gh pr checks --watch # all checks, live +gh run view --log-failed # only the failing steps +gh pr view --comments # bot and human review comments +``` + +**Never report a change as finished while checks are pending or red**, and never +describe a red PR as passing. If you ran out of turn before CI finished, say +which checks were still running. + +### The checks + +- **Go tests** — `golang-test-{linux,darwin,windows,freebsd}.yml`, sharded per + component. A failure in a component you did not touch is usually a real + interaction, not noise; read the log before assuming flake. +- **golangci-lint** — `golangci-lint.yml` runs the full repository, while + `make lint` only checks your diff. A clean local lint does not guarantee green + CI on a large change. +- **PR Title Check** — `pr-title-check.yml`, see above. +- **Codecov** — uploaded from the Linux test workflow with per-component flags + (`unit,client`, `unit,management`, `unit,relay`, `unit,proxy`, `unit,signal`, + `integration,management`). Coverage on new code should not go backwards. Add + tests for the paths you introduced; do not adjust thresholds or exclude files + to clear the report. +- **CodeRabbit** — configured in [`.coderabbit.yaml`](.coderabbit.yaml): `chill` + profile, auto-review on every non-draft PR, TypeScript/JavaScript/SVG paths + filtered out. Chat auto-reply is on, so `@coderabbitai` in a comment reaches + it. +- **SonarCloud** — project `netbirdio_netbird`, quality gate on new code (bugs, + vulnerabilities, code smells, duplication, coverage). +- **Snyk** — dependency and code scanning. + +Sonar and Snyk report as GitHub App checks rather than workflows in this +repository, so their detail lives on the PR check, not in the Actions logs. + +### Handling bot findings + +- **Read every comment and act on it.** Either fix it, or reply with the reason + it does not apply. Do not bulk-resolve threads to clear the count, and do not + silently ignore a finding because the check is advisory. +- **Bots are frequently wrong here.** NetBird has privileged, platform-specific, + and concurrency-heavy code that static analysis reads poorly. A confident + CodeRabbit or Sonar comment can still be nonsense. Verify the claim against + the code before you change anything — never edit correct code just to silence + a bot. +- **Security findings get the opposite default.** For a Snyk or Sonar + vulnerability, or a CodeRabbit comment about authentication, authorization, + certificate verification, or key handling, assume it is real until you have + disproved it. Surface it to the user rather than dismissing it yourself. +- **A new vulnerable dependency is a stop.** Bumping or replacing dependencies + needs the user's decision, as above. +- **Never change a workflow, threshold, lint exclusion, or bot config to make a + check pass.** If a check is genuinely wrong, say so and let the user decide. +- **Do not paper over flakes with blind re-runs.** Identify the failure first. If + it is a known flake, name it; if you cannot tell, report it as unresolved + rather than re-running until it goes green. + +## Discussion and support + +- Discussions: +- Slack: +- Docs: +- Security: — never in public +- Contribution process: [CONTRIBUTING.md](CONTRIBUTING.md) diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..72681748e --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,4 @@ +The agent guidelines live in [AGENTS.md](AGENTS.md). It is imported here so +every session loads it in full rather than following a pointer. + +@AGENTS.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd1c087bb..aef749cfa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to NetBird -Thanks for your interest in contributing to NetBird. +Thanks for your interest in contributing to NetBird. There are many ways that you can contribute: - Reporting issues @@ -10,12 +10,99 @@ There are many ways that you can contribute: If you haven't already, join our slack workspace [here](https://docs.netbird.io/slack-url), we would love to discuss topics that need community contribution and enhancements to existing features. +## Ticket first, PR second + +**Open a ticket and wait for feedback before you open a pull request.** Every PR +that changes behavior must link to an issue the NetBird team has agreed on. A PR +that arrives without one may be closed and redirected to a discussion, no matter +how good the code is. + +Issues in this repository are maintainer-curated work items, so the flow starts +in [Discussions](https://github.com/netbirdio/netbird/discussions): + +1. **Open a discussion.** Use + [Issue Triage](https://github.com/netbirdio/netbird/discussions/new?category=issue-triage) + for a bug, regression, or unexpected behavior, and + [Ideas & Feature Requests](https://github.com/netbirdio/netbird/discussions/new?category=ideas-feature-requests) + for a feature, enhancement, or integration idea. Setup and usage questions + belong in + [Q&A / Support](https://github.com/netbirdio/netbird/discussions/new?category=q-a-support). + Never report a security vulnerability in public — follow the + [security policy](https://github.com/netbirdio/netbird/security/policy) + instead. +2. **Wait for feedback.** DevRel validates and reproduces the report, and a + maintainer confirms the direction. We may ask for more detail or propose a + different approach. Validated discussions become issues. +3. **Then write the code**, following the approach agreed in the issue, and open + the PR linking that issue. + +Trivial fixes — a typo, a broken link, a documentation correction, or a one-line +fix that already has an issue — can go straight to a PR. Everything else starts +with a ticket. When in doubt, ask in the discussion or on +[Slack](https://docs.netbird.io/slack-url); an hour of conversation up front +regularly saves a week of rework. + +### High-risk areas + +These always need the design discussed and agreed in the issue **before** you +write code: + +- **Public API** — REST / management API, OpenAPI schema, dashboard-facing contracts +- **gRPC protocols** — management, signal, relay, and client daemon protos +- **Functionality behavior** — anything existing deployments would experience differently after an upgrade +- **Peer connectivity** — ICE and NAT traversal, relay selection, WireGuard® and Rosenpass key handling +- **Client system integration** — routing, firewall, DNS, and interface management +- **Authentication and authorization** — IdP integration, tokens, permissions, cryptography +- **CLI / service flags**, configuration file format, and daemon IPC +- **Store and database schema** — models and migrations +- **New features** + +These surfaces are NetBird's contract with operators, self-hosters, and +downstream integrators, and changes to them have compatibility, security, and +release-planning implications. Agreeing on the direction early lets the PR +review focus on implementation rather than design. + +Typical bug fixes, internal refactors, documentation updates, and tests do not +need a design discussion, but should still be tied to an issue so the work is +visible and nobody duplicates it. + +### Using AI coding agents + +We have no policy for or against using an AI agent to write NetBird code. That +choice is yours, and we are not going to interrogate anyone about their tools. + +What we do have is a lot of incoming contributions that were plainly drafted with +one, and enough experience reviewing them to see the same avoidable problems +again and again: no ticket behind the change, a diff far too large to review, a +description longer than the code it describes, an approach that was never going +to be accepted, and an author who cannot answer questions about their own PR. +None of that is caused by the tooling — it is what happens when a tool is pointed +at a repository whose expectations it has never been told. + +So rather than a rule, there is a guide. [AGENTS.md](AGENTS.md) restates the +expectations from this document in the form agents read automatically +(`CLAUDE.md` points to it), so pointing your tool at the repository is usually +enough. Among other things it tells the agent to ask you for the +discussion or issue before drafting a PR, to keep the change small and +single-purpose, to run the tests locally, to use this repository's PR template +and title tags, and to write a description a reviewer can get through. + +The guardrails are the point, and they are the same ones we apply to everyone: an +agreed ticket, a change you have actually run, a diff small enough to review with +care, and an author who can explain it. Whatever wrote the diff, you are its +author — you own every line you submit and the consequences of opening a PR with it. + +We may assess whether a contribution is maintainable and whether its merged code +aligns with our security standards and design expectations. + ## Contents - [Contributing to NetBird](#contributing-to-netbird) + - [Ticket first, PR second](#ticket-first-pr-second) + - [High-risk areas](#high-risk-areas) + - [Using AI coding agents](#using-ai-coding-agents) - [Contents](#contents) - [Code of conduct](#code-of-conduct) - - [Discuss changes with the NetBird team first](#discuss-changes-with-the-netbird-team-first) - [Directory structure](#directory-structure) - [Development setup](#development-setup) - [Requirements](#requirements) @@ -24,6 +111,8 @@ If you haven't already, join our slack workspace [here](https://docs.netbird.io/ - [Build and start](#build-and-start) - [Test suite](#test-suite) - [Checklist before submitting a PR](#checklist-before-submitting-a-pr) + - [When we close a PR](#when-we-close-a-pr) + - [Translations](#translations) - [Other project repositories](#other-project-repositories) - [Contributor License Agreement](#contributor-license-agreement) @@ -34,42 +123,66 @@ Conduct which can be found in the file [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to community@netbird.io. -## Discuss changes with the NetBird team first - -Changes to the **public API**, **gRPC protocols**, **functionality behavior**, **CLI / service flags**, or **new features** should be discussed with the NetBird team before you start the work. These surfaces are part of NetBird's contract with operators, self-hosters, and downstream integrators, and changes to them have compatibility, security, and release-planning implications that benefit from an early conversation. - -Open an issue or reach out on [Slack](https://docs.netbird.io/slack-url) to talk through what you have in mind. We'll help shape the change, flag any constraints we know about, and confirm the direction so the PR review can focus on implementation rather than design. - -Typical bug fixes, internal refactors, documentation updates, and tests do not need pre-discussion — open the PR directly. - ## Directory structure -The NetBird project monorepo is organized to maintain most of its individual dependencies code within their directories, except for a few auxiliary or shared packages. +The NetBird project monorepo keeps most of each component's code within its own +directory, except for a few auxiliary or shared packages. Protocol definitions +and the client-side service clients live under [/shared](/shared), because both +the agent and the services import them. -The most important directories are: +**Agent** -- [/.github](/.github) - Github actions workflow files and issue templates - [/client](/client) - NetBird agent code -- [/client/cmd](/client/cmd) - NetBird agent cli code +- [/client/cmd](/client/cmd) - NetBird agent CLI code - [/client/internal](/client/internal) - NetBird agent business logic code -- [/client/proto](/client/proto) - NetBird agent daemon GRPC proto files - [/client/server](/client/server) - NetBird agent daemon code for background execution -- [/client/ui](/client/ui) - NetBird agent UI code -- [/encryption](/encryption) - Contain main encryption code for agent communication -- [/iface](/iface) - Wireguard® interface code -- [/infrastructure_files](/infrastructure_files) - Getting started files containing docker and template scripts +- [/client/proto](/client/proto) - NetBird agent daemon gRPC proto files +- [/client/iface](/client/iface) - WireGuard® interface code +- [/client/firewall](/client/firewall) - Platform firewall backends (nftables, iptables, pf, WFP, userspace) +- [/client/ssh](/client/ssh) - Built-in SSH server and client +- [/client/ui](/client/ui) - NetBird agent UI code (Wails v3 + React) +- [/client/android](/client/android), [/client/ios](/client/ios) - Mobile platform bindings +- [/client/wasm](/client/wasm) - WebAssembly build of the agent +- [/client/mdm](/client/mdm) - MDM-delivered policy handling +- [/client/system](/client/system) - Host and system information collection + +**Control plane services** + - [/management](/management) - Management service code -- [/management/client](/management/client) - Management service client code which is imported by the agent code -- [/management/proto](/management/proto) - Management service GRPC proto files - [/management/server](/management/server) - Management service server code - [/management/server/http](/management/server/http) - Management service REST API code +- [/management/server/store](/management/server/store) - Persistence layer and migrations - [/management/server/idp](/management/server/idp) - Management service IDP management code -- [/release_files](/release_files) - Files that goes into release packages +- [/management/server/peer](/management/server/peer), [/management/server/groups](/management/server/groups), [/management/server/networks](/management/server/networks), [/management/server/posture](/management/server/posture), [/management/server/permissions](/management/server/permissions) - Core domain packages - [/signal](/signal) - Signal service code -- [/signal/client](/signal/client) - Signal service client code which is imported by the agent code - [/signal/peer](/signal/peer) - Signal service peer message logic -- [/signal/proto](/signal/proto) - Signal service GRPC proto files - [/signal/server](/signal/server) - Signal service server code +- [/relay](/relay) - Relay service code +- [/relay/protocol](/relay/protocol) - Relay wire protocol +- [/proxy](/proxy) - Identity-aware proxy used by Agent Network (LLM routing, ACME, access logs) +- [/agent-network](/agent-network) - Agent Network overview and documentation +- [/upload-server](/upload-server) - Debug bundle upload service + +**Shared code** + +- [/shared/management/proto](/shared/management/proto) - Management service gRPC proto files +- [/shared/management/client](/shared/management/client) - Management service client code which is imported by the agent code +- [/shared/management/http/api](/shared/management/http/api) - OpenAPI specification and generated REST API types +- [/shared/signal/proto](/shared/signal/proto) - Signal service gRPC proto files +- [/shared/signal/client](/shared/signal/client) - Signal service client code which is imported by the agent code +- [/shared/relay](/shared/relay) - Relay client and shared relay types +- [/shared/auth](/shared/auth), [/shared/sshauth](/shared/sshauth) - Shared authentication primitives +- [/encryption](/encryption) - Contain main encryption code for agent communication +- [/dns](/dns), [/route](/route), [/stun](/stun), [/sharedsock](/sharedsock), [/util](/util) - Shared networking and utility primitives +- [/flow](/flow) - Flow event protocol shared by the agent and Management + +**Build, test, and packaging** + +- [/.github](/.github) - Github actions workflow files, issue templates, and the pull request template +- [/e2e](/e2e) - End-to-end test suites and harness +- [/infrastructure_files](/infrastructure_files) - Getting started files containing docker and template scripts +- [/release_files](/release_files) - Files that goes into release packages +- [/tools](/tools) - Development and maintenance tooling ## Development setup @@ -79,13 +192,21 @@ dependencies are installed. Here is a short guide on how that can be done. ### Requirements -#### Go 1.21 +#### Go 1.26 Follow the installation guide from https://go.dev/ -#### UI client - Fyne toolkit +#### UI client - Wails v3 + React -We use the fyne toolkit in our UI client. You can follow its requirement guide to have all its dependencies installed: https://developer.fyne.io/started/#prerequisites +The desktop UI client (`client/ui`) is built with [Wails v3](https://v3.wails.io/) and a React frontend rendered in a WebView. To build it you need: + +- Go ≥ 1.26 +- Node ≥ 20 and **pnpm** (`corepack enable && corepack prepare pnpm@latest --activate`) +- The `wails3` CLI: `go install github.com/wailsapp/wails/v3/cmd/wails3@latest` +- The `task` runner: `go install github.com/go-task/task/v3/cmd/task@latest` +- Linux only: `libwebkitgtk-6.0-dev`, `libgtk-4-dev`, `libsoup-3.0-dev` + +All UI build, dev-loop, and cross-compile commands are described in the [UI client](#ui-client) section below. #### gRPC You can follow the instructions from the quickstarter guide https://grpc.io/docs/languages/go/quickstart/#prerequisites and then run the `generate.sh` files located in each `proto` directory to generate changes. @@ -214,6 +335,49 @@ To start NetBird the client in the foreground: sudo ./client up --log-level debug --log-file console ``` > On Windows use a powershell with administrator privileges + +#### UI client + +The desktop UI lives in `client/ui` and is built with Wails v3 (see [Requirements](#ui-client---wails-v3--react)). All commands run from `client/ui`. + +Live-reload development (Vite + Go binary + `*.go` watcher): + +``` +cd client/ui +task dev +``` + +Pass daemon flags after `--`, pointing the UI at the socket the daemon serves: + +``` +task dev -- --daemon-addr=unix:///var/run/netbird.sock # Linux, macOS +task dev -- --daemon-addr=npipe://netbird # Windows +``` + +On Windows the daemon serves a named pipe (`npipe://netbird`). Which path that +ends up being depends on what the daemon may create: as a service or elevated it +serves `\\.\pipe\ProtectedPrefix\Administrators\netbird`, which no unprivileged +process can take from it, and otherwise it falls back to `\\.\pipe\netbird`. +Clients try both and check who owns the pipe before using the plain one. Avoid +`tcp://127.0.0.1:41731`: loopback TCP carries no caller identity, so the daemon +refuses the operations that require an administrator and you will not exercise +those paths. + +Production build (frontend assets embedded into the binary, output in `client/ui/bin/`): + +``` +cd client/ui +task build +``` + +Cross-compile the Windows binary from Linux (requires the mingw-w64 toolchain, e.g. `sudo apt install gcc-mingw-w64-x86-64`): + +``` +CGO_ENABLED=1 task windows:build +``` + +> macOS cross-compile from Linux is not supported (signing and notarization need a real Mac). + #### Signal service To start NetBird's signal, execute: @@ -251,10 +415,10 @@ Create dist directory mkdir -p dist/netbird_windows_amd64 ``` -UI client +UI client (built with Wails v3 — see the [UI client](#ui-client) section above) ```shell -CC=x86_64-w64-mingw32-gcc CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build -o netbird-ui.exe -ldflags "-s -w -H windowsgui" ./client/ui -mv netbird-ui.exe ./dist/netbird_windows_amd64/ +(cd client/ui && CGO_ENABLED=1 task windows:build) +mv client/ui/bin/netbird-ui.exe ./dist/netbird_windows_amd64/ ``` Client @@ -283,25 +447,183 @@ The installer `netbird-installer.exe` will be created in root directory. ### Test suite -The tests can be started via: +The host-safe unit tests run as a normal user and leave host networking +untouched: ``` -cd netbird -go test -exec sudo ./... +make test-unit ``` + +Tests that need root and mutate host networking (firewall, routing, interface +management) carry the `privileged` build tag and run inside a +`--privileged --cap-add=NET_ADMIN` Docker container: + +``` +make test-privileged +``` + +Narrow a privileged run with environment variables: + +``` +PRIV_RUN=TestNftablesManager PRIV_PKGS=./client/firewall/nftables/... make test-privileged +``` + +Single packages can be run directly, adding `-race` when the change touches +shared state: + +``` +go test -race ./client/internal/dns/... +``` + > On Windows use a powershell with administrator privileges -> Non-GTK environments will need the `libayatana-appindicator3-dev` (debian/ubuntu) package installed - ## Checklist before submitting a PR -As a critical network service and open-source project, we must enforce a few things before submitting the pull-requests: + +As a critical network service and open source project, we must enforce a few +things before submitting a pull request. The +[pull request template](/.github/pull_request_template.md) mirrors this list — +fill it in rather than deleting it. + +### Link the issue + +The PR description must link the agreed issue (or the validated discussion it +came from). See [Ticket first, PR second](#ticket-first-pr-second). + +### Run it locally + +**If you can't run it, you can't submit it.** Build the affected components and +exercise the change on a real setup — see [Build and start](#build-and-start). +"CI will tell me" is not acceptable for a VPN agent that runs as root on other +people's machines. + +### Green CI, and answer the bots + +We do not start reviewing while CI is red. Get the pipeline green first — a +failing build, lint, or test means the PR is not ready for review. + +Alongside the test workflows, your PR is reviewed by CodeRabbit and scanned by +SonarCloud, Snyk, and Codecov. Read what they report and either fix it or reply +with why it does not apply; please do not resolve the threads without a +response. They are not always right — this codebase has privileged, +platform-specific, and concurrency-heavy paths that static analysis reads poorly +— so push back when a finding is wrong rather than changing correct code to +silence it. Security and dependency findings are the exception: treat those as +real until shown otherwise. Do not edit workflows, thresholds, or scanner +configuration to make a check pass. + +### One PR, one purpose + +Bug fix, refactor, feature: separate PRs. Mixed PRs are slow to review, hard to +revert, and may be closed with a request to split them. + +### Keep it small + +Size is the strongest predictor of how long a PR waits for review. Aim for under +roughly 400 changed lines across under 20 files. Past about 1000 lines or 50 +files, expect to be asked to split the change — and large PRs from outside the +core team may be blocked until the scope has been agreed in a ticket. This is +not only about reviewer time: NetBird's agent runs as root on other people's +machines, and a sprawling diff cannot be reviewed with the care that deserves. + +Measure by hand-written code, excluding generated output, `go.sum`, and +fixtures. If a change genuinely cannot be small — a protocol migration, a +cross-component rename — agree the split in the issue before you start, and land +it as a series of PRs that each build and make sense on their own. + +### Avoid force-pushing during review + +Once a PR is open, push new commits instead of rewriting history. A force-push +detaches existing review comments from their lines, throws away the +"changes since your last review" diff, and loses the CI history that showed +which commit broke what. Since we squash on merge, there is nothing to gain from +a tidy branch history. + +Force-pushing is sometimes unavoidable — rebasing to clear a real conflict, or +removing a secret or large binary committed by mistake. When that happens, leave +a comment on the PR so reviewers know their anchors moved. + +### Quality checks + +Run these from the repository root before pushing: + +```shell +go fmt ./... +make lint # golangci-lint on files changed against origin/main +make lint-all # full-repository lint, matches CI +make test-unit # host-safe unit tests +``` + +`make setup-hooks` wires `make lint` into a pre-push hook so the fast lint runs +automatically. If your change touches privileged paths (firewall, routing, +interface management), also run `make test-privileged`, which executes the +`privileged`-tagged suite inside a Docker container with `NET_ADMIN`. + +### Code standards + - Keep functions as simple as possible, with a single purpose - Use private functions and constants where possible - Comment on any new public functions - Add unit tests for any new public function +- Comment the **why**, not the **what** — explain non-obvious decisions, invariants, and constraints, not the line below +- Keep comments within 90 characters per line and roughly 250 characters per comment; when a block needs more explanation than that, extract a named function instead of writing a longer comment (see [AGENTS.md](AGENTS.md#length-budget)) + +### PR title and commits + +PR titles must start with a bracketed tag, enforced by +[pr-title-check.yml](/.github/workflows/pr-title-check.yml): + +```text +[client] Authorize daemon IPC callers by their local identity +[management,client] Add MDM policy support +``` + +Use a comma-separated list inside a single pair of brackets when a change spans +components. The `allowedTags` array in +[pr-title-check.yml](/.github/workflows/pr-title-check.yml) is the source of +truth — at the time of writing it accepts `management`, `client`, `signal`, +`proxy`, `relay`, `misc`, `infrastructure`, `self-hosted`, and `doc`. + +Commit subjects follow the same convention — keep them short and put the +reasoning in the body, why before what, with no bullet list of files changed. + +Keep the PR description itself under 1000 words on top of the template text. +Reviewers read the diff; the description explains what the diff cannot. > When pushing fixes to the PR comments, please push as separate commits; we will squash the PR before merging, so there is no need to squash it before pushing it, and we are more than okay with 10-100 commits in a single PR. This helps review the fixes to the requested changes. +### Documentation + +User-facing changes need a matching PR in +[netbirdio/docs](https://github.com/netbirdio/docs); link it in the PR +description, or state why documentation is not needed. + +## When we close a PR + +We would rather redirect early than let a PR sit. We may close one if: + +- It changes behavior with no linked issue, or the approach was never agreed with a maintainer +- The change was clearly never run or tested locally +- CI has been red without a response +- It mixes unrelated purposes, or the purpose is not clear +- It is far too large to review and the scope was never agreed in a ticket +- The author cannot answer questions about their own change — including PRs that read as unreviewed model output, where review turns into a relay between the maintainer and an LLM. Tooling is fine; unreviewed output is not, you are responsible for the code you sign your name to +- There has been no activity for 14 days after we requested changes + +A closed PR is not a rejected idea. Take it back to the +[discussion](https://github.com/netbirdio/netbird/discussions), settle the +approach, and reopen the work from there. + +## Translations + +Desktop UI translations are not contributed through pull requests. Translate on +[Crowdin](https://crowdin.com/project/netbird) instead: no ticket needed, just +join the project and pick your language. Crowdin syncs with this repository and +opens the service PRs itself, so hand-edited locale files would conflict with +the next sync. Style, terminology, and review guidance live in +[client/ui/i18n/TRANSLATING.md](client/ui/i18n/TRANSLATING.md). To request a +language the project does not offer yet, ask on the Crowdin project page or in +a [discussion](https://github.com/netbirdio/netbird/discussions). + ## Other project repositories NetBird project is composed of 3 main repositories: diff --git a/LICENSE b/LICENSE index d922f155a..4d0551de8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,9 @@ -This BSD‑3‑Clause license applies to all parts of the repository except for the directories management/, signal/, relay/ and combined/. +This BSD-3-Clause license applies to all parts of the repository except for the directories management/, signal/, relay/ and combined/. Those directories are licensed under the GNU Affero General Public License version 3.0 (AGPLv3). See the respective LICENSE files inside each directory. BSD 3-Clause License -Copyright (c) 2022 NetBird GmbH & AUTHORS +Copyright (c) 2022-2026 NetBird GmbH & AUTHORS Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/LICENSES/BSD-3-Clause.txt b/LICENSES/BSD-3-Clause.txt index 7cba76dfd..5e9a2539c 100644 --- a/LICENSES/BSD-3-Clause.txt +++ b/LICENSES/BSD-3-Clause.txt @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2022 NetBird GmbH & AUTHORS +Copyright (c) 2022-2026 NetBird GmbH & AUTHORS Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/Makefile b/Makefile index 0a4fad2f2..26c5b932e 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,8 @@ lint-install: $(GOLANGCI_LINT) # Setup git hooks for all developers setup-hooks: @git config core.hooksPath .githooks - @chmod +x .githooks/pre-push - @echo "✅ Git hooks configured! Pre-push will now run 'make lint'" + @chmod +x .githooks/pre-push .githooks/commit-msg + @echo "✅ Git hooks configured! Pre-push runs 'make lint'; commit-msg refuses attribution trailers" # Host-safe unit tests: excludes the privileged-tagged tests (root / system-mutating). # Runs as a normal user with no sudo and leaves host networking untouched. diff --git a/README.md b/README.md index 40c6b9ed5..4dbfc7bd0 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Start using NetBird at netbird.io
- See Documentation + See Documentation
Join our Slack channel or our Community forum
@@ -115,6 +115,56 @@ export NETBIRD_DOMAIN=netbird.example.com; curl -fsSL https://github.com/netbird See a complete [architecture overview](https://docs.netbird.io/about-netbird/how-netbird-works#architecture) for details. +### Reporting bugs and requesting features + +NetBird uses a discussion-first workflow. Bug reports and feature requests start in +[Discussions](https://github.com/netbirdio/netbird/discussions), not as issues. + +| What you want to do | Where to go | +| --- | --- | +| Report a bug, regression, or unexpected behavior | [Issue Triage](https://github.com/netbirdio/netbird/discussions/new?category=issue-triage) | +| Request a feature or share an idea | [Ideas & Feature Requests](https://github.com/netbirdio/netbird/discussions/new?category=ideas-feature-requests) | +| Ask about setup, configuration, or self-hosting | [Q&A / Support](https://github.com/netbirdio/netbird/discussions/new?category=q-a-support) | +| Report a security vulnerability | [Security policy](https://github.com/netbirdio/netbird/security/policy), never a public thread | + +Our team and maintainers triage discussions, ask follow-up questions, check for duplicates, +and reproduce bugs. Validated reports are promoted to issues. This keeps the issue tracker a clear +answer to one question: what is the team working on. + +Please search existing discussions and issues first, including closed ones. If something similar +already exists, upvote it and add your details there instead of opening a duplicate. + +For bug reports, include your NetBird version, operating system, deployment type (Cloud, +self-hosted, Kubernetes, or Docker), reproduction steps, expected and actual behavior, and a debug +bundle where relevant: + +```shell +netbird version +netbird status -d -A +netbird debug for 1m -A -S -U +``` + +`-U` uploads the bundle and prints a file key you can paste instead of attaching the archive. +`-A` anonymizes the output, which matters on a public thread. It masks most identifying details +but is not full redaction, so read the bundle before posting it. Two levels are available: + +| Level | How to select | What it masks | +| --- | --- | --- | +| `default` | `-A` / `--anonymize` | Public IP addresses, IPv6 ULA addresses, MAC addresses, and domains other than `netbird.io`, `netbird.cloud`, `netbird.selfhosted`, and `netbird.stage`. IPv4 private, CGNAT, and link-local ranges are kept | +| `strict` | `--anonymize-level strict` (implies `-A`) | The above, plus IPv4 private, CGNAT, and link-local ranges, peer names in front of `netbird.cloud`, `netbird.selfhosted`, and `netbird.stage`, and WireGuard public keys. Labels under `netbird.io` are kept, since it only hosts infrastructure | + +See [collecting a debug bundle](https://docs.netbird.io/help/troubleshooting-client#debug-bundle) +and the [CLI reference](https://docs.netbird.io/get-started/cli#debug-for) for details. + +See [How to use Discussions, Issues, and Pull Requests](https://github.com/netbirdio/netbird/discussions/6075) +for the full workflow, or [SUPPORT.md](SUPPORT.md) for a shorter version. + +### Contributing + +Contributions are welcome. Read [CONTRIBUTING.md](CONTRIBUTING.md) first. NetBird works ticket +first, anything that changes behavior needs an issue the team has agreed on before you open a pull +request. + ### Community projects - [NetBird installer script](https://github.com/physk/netbird-installer) - [netbird-tui](https://github.com/n0pashkov/netbird-tui) - terminal UI for managing NetBird peers, routes, and settings @@ -130,7 +180,7 @@ In November 2022, NetBird joined the [StartUpSecure program](https://www.forschu ![CISPA_Logo_BLACK_EN_RZ_RGB (1)](https://user-images.githubusercontent.com/700848/203091324-c6d311a0-22b5-4b05-a288-91cbc6cdcc46.png) ### Acknowledgements -We build on open-source technologies like [WireGuard®](https://www.wireguard.com/), [Pion ICE](https://github.com/pion/ice), and [Rosenpass](https://rosenpass.eu). We greatly appreciate the work these projects are doing, and we'd love it if you could support them too (e.g., by starring or contributing). +We build on open source technologies like [WireGuard®](https://www.wireguard.com/), [Pion ICE](https://github.com/pion/ice), and [Rosenpass](https://rosenpass.eu). We greatly appreciate the work these projects are doing, and we'd love it if you could support them too (e.g., by starring or contributing). ### Legal This repository is licensed under the BSD-3-Clause license, which applies to all parts of the repository except for the directories management/, signal/ and relay/. diff --git a/SECURITY.md b/SECURITY.md index 745c66e61..cbcc975ba 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,12 +1,70 @@ # Security Policy -NetBird's goal is to provide a secure network. If you find a vulnerability or bug, please report it by opening an issue [here](https://github.com/netbirdio/netbird/issues/new?assignees=&labels=&template=bug-issue-report.md&title=) or by contacting us by email. - -There has yet to be an official bug bounty program for the NetBird project. - -## Supported Versions -- We currently support only the latest version +NetBird's goal is to provide a secure network. The client runs as a privileged service on every machine it is installed on, +so we take reports about it seriously and we publish what we fix. ## Reporting a Vulnerability -Please report security issues to `security@netbird.io` +**Please do not open a public issue for a security vulnerability.** Public issues are visible to everyone, including before +a fix is available. + +Report security issues one of these two ways: + +- **GitHub private vulnerability reporting** — [open a private report](https://github.com/netbirdio/netbird/security/advisories/new) + on this repository. This is the preferred route: it keeps the discussion, the draft advisory, and the credit in one place. +- **Email** — `security@netbird.io`. + +If the finding affects NetBird Cloud or our hosted infrastructure rather than the open source code, email us rather than +filing a repository report. + +### What to include + +A report is easier to act on when it contains: + +- The affected component (client, management, signal, relay, dashboard) and the version or commit you tested +- The platform and configuration, where relevant — operating system, self-hosted or NetBird Cloud, container or host install +- What an attacker needs before they can exploit it: network position, an account, local access, a specific privilege level +- Steps to reproduce, and a proof of concept if you have one +- The impact you believe it has + +Partial reports are still welcome. If you are unsure whether something is a security issue, send it to `security@netbird.io` +and let us make that call. + +## What to expect from us + +- **We acknowledge your report** and tell you whether we can reproduce it. +- **We work with you on severity and scope.** If we assess it differently than you do, we will explain why rather than + silently downgrade it. +- **We fix and release**, then publish a [GitHub Security Advisory](https://github.com/netbirdio/netbird/security/advisories) + naming the affected version range and the patched version. +- **We credit reporters who want to be credited.** Tell us the name or handle you would like used, or that you would rather + stay anonymous. +- **We keep you in the loop** until the advisory is published. + +We ask that you give us a reasonable opportunity to ship a fix before disclosing the issue publicly, and that you avoid +accessing, modifying, or exfiltrating data belonging to other people while testing. Testing against your own installation +or your own account is always fine. + +## Supported Versions + +We support the latest release. Security fixes ship in the next version rather than as backports to older releases, so +upgrading to the current release is how you get them. + +Release notifications are available by watching [releases](https://github.com/netbirdio/netbird/releases). + +## Published advisories + +Every vulnerability we fix is published as a GitHub Security Advisory on the +[advisories page](https://github.com/netbirdio/netbird/security/advisories), including the affected version range, the +patched version, and the reporter's credit. Advisories for the Go module are also distributed through the Go vulnerability +database, so `govulncheck` will report them against your dependencies. + +## Bug bounty + +There is no official bug bounty program for the NetBird project. We credit reporters in advisories, and we are grateful for +the work, but we cannot currently offer payment for reports. + +## Non-security bugs + +For bugs that are not security issues, please use the +[issue tracker](https://github.com/netbirdio/netbird/discussions/new/choose). diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 000000000..fade37286 --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,121 @@ +# Getting help with NetBird + +Where to go depends on what you need. If you are not sure, start with +[Q&A / Support](https://github.com/netbirdio/netbird/discussions/new?category=q-a-support) +and we will move it. + +## Before you post + +1. Search existing [discussions](https://github.com/netbirdio/netbird/discussions) and + [issues](https://github.com/netbirdio/netbird/issues), including closed ones. +2. Check the [documentation](https://docs.netbird.io) and the troubleshooting guides for + [clients](https://docs.netbird.io/help/troubleshooting-client) and + [self-hosted deployments](https://docs.netbird.io/selfhosted/troubleshooting). +3. Remove or anonymize sensitive information from logs, screenshots, and configuration. + +If a discussion already covers your problem, upvote it and add your details there rather than +opening a duplicate. Extra reproduction detail, affected versions, and deployment notes are +useful even on an existing thread. + +## Community support + +Free, for everyone. Covers the NetBird client, open source self-hosted deployments, and general +questions. + +| What you want to do | Where to go | +| --- | --- | +| Report a bug, regression, or unexpected behavior | [Issue Triage](https://github.com/netbirdio/netbird/discussions/new?category=issue-triage) | +| Request a feature or share an idea | [Ideas & Feature Requests](https://github.com/netbirdio/netbird/discussions/new?category=ideas-feature-requests) | +| Ask about setup, configuration, or self-hosting | [Q&A / Support](https://github.com/netbirdio/netbird/discussions/new?category=q-a-support) | +| Chat with the community | [Slack](https://docs.netbird.io/slack-url) | + +## Paid support + +For NetBird Cloud customers and commercial-license self-hosted deployments, covering the +dashboard, control plane, billing, and subscriptions, see +[reporting bugs and issues](https://docs.netbird.io/help/report-bug-issues). + +## Security + +Do not report security vulnerabilities in public issues or discussions, and do not post secrets, +private keys, internal hostnames, or sensitive logs. Use the +[security policy](https://github.com/netbirdio/netbird/security/policy). + +## What makes a report we can act on + +For a bug, the most useful reports include: + +- NetBird version, and component versions where applicable +- Operating system or environment +- Deployment type: NetBird Cloud, self-hosted, Kubernetes, Docker, or local development +- Current behavior and expected behavior +- The smallest set of steps that reproduces the problem +- Logs, status output, screenshots, or a debug bundle when relevant +- Whether this worked before, and the last known working version + +For client reports, these commands usually give us what we need: + +```shell +netbird version +netbird status -d -A +netbird debug for 1m -A -S -U +``` + +`-A` (`--anonymize`) replaces sensitive values consistently across every file in the bundle, so +it stays readable while masking most identifying details. It is not a guarantee of full redaction: +internal address ranges survive at the default level, and interface names, indexes, MTUs, and +flags are never anonymized. Read the bundle before posting it publicly. Two levels are +available: + +| Level | How to select | What it masks | +| --- | --- | --- | +| `default` | `-A` / `--anonymize`, or `--anonymize-level default` | Public IP addresses, IPv6 ULA addresses, MAC addresses, and domains other than `netbird.io`, `netbird.cloud`, `netbird.selfhosted`, and `netbird.stage`. IPv4 private, CGNAT, and link-local ranges are kept, and interface names are not anonymized | +| `strict` | `--anonymize-level strict` (implies `-A`) | The above, plus IPv4 private, CGNAT, and link-local ranges, peer names in front of `netbird.cloud`, `netbird.selfhosted`, and `netbird.stage`, and WireGuard public keys. Labels under `netbird.io` are kept, since it only hosts infrastructure | + +Use `strict` when internal addressing or peer naming is itself sensitive. Either way, private +keys and SSH keys are never included, and the packet capture (`capture.pcap`) is left out of +anonymized bundles because it holds raw decrypted packets. + +`-U` (`--upload-bundle`) uploads the bundle and returns a file key you can paste into the thread +instead of attaching an archive. Retention is controlled by the upload service; check its policy +before uploading, and configure cleanup for self-hosted deployments. + +For more detail, see [troubleshooting client issues](https://docs.netbird.io/help/troubleshooting-client), +which explains [what a debug bundle contains](https://docs.netbird.io/help/troubleshooting-client#debug-bundle), +and the [CLI reference](https://docs.netbird.io/get-started/cli#debug-for). + +Intermittent problems are still worth reporting. They just need enough detail to investigate: +trigger, frequency, timing, timestamps, and any related logs. + +For a feature request, describe the problem before the solution: what you are trying to +accomplish, who is affected and how often, why the current behavior or workaround is not enough, +and what you would like to see instead. + +## What happens after you post + +Our team, maintainers, or community members may ask for missing details, link related +threads, merge duplicates, move your post to a better category, or try to reproduce the problem. + +Not every discussion becomes an issue. Some are answered in Q&A, some turn out to be +configuration problems, and some need more information before engineering can act. A +well-answered discussion is still a useful outcome. + +When a report is confirmed and actionable, a maintainer opens a validated issue linked back to +the discussion, in whichever repository the fix belongs to. You do not need to know which +repository that is. Routing is part of triage. + +## A note on issues + +Issues in this repository are maintainer-curated work items. Every open issue is something a +maintainer or contributor can pick up and act on. Issues opened without a linked validated +discussion may be closed and redirected here. + +Maintainers can still open issues directly for work found internally, such as regressions caught +during development, planned maintenance, or release blockers. + +## Related reading + +- [How to use Discussions, Issues, and Pull Requests](https://github.com/netbirdio/netbird/discussions/6075) +- [Moving to a discussion-first approach](https://github.com/netbirdio/netbird/discussions/6074) +- [CONTRIBUTING.md](CONTRIBUTING.md) for opening pull requests +- [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) diff --git a/agent-network/README.md b/agent-network/README.md index a09d3979e..35b9c6668 100644 --- a/agent-network/README.md +++ b/agent-network/README.md @@ -1,16 +1,76 @@ # NetBird Agent Network -Agent Network is NetBird's access control layer for AI agents and the people who run -them. It gives every agent a real identity, tied to your identity provider (IdP), and -governs what it can reach — the LLM APIs and AI gateways it can call, and the internal -resources it can access. Traffic flows only over the encrypted NetBird tunnel, scoped by -policy, with no API keys to leak. +Agent Network is NetBird's access control layer for AI agents and the people who run them. +It gives every agent a real identity, tied to an identity provider (IdP), and governs what it can reach: LLM APIs and +AI gateways it can call, and the internal resources it can access. Traffic flows only over the encrypted NetBird tunnel, +scoped by policy, with no API keys or other credentials to leak. It also gives you control over cost and token usage. -> **Beta.** Agent Network is open source and can be self-hosted on your own -> infrastructure. +Because every LLM request passes through an +identity-aware proxy, you can: + +- **Set spending and rate limits** per agent, per user, or per team — with hard caps + that stop requests once a budget is reached. +- **Restrict models and providers** so agents can only call approved (and cost-appropriate) + endpoints, keeping expensive models off-limits unless explicitly allowed. +- **Attribute usage** by tracking token consumption and cost per identity, group, or cost center so every + request is tied back to the agent and person responsible. +- **Reuse your existing AI gateway** — point the proxy at a gateway you already run, + keeping its routing and config in place while it adds identity on top, so you skip + API key distribution. + +https://github.com/user-attachments/assets/44d18286-d8ab-49f8-a457-98ccd66f3268 + +> **Beta.** Agent Network is in beta, but it's stable and already running in +> production environments. It's fully open source and can be self-hosted on your own +> infrastructure, with no vendor lock-in and no data leaving your environment. ## How it works +Say you have a simple use case: your Engineering or IT team needs access to Claude Code or Codex, and you want visibility into usage plus the ability to enforce budgets. +How can you do that without creating a dedicated API key for every team? + +With Agent Network you get a private endpoint inside your network, for example: https://mirror.netbird.ai +Teams configure their agents to point to that endpoint instead of using individual API keys directly. + +This endpoint is only reachable when users are connected to your NetBird network and authenticated through your IdP. Otherwise, it is not accessible from the public internet. +You can then use this private endpoint to configure your AI agents, whether that is Claude Code, Codex, or another tool. + +## Quickstart + +Full step-by-step setup: +**https://docs.netbird.io/agent-network/quickstart** + +## Client settings that don't follow the endpoint + +Most of an agent's traffic follows the base URL you hand it, but a few +client-side checks call their vendor directly and never reach the proxy. On a +network that blocks direct egress they fail even though inference works, so +they are worth setting once when you roll the endpoint out. + +For Claude Code: + +- **Fast mode** checks availability against `api.anthropic.com` rather than the + configured base URL. Set `CLAUDE_CODE_SKIP_FAST_MODE_ORG_CHECK=1` when the + agent authenticates with `ANTHROPIC_AUTH_TOKEN` alone (the usual shape when + the proxy injects the real provider key) or when a TLS-inspecting proxy + answers the check itself. Set + `CLAUDE_CODE_SKIP_FAST_MODE_NETWORK_ERRORS=1` when the network refuses the + connection outright. Fast mode is an Anthropic-API feature, so it is + unavailable on a Bedrock- or Vertex-backed endpoint whatever you set. +- **Model discovery** is off by default. Set + `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` for the picker to list the + models your policies authorise; the proxy filters the response to that set. + The client gives discovery a three-second budget and treats any redirect as + a failure, so the endpoint must serve `/v1/models` directly. +- **The WebFetch domain safety check** also calls `api.anthropic.com` directly + and is unaffected by the variables above. + +Allowing direct egress to `api.anthropic.com` covers the network cases but not +the credential one, where the check reaches Anthropic and is rejected because +the agent presents a proxy-issued key. + +## Architecture + Agent Network is built on two existing NetBird capabilities: - **Overlay network** — the encrypted WireGuard mesh between peers. @@ -22,6 +82,9 @@ LLM traffic is routed through the proxy's identity-aware pipeline, while interna resources (databases, internal APIs, self-hosted models) are reached directly over peer-to-peer WireGuard tunnels, governed by the same identities and access policies. +image + + ## Where the code lives There is no separate "agent-network" service — it reuses the reverse-proxy and management @@ -33,6 +96,42 @@ components: — the management-side control plane: providers, policies, guardrails, limits, routing, and usage/access logs. +## Access roles + +Agent Network permissions build on the account permission matrix +([`management/server/permissions/`](../management/server/permissions)). The +`agent_network` area is split into dotted submodules (`agent_network.providers`, +`.policies`, `.guardrails`, `.budgets`, `.usage`, `.logs`, `.settings`); a role may +grant a single submodule or the parent, which cascades to all of them. + +Two roles delegate Agent Network access without account-admin rights: + +- **`agent_network_admin`** — full control over the whole `agent_network` area plus + read-only users, groups, peers, and account info (needed to build policies). + Nothing else in the account. +- **`usage_viewer`** — the regular User baseline plus read on + `agent_network.usage` (the aggregated usage and cost overview) and + `agent_network.logs` (the account-wide request-level access logs, which can + contain captured prompts), and read-only access to the resources those + filters resolve against: users, groups, peers, and the provider list + (connection config redacted — no upstream URLs or operator-supplied header + values). No policies, guardrails, budgets, or settings. + +Every authenticated user, regardless of role, can read the caller-scoped +self-service endpoint `GET /api/agent-network/agent-config` (the endpoint, providers, +and models the caller's own policies allow — what a local AI tool needs and nothing +more). The regular usage and access-log endpoints self-scope instead of denying: +a caller without the account-wide grant gets their own rows back, so "my usage" +and "my requests" are the same endpoints the admin dashboard uses. The provider +list self-scopes the same way — a caller without the providers grant gets the +providers their own policies authorize, reduced to the display surface, with +each provider's model list cut to what the caller's policy guardrails and the +provider's declared models effectively permit (the same computation the setup +answer and the proxy use). This feeds the dashboard's provider and model +filters. Role +definitions live in +[`management/server/permissions/roles/`](../management/server/permissions/roles). + ## Documentation Full documentation, architecture, and quickstart: diff --git a/base62/base62.go b/base62/base62.go index efafbc768..1a02e98e2 100644 --- a/base62/base62.go +++ b/base62/base62.go @@ -3,56 +3,75 @@ package base62 import ( "fmt" "math" - "strings" ) const ( - alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - base = uint32(len(alphabet)) + alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + base = uint32(len(alphabet)) + maxBase62Digits = 6 // max number of digits required to encode MaxUint32 + ) +var ( + ErrEmptyString = fmt.Errorf("empty string") + ErrInvalidChar = fmt.Errorf("invalid character") + ErrOverflow = fmt.Errorf("integer overflow") +) + +// Fixed-size arrays have better performance and lower memory overhead compared to maps for small static sets of data +var charToIndex [123]int8 // Assuming ASCII from '\0' - 'z' + +func init() { + for i := range charToIndex { + charToIndex[i] = -1 + } + for i, c := range alphabet { + charToIndex[c] = int8(i) + } +} + // Encode encodes a uint32 value to a base62 string. -func Encode(num uint32) string { - if num == 0 { - return string(alphabet[0]) +// The returned string will be between 1-6 characters long. +func Encode(n uint32) string { + if n < base { + return string(alphabet[n]) + } + // avoid dynamic memory usage for small, fixed size data + buf := [maxBase62Digits]byte{} + idx := len(buf) + + for n > 0 { + idx-- + buf[idx] = alphabet[n%base] + n /= base } - var encoded strings.Builder - - for num > 0 { - remainder := num % base - encoded.WriteByte(alphabet[remainder]) - num /= base - } - - // Reverse the encoded string - encodedString := encoded.String() - reversed := reverse(encodedString) - return reversed + return string(buf[idx:]) } // Decode decodes a base62 string to a uint32 value. +// Returns an error if the input string is empty, contains invalid characters, +// or would result in integer overflow. func Decode(encoded string) (uint32, error) { + if len(encoded) == 0 { + return 0, ErrEmptyString + } var decoded uint32 - strLen := len(encoded) - - for i, char := range encoded { - index := strings.IndexRune(alphabet, char) + for _, char := range encoded { + index := int8(-1) + if int(char) < len(charToIndex) { + index = charToIndex[char] + } if index < 0 { - return 0, fmt.Errorf("invalid character: %c", char) + return 0, fmt.Errorf("%w: %c", ErrInvalidChar, char) + } + // Add overflow check when calculating the decoded value to prevent silent overflow of uint32 + if decoded > (math.MaxUint32-uint32(index))/base { + return 0, fmt.Errorf("%w: %s", ErrOverflow, encoded) } - decoded += uint32(index) * uint32(math.Pow(float64(base), float64(strLen-i-1))) + decoded = decoded*base + uint32(index) } return decoded, nil } - -// Reverse a string. -func reverse(s string) string { - runes := []rune(s) - for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { - runes[i], runes[j] = runes[j], runes[i] - } - return string(runes) -} diff --git a/base62/base62_test.go b/base62/base62_test.go index 00da2124a..f2ad06d6f 100644 --- a/base62/base62_test.go +++ b/base62/base62_test.go @@ -1,31 +1,67 @@ package base62 import ( + "errors" + "math" "testing" ) func TestEncodeDecode(t *testing.T) { - tests := []struct { - num uint32 + testCases := []struct { + input uint32 + expected string }{ - {0}, - {1}, - {42}, - {12345}, - {99999}, - {123456789}, + {0, "0"}, + {1, "1"}, + {5, "5"}, + {9, "9"}, + {10, "A"}, + {42, "g"}, + {61, "z"}, + {62, "10"}, + {'0', "m"}, + {'9', "v"}, + {'A', "13"}, + {'Z', "1S"}, + {'a', "1Z"}, + {'z', "1y"}, + {99999, "Q0t"}, + {12345, "3D7"}, + {123456789, "8M0kX"}, + {math.MaxUint32, "4gfFC3"}, } - for _, tt := range tests { - encoded := Encode(tt.num) + for _, tc := range testCases { + encoded := Encode(tc.input) + if encoded != tc.expected { + t.Errorf("Encode(%d) = %s; want %s", tc.input, encoded, tc.expected) + } decoded, err := Decode(encoded) - if err != nil { - t.Errorf("Decode error: %v", err) + t.Errorf("Expected error nil, got %v", err) } - if decoded != tt.num { - t.Errorf("Decode(%v) = %v, want %v", encoded, decoded, tt.num) + if decoded != tc.input { + t.Errorf("Decode(%v) = %v, want %v", encoded, decoded, tc.input) } } } + +// Decode handles empty string input with appropriate error +func TestDecodeEmptyString(t *testing.T) { + if _, err := Decode(""); !errors.Is(err, ErrEmptyString) { + t.Errorf("Expected error %v, got %v", ErrEmptyString, err) + } +} + +func TestDecodeOverflow(t *testing.T) { + if _, err := Decode("4gfFC4"); !errors.Is(err, ErrOverflow) { + t.Errorf("Expected error %v, got %v", ErrOverflow, err) + } +} + +func TestDecodeInvalid(t *testing.T) { + if _, err := Decode("/"); !errors.Is(err, ErrInvalidChar) { + t.Errorf("Expected error %v, got %v", ErrInvalidChar, err) + } +} diff --git a/client/Dockerfile-nblink b/client/Dockerfile-nblink new file mode 100644 index 000000000..63177a6c0 --- /dev/null +++ b/client/Dockerfile-nblink @@ -0,0 +1,34 @@ +# nblink runs unprivileged: no TUN device, no capabilities, no host networking. +# +# The image is built for platforms that assign an arbitrary UID at run time, so +# it must not depend on a passwd entry and its state directory has to be +# writable by the root group rather than by one named user. +FROM alpine:3.22 AS build + +# Group 0 owns the state directory so a container started as any UID in that +# group can still persist its identity. Mode 770 rather than g=u, because the +# directory holds the peer's keys and nothing outside owner and group should +# read them, including when the path is bind-mounted from a host. +RUN mkdir -p /tmp/state/nblink \ + && chgrp -R 0 /tmp/state/nblink \ + && chmod -R 770 /tmp/state/nblink + +FROM gcr.io/distroless/static-debian13:nonroot + +ARG TARGETPLATFORM +COPY ${TARGETPLATFORM}/nblink /usr/local/bin/nblink +# COPY of a directory copies its contents and creates the target with mode +# 755, which group 0 cannot write. Copying the parent makes nblink/ itself the +# copied entry, so its mode survives. +COPY --from=build --chown=65532:0 /tmp/state/ /var/lib/ + +# Numeric so the runtime can substitute its own UID without a passwd lookup. +USER 65532:0 + +# No NB_STATE_DIR by default: state stays in memory, so the image runs on a +# read-only root filesystem without a volume. Set NB_STATE_DIR=/var/lib/nblink +# and mount a volume there to keep the peer identity; a volume mounted over +# the directory starts out with its group-0 ownership. +ENV NB_LOG_LEVEL=info + +ENTRYPOINT ["/usr/local/bin/nblink"] diff --git a/client/Dockerfile-rootless.ubi b/client/Dockerfile-rootless.ubi new file mode 100644 index 000000000..4701728c1 --- /dev/null +++ b/client/Dockerfile-rootless.ubi @@ -0,0 +1,45 @@ +FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:7fbeae18dc9476399f565e68255f602a3374ea8614ba3d14843565131a13ff93 + +ARG TARGETPLATFORM +ARG NETBIRD_BINARY=$TARGETPLATFORM/netbird +ARG VERSION=dev +ARG RELEASE=1 + +LABEL name="netbird-rootless" \ + maintainer="NetBird " \ + vendor="NetBird GmbH" \ + version="${VERSION}" \ + release="${RELEASE}" \ + summary="NetBird Rootless Client" \ + description="NetBird connects devices through an encrypted overlay using userspace networking without a TUN device or network administration capabilities." + +RUN microdnf install -y bash ca-certificates && microdnf clean all + +COPY --chmod=0555 client/netbird-entrypoint.sh /usr/local/bin/netbird-entrypoint.sh +COPY --chmod=0555 ${NETBIRD_BINARY} /usr/local/bin/netbird +COPY licenses/ /licenses/ +# Only application storage is group-writable for arbitrary non-root UIDs. +# Runtime-created credentials keep the client's restrictive file modes. +RUN mkdir -p /var/lib/netbird && \ + chown 1000:0 /var/lib/netbird && \ + chmod 0770 /var/lib/netbird && \ + chmod -R a+rX /licenses + +WORKDIR /var/lib/netbird +USER 1000:0 + +ENV \ + HOME="/var/lib/netbird" \ + NETBIRD_BIN="/usr/local/bin/netbird" \ + NB_USE_NETSTACK_MODE="true" \ + NB_ENABLE_NETSTACK_LOCAL_FORWARDING="true" \ + NB_CONFIG="/var/lib/netbird/config.json" \ + NB_STATE_DIR="/var/lib/netbird" \ + NB_DAEMON_ADDR="unix:///var/lib/netbird/netbird.sock" \ + NB_LOG_FILE="console,/var/lib/netbird/client.log" \ + NB_DISABLE_DNS="true" \ + NB_ENABLE_CAPTURE="false" \ + NB_ENTRYPOINT_SERVICE_TIMEOUT="30" + +STOPSIGNAL SIGTERM +ENTRYPOINT ["/usr/local/bin/netbird-entrypoint.sh"] diff --git a/client/android/client.go b/client/android/client.go index 99ccdf393..9705db8e0 100644 --- a/client/android/client.go +++ b/client/android/client.go @@ -7,13 +7,16 @@ import ( "fmt" "os" "slices" + "strings" "sync" + "sync/atomic" "time" "golang.org/x/exp/maps" log "github.com/sirupsen/logrus" + nbAnonymize "github.com/netbirdio/netbird/client/anonymize" "github.com/netbirdio/netbird/client/iface/device" "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/debug" @@ -24,6 +27,7 @@ import ( "github.com/netbirdio/netbird/client/internal/routemanager" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/net" + "github.com/netbirdio/netbird/client/netevents" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/formatter" "github.com/netbirdio/netbird/route" @@ -31,10 +35,12 @@ import ( types "github.com/netbirdio/netbird/upload-server/types" ) -// ConnectionListener export internal Listener for mobile -type ConnectionListener interface { - peer.Listener -} +// AnonymizeLevelDefault and AnonymizeLevelStrict are the accepted +// anonymizeLevel values for DebugBundle. +const ( + AnonymizeLevelDefault = nbAnonymize.LevelDefaultString + AnonymizeLevelStrict = nbAnonymize.LevelStrictString +) // TunAdapter export internal TunAdapter for mobile type TunAdapter interface { @@ -56,6 +62,12 @@ type DnsReadyListener interface { dns.ReadyListener } +// TunSettings is a snapshot of the settings the TUN device is rebuilt with +type TunSettings struct { + Routes string + SearchDomains string +} + func init() { formatter.SetLogcatFormatter(log.StandardLogger()) } @@ -70,18 +82,50 @@ type Client struct { deviceName string uiVersion string networkChangeListener listener.NetworkChangeListener + // netMgr outlives engine restarts: it mirrors the OS connectivity, not + // the engine lifecycle. Run and RunWithoutLogin inject its state and + // sweeper into each new ConnectClient. + netMgr *netevents.Manager stateMu sync.RWMutex connectClient *internal.ConnectClient config *profilemanager.Config cacheDir string + + // mdmSource holds the per-Client MDM policy source and its change + // detector as one unit. Set by SetMDMPolicyFetcher (called from the + // Kotlin side). Each Run passes the loader to the resolved Config so + // applyMDMPolicy picks up the active overlay. Nil means "MDM + // enforcement off for this Client". + mdmSource atomic.Pointer[mdmSource] + + // Identifies the running profile for the SSO login hint; see profile_state.go. + cfgPath string + + stateChangeMu sync.Mutex + stateChangeSubID string + // Closed to stop the watch goroutine from delivering buffered ticks to a + // listener that has been removed or replaced. See stopStateChangeWatchLocked. + stateChangeDone chan struct{} + + // Latched "the server wants an interactive login": survives the engine + // restarts that replace the run loop's context state. See Client.Status. + // Guarded by loginRequiredMu together with loginCleared, which counts + // clears so a stale observation cannot re-latch over one. + loginRequiredMu sync.Mutex + loginRequired bool + loginCleared uint64 + + extendMu sync.Mutex + extendCancel context.CancelFunc } -func (c *Client) setState(cfg *profilemanager.Config, cacheDir string, cc *internal.ConnectClient) { +func (c *Client) setState(cfg *profilemanager.Config, cacheDir string, cfgPath string, cc *internal.ConnectClient) { c.stateMu.Lock() defer c.stateMu.Unlock() c.config = cfg c.cacheDir = cacheDir + c.cfgPath = cfgPath c.connectClient = cc } @@ -91,6 +135,16 @@ func (c *Client) stateSnapshot() (*profilemanager.Config, string, *internal.Conn return c.config, c.cacheDir, c.connectClient } +// authSnapshot returns the config together with the path it was loaded from, in +// one lock: the path identifies the profile whose account email backs the login +// hint, so reading it separately could pair one profile's config with another's +// hint when a profile switch lands in between. +func (c *Client) authSnapshot() (*profilemanager.Config, string, *internal.ConnectClient) { + c.stateMu.RLock() + defer c.stateMu.RUnlock() + return c.config, c.cfgPath, c.connectClient +} + func (c *Client) getConnectClient() *internal.ConnectClient { c.stateMu.RLock() defer c.stateMu.RUnlock() @@ -102,14 +156,17 @@ func NewClient(androidSDKVersion int, deviceName string, uiVersion string, tunAd execWorkaround(androidSDKVersion) net.SetAndroidProtectSocketFn(tunAdapter.ProtectSocket) + system.SetIFaceDiscover(iFaceDiscover) + recorder := peer.NewRecorder("") return &Client{ deviceName: deviceName, uiVersion: uiVersion, tunAdapter: tunAdapter, iFaceDiscover: iFaceDiscover, - recorder: peer.NewRecorder(""), + recorder: recorder, ctxCancelLock: &sync.Mutex{}, networkChangeListener: networkChangeListener, + netMgr: netevents.NewManager(recorder), } } @@ -129,6 +186,7 @@ func (c *Client) Run(platformFiles PlatformFiles, urlOpener URLOpener, isAndroid if err != nil { return err } + c.applyMDMOverlay(cfg) c.recorder.UpdateManagementAddress(cfg.ManagementURL.String()) c.recorder.UpdateRosenpass(cfg.RosenpassEnabled, cfg.RosenpassPermissive) @@ -143,16 +201,24 @@ func (c *Client) Run(platformFiles PlatformFiles, urlOpener URLOpener, isAndroid defer c.ctxCancel() c.ctxCancelLock.Unlock() - auth := NewAuthWithConfig(ctx, cfg) + auth := NewAuthWithConfig(ctx, cfg, cfgFile) err = auth.login(urlOpener, isAndroidTV) if err != nil { return err } - // todo do not throw error in case of cancelled context ctx = internal.CtxInitState(ctx) - connectClient := internal.NewConnectClient(ctx, cfg, c.recorder) - c.setState(cfg, cacheDir, connectClient) + + connectClient := internal.NewConnectClient(ctx, cfg, c.recorder, + internal.WithNetEvents(c.netMgr)) + c.setState(cfg, cacheDir, cfgFile, connectClient) + connectClient.SetSyncResponsePersistence(true) + // This path runs the interactive SSO flow, so reaching here means the peer + // is authenticated again — release the latch Status() reports from. Clear + // only once the fresh connect client is installed: until then Status() + // still reads the previous run's context state, which holds the NeedsLogin + // that prompted this login, and would re-latch what was just cleared. + c.clearLoginRequired() return connectClient.RunOnAndroid(c.tunAdapter, c.iFaceDiscover, c.networkChangeListener, slices.Clone(dns.items), dnsReadyListener, stateFile, cacheDir) } @@ -173,6 +239,7 @@ func (c *Client) RunWithoutLogin(platformFiles PlatformFiles, dns *DNSList, dnsR if err != nil { return err } + c.applyMDMOverlay(cfg) c.recorder.UpdateManagementAddress(cfg.ManagementURL.String()) c.recorder.UpdateRosenpass(cfg.RosenpassEnabled, cfg.RosenpassPermissive) @@ -186,8 +253,10 @@ func (c *Client) RunWithoutLogin(platformFiles PlatformFiles, dns *DNSList, dnsR // todo do not throw error in case of cancelled context ctx = internal.CtxInitState(ctx) - connectClient := internal.NewConnectClient(ctx, cfg, c.recorder) - c.setState(cfg, cacheDir, connectClient) + connectClient := internal.NewConnectClient(ctx, cfg, c.recorder, + internal.WithNetEvents(c.netMgr)) + c.setState(cfg, cacheDir, cfgFile, connectClient) + connectClient.SetSyncResponsePersistence(true) return connectClient.RunOnAndroid(c.tunAdapter, c.iFaceDiscover, c.networkChangeListener, slices.Clone(dns.items), dnsReadyListener, stateFile, cacheDir) } @@ -216,9 +285,62 @@ func (c *Client) RenewTun(fd int) error { return e.RenewTun(fd) } +func (c *Client) GetTunSettings() (*TunSettings, error) { + cc := c.getConnectClient() + if cc == nil { + return nil, fmt.Errorf("engine not running") + } + + e := cc.Engine() + if e == nil { + return nil, fmt.Errorf("engine not initialized") + } + + routes, searchDomains := e.TunSettings() + return &TunSettings{ + Routes: strings.Join(routes, ";"), + SearchDomains: strings.Join(searchDomains, ";"), + }, nil +} + +// SetNetworkAvailable feeds OS-reported network availability into the client. +// While unavailable, the internal reconnect loops suspend their attempts and +// the connection listener reports NoNetwork instead of Connecting; when +// availability returns, the loops resume immediately with a fresh backoff. +// Losing the last network also sweeps the registered connections: nothing can +// redial while offline, so the stale sockets would otherwise stay silently +// "connected" until their own timeouts and the client would keep reporting +// Connected with no network at all. +func (c *Client) SetNetworkAvailable(available bool) { + c.netMgr.SetNetworkAvailable(available) +} + +// NotifyNetworkChange marks the management, signal and relay connections +// stale after the OS switched networks and schedules a sweep that cuts +// whatever has not redialed on the new network by then. The engine and the +// TUN device stay untouched. +func (c *Client) NotifyNetworkChange() { + c.netMgr.NotifyNetworkChange() +} + // DebugBundle generates a debug bundle, uploads it, and returns the upload key. -// It works both with and without a running engine. -func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (string, error) { +// It works both with and without a running engine. anonymizeLevel is "default" +// or "strict"; strict also anonymizes internal IP ranges, peer names, and +// WireGuard public keys, and implies anonymize. +func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool, anonymizeLevel string) (string, error) { + return c.debugBundle(platformFiles, anonymize, anonymizeLevel, true) +} + +// DebugBundleFile generates a debug bundle and returns the path of the zip in +// the cache directory instead of uploading it, so the app can hand the file to +// the user for inspection. The caller owns the file and removes it once done; +// the stale-bundle cleanup of later runs removes it only after a day. +// anonymize and anonymizeLevel behave as in DebugBundle. +func (c *Client) DebugBundleFile(platformFiles PlatformFiles, anonymize bool, anonymizeLevel string) (string, error) { + return c.debugBundle(platformFiles, anonymize, anonymizeLevel, false) +} + +func (c *Client) debugBundle(platformFiles PlatformFiles, anonymize bool, anonymizeLevel string, upload bool) (string, error) { cfg, cacheDir, cc := c.stateSnapshot() // If the engine hasn't been started, load config from disk @@ -230,13 +352,20 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (strin if err != nil { return "", fmt.Errorf("load config: %w", err) } + c.applyMDMOverlay(cfg) cacheDir = platformFiles.CacheDir() } + // Clear what an interrupted earlier run may have left in the cache before + // adding to it. Remote debug jobs write to the same directory, so anything + // younger than an hour is treated as possibly still in use. + debug.RemoveStaleBundles(cacheDir, time.Hour) + deps := debug.GeneratorDependencies{ InternalConfig: cfg, StatusRecorder: c.recorder, TempDir: cacheDir, + StatePath: platformFiles.StateFilePath(), } if cc != nil { @@ -247,6 +376,9 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (strin deps.SyncResponse = resp if e := cc.Engine(); e != nil { + deps.RefreshStatus = func() { + e.RunHealthProbes(context.Background(), true) + } if cm := e.GetClientMetrics(); cm != nil { deps.ClientMetrics = cm } @@ -257,6 +389,7 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (strin deps, debug.BundleConfig{ Anonymize: anonymize, + AnonymizeLevel: nbAnonymize.ParseLevel(anonymizeLevel), IncludeSystemInfo: true, }, ) @@ -265,6 +398,9 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (strin if err != nil { return "", fmt.Errorf("generate debug bundle: %w", err) } + if !upload { + return debug.ExportBundle(path) + } defer func() { if err := os.Remove(path); err != nil { log.Errorf("failed to remove debug bundle file: %v", err) @@ -274,7 +410,7 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (strin uploadCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - key, err := debug.UploadDebugBundle(uploadCtx, types.DefaultBundleURL, cfg.ManagementURL.String(), path) + key, err := debug.UploadDebugBundle(uploadCtx, types.DefaultBundleURL, cfg.ManagementURL.String(), path, false) if err != nil { return "", fmt.Errorf("upload debug bundle: %w", err) } @@ -296,6 +432,13 @@ func (c *Client) SetInfoLogLevel() { // PeersList return with the list of the PeerInfos func (c *Client) PeersList() *PeerInfoArray { + // The recorder only caches transfer counters and handshake times; nothing + // refreshes them on its own, so without this they read as zero. The desktop + // daemon does the same before serving a full peer status. + if err := c.recorder.RefreshWireGuardStats(); err != nil { + log.Debugf("failed to refresh WireGuard stats: %v", err) + } + fullStatus := c.recorder.GetFullStatus() peerInfos := make([]PeerInfo, len(fullStatus.Peers)) @@ -306,6 +449,20 @@ func (c *Client) PeersList() *PeerInfoArray { FQDN: p.FQDN, ConnStatus: int(p.ConnStatus), Routes: PeerRoutes{routes: maps.Keys(p.GetRoutes())}, + + PubKey: p.PubKey, + Latency: formatDuration(p.Latency), + LatencyMs: p.Latency.Milliseconds(), + BytesRx: p.BytesRx, + BytesTx: p.BytesTx, + ConnStatusUpdate: formatTime(p.ConnStatusUpdate), + Relayed: p.Relayed, + RosenpassEnabled: p.RosenpassEnabled, + LastWireguardHandshake: formatTime(p.LastWireguardHandshake), + LocalIceCandidateType: p.LocalIceCandidateType, + RemoteIceCandidateType: p.RemoteIceCandidateType, + LocalIceCandidateEndpoint: p.LocalIceCandidateEndpoint, + RemoteIceCandidateEndpoint: p.RemoteIceCandidateEndpoint, } peerInfos[n] = pi } @@ -340,6 +497,7 @@ func (c *Client) Networks() *NetworkArray { routesMap := routeManager.GetClientRoutesWithNetID() v6Merged := route.V6ExitMergeSet(routesMap) resolvedDomains := c.recorder.GetResolvedDomainsStates() + activeRoutePeers := c.recorder.GetActiveRoutePeers() networkArray := &NetworkArray{ items: make([]Network, 0), @@ -353,7 +511,7 @@ func (c *Client) Networks() *NetworkArray { continue } - network := c.buildNetwork(id, routes, routeSelector.IsSelected(id), resolvedDomains, v6Merged) + network := c.buildNetwork(id, routes, routeSelector.IsSelected(id), resolvedDomains, v6Merged, activeRoutePeers) if network == nil { continue } @@ -362,14 +520,14 @@ func (c *Client) Networks() *NetworkArray { return networkArray } -func (c *Client) buildNetwork(id route.NetID, routes []*route.Route, selected bool, resolvedDomains map[domain.Domain]peer.ResolvedDomainInfo, v6Merged map[route.NetID]struct{}) *Network { +func (c *Client) buildNetwork(id route.NetID, routes []*route.Route, selected bool, resolvedDomains map[domain.Domain]peer.ResolvedDomainInfo, v6Merged map[route.NetID]struct{}, activeRoutePeers map[route.HAUniqueID]string) *Network { r := routes[0] netStr := r.Network.String() if r.IsDynamic() { netStr = r.Domains.SafeString() } - routePeer, err := c.findBestRoutePeer(routes) + routePeer, err := c.findBestRoutePeer(routes, activeRoutePeers) if err != nil { log.Errorf("could not get peer info for route %s: %v", id, err) return nil @@ -393,12 +551,9 @@ func (c *Client) buildNetwork(id route.NetID, routes []*route.Route, selected bo // findBestRoutePeer returns the peer actively routing traffic for the given // HA route group. Falls back to the first connected peer, then the first peer. -func (c *Client) findBestRoutePeer(routes []*route.Route) (peer.State, error) { - netStr := routes[0].Network.String() - - fullStatus := c.recorder.GetFullStatus() - for _, p := range fullStatus.Peers { - if _, ok := p.GetRoutes()[netStr]; ok { +func (c *Client) findBestRoutePeer(routes []*route.Route, activeRoutePeers map[route.HAUniqueID]string) (peer.State, error) { + if peerKey, ok := activeRoutePeers[routes[0].GetHAUniqueID()]; ok { + if p, err := c.recorder.GetPeer(peerKey); err == nil { return p, nil } } @@ -428,7 +583,11 @@ func (c *Client) OnUpdatedHostDNS(list *DNSList) error { // SetConnectionListener set the network connection listener func (c *Client) SetConnectionListener(listener ConnectionListener) { - c.recorder.SetConnectionListener(listener) + if listener == nil { + c.recorder.RemoveConnectionListener() + return + } + c.recorder.SetConnectionListener(connectionListenerAdapter{listener}) } // RemoveConnectionListener remove connection listener @@ -436,10 +595,6 @@ func (c *Client) RemoveConnectionListener() { c.recorder.RemoveConnectionListener() } -func (c *Client) toggleRoute(command routeCommand) error { - return command.toggleRoute() -} - func (c *Client) getRouteManager() (routemanager.Manager, error) { client := c.getConnectClient() if client == nil { @@ -459,22 +614,22 @@ func (c *Client) getRouteManager() (routemanager.Manager, error) { return manager, nil } -func (c *Client) SelectRoute(route string) error { +func (c *Client) SelectRoute(id string) error { manager, err := c.getRouteManager() if err != nil { return err } - return c.toggleRoute(selectRouteCommand{route: route, manager: manager}) + return manager.SelectRoutes([]route.NetID{route.NetID(id)}, true) } -func (c *Client) DeselectRoute(route string) error { +func (c *Client) DeselectRoute(id string) error { manager, err := c.getRouteManager() if err != nil { return err } - return c.toggleRoute(deselectRouteCommand{route: route, manager: manager}) + return manager.DeselectRoutes([]route.NetID{route.NetID(id)}) } // getNetworkDomainsFromRoute extracts domains from a route and enriches each domain @@ -509,3 +664,28 @@ func exportEnvList(list *EnvList) { } } } + +// formatDuration renders a duration for display, trimming the fractional part +// to two digits so latencies read as "12.34ms" rather than "12.345678ms". +func formatDuration(d time.Duration) string { + ds := d.String() + dotIndex := strings.Index(ds, ".") + if dotIndex == -1 { + return ds + } + + endIndex := min(dotIndex+3, len(ds)) + + // Skip the remaining digits so only the unit suffix is appended back. + unitStart := endIndex + for unitStart < len(ds) && ds[unitStart] >= '0' && ds[unitStart] <= '9' { + unitStart++ + } + return ds[:endIndex] + ds[unitStart:] +} + +// formatTime renders a timestamp in UTC using a fixed layout. The zero time is +// passed through as-is so the UI can recognise it and show "never" instead. +func formatTime(t time.Time) string { + return t.UTC().Format("2006-01-02 15:04:05") +} diff --git a/client/android/client_mdm.go b/client/android/client_mdm.go new file mode 100644 index 000000000..d043b85d3 --- /dev/null +++ b/client/android/client_mdm.go @@ -0,0 +1,52 @@ +//go:build android + +package android + +import ( + "github.com/netbirdio/netbird/client/internal/profilemanager" + "github.com/netbirdio/netbird/client/mdm" +) + +type mdmSource struct { + loader *mdm.Loader + detector *mdm.ChangeDetector +} + +// SetMDMPolicyFetcher registers the native-provided MDM policy fetcher on +// this Client; passing nil disables MDM enforcement. +func (c *Client) SetMDMPolicyFetcher(p PolicyFetcher) { + loader := loaderFor(p) + c.mdmSource.Store(&mdmSource{loader: loader, detector: mdm.NewChangeDetector(loader)}) +} + +// HasMDMPolicyChanged re-reads the managed configuration and reports whether +// it changed since the last observation; call it from the native OS-change +// notification and restart the engine only on true. +func (c *Client) HasMDMPolicyChanged() bool { + src := c.mdmSource.Load() + if src == nil { + return false + } + return src.detector.Changed() +} + +// GetRestrictionsJSON returns the UI enforcement snapshot derived from the +// active MDM policy, in the JSON shape shared with the desktop frontend. +func (c *Client) GetRestrictionsJSON() (string, error) { + return mdm.BuildRestrictions(c.mdmLoader().Load()).JSON() +} + +func (c *Client) applyMDMOverlay(cfg *profilemanager.Config) { + loader := c.mdmLoader() + if cfg == nil || loader == nil { + return + } + cfg.ApplyMDMPolicy(loader.Load()) +} + +func (c *Client) mdmLoader() *mdm.Loader { + if src := c.mdmSource.Load(); src != nil { + return src.loader + } + return nil +} diff --git a/client/android/connection_listener.go b/client/android/connection_listener.go new file mode 100644 index 000000000..77c47574b --- /dev/null +++ b/client/android/connection_listener.go @@ -0,0 +1,41 @@ +//go:build android + +package android + +import ( + "github.com/netbirdio/netbird/client/internal/peer" +) + +// Client state values delivered via ConnectionListener.OnStateChanged, +// re-exported as basic constants so gomobile emits them into the generated +// Java bindings. They mirror peer.ClientState*: append-only, never reorder. +const ( + ClientStateDisconnected = int(peer.ClientStateDisconnected) + ClientStateConnected = int(peer.ClientStateConnected) + ClientStateConnecting = int(peer.ClientStateConnecting) + ClientStateDisconnecting = int(peer.ClientStateDisconnecting) + ClientStateNoNetwork = int(peer.ClientStateNoNetwork) +) + +// ConnectionListener export internal Listener for mobile. It mirrors +// peer.Listener with OnStateChanged taking a plain int (one of the +// ClientState* constants), because gomobile cannot bind named types. +type ConnectionListener interface { + OnStateChanged(state int) + OnConnected() + OnDisconnected() + OnConnecting() + OnDisconnecting() + OnAddressChanged(string, string) + OnPeersListChanged(int) +} + +// connectionListenerAdapter adapts the gomobile-facing ConnectionListener to +// peer.Listener, converting the typed state to the int the binding carries. +type connectionListenerAdapter struct { + ConnectionListener +} + +func (a connectionListenerAdapter) OnStateChanged(state peer.ClientState) { + a.ConnectionListener.OnStateChanged(int(state)) +} diff --git a/client/android/login.go b/client/android/login.go index a9422cdbf..b9ec21b39 100644 --- a/client/android/login.go +++ b/client/android/login.go @@ -4,8 +4,12 @@ import ( "context" "fmt" + log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/client/internal/auth" "github.com/netbirdio/netbird/client/internal/profilemanager" + "github.com/netbirdio/netbird/client/mdm" + "github.com/netbirdio/netbird/client/mobile" "github.com/netbirdio/netbird/client/system" ) @@ -36,15 +40,31 @@ type Auth struct { } // NewAuth instantiate Auth struct and validate the management URL -func NewAuth(cfgPath string, mgmURL string) (*Auth, error) { - inputCfg := profilemanager.ConfigInput{ - ManagementURL: mgmURL, +// +// The configuration at cfgPath is reused when one is already there, and only created when it is +// not. Building a fresh in-memory config unconditionally gives the client a new WireGuard key on +// every call: the peer registers under that key, the key is written out, and any peer registered by +// an earlier call is orphaned on the server. It also breaks a client that enrols and then runs from +// the persisted config, because the identity it registered is not the one it runs with — the +// management stream rejects it with "no peer auth method provided". +// +// Auth is constructed under the active MDM policy: the policy is overlaid on +// the resolved config so the login runs against the enforced values, while +// the persisted config keeps the caller-supplied ones; a caller-supplied +// management URL is ignored while MDM manages that key. A nil fetcher +// disables MDM enforcement. +func NewAuth(cfgPath string, mgmURL string, fetcher PolicyFetcher) (*Auth, error) { + policy := loaderFor(fetcher).Load() + inputCfg := profilemanager.ConfigInput{ConfigPath: cfgPath} + if _, managed := policy.GetString(mdm.KeyManagementURL); !managed { + inputCfg.ManagementURL = mgmURL } - cfg, err := profilemanager.CreateInMemoryConfig(inputCfg) + cfg, err := profilemanager.UpdateOrCreateConfig(inputCfg) if err != nil { return nil, err } + cfg.ApplyMDMPolicy(policy) return &Auth{ ctx: context.Background(), @@ -53,17 +73,18 @@ func NewAuth(cfgPath string, mgmURL string) (*Auth, error) { }, nil } -// NewAuthWithConfig instantiate Auth based on existing config -func NewAuthWithConfig(ctx context.Context, config *profilemanager.Config) *Auth { +// NewAuthWithConfig instantiate Auth based on existing config. cfgPath is the +// file the config was loaded from; it identifies the profile whose account email +// backs the login_hint. +func NewAuthWithConfig(ctx context.Context, config *profilemanager.Config, cfgPath string) *Auth { return &Auth{ - ctx: ctx, - config: config, + ctx: ctx, + config: config, + cfgPath: cfgPath, } } -// SaveConfigIfSSOSupported test the connectivity with the management server by retrieving the server device flow info. -// If it returns a flow info than save the configuration and return true. If it gets a codes.NotFound, it means that SSO -// is not supported and returns false without saving the configuration. For other errors return false. +// SaveConfigIfSSOSupported reports whether the management server supports SSO; the config is already persisted by NewAuth. func (a *Auth) SaveConfigIfSSOSupported(listener SSOListener) { go func() { sso, err := a.saveConfigIfSSOSupported() @@ -87,15 +108,10 @@ func (a *Auth) saveConfigIfSSOSupported() (bool, error) { return false, fmt.Errorf("failed to check SSO support: %v", err) } - if !supportsSSO { - return false, nil - } - - err = profilemanager.WriteOutConfig(a.cfgPath, a.config) - return true, err + return supportsSSO, nil } -// LoginWithSetupKeyAndSaveConfig test the connectivity with the management server with the setup key. +// LoginWithSetupKeyAndSaveConfig registers the peer with the setup key; the config is already persisted by NewAuth. func (a *Auth) LoginWithSetupKeyAndSaveConfig(resultListener ErrListener, setupKey string, deviceName string) { go func() { err := a.loginWithSetupKeyAndSaveConfig(setupKey, deviceName) @@ -120,8 +136,7 @@ func (a *Auth) loginWithSetupKeyAndSaveConfig(setupKey string, deviceName string if err != nil { return fmt.Errorf("login failed: %v", err) } - - return profilemanager.WriteOutConfig(a.cfgPath, a.config) + return nil } // Login try register the client on the server @@ -150,12 +165,14 @@ func (a *Auth) login(urlOpener URLOpener, isAndroidTV bool) error { } jwtToken := "" + email := "" if needsLogin { tokenInfo, err := a.foregroundGetTokenInfo(authClient, urlOpener, isAndroidTV) if err != nil { return fmt.Errorf("interactive sso login failed: %v", err) } jwtToken = tokenInfo.GetTokenToUse() + email = tokenInfo.Email } err, _ = authClient.Login(a.ctx, "", jwtToken) @@ -163,27 +180,99 @@ func (a *Auth) login(urlOpener URLOpener, isAndroidTV bool) error { return fmt.Errorf("login failed: %v", err) } + // Stored after Login, not before: a rejected token must not leave a hint + // pointing at an account that cannot be used. + if email != "" && a.cfgPath != "" { + if err := mobile.WriteProfileEmail(a.cfgPath, email); err != nil { + log.Warnf("failed to store profile account email: %v", err) + } + } + go urlOpener.OnLoginSuccess() return nil } func (a *Auth) foregroundGetTokenInfo(authClient *auth.Auth, urlOpener URLOpener, isAndroidTV bool) (*auth.TokenInfo, error) { - oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, isAndroidTV) + return a.foregroundGetTokenInfoFlow(authClient, urlOpener, isAndroidTV, false) +} + +// foregroundGetTokenInfoFlow runs the interactive flow. sessionExtend tells the +// server the token will renew this peer's session rather than log a peer in, so +// it can rule out a silent authorization the IdP could answer from an unrelated +// account. See PKCEAuthorizationFlowRequest. +func (a *Auth) foregroundGetTokenInfoFlow(authClient *auth.Auth, urlOpener URLOpener, isAndroidTV bool, sessionExtend bool) (*auth.TokenInfo, error) { + hint := profileLoginHint(a.cfgPath) + + oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, isAndroidTV, sessionExtend, hint) if err != nil { return nil, fmt.Errorf("failed to get OAuth flow: %v", err) } - flowInfo, err := oAuthFlow.RequestAuthInfo(context.TODO()) + tokenInfo, err := runOAuthFlow(a.ctx, oAuthFlow, urlOpener, nil) if err != nil { - return nil, fmt.Errorf("getting a request OAuth flow info failed: %v", err) + return nil, err } - go urlOpener.Open(flowInfo.VerificationURIComplete, flowInfo.UserCode) + if tokenInfo.MatchesAccount(hint) { + return tokenInfo, nil + } - tokenInfo, err := oAuthFlow.WaitToken(a.ctx, flowInfo) + // The IdP answered from a session belonging to another account. Retrying is + // what makes this recoverable: on a peer already registered the server would + // reject the token, and on a fresh one it would silently register the peer + // under the wrong account and bind the profile to it. + log.Infof("login returned an account other than the one this profile is bound to, retrying with an account prompt") + retryFlow := auth.RetryFlowForAccount(oAuthFlow) + if retryFlow == nil { + return tokenInfo, nil + } + + retryToken, err := runOAuthFlow(a.ctx, retryFlow, urlOpener, nil) if err != nil { - return nil, fmt.Errorf("waiting for browser login failed: %v", err) + return nil, err + } + if !retryToken.MatchesAccount(hint) { + log.Warnf("login still returned a different account after the prompt, continuing with it") + } + + return retryToken, nil +} + +// profileLoginHint returns the stored account email for the profile at cfgPath. +// An empty hint is deliberate, not a fallback: a fresh profile leaves the +// choice to the IdP. Switching accounts is done by switching or removing +// profiles, not by logging out — logout keeps the email. +func profileLoginHint(cfgPath string) string { + if cfgPath == "" { + return "" + } + return mobile.ReadProfileEmail(cfgPath) +} + +// runOAuthFlow drives an already acquired OAuth flow to a token: requests the +// flow info, presents the verification URL through the opener and waits for +// the browser round-trip. Open is called synchronously — it is what marks the +// surface as opened on the client side, and a fast token's OnLoginSuccess is +// a no-op until it has, so the dismissal would be dropped rather than +// delayed. Openers must therefore not block: they post their UI work and +// return. onWaiting, when set, runs after the URL is shown, right before the +// blocking wait. +func runOAuthFlow(ctx context.Context, flow auth.OAuthFlow, urlOpener URLOpener, onWaiting func()) (*auth.TokenInfo, error) { + flowInfo, err := flow.RequestAuthInfo(ctx) + if err != nil { + return nil, fmt.Errorf("request auth info: %w", err) + } + + urlOpener.Open(flowInfo.VerificationURIComplete, flowInfo.UserCode) + + if onWaiting != nil { + onWaiting() + } + + tokenInfo, err := flow.WaitToken(ctx, flowInfo) + if err != nil { + return nil, fmt.Errorf("wait for token: %w", err) } return &tokenInfo, nil diff --git a/client/android/login_test.go b/client/android/login_test.go new file mode 100644 index 000000000..130a846fc --- /dev/null +++ b/client/android/login_test.go @@ -0,0 +1,51 @@ +package android + +import ( + "path/filepath" + "testing" +) + +// NewAuth must reuse the configuration already at cfgPath rather than building a fresh one. +// +// Creating a new in-memory config on every call gives the client a new WireGuard private key each +// time. The peer registers under that key and the key is written out, so a peer registered by an +// earlier call is orphaned on the server — a client that enrols twice leaves two entries and owns +// neither. It also breaks enrol-then-run: RunWithoutLogin reloads the configuration from disk, so +// the identity that registered is not the identity that runs, and the management stream rejects it +// with "no peer auth method provided, please use a setup key or interactive SSO login". +func TestNewAuth_ReusesPersistedIdentity(t *testing.T) { + cfgPath := filepath.Join(t.TempDir(), "config.json") + + first, err := NewAuth(cfgPath, "https://api.example.com:443", nil) + if err != nil { + t.Fatalf("first NewAuth: %v", err) + } + if first.config.PrivateKey == "" { + t.Fatal("first NewAuth produced no private key") + } + + second, err := NewAuth(cfgPath, "https://api.example.com:443", nil) + if err != nil { + t.Fatalf("second NewAuth: %v", err) + } + + if second.config.PrivateKey != first.config.PrivateKey { + t.Errorf("private key changed between calls: a second enrolment would orphan the peer registered by the first") + } +} + +// A missing configuration is still created, so a first enrolment works unchanged. +func TestNewAuth_CreatesConfigWhenAbsent(t *testing.T) { + cfgPath := filepath.Join(t.TempDir(), "config.json") + + auth, err := NewAuth(cfgPath, "https://api.example.com:443", nil) + if err != nil { + t.Fatalf("NewAuth: %v", err) + } + if auth.config == nil || auth.config.PrivateKey == "" { + t.Fatal("NewAuth did not create a usable configuration") + } + if auth.cfgPath != cfgPath { + t.Errorf("cfgPath = %q, want %q", auth.cfgPath, cfgPath) + } +} diff --git a/client/android/mdm.go b/client/android/mdm.go new file mode 100644 index 000000000..617d8f7cb --- /dev/null +++ b/client/android/mdm.go @@ -0,0 +1,19 @@ +package android + +import ( + "github.com/netbirdio/netbird/client/mdm" +) + +// PolicyFetcher is implemented by the native layer to return the current +// managed configuration as a JSON-encoded object string; "" means no MDM +// source is present. +type PolicyFetcher interface { + FetchJSON() string +} + +func loaderFor(p PolicyFetcher) *mdm.Loader { + if p == nil { + return mdm.NewJSONLoader(nil) + } + return mdm.NewJSONLoader(p.FetchJSON) +} diff --git a/client/android/peer_notifier.go b/client/android/peer_notifier.go index c2595e574..f525055bb 100644 --- a/client/android/peer_notifier.go +++ b/client/android/peer_notifier.go @@ -12,12 +12,30 @@ const ( ) // PeerInfo describe information about the peers. It designed for the UI usage +// +// The fields below ConnStatus back the peer detail screen. Durations and times +// are pre-formatted into strings so the UI does not have to know Go's layouts; +// Latency is additionally exposed as LatencyMs for colour coding. type PeerInfo struct { IP string IPv6 string FQDN string ConnStatus int Routes PeerRoutes + + PubKey string + Latency string + LatencyMs int64 + BytesRx int64 + BytesTx int64 + ConnStatusUpdate string + Relayed bool + RosenpassEnabled bool + LastWireguardHandshake string + LocalIceCandidateType string + RemoteIceCandidateType string + LocalIceCandidateEndpoint string + RemoteIceCandidateEndpoint string } func (p *PeerInfo) GetPeerRoutes() *PeerRoutes { diff --git a/client/android/preferences.go b/client/android/preferences.go index 066477293..3623de23f 100644 --- a/client/android/preferences.go +++ b/client/android/preferences.go @@ -1,12 +1,16 @@ package android import ( + "sync/atomic" + "github.com/netbirdio/netbird/client/internal/profilemanager" + "github.com/netbirdio/netbird/client/mdm" ) // Preferences exports a subset of the internal config for gomobile type Preferences struct { configInput profilemanager.ConfigInput + mdmLoader atomic.Pointer[mdm.Loader] } // NewPreferences creates a new Preferences instance @@ -14,20 +18,39 @@ func NewPreferences(configPath string) *Preferences { ci := profilemanager.ConfigInput{ ConfigPath: configPath, } - return &Preferences{ci} + return &Preferences{configInput: ci} +} + +// SetMDMPolicyFetcher registers the native-provided MDM policy fetcher on +// this Preferences instance; passing nil disables MDM enforcement. +func (p *Preferences) SetMDMPolicyFetcher(f PolicyFetcher) { + p.mdmLoader.Store(loaderFor(f)) +} + +// GetRestrictionsJSON returns the UI enforcement snapshot derived from the +// active MDM policy, in the JSON shape shared with the desktop frontend. +func (p *Preferences) GetRestrictionsJSON() (string, error) { + return mdm.BuildRestrictions(p.policy()).JSON() +} + +func (p *Preferences) policy() *mdm.Policy { + return p.mdmLoader.Load().Load() } // GetManagementURL reads URL from config file func (p *Preferences) GetManagementURL() (string, error) { + if v, ok := p.policy().GetString(mdm.KeyManagementURL); ok { + return mdm.CanonicalURL(v), nil + } if p.configInput.ManagementURL != "" { return p.configInput.ManagementURL, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return "", err } - return cfg.ManagementURL.String(), err + return cfg.ManagementURL.String(), nil } // SetManagementURL stores the given URL and waits for commit @@ -41,7 +64,7 @@ func (p *Preferences) GetAdminURL() (string, error) { return p.configInput.AdminURL, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return "", err } @@ -53,17 +76,21 @@ func (p *Preferences) SetAdminURL(url string) { p.configInput.AdminURL = url } -// GetPreSharedKey reads pre-shared key from config file -func (p *Preferences) GetPreSharedKey() (string, error) { +// HasPreSharedKey reports whether a pre-shared key is staged, persisted, or +// enforced by MDM; the key itself is never handed to the native layer. +func (p *Preferences) HasPreSharedKey() (bool, error) { + if _, ok := p.policy().GetString(mdm.KeyPreSharedKey); ok { + return true, nil + } if p.configInput.PreSharedKey != nil { - return *p.configInput.PreSharedKey, nil + return *p.configInput.PreSharedKey != "", nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { - return "", err + return false, err } - return cfg.PreSharedKey, err + return cfg.PreSharedKey != "", nil } // SetPreSharedKey stores the given key and waits for commit @@ -78,11 +105,14 @@ func (p *Preferences) SetRosenpassEnabled(enabled bool) { // GetRosenpassEnabled reads Rosenpass enabled status from config file func (p *Preferences) GetRosenpassEnabled() (bool, error) { + if v, ok := p.policy().GetBool(mdm.KeyRosenpassEnabled); ok { + return v, nil + } if p.configInput.RosenpassEnabled != nil { return *p.configInput.RosenpassEnabled, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -96,11 +126,14 @@ func (p *Preferences) SetRosenpassPermissive(permissive bool) { // GetRosenpassPermissive reads Rosenpass permissive setting from config file func (p *Preferences) GetRosenpassPermissive() (bool, error) { + if v, ok := p.policy().GetBool(mdm.KeyRosenpassPermissive); ok { + return v, nil + } if p.configInput.RosenpassPermissive != nil { return *p.configInput.RosenpassPermissive, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -109,11 +142,14 @@ func (p *Preferences) GetRosenpassPermissive() (bool, error) { // GetDisableClientRoutes reads disable client routes setting from config file func (p *Preferences) GetDisableClientRoutes() (bool, error) { + if v, ok := p.policy().GetBool(mdm.KeyDisableClientRoutes); ok { + return v, nil + } if p.configInput.DisableClientRoutes != nil { return *p.configInput.DisableClientRoutes, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -127,11 +163,14 @@ func (p *Preferences) SetDisableClientRoutes(disable bool) { // GetDisableServerRoutes reads disable server routes setting from config file func (p *Preferences) GetDisableServerRoutes() (bool, error) { + if v, ok := p.policy().GetBool(mdm.KeyDisableServerRoutes); ok { + return v, nil + } if p.configInput.DisableServerRoutes != nil { return *p.configInput.DisableServerRoutes, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -149,7 +188,7 @@ func (p *Preferences) GetDisableDNS() (bool, error) { return *p.configInput.DisableDNS, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -167,7 +206,7 @@ func (p *Preferences) GetDisableFirewall() (bool, error) { return *p.configInput.DisableFirewall, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -181,11 +220,14 @@ func (p *Preferences) SetDisableFirewall(disable bool) { // GetServerSSHAllowed reads server SSH allowed setting from config file func (p *Preferences) GetServerSSHAllowed() (bool, error) { + if v, ok := p.policy().GetBool(mdm.KeyAllowServerSSH); ok { + return v, nil + } if p.configInput.ServerSSHAllowed != nil { return *p.configInput.ServerSSHAllowed, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -207,7 +249,7 @@ func (p *Preferences) GetEnableSSHRoot() (bool, error) { return *p.configInput.EnableSSHRoot, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -229,7 +271,7 @@ func (p *Preferences) GetEnableSSHSFTP() (bool, error) { return *p.configInput.EnableSSHSFTP, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -251,7 +293,7 @@ func (p *Preferences) GetEnableSSHLocalPortForwarding() (bool, error) { return *p.configInput.EnableSSHLocalPortForwarding, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -273,7 +315,7 @@ func (p *Preferences) GetEnableSSHRemotePortForwarding() (bool, error) { return *p.configInput.EnableSSHRemotePortForwarding, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -291,11 +333,14 @@ func (p *Preferences) SetEnableSSHRemotePortForwarding(enabled bool) { // GetBlockInbound reads block inbound setting from config file func (p *Preferences) GetBlockInbound() (bool, error) { + if v, ok := p.policy().GetBool(mdm.KeyBlockInbound); ok { + return v, nil + } if p.configInput.BlockInbound != nil { return *p.configInput.BlockInbound, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -313,7 +358,7 @@ func (p *Preferences) GetDisableIPv6() (bool, error) { return *p.configInput.DisableIPv6, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -325,8 +370,34 @@ func (p *Preferences) SetDisableIPv6(disable bool) { p.configInput.DisableIPv6 = &disable } +// GetRemoteJobsAllowed reads the remote jobs opt-in from config file +func (p *Preferences) GetRemoteJobsAllowed() (bool, error) { + policy := p.policy() + if !policy.HasKey(mdm.KeyRemoteJobsAllowed) && p.configInput.RemoteJobsAllowed != nil { + return *p.configInput.RemoteJobsAllowed, nil + } + + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) + if err != nil { + return false, err + } + cfg.ApplyMDMPolicy(policy) + if cfg.RemoteJobsAllowed == nil { + return false, nil + } + return *cfg.RemoteJobsAllowed, nil +} + +// SetRemoteJobsAllowed stores the given value and waits for commit +func (p *Preferences) SetRemoteJobsAllowed(allowed bool) { + p.configInput.RemoteJobsAllowed = &allowed +} + // Commit writes out the changes to the config file func (p *Preferences) Commit() error { + if err := profilemanager.CheckMDMConflicts(p.configInput, p.policy()); err != nil { + return err + } _, err := profilemanager.UpdateOrCreateConfig(p.configInput) return err } diff --git a/client/android/preferences_test.go b/client/android/preferences_test.go index 2bbccef86..d9f5b1918 100644 --- a/client/android/preferences_test.go +++ b/client/android/preferences_test.go @@ -28,14 +28,13 @@ func TestPreferences_DefaultValues(t *testing.T) { t.Errorf("invalid default management url: %s", defaultVar) } - var preSharedKey string - preSharedKey, err = p.GetPreSharedKey() + hasPSK, err := p.HasPreSharedKey() if err != nil { - t.Fatalf("failed to read default preshared key: %s", err) + t.Fatalf("failed to read default preshared key presence: %s", err) } - if preSharedKey != "" { - t.Errorf("invalid preshared key: %s", preSharedKey) + if hasPSK { + t.Errorf("unexpected preshared key presence on fresh config") } } @@ -65,13 +64,13 @@ func TestPreferences_ReadUncommitedValues(t *testing.T) { } p.SetPreSharedKey(exampleString) - resp, err = p.GetPreSharedKey() + hasPSK, err := p.HasPreSharedKey() if err != nil { - t.Fatalf("failed to read preshared key: %s", err) + t.Fatalf("failed to read preshared key presence: %s", err) } - if resp != exampleString { - t.Errorf("unexpected preshared key: %s", resp) + if !hasPSK { + t.Errorf("expected preshared key presence after staging one") } } @@ -109,12 +108,12 @@ func TestPreferences_Commit(t *testing.T) { t.Errorf("unexpected management url: %s", resp) } - resp, err = p.GetPreSharedKey() + hasPSK, err := p.HasPreSharedKey() if err != nil { - t.Fatalf("failed to read preshared key: %s", err) + t.Fatalf("failed to read preshared key presence: %s", err) } - if resp != examplePresharedKey { - t.Errorf("unexpected preshared key: %s", resp) + if !hasPSK { + t.Errorf("expected preshared key presence after commit") } } diff --git a/client/android/profile_manager.go b/client/android/profile_manager.go index 87c001396..4bc60c453 100644 --- a/client/android/profile_manager.go +++ b/client/android/profile_manager.go @@ -3,42 +3,37 @@ package android import ( - "fmt" - "os" - "path/filepath" - - log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/client/internal/profilemanager" + "github.com/netbirdio/netbird/client/mobile" ) const ( - // Android-specific config filename (different from desktop default.json) - defaultConfigFilename = "netbird.cfg" - // Subdirectory for non-default profiles (must match Java Preferences.java) - profilesSubdir = "profiles" - // Android uses a single user context per app (non-empty username required by ServiceManager) + // Android uses a single user context per app. androidUsername = "android" ) -// Profile represents a profile for gomobile +// Profile represents a profile for gomobile. type Profile struct { - ID string - Name string + ID string + Name string + // Email is the account this profile last logged in with, "" if it never + // completed an SSO login. Kept across logouts; cleared when the profile is + // removed. See client/mobile/profile_state.go. + Email string IsActive bool } -// ProfileArray wraps profiles for gomobile compatibility +// ProfileArray wraps profiles for gomobile compatibility (gomobile cannot +// bind Go slices directly). type ProfileArray struct { items []*Profile } -// Length returns the number of profiles +// Length returns the number of profiles. func (p *ProfileArray) Length() int { return len(p.items) } -// Get returns the profile at index i +// Get returns the profile at index i, or nil if out of range. func (p *ProfileArray) Get(i int) *Profile { if i < 0 || i >= len(p.items) { return nil @@ -46,214 +41,104 @@ func (p *ProfileArray) Get(i int) *Profile { return p.items[i] } -/* - -/data/data/io.netbird.client/files/ ← configDir parameter -├── netbird.cfg ← Default profile config -├── state.json ← Default profile state -├── active_profile.json ← Active profile tracker (JSON with Name + Username) -└── profiles/ ← Subdirectory for non-default profiles - ├── work.json ← Legacy work profile config - ├── work.state.json ← Legacy work profile state - ├── 4c5f5c8198c3989cffb5b5394f5a7ae0.json ← ID profile config - ├── 4c5f5c8198c3989cffb5b5394f5a7ae0.state.json ← ID profile state -*/ - -// ProfileManager manages profiles for Android -// It wraps the internal profilemanager to provide Android-specific behavior +// ProfileManager adapts the shared mobile profile manager (client/mobile) to +// gomobile-friendly types. See that package for the on-disk layout and +// semantics. type ProfileManager struct { - configDir string - serviceMgr *profilemanager.ServiceManager + impl *mobile.ProfileManager } -// NewProfileManager creates a new profile manager for Android +// NewProfileManager creates a new profile manager for Android. configDir is +// the app's files directory. func NewProfileManager(configDir string) *ProfileManager { - // Set the default config path for Android (stored in root configDir, not profiles/) - defaultConfigPath := filepath.Join(configDir, defaultConfigFilename) - - // Set global paths for Android - profilemanager.DefaultConfigPathDir = configDir - profilemanager.DefaultConfigPath = defaultConfigPath - profilemanager.ActiveProfileStatePath = filepath.Join(configDir, "active_profile.json") - - // Create ServiceManager with profiles/ subdirectory - // This avoids modifying the global ConfigDirOverride for profile listing - profilesDir := filepath.Join(configDir, profilesSubdir) - serviceMgr := profilemanager.NewServiceManagerWithProfilesDir(defaultConfigPath, profilesDir) - - return &ProfileManager{ - configDir: configDir, - serviceMgr: serviceMgr, - } + return &ProfileManager{impl: mobile.NewProfileManager(configDir, androidUsername)} } -// ListProfiles returns all available profiles +// SetMDMPolicyFetcher registers the native-provided MDM policy fetcher on +// this ProfileManager; passing nil disables MDM enforcement. +func (pm *ProfileManager) SetMDMPolicyFetcher(f PolicyFetcher) { + pm.impl.SetMDMLoader(loaderFor(f)) +} + +// ListProfiles returns all available profiles, including the default profile, +// with their active status set. func (pm *ProfileManager) ListProfiles() (*ProfileArray, error) { - // Use ServiceManager (looks in profiles/ directory, checks active_profile.json for IsActive) - internalProfiles, err := pm.serviceMgr.ListProfiles(androidUsername) + profiles, err := pm.impl.ListProfiles() if err != nil { - return nil, fmt.Errorf("failed to list profiles: %w", err) + return nil, err } - // Convert internal profiles to Android Profile type - var profiles []*Profile - for _, p := range internalProfiles { - profiles = append(profiles, &Profile{ - ID: p.ID.String(), - Name: p.Name, - IsActive: p.IsActive, - }) + items := make([]*Profile, 0, len(profiles)) + for i := range profiles { + items = append(items, fromMobileProfile(&profiles[i])) } - - return &ProfileArray{items: profiles}, nil + return &ProfileArray{items: items}, nil } -// GetActiveProfile returns the currently active profile name +// GetActiveProfile returns the currently active profile. func (pm *ProfileManager) GetActiveProfile() (*Profile, error) { - // Use ServiceManager to stay consistent with ListProfiles - // ServiceManager uses active_profile.json - activeState, err := pm.serviceMgr.GetActiveProfileState() + p, err := pm.impl.GetActiveProfile() if err != nil { - return nil, fmt.Errorf("failed to get active profile: %w", err) + return nil, err } - - // ActiveProfileState only stores the ID (and username), not the display - // name. Resolve the ID to the full profile so callers get the real Name. - prof, err := pm.serviceMgr.ResolveProfile(activeState.ID.String(), androidUsername) - if err != nil { - return nil, fmt.Errorf("failed to resolve active profile %q: %w", activeState.ID, err) - } - return &Profile{ID: prof.ID.String(), Name: prof.Name, IsActive: true}, nil + return fromMobileProfile(p), nil } -// SwitchProfile switches to a different profile +// SwitchProfile records the given profile ID as the active profile. The caller +// must stop the VPN tunnel before switching. func (pm *ProfileManager) SwitchProfile(id string) error { - // Use ServiceManager to stay consistent with ListProfiles - // ServiceManager uses active_profile.json - err := pm.serviceMgr.SetActiveProfileState(&profilemanager.ActiveProfileState{ - ID: profilemanager.ID(id), - Username: androidUsername, - }) - if err != nil { - return fmt.Errorf("failed to switch profile: %w", err) - } - - log.Infof("switched to profile: %s", id) - return nil + return pm.impl.SwitchProfile(id) } -// AddProfile creates a new profile +// AddProfile creates a new profile with the given display name and a +// generated ID. func (pm *ProfileManager) AddProfile(profileName string) error { - // Use ServiceManager (creates profile in profiles/ directory) - profile, err := pm.serviceMgr.AddProfile(profileName, androidUsername) - if err != nil { - return fmt.Errorf("failed to add profile: %w", err) - } - - log.Infof("created new profile: %s", profile.ID) - return nil + _, err := pm.impl.AddProfile(profileName) + return err } -// LogoutProfile logs out from a profile (clears authentication) +// RenameProfile changes the display name of the profile identified by id. The +// on-disk filename (the ID) is left unchanged. +func (pm *ProfileManager) RenameProfile(id string, newName string) error { + return pm.impl.RenameProfile(id, newName) +} + +// LogoutProfile clears authentication data for a profile, forcing a re-login. +// The management URL and other settings are preserved. func (pm *ProfileManager) LogoutProfile(id string) error { - configPath, err := pm.getProfileConfigPath(id) - if err != nil { - return err - } - - if !profilemanager.IsValidProfileFilenameStem(profilemanager.ID(id)) { - return fmt.Errorf("id '%s' is not valid", id) - } - - // Check if profile exists - if _, err := os.Stat(configPath); os.IsNotExist(err) { - return fmt.Errorf("profile '%s' does not exist", id) - } - - // Read current config using internal profilemanager - config, err := profilemanager.ReadConfig(configPath) - if err != nil { - return fmt.Errorf("failed to read profile config: %w", err) - } - - // Clear authentication by removing private key and SSH key - config.PrivateKey = "" - config.SSHKey = "" - - // Save config using internal profilemanager - if err := profilemanager.WriteOutConfig(configPath, config); err != nil { - return fmt.Errorf("failed to save config: %w", err) - } - - log.Infof("logged out from profile: %s", id) - return nil + return pm.impl.LogoutProfile(id) } -// RemoveProfile deletes a profile +// RemoveProfile deletes a profile. The default profile and the active profile +// cannot be removed. func (pm *ProfileManager) RemoveProfile(id string) error { - // Use ServiceManager (removes profile from profiles/ directory) - if err := pm.serviceMgr.RemoveProfile(profilemanager.ID(id), androidUsername); err != nil { - return fmt.Errorf("failed to remove profile: %w", err) - } - - log.Infof("removed profile: %s", id) - return nil + return pm.impl.RemoveProfile(id) } -// getProfileConfigPath returns the config file path for a profile -// This is needed for Android-specific path handling (netbird.cfg for default profile) -func (pm *ProfileManager) getProfileConfigPath(id string) (string, error) { - if !profilemanager.IsValidProfileFilenameStem(profilemanager.ID(id)) { - return "", fmt.Errorf("id %q is not valid", id) - } - - if id == profilemanager.DefaultProfileName { - // Android uses netbird.cfg for default profile instead of default.json - // Default profile is stored in root configDir, not in profiles/ - return filepath.Join(pm.configDir, defaultConfigFilename), nil - } - - profilesDir := filepath.Join(pm.configDir, profilesSubdir) - return filepath.Join(profilesDir, id+".json"), nil -} - -// GetConfigPath returns the config file path for a given profile id -// Java should call this instead of constructing paths with Preferences.configFile() +// GetConfigPath returns the config file path for the given profile ID. Java +// should call this instead of constructing paths with Preferences.configFile(). func (pm *ProfileManager) GetConfigPath(id string) (string, error) { - return pm.getProfileConfigPath(id) + return pm.impl.GetConfigPath(id) } -// GetStateFilePath returns the state file path for a given profile -// Java should call this instead of constructing paths with Preferences.stateFile() +// GetStateFilePath returns the state file path for the given profile ID. Java +// should call this instead of constructing paths with Preferences.stateFile(). func (pm *ProfileManager) GetStateFilePath(id string) (string, error) { - if id == "" || id == profilemanager.DefaultProfileName { - return filepath.Join(pm.configDir, "state.json"), nil - } - - if !profilemanager.IsValidProfileFilenameStem(profilemanager.ID(id)) { - return "", fmt.Errorf("id %q is not valid", id) - } - - profilesDir := filepath.Join(pm.configDir, profilesSubdir) - return filepath.Join(profilesDir, id+".state.json"), nil + return pm.impl.GetStateFilePath(id) } -// GetActiveConfigPath returns the config file path for the currently active profile -// Java should call this instead of Preferences.getActiveProfileName() + Preferences.configFile() +// GetActiveConfigPath returns the config file path for the currently active +// profile. func (pm *ProfileManager) GetActiveConfigPath() (string, error) { - activeProfile, err := pm.GetActiveProfile() - if err != nil { - return "", fmt.Errorf("failed to get active profile: %w", err) - } - return pm.GetConfigPath(activeProfile.ID) + return pm.impl.GetActiveConfigPath() } -// GetActiveStateFilePath returns the state file path for the currently active profile -// Java should call this instead of Preferences.getActiveProfileName() + Preferences.stateFile() +// GetActiveStateFilePath returns the state file path for the currently active +// profile. func (pm *ProfileManager) GetActiveStateFilePath() (string, error) { - activeProfile, err := pm.GetActiveProfile() - if err != nil { - return "", fmt.Errorf("failed to get active profile: %w", err) - } - return pm.GetStateFilePath(activeProfile.ID) + return pm.impl.GetActiveStateFilePath() +} + +func fromMobileProfile(p *mobile.Profile) *Profile { + return &Profile{ID: p.ID, Name: p.Name, Email: p.Email, IsActive: p.IsActive} } diff --git a/client/android/profile_prefs.go b/client/android/profile_prefs.go new file mode 100644 index 000000000..a761ebbcf --- /dev/null +++ b/client/android/profile_prefs.go @@ -0,0 +1,37 @@ +//go:build android + +package android + +import ( + "fmt" + + "github.com/netbirdio/netbird/client/internal/profilemanager" +) + +type prefsStore interface { + Get(namespace string, v any) (bool, error) + Put(namespace string, v any) error +} + +type profilePrefs struct { + prefs *profilemanager.Prefs +} + +func newProfilePrefs(configDir, profileID string) (*profilePrefs, error) { + if configDir == "" || profileID == "" { + return nil, fmt.Errorf("profile prefs require a config dir and profile ID") + } + prefs, err := NewProfileManager(configDir).impl.ProfilePrefs(profileID) + if err != nil { + return nil, err + } + return &profilePrefs{prefs: prefs}, nil +} + +func (p *profilePrefs) Get(namespace string, v any) (bool, error) { + return p.prefs.Get(namespace, v) +} + +func (p *profilePrefs) Put(namespace string, v any) error { + return p.prefs.Put(namespace, v) +} diff --git a/client/android/route_command.go b/client/android/route_command.go deleted file mode 100644 index 5e7357335..000000000 --- a/client/android/route_command.go +++ /dev/null @@ -1,70 +0,0 @@ -//go:build android - -package android - -import ( - "fmt" - - log "github.com/sirupsen/logrus" - "golang.org/x/exp/maps" - - "github.com/netbirdio/netbird/client/internal/routemanager" - "github.com/netbirdio/netbird/route" -) - -func executeRouteToggle(id string, manager routemanager.Manager, - operationName string, - routeOperation func(routes []route.NetID, allRoutes []route.NetID) error) error { - netID := route.NetID(id) - routes := []route.NetID{netID} - - routesMap := manager.GetClientRoutesWithNetID() - routes = route.ExpandV6ExitPairs(routes, routesMap) - - log.Debugf("%s with ids: %v", operationName, routes) - - if err := routeOperation(routes, maps.Keys(routesMap)); err != nil { - log.Debugf("error when %s: %s", operationName, err) - return fmt.Errorf("error %s: %w", operationName, err) - } - - manager.TriggerSelection(manager.GetClientRoutes()) - - return nil -} - -type routeCommand interface { - toggleRoute() error -} - -type selectRouteCommand struct { - route string - manager routemanager.Manager -} - -func (s selectRouteCommand) toggleRoute() error { - routeSelector := s.manager.GetRouteSelector() - if routeSelector == nil { - return fmt.Errorf("no route selector available") - } - - routeOperation := func(routes []route.NetID, allRoutes []route.NetID) error { - return routeSelector.SelectRoutes(routes, true, allRoutes) - } - - return executeRouteToggle(s.route, s.manager, "selecting route", routeOperation) -} - -type deselectRouteCommand struct { - route string - manager routemanager.Manager -} - -func (d deselectRouteCommand) toggleRoute() error { - routeSelector := d.manager.GetRouteSelector() - if routeSelector == nil { - return fmt.Errorf("no route selector available") - } - - return executeRouteToggle(d.route, d.manager, "deselecting route", routeSelector.DeselectRoutes) -} diff --git a/client/android/session.go b/client/android/session.go new file mode 100644 index 000000000..b2de8dadb --- /dev/null +++ b/client/android/session.go @@ -0,0 +1,243 @@ +//go:build android + +package android + +import ( + "context" + "fmt" + + "github.com/netbirdio/netbird/client/internal" + "github.com/netbirdio/netbird/client/internal/auth" +) + +// StateChangeListener receives client state notifications. +// +// OnStateChanged is a payload-free wake-up whenever the state snapshot +// changed: connection state, the run-loop status label (e.g. NeedsLogin) or +// the session deadline. It mirrors the daemon's SubscribeStatus stream +// trigger — on each signal the consumer pulls the fresh values via +// Status() / SessionExpiresAtUnix(). The engine arms no expiry-warning +// timers on Android; the app schedules the warnings from the deadline it +// reads here. +type StateChangeListener interface { + OnStateChanged() +} + +// Status returns the connect run-loop's status label — the same value the +// desktop daemon serves in StatusResponse.Status. "NeedsLogin" means the +// management server rejected the peer and an interactive login is required. +// +// The label is latched: the run loop keeps its status in a per-run context +// state, which a restart replaces with a fresh Idle one, so an engine restart +// (network change, always-on) would otherwise erase the fact that the peer +// still needs to log in. Only a successful interactive login or extend clears +// it — see clearLoginRequired. +func (c *Client) Status() string { + latched, generation := c.loginRequiredState() + if latched { + return string(internal.StatusNeedsLogin) + } + cc := c.getConnectClient() + if cc == nil { + return string(internal.StatusIdle) + } + status := cc.Status() + if status == internal.StatusNeedsLogin { + c.latchLoginRequired(generation) + } + return string(status) +} + +func (c *Client) loginRequiredState() (bool, uint64) { + c.loginRequiredMu.Lock() + defer c.loginRequiredMu.Unlock() + return c.loginRequired, c.loginCleared +} + +// latchLoginRequired records a NeedsLogin observation, unless a clear landed +// while the caller was reading the run loop's status: cc.Status() is read +// outside the lock, so a login or extend completing in that window would +// otherwise be undone by this stale observation, stranding the UI on +// "login required" over a healthy session. +func (c *Client) latchLoginRequired(observedGeneration uint64) { + c.loginRequiredMu.Lock() + defer c.loginRequiredMu.Unlock() + if c.loginCleared != observedGeneration { + return + } + c.loginRequired = true +} + +// clearLoginRequired releases the latch after a successful interactive login +// or session extend, and invalidates any observation already in flight. +func (c *Client) clearLoginRequired() { + c.loginRequiredMu.Lock() + defer c.loginRequiredMu.Unlock() + c.loginRequired = false + c.loginCleared++ +} + +// SessionExpiresAtUnix returns the SSO session deadline as unix seconds, or 0 +// when no deadline is known (not SSO-registered, expiry disabled, or the +// engine has not received one yet). A past value means the session expired. +// Mirror of StatusResponse.sessionExpiresAt on the desktop daemon. +func (c *Client) SessionExpiresAtUnix() int64 { + deadline := c.recorder.GetSessionExpiresAt() + if deadline.IsZero() { + return 0 + } + return deadline.Unix() +} + +// SetStateChangeListener registers the state notification listener. +// Replaces any previously registered listener; remove it with +// RemoveStateChangeListener. +func (c *Client) SetStateChangeListener(listener StateChangeListener) { + c.stateChangeMu.Lock() + defer c.stateChangeMu.Unlock() + c.stopStateChangeWatchLocked() + if listener == nil { + return + } + + // The subscription is buffered (one pending tick), so unsubscribing is + // not enough to stop callbacks: the loop would drain what is already + // queued and deliver it to a listener the caller has already removed or + // replaced. Gate every callback on this registration's own signal, which + // is closed before unsubscribing. + done := make(chan struct{}) + c.stateChangeDone = done + + id, ch := c.recorder.SubscribeToStateChanges() + c.stateChangeSubID = id + // The channel is closed by UnsubscribeFromStateChanges, which ends the + // goroutine. Ticks are coalesced (buffer of one), so a burst of changes + // wakes the listener once. + go func() { + for range ch { + select { + case <-done: + return + default: + } + listener.OnStateChanged() + } + }() +} + +// RemoveStateChangeListener unregisters the state notification listener. +func (c *Client) RemoveStateChangeListener() { + c.stateChangeMu.Lock() + defer c.stateChangeMu.Unlock() + c.stopStateChangeWatchLocked() +} + +// ExtendAuthSession runs the interactive SSO flow to obtain a fresh JWT and +// asks the management server to extend the session deadline. The tunnel is +// untouched: no resync, no reconnect. Async; the result arrives on the +// listener. Mirror of the daemon's RequestExtendAuthSession / +// WaitExtendAuthSession RPC pair, with URLOpener playing the "UI opens the +// browser" role. +// +// Only one flow may be in flight: the PKCE step binds a fixed loopback port, +// so a second concurrent flow would fail on that bind. Call +// CancelExtendAuthSession when the user abandons the browser. +func (c *Client) ExtendAuthSession(urlOpener URLOpener, isAndroidTV bool, resultListener ErrListener) { + ctx, err := c.beginExtend() + if err != nil { + resultListener.OnError(err) + return + } + + go func() { + defer c.endExtend() + if err := c.extendAuthSession(ctx, urlOpener, isAndroidTV); err != nil { + resultListener.OnError(err) + return + } + resultListener.OnSuccess() + }() +} + +// CancelExtendAuthSession aborts an in-flight ExtendAuthSession. The tunnel is +// left alone — unlike the login flow, which cancels the whole client context +// by stopping the engine. Without this the abandoned PKCE wait keeps its +// loopback port for the full flow timeout and blocks every later attempt. +// No-op when no flow is running. +func (c *Client) CancelExtendAuthSession() { + c.extendMu.Lock() + defer c.extendMu.Unlock() + if c.extendCancel != nil { + c.extendCancel() + } +} + +func (c *Client) stopStateChangeWatchLocked() { + // Signal first, unsubscribe second: closing the channel only stops new + // items, and the loop would still hand whatever is buffered to a listener + // that is no longer registered. + if c.stateChangeDone != nil { + close(c.stateChangeDone) + c.stateChangeDone = nil + } + if c.stateChangeSubID != "" { + c.recorder.UnsubscribeFromStateChanges(c.stateChangeSubID) + c.stateChangeSubID = "" + } +} + +func (c *Client) beginExtend() (context.Context, error) { + c.extendMu.Lock() + defer c.extendMu.Unlock() + if c.extendCancel != nil { + return nil, fmt.Errorf("session extend already in progress") + } + ctx, cancel := context.WithCancel(context.Background()) + c.extendCancel = cancel + return ctx, nil +} + +func (c *Client) endExtend() { + c.extendMu.Lock() + defer c.extendMu.Unlock() + if c.extendCancel != nil { + c.extendCancel() + c.extendCancel = nil + } +} + +func (c *Client) extendAuthSession(ctx context.Context, urlOpener URLOpener, isAndroidTV bool) error { + cfg, cfgPath, cc := c.authSnapshot() + if cfg == nil || cc == nil { + return fmt.Errorf("engine is not running") + } + engine := cc.Engine() + if engine == nil { + return fmt.Errorf("engine is not initialized") + } + + authClient, err := auth.NewAuth(ctx, cfg.PrivateKey, cfg.ManagementURL, cfg) + if err != nil { + return fmt.Errorf("failed to create auth client: %v", err) + } + defer authClient.Close() + + // Passing the config path makes the flow pick up the login_hint. That alone + // cannot keep the IdP on this profile's account though — a hint is only a + // suggestion, and a silent authorization is answered from whatever session the + // IdP already has, which need not be this peer's when several accounts are + // signed in. Marking the flow as an extend lets the server rule that out. + a := NewAuthWithConfig(ctx, cfg, cfgPath) + tokenInfo, err := a.foregroundGetTokenInfoFlow(authClient, urlOpener, isAndroidTV, true) + if err != nil { + return fmt.Errorf("interactive sso login failed: %v", err) + } + + if _, err := engine.ExtendAuthSession(ctx, tokenInfo.GetTokenToUse()); err != nil { + return err + } + c.clearLoginRequired() + + go urlOpener.OnLoginSuccess() + return nil +} diff --git a/client/android/split_tunnel.go b/client/android/split_tunnel.go new file mode 100644 index 000000000..60e1ceeb8 --- /dev/null +++ b/client/android/split_tunnel.go @@ -0,0 +1,125 @@ +package android + +// SplitTunnelMode is which of the two selections, if either, the tunnel applies. +// Its values land in the profile's stored preferences, so the constants below +// are append-only and must never be reordered. +type SplitTunnelMode int + +const ( + modeOff SplitTunnelMode = iota + modeExclude + modeInclude +) + +// The same modes as basic ints. gomobile drops a constant whose type is not a +// basic one, so these are what reaches the generated Java bindings, and they +// keep the Android side tied to the values above instead of repeating 0, 1, 2. +const ( + SplitTunnelModeOff = int(modeOff) + SplitTunnelModeExclude = int(modeExclude) + SplitTunnelModeInclude = int(modeInclude) +) + +type splitTunnelSection struct { + Mode SplitTunnelMode `json:"mode"` + Excluded []string `json:"excluded"` + Included []string `json:"included"` +} + +// PackageList wraps []string for gomobile compatibility. +type PackageList struct { + items []string +} + +// NewPackageList creates an empty list to fill via Add. +func NewPackageList() *PackageList { + return &PackageList{} +} + +// Add appends a package name, ignoring empty ones. +func (l *PackageList) Add(s string) { + if s == "" { + return + } + l.items = append(l.items, s) +} + +// Size returns the number of entries. +func (l *PackageList) Size() int { + return len(l.items) +} + +// Get returns the entry at index i, or an empty string when out of range. +func (l *PackageList) Get(i int) string { + if i < 0 || i >= len(l.items) { + return "" + } + return l.items[i] +} + +// SplitTunnelSettings is one profile's choice of which applications the tunnel +// carries. The two selections are kept apart because the platform applies one +// or the other and never both, and so that switching mode does not throw away +// the picks made in the other one. +// +// Mode is an int rather than a SplitTunnelMode because gomobile carries only +// basic types across the binding. It holds one of the SplitTunnelMode* +// constants. +type SplitTunnelSettings struct { + Mode int + Excluded *PackageList + Included *PackageList +} + +// NewSplitTunnelSettings creates settings that carry every application. +func NewSplitTunnelSettings() *SplitTunnelSettings { + return &SplitTunnelSettings{ + Mode: SplitTunnelModeOff, + Excluded: NewPackageList(), + Included: NewPackageList(), + } +} + +func packagesOf(list *PackageList) []string { + if list == nil { + return nil + } + out := make([]string, 0, len(list.items)) + out = append(out, list.items...) + return out +} + +// normalizeSplitTunnelMode maps anything outside the known set to off, so a mode +// written by a newer build degrades to carrying every application rather than to +// some other mode's behaviour. +func normalizeSplitTunnelMode(mode SplitTunnelMode) SplitTunnelMode { + switch mode { + case modeExclude, modeInclude: + return mode + default: + return modeOff + } +} + +func settingsFromSection(section splitTunnelSection) *SplitTunnelSettings { + out := NewSplitTunnelSettings() + out.Mode = int(normalizeSplitTunnelMode(section.Mode)) + for _, pkg := range section.Excluded { + out.Excluded.Add(pkg) + } + for _, pkg := range section.Included { + out.Included.Add(pkg) + } + return out +} + +func sectionFromSettings(settings *SplitTunnelSettings) splitTunnelSection { + if settings == nil { + settings = NewSplitTunnelSettings() + } + return splitTunnelSection{ + Mode: normalizeSplitTunnelMode(SplitTunnelMode(settings.Mode)), + Excluded: packagesOf(settings.Excluded), + Included: packagesOf(settings.Included), + } +} diff --git a/client/android/split_tunnel_store.go b/client/android/split_tunnel_store.go new file mode 100644 index 000000000..f54e0c8ef --- /dev/null +++ b/client/android/split_tunnel_store.go @@ -0,0 +1,34 @@ +//go:build android + +package android + +const splitTunnelNamespace = "split-tunnel" + +// SplitTunnelStore reads and writes a profile's split tunnelling settings. +type SplitTunnelStore struct { + prefs prefsStore +} + +// NewSplitTunnelStore opens the split tunnelling store of the given profile. +func NewSplitTunnelStore(configDir, profileID string) (*SplitTunnelStore, error) { + prefs, err := newProfilePrefs(configDir, profileID) + if err != nil { + return nil, err + } + return &SplitTunnelStore{prefs: prefs}, nil +} + +// Load returns the stored settings, or settings that carry every application +// when the profile has none saved. +func (s *SplitTunnelStore) Load() (*SplitTunnelSettings, error) { + var section splitTunnelSection + if _, err := s.prefs.Get(splitTunnelNamespace, §ion); err != nil { + return nil, err + } + return settingsFromSection(section), nil +} + +// Save replaces the stored settings. +func (s *SplitTunnelStore) Save(settings *SplitTunnelSettings) error { + return s.prefs.Put(splitTunnelNamespace, sectionFromSettings(settings)) +} diff --git a/client/android/split_tunnel_test.go b/client/android/split_tunnel_test.go new file mode 100644 index 000000000..bf6e0267e --- /dev/null +++ b/client/android/split_tunnel_test.go @@ -0,0 +1,151 @@ +package android + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestNormalizeSplitTunnelMode(t *testing.T) { + tests := []struct { + name string + mode SplitTunnelMode + want SplitTunnelMode + }{ + {name: "exclude is kept", mode: modeExclude, want: modeExclude}, + {name: "include is kept", mode: modeInclude, want: modeInclude}, + {name: "off is kept", mode: modeOff, want: modeOff}, + {name: "a mode from a newer build falls back to off", mode: SplitTunnelMode(7), want: modeOff}, + {name: "a negative mode falls back to off", mode: SplitTunnelMode(-1), want: modeOff}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := normalizeSplitTunnelMode(tt.mode); got != tt.want { + t.Errorf("normalizeSplitTunnelMode(%d) = %d, want %d", tt.mode, got, tt.want) + } + }) + } +} + +// The constants the Android side reads must stay the values the store writes: +// gomobile carries the ints below, not the typed constants they mirror. +func TestSplitTunnelModeConstantsMirrorTheTypedOnes(t *testing.T) { + if SplitTunnelModeOff != int(modeOff) { + t.Errorf("off = %d, want %d", SplitTunnelModeOff, modeOff) + } + if SplitTunnelModeExclude != int(modeExclude) { + t.Errorf("exclude = %d, want %d", SplitTunnelModeExclude, modeExclude) + } + if SplitTunnelModeInclude != int(modeInclude) { + t.Errorf("include = %d, want %d", SplitTunnelModeInclude, modeInclude) + } +} + +func TestSettingsFromSection(t *testing.T) { + got := settingsFromSection(splitTunnelSection{ + Mode: modeExclude, + Excluded: []string{"com.example.a", "com.example.b"}, + Included: []string{"com.example.c"}, + }) + + if got.Mode != SplitTunnelModeExclude { + t.Errorf("mode = %d, want %d", got.Mode, SplitTunnelModeExclude) + } + if got.Excluded.Size() != 2 || got.Excluded.Get(0) != "com.example.a" { + t.Errorf("excluded = %v, want the two stored packages", packagesOf(got.Excluded)) + } + if got.Included.Size() != 1 || got.Included.Get(0) != "com.example.c" { + t.Errorf("included = %v, want the stored package", packagesOf(got.Included)) + } +} + +// A profile that has never stored anything decodes into an empty section, and +// must come back as settings that carry every application rather than as nil +// lists the caller would have to guard against. +func TestSettingsFromEmptySectionCarriesEverything(t *testing.T) { + got := settingsFromSection(splitTunnelSection{}) + + if got.Mode != SplitTunnelModeOff { + t.Errorf("mode = %d, want %d", got.Mode, SplitTunnelModeOff) + } + if got.Excluded == nil || got.Included == nil { + t.Fatal("both selections must be usable lists, not nil") + } + if got.Excluded.Size() != 0 || got.Included.Size() != 0 { + t.Errorf("selections = %v/%v, want both empty", packagesOf(got.Excluded), packagesOf(got.Included)) + } +} + +// The section is what the profile's preference file holds, so the mode has to +// survive a JSON round trip as the number the constants name. +func TestSectionEncodesTheModeAsItsNumber(t *testing.T) { + raw, err := json.Marshal(sectionFromSettings(&SplitTunnelSettings{Mode: SplitTunnelModeInclude})) + if err != nil { + t.Fatalf("marshal section: %v", err) + } + + var back splitTunnelSection + if err := json.Unmarshal(raw, &back); err != nil { + t.Fatalf("unmarshal section: %v", err) + } + if back.Mode != modeInclude { + t.Errorf("mode = %d, want %d, from %s", back.Mode, modeInclude, raw) + } +} + +func TestSectionFromSettingsRoundTrip(t *testing.T) { + settings := NewSplitTunnelSettings() + settings.Mode = SplitTunnelModeInclude + settings.Included.Add("com.example.a") + settings.Excluded.Add("com.example.b") + + section := sectionFromSettings(settings) + back := settingsFromSection(section) + + if back.Mode != SplitTunnelModeInclude { + t.Errorf("mode = %d, want %d", back.Mode, SplitTunnelModeInclude) + } + if !reflect.DeepEqual(packagesOf(back.Included), []string{"com.example.a"}) { + t.Errorf("included = %v, want [com.example.a]", packagesOf(back.Included)) + } + // The inactive selection survives, so switching mode back does not make the + // user pick their applications again. + if !reflect.DeepEqual(packagesOf(back.Excluded), []string{"com.example.b"}) { + t.Errorf("excluded = %v, want [com.example.b]", packagesOf(back.Excluded)) + } +} + +// A mode the Java side never sets, such as one left by a newer build, must not +// reach the stored section either. +func TestSectionFromSettingsNormalizesAnUnknownMode(t *testing.T) { + section := sectionFromSettings(&SplitTunnelSettings{Mode: 7}) + + if section.Mode != modeOff { + t.Errorf("mode = %d, want %d", section.Mode, modeOff) + } +} + +func TestSectionFromNilSettings(t *testing.T) { + section := sectionFromSettings(nil) + + if section.Mode != modeOff { + t.Errorf("mode = %d, want %d", section.Mode, modeOff) + } + if len(section.Excluded) != 0 || len(section.Included) != 0 { + t.Errorf("selections = %v/%v, want both empty", section.Excluded, section.Included) + } +} + +func TestPackageListIgnoresEmptyAndBounds(t *testing.T) { + list := NewPackageList() + list.Add("com.example.a") + list.Add("") + + if list.Size() != 1 { + t.Errorf("size = %d, want 1", list.Size()) + } + if list.Get(-1) != "" || list.Get(5) != "" { + t.Error("out of range access must return an empty string") + } +} diff --git a/client/android/ssh_client.go b/client/android/ssh_client.go new file mode 100644 index 000000000..e05249e98 --- /dev/null +++ b/client/android/ssh_client.go @@ -0,0 +1,651 @@ +//go:build android + +package android + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strconv" + "strings" + "sync" + "time" + + log "github.com/sirupsen/logrus" + gossh "golang.org/x/crypto/ssh" + + "github.com/netbirdio/netbird/client/internal" + "github.com/netbirdio/netbird/client/internal/auth" + "github.com/netbirdio/netbird/client/internal/profilemanager" + nbssh "github.com/netbirdio/netbird/client/ssh" + "github.com/netbirdio/netbird/client/ssh/detection" +) + +const ( + sshDialTimeout = 30 * time.Second + sshDetectionTimeout = 5 * time.Second +) + +// PasswordRequiredMarker tells Java to prompt for a password and retry. It is +// a string because gomobile flattens errors to their message, so a sentinel +// value would not survive the binding. +// +//nolint:gosec // G101 false positive: a sentinel marker, not a credential +const PasswordRequiredMarker = "netbird-ssh-password-required" + +// HostKeyUnknownMarker tells Java to show the fingerprint and, on confirmation, +// retry with TrustHostKey set. The presented fingerprint is appended after the +// marker so the prompt can display it and the retry can guard against a key +// that changed between the two connects. Only regular (non-NetBird) servers +// reach this: NetBird peers verify against the registry. +const HostKeyUnknownMarker = "netbird-ssh-hostkey-unknown" + +var ( + errPasswordRequired = errors.New(PasswordRequiredMarker) + errClientClosed = errors.New("ssh client closed") +) + +// errHostKeyUnknown carries the presented fingerprint so Connect can build the +// marker message the Java side parses. +type errHostKeyUnknown struct { + fingerprint string +} + +func (e *errHostKeyUnknown) Error() string { + return HostKeyUnknownMarker + ":" + e.fingerprint +} + +// SSHTerminalListener receives SSH session events. It is implemented in Java. +// +// All callbacks are invoked from goroutines and may run concurrently with each +// other; the implementation must be safe to call from any thread. +type SSHTerminalListener interface { + OnConnected() + OnData(data []byte) + OnClose(reason string) + OnError(message string) +} + +// SSHClient is a NetBird-aware SSH client exposed to Java via gomobile. +// +// It dials through the running NetBird tunnel and runs a standard SSH session +// on top with PTY enabled. Host-key verification uses the NetBird-provided +// peer SSH host keys, identical to the desktop client. +type SSHClient struct { + nb *Client + mu sync.Mutex + listener SSHTerminalListener + urlOpener URLOpener + + sshClient *gossh.Client + session *gossh.Session + stdin io.WriteCloser + closed bool + + // gen identifies the current connection attempt. Connect and Close bump it, + // so an in-flight dial or a reader left over from a previous connection + // finds itself stale and stays silent instead of publishing OnConnected or + // OnClose for a connection the caller already abandoned. + gen uint64 + dialCancel context.CancelFunc + + // knownHostsConfigDir and knownHostsProfile locate the TOFU store for + // regular SSH servers in the profile's preferences. Java supplies them, + // since an overlay IP is a different host under a different profile. Empty + // until set: without them a regular server cannot be verified and Connect + // refuses one. + knownHostsConfigDir string + knownHostsProfile string + // trustHostKey carries the fingerprint the user confirmed on a previous + // attempt, so the retry accepts exactly that key and persists it. + trustHostKey string +} + +// NewSSHClient creates a new SSH client bound to the running NetBird Client. +func NewSSHClient(c *Client) *SSHClient { + return &SSHClient{nb: c} +} + +// SetListener registers the Java listener. Must be called before Connect to +// receive any events. +func (s *SSHClient) SetListener(l SSHTerminalListener) { + s.mu.Lock() + s.listener = l + s.mu.Unlock() +} + +// SetURLOpener registers the Java URL opener used to display the device-code +// authorization page in a Custom Tabs window when the target peer requires +// JWT authentication. Must be set before Connect to be effective. +func (s *SSHClient) SetURLOpener(opener URLOpener) { + s.mu.Lock() + s.urlOpener = opener + s.mu.Unlock() +} + +// SetKnownHostsStore points the TOFU host-key store at a profile's preferences. +// Must be set before connecting to a regular SSH server; without it such a +// server cannot be verified and Connect refuses one. +func (s *SSHClient) SetKnownHostsStore(configDir, profileID string) { + s.mu.Lock() + s.knownHostsConfigDir = configDir + s.knownHostsProfile = profileID + s.mu.Unlock() +} + +// TrustHostKey records the fingerprint the user confirmed for a regular server, +// so the next Connect accepts that exact key and adds it to the known-hosts +// store. Passing a fingerprint that no longer matches makes the connect fail +// rather than trust a key that changed since the prompt. +func (s *SSHClient) TrustHostKey(fingerprint string) { + s.mu.Lock() + s.trustHostKey = fingerprint + s.mu.Unlock() +} + +// Connect dials the SSH server through the NetBird tunnel and performs the +// SSH handshake. It auto-detects the server type via SSH banner inspection +// and selects the appropriate authentication path: +// +// - NetBird-SSH server requiring JWT: launches the OAuth 2.0 device-code +// flow, opens the verification URL through the registered URLOpener, and +// uses the resulting token as the SSH password. Host-key verification +// uses the NetBird peer registry. +// - NetBird-SSH server without JWT: authenticates with the NetBird SSH +// private key. Host-key verification uses the NetBird peer registry. +// - Regular SSH server (e.g. OpenSSH): authenticates with the NetBird key +// first (so a user-installed NetBird public key works), then falls back +// to the supplied password if non-empty. Host-key verification is +// trust-on-first-use against the per-profile known-hosts store. +// +// The password parameter is only consulted for regular SSH servers. +func (s *SSHClient) Connect(host string, port int, user, password string) error { + if port < 1 || port > 65535 { + return fmt.Errorf("invalid port: %d", port) + } + + cfg, cfgPath, cc := s.nb.authSnapshot() + if cc == nil { + return errors.New("netbird client not running") + } + if cfg == nil { + return errors.New("netbird config not loaded") + } + engine := cc.Engine() + if engine == nil { + return errors.New("netbird engine not available") + } + + s.mu.Lock() + s.gen++ + gen := s.gen + s.mu.Unlock() + + serverType := detectServerType(host, port) + log.Debugf("SSH server type: %s", serverType) + + authMethods, hostKeyCallback, err := s.buildAuth(cfg, cfgPath, engine, serverType, password) + if err != nil { + return err + } + + clientConfig := &gossh.ClientConfig{ + User: user, + Auth: authMethods, + HostKeyCallback: hostKeyCallback, + Timeout: sshDialTimeout, + } + err = s.dialAndHandshake(gen, host, port, clientConfig) + + // An unknown host key is a prompt, not a failure: return the marker intact + // (rootCause would unwrap it) so Java can show the fingerprint and retry. + var unknownHost *errHostKeyUnknown + if errors.As(err, &unknownHost) { + return errors.New(unknownHost.Error()) + } + + // A regular server may still accept a password, so let the caller ask for + // one instead of failing. NetBird servers never use a password, so a + // failure there is genuine. + if err != nil && serverType != detection.ServerTypeNetBirdJWT && + serverType != detection.ServerTypeNetBirdNoJWT && isAuthFailure(err) && + passwordCouldHelp(err, password != "") { + return errPasswordRequired + } + if err != nil { + return rootCause(err) + } + return nil +} + +// StartSession requests a PTY and starts an interactive shell. Output from +// the session is forwarded to the listener via OnData. +func (s *SSHClient) StartSession(cols, rows int) error { + err := s.startSession(cols, rows) + if err != nil { + log.Infof("SSH: start session failed: %v", err) + return rootCause(err) + } + return nil +} + +// Write sends data to the SSH session stdin. +func (s *SSHClient) Write(data []byte) error { + s.mu.Lock() + stdin := s.stdin + s.mu.Unlock() + if stdin == nil { + return errors.New("ssh session not started") + } + if _, err := stdin.Write(data); err != nil { + return fmt.Errorf("write stdin: %w", err) + } + return nil +} + +// Resize updates the PTY window size. +func (s *SSHClient) Resize(cols, rows int) error { + s.mu.Lock() + session := s.session + s.mu.Unlock() + if session == nil { + return errors.New("ssh session not started") + } + return session.WindowChange(rows, cols) +} + +// Reset makes a closed client usable for another Connect: Close leaves the +// one-shot guard set, and clearing it lets the same client back a reconnect. +func (s *SSHClient) Reset() { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = false +} + +// Close terminates the SSH session and underlying connection. Safe to call +// multiple times. +func (s *SSHClient) Close() error { + s.mu.Lock() + s.gen++ + if s.dialCancel != nil { + s.dialCancel() + s.dialCancel = nil + } + sshClient := s.sshClient + session := s.session + stdin := s.stdin + s.sshClient = nil + s.session = nil + s.stdin = nil + notify := !s.closed + s.closed = true + listener := s.listener + s.mu.Unlock() + + if stdin != nil { + if err := stdin.Close(); err != nil { + log.Debugf("ssh: stdin close: %v", err) + } + } + if session != nil { + if err := session.Close(); err != nil && !errors.Is(err, io.EOF) { + log.Debugf("ssh: session close: %v", err) + } + } + var firstErr error + if sshClient != nil { + if err := sshClient.Close(); err != nil { + firstErr = err + } + } + if notify && listener != nil { + listener.OnClose("closed by client") + } + return firstErr +} + +func (s *SSHClient) startSession(cols, rows int) error { + log.Debugf("SSH: starting session %dx%d", cols, rows) + s.mu.Lock() + sshClient := s.sshClient + gen := s.gen + s.mu.Unlock() + + if sshClient == nil { + return errors.New("ssh client not connected") + } + + pty, err := nbssh.StartPTYSession(sshClient, cols, rows) + if err != nil { + return err + } + + s.mu.Lock() + if gen != s.gen { + s.mu.Unlock() + closeQuiet(pty.Session, "stale session") + return errClientClosed + } + s.session = pty.Session + s.stdin = pty.Stdin + s.mu.Unlock() + + readerDone := make(chan string, 2) + go func() { readerDone <- s.readLoop(pty.Stdout, "stdout") }() + go func() { readerDone <- s.readLoop(pty.Stderr, "stderr") }() + go func() { + reason := <-readerDone + if second := <-readerDone; reason == "" { + reason = second + } + s.notifyClose(gen, reason) + }() + log.Debug("SSH: session started, shell running") + return nil +} + +func (s *SSHClient) buildAuth(cfg *profilemanager.Config, cfgPath string, engine *internal.Engine, + serverType detection.ServerType, password string) ([]gossh.AuthMethod, gossh.HostKeyCallback, error) { + + switch serverType { + case detection.ServerTypeNetBirdJWT: + token, err := s.requestJWTToken(cfg, cfgPath) + if err != nil { + return nil, nil, fmt.Errorf("jwt: %w", err) + } + auths := []gossh.AuthMethod{gossh.Password(token)} + return auths, nbssh.CreateHostKeyCallback(nbssh.PeerKeyLookup(engine.GetPeerSSHKey)), nil + + case detection.ServerTypeNetBirdNoJWT: + if cfg.SSHKey == "" { + return nil, nil, errors.New("no NetBird SSH key available") + } + signer, err := gossh.ParsePrivateKey([]byte(cfg.SSHKey)) + if err != nil { + return nil, nil, fmt.Errorf("parse netbird ssh key: %w", err) + } + auths := []gossh.AuthMethod{gossh.PublicKeys(signer)} + return auths, nbssh.CreateHostKeyCallback(nbssh.PeerKeyLookup(engine.GetPeerSSHKey)), nil + + case detection.ServerTypeRegular: + var auths []gossh.AuthMethod + if cfg.SSHKey != "" { + if signer, err := gossh.ParsePrivateKey([]byte(cfg.SSHKey)); err == nil { + auths = append(auths, gossh.PublicKeys(signer)) + } else { + log.Debugf("ssh: parse netbird key for regular auth: %v", err) + } + } + if password != "" { + pw := password + auths = append(auths, gossh.Password(pw)) + auths = append(auths, gossh.KeyboardInteractive(func(_, _ string, questions []string, _ []bool) ([]string, error) { + answers := make([]string, len(questions)) + for i := range questions { + answers[i] = pw + } + return answers, nil + })) + } + if len(auths) == 0 { + // Nothing to offer at all: ask for a password rather than failing, + // so the caller can retry once the user supplies one. + return nil, nil, errPasswordRequired + } + callback, err := s.tofuHostKeyCallback() + if err != nil { + return nil, nil, err + } + return auths, callback, nil + + default: + return nil, nil, fmt.Errorf("unsupported SSH server type: %v", serverType) + } +} + +// tofuHostKeyCallback verifies a regular server's host key against the +// per-profile known-hosts store. An unknown host returns errHostKeyUnknown so +// Java can show the fingerprint and, once confirmed, retry with the key +// trusted; a changed key is rejected outright, as OpenSSH does. When the user +// has confirmed a fingerprint, the callback accepts exactly that key and +// appends it to the store. +func (s *SSHClient) tofuHostKeyCallback() (gossh.HostKeyCallback, error) { + s.mu.Lock() + configDir := s.knownHostsConfigDir + profileID := s.knownHostsProfile + trusted := s.trustHostKey + s.mu.Unlock() + + if configDir == "" || profileID == "" { + return nil, errors.New("no known-hosts store configured for regular SSH") + } + + store, err := openKnownHostsStore(configDir, profileID) + if err != nil { + return nil, fmt.Errorf("load known-hosts store: %w", err) + } + + return func(hostname string, remote net.Addr, key gossh.PublicKey) error { + verdict, err := store.verify(hostname, remote, key) + if err != nil { + return err + } + if verdict == hostKeyMatched { + return nil + } + if verdict == hostKeyChanged { + return fmt.Errorf("SSH host key changed for %s (possible attack)", hostname) + } + + fingerprint := gossh.FingerprintSHA256(key) + if trusted == "" { + return &errHostKeyUnknown{fingerprint: fingerprint} + } + if trusted != fingerprint { + return fmt.Errorf("SSH host key changed since it was confirmed for %s", hostname) + } + if err := store.append(hostname, remote, key); err != nil { + return fmt.Errorf("persist trusted host key: %w", err) + } + // The confirmation is spent: now that the key is stored, a later + // reconnect must verify against the file, not re-accept this fingerprint. + s.mu.Lock() + s.trustHostKey = "" + s.mu.Unlock() + return nil + }, nil +} + +func (s *SSHClient) requestJWTToken(cfg *profilemanager.Config, cfgPath string) (string, error) { + s.mu.Lock() + urlOpener := s.urlOpener + s.mu.Unlock() + if urlOpener == nil { + return "", errors.New("URL opener not configured for JWT auth") + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + flow, err := auth.NewOAuthFlow(ctx, cfg, false, true, profileLoginHint(cfgPath), false) + if err != nil { + return "", fmt.Errorf("create oauth flow: %w", err) + } + + // The status callback covers the browser round-trip, which would + // otherwise leave the terminal blank. + tokenInfo, err := runOAuthFlow(ctx, flow, urlOpener, func() { + s.notifyStatus("Waiting for browser authentication...") + }) + if err != nil { + return "", err + } + + token := tokenInfo.GetTokenToUse() + if token == "" { + return "", errors.New("empty token returned by IdP") + } + + // Tells the client the browser round-trip is over so it can dismiss the + // surface it opened, the same way the login and session-extend flows do. + // Without it the Custom Tab stays in front of the terminal even though the + // token has already been collected. + urlOpener.OnLoginSuccess() + + return token, nil +} + +func (s *SSHClient) dialAndHandshake(gen uint64, host string, port int, clientConfig *gossh.ClientConfig) error { + addr := net.JoinHostPort(host, strconv.Itoa(port)) + ctx, cancel := context.WithTimeout(context.Background(), sshDialTimeout) + defer cancel() + + s.mu.Lock() + if gen != s.gen { + s.mu.Unlock() + return errClientClosed + } + s.dialCancel = cancel + s.mu.Unlock() + + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "tcp", addr) + if err != nil { + return fmt.Errorf("dial %s: %w", addr, err) + } + + client, err := nbssh.Handshake(ctx, conn, addr, clientConfig) + if err != nil { + return err + } + + s.mu.Lock() + if gen != s.gen { + s.mu.Unlock() + closeQuiet(client, "stale ssh client") + return errClientClosed + } + s.sshClient = client + listener := s.listener + s.mu.Unlock() + + if listener != nil { + listener.OnConnected() + } + return nil +} + +func (s *SSHClient) readLoop(r io.Reader, name string) string { + buf := make([]byte, 4096) + for { + n, err := r.Read(buf) + if n > 0 { + s.mu.Lock() + listener := s.listener + s.mu.Unlock() + if listener != nil { + chunk := make([]byte, n) + copy(chunk, buf[:n]) + listener.OnData(chunk) + } + } + if err != nil { + // EOF is a normal shell exit, so report it without a reason. + if errors.Is(err, io.EOF) { + return "" + } + log.Debugf("ssh %s read: %v", name, err) + return rootCause(err).Error() + } + } +} + +// notifyStatus writes a progress line to the terminal through the normal +// output path, so long steps are visible while nothing else is arriving. +func (s *SSHClient) notifyStatus(text string) { + s.mu.Lock() + listener := s.listener + s.mu.Unlock() + if listener != nil { + listener.OnData([]byte("\r\n\x1b[33m" + text + "\x1b[0m\r\n")) + } +} + +func (s *SSHClient) notifyClose(gen uint64, reason string) { + s.mu.Lock() + if gen != s.gen || s.closed { + s.mu.Unlock() + return + } + s.closed = true + listener := s.listener + s.mu.Unlock() + if listener != nil { + listener.OnClose(reason) + } +} + +func closeQuiet(c io.Closer, label string) { + if c == nil { + return + } + if err := c.Close(); err != nil && !errors.Is(err, io.EOF) { + log.Debugf("ssh: close %s: %v", label, err) + } +} + +func detectServerType(host string, port int) detection.ServerType { + ctx, cancel := context.WithTimeout(context.Background(), sshDetectionTimeout) + defer cancel() + + dialer := &net.Dialer{} + serverType, err := detection.DetectSSHServerType(ctx, dialer, host, port) + if err != nil { + log.Debugf("ssh: server detection failed: %v (assuming regular SSH)", err) + return detection.ServerTypeRegular + } + return serverType +} + +// rootCause returns the innermost error of a %w chain, so the terminal shows +// "i/o timeout" rather than every layer that added context on the way up. +func rootCause(err error) error { + for { + // A joined error has no single root, so keep it as-is. + if _, ok := err.(interface{ Unwrap() []error }); ok { + return err + } + next := errors.Unwrap(err) + if next == nil { + return err + } + err = next + } +} + +// isAuthFailure distinguishes credential rejection from dial, timeout and +// host-key errors, which retrying with a password would not fix. +func isAuthFailure(err error) bool { + if errors.Is(err, errPasswordRequired) { + return true + } + var partial *gossh.PartialSuccessError + if errors.As(err, &partial) { + return true + } + return strings.Contains(err.Error(), "unable to authenticate") +} + +// passwordCouldHelp reports whether prompting for a password again can change +// the outcome. gossh lists a method under "attempted methods" only when the +// server offered it, so a supplied password that was never attempted means the +// server does not accept passwords and the real error should surface instead. +func passwordCouldHelp(err error, passwordOffered bool) bool { + if !passwordOffered { + return true + } + msg := err.Error() + return strings.Contains(msg, "password") || strings.Contains(msg, "keyboard-interactive") +} diff --git a/client/android/ssh_known_hosts.go b/client/android/ssh_known_hosts.go new file mode 100644 index 000000000..eea90fd32 --- /dev/null +++ b/client/android/ssh_known_hosts.go @@ -0,0 +1,168 @@ +//go:build android + +package android + +import ( + "bytes" + "net" + "strconv" + "strings" + "sync" + + gossh "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +const knownHostsNamespace = "ssh" + +const ( + hostKeyUnknown hostKeyVerdict = iota + hostKeyMatched + hostKeyChanged +) + +var knownHostsMu sync.Mutex + +type hostKeyVerdict uint8 + +type knownHostsSection struct { + KnownHosts []string `json:"knownHosts"` +} + +type knownHostsStore struct { + prefs prefsStore +} + +// RemoveKnownHost deletes every known-hosts entry for host:port from the +// profile's store, so a host trusted for a session that is being deleted does +// not linger. Java calls this only once no session targets that host, so a +// shared host stays trusted. A missing entry is not an error: the goal state +// is "absent". +func RemoveKnownHost(configDir, profileID, host string, port int) error { + store, err := openKnownHostsStore(configDir, profileID) + if err != nil { + return err + } + return store.removeHost(host, port) +} + +func openKnownHostsStore(configDir, profileID string) (*knownHostsStore, error) { + prefs, err := newProfilePrefs(configDir, profileID) + if err != nil { + return nil, err + } + return &knownHostsStore{prefs: prefs}, nil +} + +func (st *knownHostsStore) verify(hostname string, remote net.Addr, key gossh.PublicKey) (hostKeyVerdict, error) { + lines, err := st.lines() + if err != nil { + return hostKeyUnknown, err + } + targets := knownHostsTargets(hostname, remote) + + verdict := hostKeyUnknown + for _, line := range lines { + pubKey, ok := knownHostsLineKey(line, targets) + if !ok { + continue + } + if pubKey.Type() == key.Type() && bytes.Equal(pubKey.Marshal(), key.Marshal()) { + return hostKeyMatched, nil + } + verdict = hostKeyChanged + } + return verdict, nil +} + +func (st *knownHostsStore) append(hostname string, remote net.Addr, key gossh.PublicKey) error { + line := knownhosts.Line(knownHostsTargets(hostname, remote), key) + + knownHostsMu.Lock() + defer knownHostsMu.Unlock() + + lines, err := st.lines() + if err != nil { + return err + } + return st.prefs.Put(knownHostsNamespace, knownHostsSection{KnownHosts: append(lines, line)}) +} + +func (st *knownHostsStore) removeHost(host string, port int) error { + target := knownhosts.Normalize(net.JoinHostPort(host, strconv.Itoa(port))) + + knownHostsMu.Lock() + defer knownHostsMu.Unlock() + + lines, err := st.lines() + if err != nil { + return err + } + kept := make([]string, 0, len(lines)) + for _, line := range lines { + if knownHostsLineMatches(line, target) { + continue + } + kept = append(kept, line) + } + if len(kept) == len(lines) { + return nil + } + return st.prefs.Put(knownHostsNamespace, knownHostsSection{KnownHosts: kept}) +} + +func (st *knownHostsStore) lines() ([]string, error) { + var section knownHostsSection + if _, err := st.prefs.Get(knownHostsNamespace, §ion); err != nil { + return nil, err + } + return section.KnownHosts, nil +} + +func knownHostsTargets(hostname string, remote net.Addr) []string { + targets := []string{knownhosts.Normalize(hostname)} + if remote != nil { + if normalized := knownhosts.Normalize(remote.String()); normalized != targets[0] { + targets = append(targets, normalized) + } + } + return targets +} + +func knownHostsLineKey(line string, targets []string) (gossh.PublicKey, bool) { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + return nil, false + } + _, hosts, pubKey, _, _, err := gossh.ParseKnownHosts([]byte(trimmed)) + if err != nil { + return nil, false + } + for _, host := range hosts { + for _, target := range targets { + if host == target { + return pubKey, true + } + } + } + return nil, false +} + +// knownHostsLineMatches reports whether a known-hosts line's address list +// contains the normalized target. Comment and blank lines never match. +func knownHostsLineMatches(line, target string) bool { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + return false + } + fields := strings.Fields(trimmed) + if len(fields) == 0 { + return false + } + for _, addr := range strings.Split(fields[0], ",") { + if addr == target { + return true + } + } + return false +} diff --git a/client/android/ssh_sessions.go b/client/android/ssh_sessions.go new file mode 100644 index 000000000..44b5464e9 --- /dev/null +++ b/client/android/ssh_sessions.go @@ -0,0 +1,104 @@ +//go:build android + +package android + +const ( + sshSessionsNamespace = "ssh-sessions" + maxStoredSSHSessions = 50 +) + +type sshSessionRecord struct { + ID string `json:"id"` + Host string `json:"host"` + Port int `json:"port"` + User string `json:"user"` +} + +type sshSessionsSection struct { + Sessions []sshSessionRecord `json:"sessions"` +} + +// SSHSessionEntry is one stored SSH session, without any credential. +type SSHSessionEntry struct { + ID string + Host string + Port int + User string +} + +// SSHSessionArray wraps stored SSH sessions for gomobile compatibility. +type SSHSessionArray struct { + items []*SSHSessionEntry +} + +// NewSSHSessionArray creates an empty session array to fill via Add. +func NewSSHSessionArray() *SSHSessionArray { + return &SSHSessionArray{} +} + +// Add appends a session entry, oldest first. +func (a *SSHSessionArray) Add(id, host string, port int, user string) { + a.items = append(a.items, &SSHSessionEntry{ID: id, Host: host, Port: port, User: user}) +} + +// Length returns the number of entries. +func (a *SSHSessionArray) Length() int { + return len(a.items) +} + +// Get returns the entry at index i, or nil when out of range. +func (a *SSHSessionArray) Get(i int) *SSHSessionEntry { + if i < 0 || i >= len(a.items) { + return nil + } + return a.items[i] +} + +// SSHSessionStore reads and writes a profile's stored SSH sessions. +type SSHSessionStore struct { + prefs prefsStore +} + +// NewSSHSessionStore opens the session store of the given profile. +func NewSSHSessionStore(configDir, profileID string) (*SSHSessionStore, error) { + prefs, err := newProfilePrefs(configDir, profileID) + if err != nil { + return nil, err + } + return &SSHSessionStore{prefs: prefs}, nil +} + +// Load returns the stored sessions, oldest first. +func (s *SSHSessionStore) Load() (*SSHSessionArray, error) { + var section sshSessionsSection + if _, err := s.prefs.Get(sshSessionsNamespace, §ion); err != nil { + return nil, err + } + + out := NewSSHSessionArray() + for _, record := range section.Sessions { + if record.ID == "" || record.Host == "" { + continue + } + out.Add(record.ID, record.Host, record.Port, record.User) + } + return out, nil +} + +// Save replaces the stored sessions, keeping only the newest entries when the +// list exceeds the storage cap. +func (s *SSHSessionStore) Save(sessions *SSHSessionArray) error { + var items []*SSHSessionEntry + if sessions != nil { + items = sessions.items + } + if len(items) > maxStoredSSHSessions { + items = items[len(items)-maxStoredSSHSessions:] + } + + records := make([]sshSessionRecord, 0, len(items)) + for _, item := range items { + records = append(records, sshSessionRecord{ID: item.ID, Host: item.Host, Port: item.Port, User: item.User}) + } + return s.prefs.Put(sshSessionsNamespace, sshSessionsSection{Sessions: records}) +} diff --git a/client/anonymize/anonymize.go b/client/anonymize/anonymize.go index c140cef89..c5d43ed55 100644 --- a/client/anonymize/anonymize.go +++ b/client/anonymize/anonymize.go @@ -2,6 +2,7 @@ package anonymize import ( "crypto/rand" + "encoding/base64" "fmt" "math/big" "net" @@ -15,13 +16,88 @@ import ( const anonTLD = ".domain" +// Level selects how much the anonymizer redacts. Levels are ordered: a higher +// level redacts strictly more. On the wire (protos, flags) levels travel as +// their string form. +type Level int + +const ( + // LevelDefault anonymizes public IP addresses, IPv6 ULA, domains, and MAC + // addresses. Internal IPv4 ranges (RFC 1918, CGNAT, link-local) are + // preserved so support can reason about the real topology. + LevelDefault Level = iota + // LevelStrict additionally anonymizes internal IP ranges, peer names, and + // WireGuard public keys. + LevelStrict +) + +// LevelDefaultString and LevelStrictString are the wire forms of the levels, +// for boundaries that pass levels as strings (flags, protos, mobile bindings). +const ( + LevelDefaultString = "default" + LevelStrictString = "strict" +) + +// ParseLevel maps s to a Level. Empty means LevelDefault; anything +// unrecognized maps to LevelStrict so an unknown request never yields less +// anonymization than intended. +func ParseLevel(s string) Level { + switch strings.ToLower(s) { + case "", LevelDefaultString: + return LevelDefault + default: + return LevelStrict + } +} + +// String returns the wire form of the level: "default" or "strict". +func (l Level) String() string { + if l >= LevelStrict { + return LevelStrictString + } + return LevelDefaultString +} + +// protectedDomains are NetBird-operated suffixes that stay recognizable in an +// anonymized bundle. At LevelStrict the labels in front of them (the peer +// name) are still replaced, except under netbird.io, which only hosts +// NetBird infrastructure (api, signal, flow), never peer names. +var protectedDomains = []string{"netbird.io", "netbird.selfhosted", "netbird.cloud", "netbird.stage"} + +const infraDomain = "netbird.io" + +var ( + macColonRegex = regexp.MustCompile(`\b[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}\b`) + macDashRegex = regexp.MustCompile(`\b[0-9a-fA-F]{2}(?:-[0-9a-fA-F]{2}){5}\b`) + wgKeyRegex = regexp.MustCompile(`\b[A-Za-z0-9+/]{43}=`) +) + type Anonymizer struct { ipAnonymizer map[netip.Addr]netip.Addr domainAnonymizer map[string]string - currentAnonIPv4 netip.Addr - currentAnonIPv6 netip.Addr - startAnonIPv4 netip.Addr - startAnonIPv6 netip.Addr + // domainOrder caches the keys of domainAnonymizer sorted longest-first + // for AnonymizeString; it is rebuilt when the map gains entries. + domainOrder []string + labelAnonymizer map[string]string + labelAnonymized map[string]struct{} + labelCounter uint32 + macAnonymizer map[string]string + macCounter uint32 + wgKeyAnonymizer map[string]string + wgKeyAnonymized map[string]struct{} + currentAnonIPv4 netip.Addr + currentAnonIPv6 netip.Addr + startAnonIPv4 netip.Addr + startAnonIPv6 netip.Addr + + // LevelStrict also anonymizes internal ranges (RFC 1918, CGNAT, + // link-local), replacing them from the dedicated internal pools below so + // a reader can still tell an internal address from a public one. + level Level + currentAnonInternalIPv4 netip.Addr + currentAnonInternalIPv6 netip.Addr + startAnonInternalIPv4 netip.Addr + startAnonInternalIPv6 netip.Addr domainKeyRegex *regexp.Regexp } @@ -32,25 +108,50 @@ func DefaultAddresses() (netip.Addr, netip.Addr) { return netip.AddrFrom4([4]byte{198, 51, 100, 0}), netip.MustParseAddr("2001:db8:ffff::") } +// InternalAddresses returns the pool starts used in strict mode for internal +// ranges. Both are reserved ranges that cannot collide with real addressing: +// 198.18.0.0 (RFC 2544 benchmarking), 2001:db8:1:: (RFC 3849 documentation). +func InternalAddresses() (netip.Addr, netip.Addr) { + return netip.AddrFrom4([4]byte{198, 18, 0, 0}), netip.MustParseAddr("2001:db8:1::") +} + func NewAnonymizer(startIPv4, startIPv6 netip.Addr) *Anonymizer { + internalIPv4, internalIPv6 := InternalAddresses() return &Anonymizer{ ipAnonymizer: map[netip.Addr]netip.Addr{}, domainAnonymizer: map[string]string{}, + labelAnonymizer: map[string]string{}, + labelAnonymized: map[string]struct{}{}, + macAnonymizer: map[string]string{}, + wgKeyAnonymizer: map[string]string{}, + wgKeyAnonymized: map[string]struct{}{}, currentAnonIPv4: startIPv4, currentAnonIPv6: startIPv6, startAnonIPv4: startIPv4, startAnonIPv6: startIPv6, + level: LevelDefault, + currentAnonInternalIPv4: internalIPv4, + currentAnonInternalIPv6: internalIPv6, + startAnonInternalIPv4: internalIPv4, + startAnonInternalIPv6: internalIPv6, + domainKeyRegex: regexp.MustCompile(`\bdomain=([^\s,:"]+)`), } } +// SetLevel selects the anonymization level. The zero value of a new +// Anonymizer is LevelDefault. +func (a *Anonymizer) SetLevel(level Level) { + a.level = level +} + func (a *Anonymizer) AnonymizeIP(ip netip.Addr) netip.Addr { + // Normalize 4-in-6 addresses so ::ffff:192.168.1.1 classifies and maps + // like 192.168.1.1. + ip = ip.Unmap() + if ip.IsLoopback() || - ip.IsLinkLocalUnicast() || - ip.IsLinkLocalMulticast() || - ip.IsInterfaceLocalMulticast() || - (ip.Is4() && ip.IsPrivate()) || ip.IsUnspecified() || ip.IsMulticast() || isWellKnown(ip) || @@ -59,18 +160,100 @@ func (a *Anonymizer) AnonymizeIP(ip netip.Addr) netip.Addr { return ip } + if isInternal(ip) && a.level < LevelStrict { + return ip + } + if _, ok := a.ipAnonymizer[ip]; !ok { - if ip.Is4() { - a.ipAnonymizer[ip] = a.currentAnonIPv4 - a.currentAnonIPv4 = a.currentAnonIPv4.Next() - } else { - a.ipAnonymizer[ip] = a.currentAnonIPv6 - a.currentAnonIPv6 = a.currentAnonIPv6.Next() - } + a.ipAnonymizer[ip] = a.nextAnonIP(ip) } return a.ipAnonymizer[ip] } +func (a *Anonymizer) nextAnonIP(ip netip.Addr) netip.Addr { + // At the strict level, internal addresses (including IPv6 ULA, matched + // by IsPrivate) come from the internal pools so they remain recognizable + // as internal without disclosing the real values. + if a.level >= LevelStrict && (isInternal(ip) || ip.IsPrivate()) { + if ip.Is4() { + anon := a.currentAnonInternalIPv4 + a.currentAnonInternalIPv4 = a.currentAnonInternalIPv4.Next() + return anon + } + anon := a.currentAnonInternalIPv6 + a.currentAnonInternalIPv6 = a.currentAnonInternalIPv6.Next() + return anon + } + + if ip.Is4() { + anon := a.currentAnonIPv4 + a.currentAnonIPv4 = a.currentAnonIPv4.Next() + return anon + } + anon := a.currentAnonIPv6 + a.currentAnonIPv6 = a.currentAnonIPv6.Next() + return anon +} + +// AnonymizeMAC replaces a MAC address with a consistent placeholder from the +// locally administered range starting at 02:00:00:00:00:01, at every +// anonymization level. Broadcast, multicast, all-zero, and already assigned +// placeholder addresses are preserved. The colon and dash spellings of the +// same address share one placeholder; the output keeps the input's separator. +func (a *Anonymizer) AnonymizeMAC(mac string) string { + hw, err := net.ParseMAC(mac) + if err != nil || len(hw) != 6 { + return mac + } + + if isWellKnownMAC(hw) || a.isAnonymizedMAC(hw) { + return mac + } + + key := hw.String() + anon, ok := a.macAnonymizer[key] + if !ok { + a.macCounter++ + anon = fmt.Sprintf("02:00:00:%02x:%02x:%02x", byte(a.macCounter>>16), byte(a.macCounter>>8), byte(a.macCounter)) + a.macAnonymizer[key] = anon + } + + if strings.Contains(mac, "-") { + anon = strings.ReplaceAll(anon, ":", "-") + } + return anon +} + +// isAnonymizedMAC reports whether hw is a placeholder this anonymizer already +// handed out, so a second pass over anonymized output leaves it unchanged. +func (a *Anonymizer) isAnonymizedMAC(hw net.HardwareAddr) bool { + if hw[0] != 0x02 || hw[1] != 0 || hw[2] != 0 { + return false + } + value := uint32(hw[3])<<16 | uint32(hw[4])<<8 | uint32(hw[5]) + return value <= a.macCounter +} + +// AnonymizeWGKey replaces a WireGuard public key with a consistent random +// placeholder of the same shape. Keys are only anonymized at LevelStrict; +// placeholders already handed out pass through unchanged. +func (a *Anonymizer) AnonymizeWGKey(key string) string { + if a.level < LevelStrict || !looksLikeWGKey(key) { + return key + } + if _, ok := a.wgKeyAnonymized[key]; ok { + return key + } + + anon, ok := a.wgKeyAnonymizer[key] + if !ok { + anon = generateAnonymousKey() + a.wgKeyAnonymizer[key] = anon + a.wgKeyAnonymized[anon] = struct{}{} + } + return anon +} + func (a *Anonymizer) AnonymizeUDPAddr(addr net.UDPAddr) net.UDPAddr { // Convert IP to netip.Addr ip, ok := netip.AddrFromSlice(addr.IP) @@ -89,12 +272,12 @@ func (a *Anonymizer) AnonymizeUDPAddr(addr net.UDPAddr) net.UDPAddr { // isInAnonymizedRange checks if an IP is within the range of already assigned anonymized IPs func (a *Anonymizer) isInAnonymizedRange(ip netip.Addr) bool { - if ip.Is4() && ip.Compare(a.startAnonIPv4) >= 0 && ip.Compare(a.currentAnonIPv4) <= 0 { - return true - } else if !ip.Is4() && ip.Compare(a.startAnonIPv6) >= 0 && ip.Compare(a.currentAnonIPv6) <= 0 { - return true + if ip.Is4() { + return inPoolRange(ip, a.startAnonIPv4, a.currentAnonIPv4) || + inPoolRange(ip, a.startAnonInternalIPv4, a.currentAnonInternalIPv4) } - return false + return inPoolRange(ip, a.startAnonIPv6, a.currentAnonIPv6) || + inPoolRange(ip, a.startAnonInternalIPv6, a.currentAnonInternalIPv6) } func (a *Anonymizer) AnonymizeIPString(ip string) string { @@ -118,14 +301,23 @@ func (a *Anonymizer) AnonymizeDomain(domain string) string { baseDomain = domain[:len(domain)-1] } - if strings.HasSuffix(baseDomain, "netbird.io") || - strings.HasSuffix(baseDomain, "netbird.selfhosted") || - strings.HasSuffix(baseDomain, "netbird.cloud") || - strings.HasSuffix(baseDomain, "netbird.stage") || - strings.HasSuffix(baseDomain, anonTLD) { + if strings.HasSuffix(baseDomain, anonTLD) { return domain } + // A reverse zone names an address prefix, so it follows the address rules, + // which also keeps its digit labels intact. + if zone, ok := a.anonymizeReverseZone(baseDomain); ok { + return withTrailingDot(zone, hasDot) + } + + if suffix := protectedSuffix(baseDomain); suffix != "" { + if a.level < LevelStrict || baseDomain == suffix || suffix == infraDomain { + return domain + } + return withTrailingDot(a.anonymizePeerName(baseDomain, suffix), hasDot) + } + parts := strings.Split(baseDomain, ".") if len(parts) < 2 { return domain @@ -141,12 +333,53 @@ func (a *Anonymizer) AnonymizeDomain(domain string) string { } result := strings.Replace(baseDomain, baseForLookup, anonymized, 1) - if hasDot { - result += "." + if a.level >= LevelStrict && len(parts) > 2 { + prefix := strings.TrimSuffix(baseDomain, "."+baseForLookup) + result = a.anonymizeLabels(prefix, "host") + "." + anonymized + // The full mapping feeds AnonymizeString so seeded FQDNs are caught + // in log lines as a whole, labels included. + a.domainAnonymizer[baseDomain] = result + } + return withTrailingDot(result, hasDot) +} + +// anonymizePeerName replaces the labels in front of a protected suffix with +// numbered peer placeholders, keeping the suffix, and records the full +// mapping for string replacement in logs. The numbering keeps a peer +// recognizable across the whole bundle without disclosing its name. +func (a *Anonymizer) anonymizePeerName(baseDomain, suffix string) string { + prefix := strings.TrimSuffix(baseDomain, "."+suffix) + result := a.anonymizeLabels(prefix, "peer") + "." + suffix + if result != baseDomain { + a.domainAnonymizer[baseDomain] = result } return result } +// anonymizeLabels replaces each dot-separated label with a consistent +// numbered placeholder ("-"). Wildcard labels and +// placeholders already handed out pass through unchanged. +func (a *Anonymizer) anonymizeLabels(prefix, placeholder string) string { + labels := strings.Split(prefix, ".") + for i, label := range labels { + if label == "*" { + continue + } + if _, ok := a.labelAnonymized[label]; ok { + continue + } + anon, ok := a.labelAnonymizer[label] + if !ok { + a.labelCounter++ + anon = fmt.Sprintf("%s-%d", placeholder, a.labelCounter) + a.labelAnonymizer[label] = anon + a.labelAnonymized[anon] = struct{}{} + } + labels[i] = anon + } + return strings.Join(labels, ".") +} + func (a *Anonymizer) AnonymizeURI(uri string) string { u, err := url.Parse(uri) if err != nil { @@ -178,17 +411,75 @@ func (a *Anonymizer) AnonymizeString(str string) string { ipv4Regex := regexp.MustCompile(`\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b`) ipv6Regex := regexp.MustCompile(`\b([0-9a-fA-F:]+:+[0-9a-fA-F]{0,4})(?:%[0-9a-zA-Z]+)?(?:\/[0-9]{1,3})?(?::[0-9]{1,5})?\b`) + // Reverse zones go first and are then held out of the passes below: their + // labels are digits, which the address patterns would otherwise consume. + str, restoreZones := a.replaceReverseZones(str) + str = ipv4Regex.ReplaceAllStringFunc(str, a.AnonymizeIPString) str = ipv6Regex.ReplaceAllStringFunc(str, a.AnonymizeIPString) - for domain, anonDomain := range a.domainAnonymizer { - str = strings.ReplaceAll(str, domain, anonDomain) + for _, domain := range a.sortedDomains() { + str = strings.ReplaceAll(str, domain, a.domainAnonymizer[domain]) } str = a.AnonymizeSchemeURI(str) str = a.AnonymizeDNSLogLine(str) - return str + // MAC handling runs after the IP passes so preserved IPv6 addresses are + // already out of the way; the separator guard skips matches embedded in a + // longer colon- or dash-separated sequence (such as an IPv6 tail). + str = a.anonymizeMACsInString(str, macColonRegex, ':') + str = a.anonymizeMACsInString(str, macDashRegex, '-') + + if a.level >= LevelStrict { + str = wgKeyRegex.ReplaceAllStringFunc(str, a.AnonymizeWGKey) + } + + return restoreZones(str) +} + +// sortedDomains returns the domain mappings longest-first, so a full-FQDN +// mapping (strict level) is applied before the base-domain mapping it +// contains. The order is rebuilt only when domainAnonymizer has grown. +func (a *Anonymizer) sortedDomains() []string { + if len(a.domainOrder) == len(a.domainAnonymizer) { + return a.domainOrder + } + + a.domainOrder = a.domainOrder[:0] + for domain := range a.domainAnonymizer { + a.domainOrder = append(a.domainOrder, domain) + } + slices.SortFunc(a.domainOrder, func(x, y string) int { + if d := len(y) - len(x); d != 0 { + return d + } + return strings.Compare(x, y) + }) + return a.domainOrder +} + +// anonymizeMACsInString replaces MAC addresses matched by re, skipping +// matches that directly adjoin another sep so a six-group run inside a longer +// separated sequence is left alone. +func (a *Anonymizer) anonymizeMACsInString(str string, re *regexp.Regexp, sep byte) string { + matches := re.FindAllStringIndex(str, -1) + if len(matches) == 0 { + return str + } + + var b strings.Builder + last := 0 + for _, m := range matches { + if (m[0] > 0 && str[m[0]-1] == sep) || (m[1] < len(str) && str[m[1]] == sep) { + continue + } + b.WriteString(str[last:m[0]]) + b.WriteString(a.AnonymizeMAC(str[m[0]:m[1]])) + last = m[1] + } + b.WriteString(str[last:]) + return b.String() } // AnonymizeSchemeURI finds and anonymizes URIs with ws, wss, rel, rels, stun, stuns, turn, and turns schemes. @@ -239,10 +530,79 @@ func isWellKnown(addr netip.Addr) bool { "128.0.0.0", "8000::", // 2nd split subnet for default routes } - if slices.Contains(wellKnown, addr.String()) { + return slices.Contains(wellKnown, addr.String()) +} + +// isInternal reports whether ip identifies a host only within the local +// network: IPv4 private (RFC 1918), CGNAT (RFC 6598), and link-local (v4 and +// v6). These are preserved at the default level so support can reason about +// the real topology, and replaced from the internal pools at the strict +// level. IPv6 ULA is deliberately not internal: its random global ID uniquely +// fingerprints the network, so it is anonymized at every level. +func isInternal(ip netip.Addr) bool { + return (ip.Is4() && ip.IsPrivate()) || + ip.IsLinkLocalUnicast() || + isCGNAT(ip) +} + +func inPoolRange(ip, start, current netip.Addr) bool { + return ip.Compare(start) >= 0 && ip.Compare(current) <= 0 +} + +// isWellKnownMAC reports whether hw carries no stable host identity: all-zero +// or a group address (broadcast and multicast). +func isWellKnownMAC(hw net.HardwareAddr) bool { + if hw[0]&1 == 1 { return true } + for _, b := range hw { + if b != 0 { + return false + } + } + return true +} +// looksLikeWGKey reports whether s has the shape of a WireGuard key: +// 44 base64 characters decoding to 32 bytes. +func looksLikeWGKey(s string) bool { + if len(s) != 44 || s[43] != '=' { + return false + } + decoded, err := base64.StdEncoding.DecodeString(s) + return err == nil && len(decoded) == 32 +} + +func generateAnonymousKey() string { + buf := make([]byte, 32) + if _, err := rand.Read(buf); err != nil { + return strings.Repeat("A", 43) + "=" + } + return base64.StdEncoding.EncodeToString(buf) +} + +// protectedSuffix returns the protected NetBird suffix baseDomain ends with, +// or empty. The match is label-anchored so an unrelated domain that merely +// ends in the same characters is not preserved. +func protectedSuffix(baseDomain string) string { + for _, d := range protectedDomains { + if baseDomain == d || strings.HasSuffix(baseDomain, "."+d) { + return d + } + } + return "" +} + +func withTrailingDot(domain string, hasDot bool) string { + if hasDot { + return domain + "." + } + return domain +} + +// isCGNAT reports whether addr is in 100.64.0.0/10 (RFC 6598), the range +// NetBird assigns overlay peer addresses from. +func isCGNAT(addr netip.Addr) bool { cgnatRangeStart := netip.AddrFrom4([4]byte{100, 64, 0, 0}) cgnatRange := netip.PrefixFrom(cgnatRangeStart, 10) diff --git a/client/anonymize/anonymize_test.go b/client/anonymize/anonymize_test.go index 852315fa1..7c3c7bcf8 100644 --- a/client/anonymize/anonymize_test.go +++ b/client/anonymize/anonymize_test.go @@ -1,8 +1,11 @@ package anonymize_test import ( + "bytes" + "encoding/base64" "net/netip" "regexp" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -44,6 +47,301 @@ func TestAnonymizeIP(t *testing.T) { } } +func TestParseLevel(t *testing.T) { + tests := []struct { + input string + expect anonymize.Level + }{ + {"", anonymize.LevelDefault}, + {"default", anonymize.LevelDefault}, + {"DEFAULT", anonymize.LevelDefault}, + {"strict", anonymize.LevelStrict}, + {"STRICT", anonymize.LevelStrict}, + // Unknown values must never yield less anonymization than requested. + {"garbage", anonymize.LevelStrict}, + } + + for _, tc := range tests { + t.Run("input="+tc.input, func(t *testing.T) { + assert.Equal(t, tc.expect, anonymize.ParseLevel(tc.input), "parsed level should match") + }) + } +} + +func TestAnonymizeIP_DefaultLevelInternalRanges(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + tests := []struct { + name string + ip string + expect string + }{ + {"RFC1918 10/8", "10.1.2.3", "10.1.2.3"}, + {"RFC1918 172.16/12", "172.16.5.5", "172.16.5.5"}, + {"RFC1918 192.168/16", "192.168.1.1", "192.168.1.1"}, + {"CGNAT", "100.64.0.5", "100.64.0.5"}, + {"IPv4 link-local", "169.254.1.1", "169.254.1.1"}, + {"IPv6 link-local", "fe80::1", "fe80::1"}, + // ULA is anonymized even at the default level: its random global ID + // uniquely fingerprints the network, unlike shared RFC 1918 space. + {"IPv6 ULA", "fd12:3456:789a::1", "2001:db8:ffff::"}, + // 4-in-6 addresses classify like their unmapped IPv4 form. + {"4-in-6 RFC1918", "::ffff:192.168.1.1", "192.168.1.1"}, + {"4-in-6 CGNAT", "::ffff:100.64.0.5", "100.64.0.5"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := anonymizer.AnonymizeIP(netip.MustParseAddr(tc.ip)) + assert.Equal(t, tc.expect, result.String(), "default level should preserve internal ranges except ULA") + }) + } +} + +func TestAnonymizeIP_StrictLevel(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + // Order matters: internal pool addresses are assigned sequentially. + tests := []struct { + name string + ip string + expect string + }{ + {"RFC1918 192.168/16", "192.168.1.1", "198.18.0.0"}, + {"Second RFC1918", "192.168.1.2", "198.18.0.1"}, + {"Repeated RFC1918", "192.168.1.1", "198.18.0.0"}, + {"RFC1918 10/8", "10.1.2.3", "198.18.0.2"}, + {"RFC1918 172.16/12", "172.16.5.5", "198.18.0.3"}, + {"CGNAT", "100.64.0.5", "198.18.0.4"}, + {"IPv4 link-local", "169.254.1.1", "198.18.0.5"}, + {"Public IPv4 uses public pool", "1.2.3.4", "198.51.100.0"}, + {"IPv6 link-local", "fe80::1", "2001:db8:1::"}, + {"IPv6 ULA", "fd12:3456:789a::1", "2001:db8:1::1"}, + {"Public IPv6 uses public pool", "2607:f8b0:4005:805::200e", "2001:db8:ffff::"}, + {"Loopback IPv4", "127.0.0.1", "127.0.0.1"}, + {"Loopback IPv6", "::1", "::1"}, + {"Unspecified", "0.0.0.0", "0.0.0.0"}, + {"Multicast", "224.0.0.251", "224.0.0.251"}, + {"Well known resolver", "8.8.8.8", "8.8.8.8"}, + {"Well known split marker", "128.0.0.0", "128.0.0.0"}, + {"In internal pool range", "198.18.0.3", "198.18.0.3"}, + {"In public pool range", "198.51.100.0", "198.51.100.0"}, + {"4-in-6 repeated RFC1918", "::ffff:192.168.1.1", "198.18.0.0"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := anonymizer.AnonymizeIP(netip.MustParseAddr(tc.ip)) + assert.Equal(t, tc.expect, result.String(), "strict level should replace internal ranges from the internal pools") + }) + } +} + +func TestAnonymizeString_StrictInternalIPs(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + input := "route 10.20.30.0/24 via 192.168.1.1 dev eth0 src 100.64.0.7" + firstPass := anonymizer.AnonymizeString(input) + secondPass := anonymizer.AnonymizeString(firstPass) + + assert.NotContains(t, firstPass, "10.20.30.0", "private network address should be anonymized") + assert.NotContains(t, firstPass, "192.168.1.1", "private gateway should be anonymized") + assert.NotContains(t, firstPass, "100.64.0.7", "CGNAT address should be anonymized") + assert.Contains(t, firstPass, "/24", "prefix length should be preserved") + assert.Equal(t, firstPass, secondPass, "second pass should not further anonymize the string") +} + +func TestAnonymizeMAC(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + first := anonymizer.AnonymizeMAC("aa:bb:cc:dd:ee:0f") + assert.Equal(t, "02:00:00:00:00:01", first, "first MAC should get the first placeholder") + assert.Equal(t, first, anonymizer.AnonymizeMAC("aa:bb:cc:dd:ee:0f"), "repeated MAC should map to the same placeholder") + assert.Equal(t, first, anonymizer.AnonymizeMAC("AA:BB:CC:DD:EE:0F"), "case should not affect the mapping") + assert.Equal(t, "02-00-00-00-00-01", anonymizer.AnonymizeMAC("AA-BB-CC-DD-EE-0F"), "dash form should keep its separator but share the mapping") + + second := anonymizer.AnonymizeMAC("10:22:33:44:55:66") + assert.Equal(t, "02:00:00:00:00:02", second, "second distinct MAC should get the next placeholder") + + tests := []struct { + name string + mac string + }{ + {"Broadcast", "ff:ff:ff:ff:ff:ff"}, + {"IPv4 multicast", "01:00:5e:00:00:fb"}, + {"IPv6 multicast", "33:33:00:00:00:01"}, + {"All zero", "00:00:00:00:00:00"}, + {"Assigned placeholder", "02:00:00:00:00:01"}, + {"Invalid", "not-a-mac"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.mac, anonymizer.AnonymizeMAC(tc.mac), "should be preserved") + }) + } +} + +func TestAnonymizeString_MACAddresses(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + tests := []struct { + name string + input string + expect string + }{ + { + name: "nftables ether rule", + input: "ether saddr aa:bb:cc:dd:ee:ff drop", + expect: "ether saddr 02:00:00:00:00:01 drop", + }, + { + name: "Windows dash form", + input: "Physical Address : AA-BB-CC-DD-EE-FF", + expect: "Physical Address : 02-00-00-00-00-01", + }, + { + name: "IPv6 address tail is not treated as MAC", + input: "addr fe80:0:11:22:33:44:55:66 scope link", + expect: "addr fe80:0:11:22:33:44:55:66 scope link", + }, + { + name: "broadcast MAC preserved", + input: "dst ff:ff:ff:ff:ff:ff type ARP", + expect: "dst ff:ff:ff:ff:ff:ff type ARP", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := anonymizer.AnonymizeString(tc.input) + assert.Equal(t, tc.expect, result, "MAC addresses should be anonymized at every level") + assert.Equal(t, result, anonymizer.AnonymizeString(result), "second pass should not change the result") + }) + } +} + +func TestAnonymizeWGKey(t *testing.T) { + key := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0x42}, 32)) + + t.Run("default level preserves keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + assert.Equal(t, key, anonymizer.AnonymizeWGKey(key), "default level should not touch WireGuard keys") + }) + + t.Run("strict level replaces keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + anon := anonymizer.AnonymizeWGKey(key) + assert.NotEqual(t, key, anon, "strict level should replace the key") + assert.Regexp(t, `^[A-Za-z0-9+/]{43}=$`, anon, "placeholder should keep the WireGuard key shape") + assert.Equal(t, anon, anonymizer.AnonymizeWGKey(key), "repeated key should map to the same placeholder") + assert.Equal(t, anon, anonymizer.AnonymizeWGKey(anon), "an assigned placeholder should pass through unchanged") + + assert.Equal(t, "not-a-key", anonymizer.AnonymizeWGKey("not-a-key"), "non-key values should be preserved") + }) +} + +func TestAnonymizeString_WGKeys(t *testing.T) { + key := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0x42}, 32)) + input := "peer " + key + " handshake completed" + + t.Run("default level preserves keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + assert.Equal(t, input, anonymizer.AnonymizeString(input), "default level should not touch WireGuard keys in strings") + }) + + t.Run("strict level replaces keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + firstPass := anonymizer.AnonymizeString(input) + assert.NotContains(t, firstPass, key, "the key should not survive strict anonymization") + assert.Equal(t, anonymizer.AnonymizeWGKey(key), extractKey(t, firstPass), "string replacement should be consistent with AnonymizeWGKey") + assert.Equal(t, firstPass, anonymizer.AnonymizeString(firstPass), "second pass should not change the result") + }) +} + +func extractKey(t *testing.T, logLine string) string { + t.Helper() + fields := strings.Fields(logLine) + require.Len(t, fields, 4, "log line should keep its structure") + return fields[1] +} + +func TestAnonymizeDomain_StrictLevel(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + t.Run("netbird peer name", func(t *testing.T) { + result := anonymizer.AnonymizeDomain("my-laptop.netbird.cloud") + assert.Regexp(t, `^peer-\d+\.netbird\.cloud$`, result, "peer name should be anonymized, suffix kept") + assert.NotContains(t, result, "my-laptop", "the peer name should not survive") + assert.Equal(t, result, anonymizer.AnonymizeDomain("my-laptop.netbird.cloud"), "repeated domain should map consistently") + assert.Equal(t, result, anonymizer.AnonymizeDomain(result), "an anonymized domain should pass through unchanged") + }) + + t.Run("bare netbird domain", func(t *testing.T) { + assert.Equal(t, "netbird.cloud", anonymizer.AnonymizeDomain("netbird.cloud"), "the bare protected suffix should be preserved") + }) + + t.Run("netbird infrastructure preserved", func(t *testing.T) { + assert.Equal(t, "api.netbird.io", anonymizer.AnonymizeDomain("api.netbird.io"), + "netbird.io hosts infrastructure, not peer names, and should stay readable") + }) + + t.Run("leading labels of other domains", func(t *testing.T) { + result := anonymizer.AnonymizeDomain("host1.corp.example.com") + assert.Regexp(t, `^host-\d+\.host-\d+\.anon-[a-zA-Z0-9]+\.domain$`, result, "every label should be anonymized") + for _, label := range []string{"host1", "corp", "example"} { + assert.NotContains(t, result, label, "no original label should survive") + } + assert.Equal(t, result, anonymizer.AnonymizeDomain("host1.corp.example.com"), "repeated domain should map consistently") + }) + + t.Run("same label maps consistently across domains", func(t *testing.T) { + first := anonymizer.AnonymizeDomain("shared.one.com") + second := anonymizer.AnonymizeDomain("shared.two.com") + assert.Equal(t, strings.Split(first, ".")[0], strings.Split(second, ".")[0], "the shared host label should get one placeholder") + }) + + t.Run("wildcard label preserved", func(t *testing.T) { + result := anonymizer.AnonymizeDomain("*.example.com") + assert.Regexp(t, `^\*\.anon-[a-zA-Z0-9]+\.domain$`, result, "the wildcard label should stay a wildcard") + }) +} + +func TestAnonymizeDomain_DefaultLevelKeepsPeerNames(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + assert.Equal(t, "my-laptop.netbird.cloud", anonymizer.AnonymizeDomain("my-laptop.netbird.cloud"), + "default level should preserve netbird FQDNs including the peer name") + assert.Regexp(t, `^sub\.anon-[a-zA-Z0-9]+\.domain$`, anonymizer.AnonymizeDomain("sub.example.com"), + "default level should keep subdomain labels") +} + +func TestAnonymizeString_StrictPeerNames(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + // Seed like the bundle generator does from the status: base first, then + // the full FQDN, so replacement must prefer the longer mapping. + anonBase := anonymizer.AnonymizeDomain("example.com") + anonPeer := anonymizer.AnonymizeDomain("peer1.netbird.cloud") + anonHost := anonymizer.AnonymizeDomain("host1.example.com") + + logLine := "connected to peer1.netbird.cloud via host1.example.com endpoint" + firstPass := anonymizer.AnonymizeString(logLine) + assert.NotContains(t, firstPass, "peer1", "the peer name should not survive in logs") + assert.NotContains(t, firstPass, "host1", "the host label should not survive in logs") + assert.Contains(t, firstPass, anonPeer, "the seeded peer mapping should be applied") + assert.Contains(t, firstPass, anonHost, "the seeded host mapping should be applied, not just the base mapping") + assert.NotContains(t, firstPass, "host1."+anonBase, "the base mapping must not preempt the longer FQDN mapping") + assert.Equal(t, firstPass, anonymizer.AnonymizeString(firstPass), "second pass should not change the result") +} + func TestAnonymizeDNSLogLine(t *testing.T) { anonymizer := anonymize.NewAnonymizer(netip.Addr{}, netip.Addr{}) tests := []struct { diff --git a/client/anonymize/reverse_zone.go b/client/anonymize/reverse_zone.go new file mode 100644 index 000000000..b521b71b7 --- /dev/null +++ b/client/anonymize/reverse_zone.go @@ -0,0 +1,174 @@ +package anonymize + +import ( + "encoding/hex" + "net/netip" + "regexp" + "strconv" + "strings" +) + +const ( + reverseZoneSuffixV4 = ".in-addr.arpa" + reverseZoneSuffixV6 = ".ip6.arpa" + + v6Nibbles = 32 + v4Octets = 4 +) + +// reverseZoneRegexes match a reverse zone or a full reverse name in free text. +// They are applied before the address passes of AnonymizeString, whose IPv4 +// pattern would otherwise consume the digit labels of a zone and replace parts +// of it with unrelated addresses. +var reverseZoneRegexes = []*regexp.Regexp{ + regexp.MustCompile(`(?:[0-9]{1,3}\.){1,4}in-addr\.arpa\b`), + regexp.MustCompile(`(?:[0-9a-fA-F]\.){1,32}ip6\.arpa\b`), +} + +// anonymizeReverseZone maps a reverse zone to the zone of the anonymized form +// of the prefix it encodes, so it follows the address rules rather than the +// domain ones: the zone of an address that is preserved is preserved too, and +// the zone of one that is replaced names the replacement. This keeps a reverse +// zone recognizable as such, and consistent with the addresses it belongs to +// elsewhere in the same output. It reports false for anything that is not a +// reverse zone. +func (a *Anonymizer) anonymizeReverseZone(domain string) (string, bool) { + prefix, labelCount, suffix, ok := parseReverseZone(domain) + if !ok { + return "", false + } + + anonymized := a.AnonymizeIP(prefix) + if anonymized == prefix { + return domain, true + } + + return reverseZoneName(anonymized, labelCount) + suffix, true +} + +// replaceReverseZones anonymizes every reverse zone in str and swaps each one +// for a placeholder, returning a function that puts the anonymized zones back. +// The placeholders carry no dots, digits or colons, so no later pass matches +// them. +func (a *Anonymizer) replaceReverseZones(str string) (string, func(string) string) { + var zones []string + + for _, re := range reverseZoneRegexes { + str = re.ReplaceAllStringFunc(str, func(match string) string { + zone, ok := a.anonymizeReverseZone(match) + if !ok { + return match + } + + zones = append(zones, zone) + return reverseZonePlaceholder(len(zones) - 1) + }) + } + + if len(zones) == 0 { + return str, func(s string) string { return s } + } + + return str, func(s string) string { + for i, zone := range zones { + s = strings.ReplaceAll(s, reverseZonePlaceholder(i), zone) + } + return s + } +} + +func reverseZonePlaceholder(index int) string { + return "\x00reversezone" + strconv.Itoa(index) + "\x00" +} + +// parseReverseZone turns a reverse zone into the address of the prefix its +// labels spell backwards, padding the absent low-order part with zeroes, and +// returns the label count and zone suffix so the name can be rebuilt. +func parseReverseZone(domain string) (netip.Addr, int, string, bool) { + lower := strings.ToLower(domain) + + switch { + case strings.HasSuffix(lower, reverseZoneSuffixV4): + labels := strings.Split(strings.TrimSuffix(lower, reverseZoneSuffixV4), ".") + addr, ok := reverseZoneAddrV4(labels) + return addr, len(labels), reverseZoneSuffixV4, ok + case strings.HasSuffix(lower, reverseZoneSuffixV6): + labels := strings.Split(strings.TrimSuffix(lower, reverseZoneSuffixV6), ".") + addr, ok := reverseZoneAddrV6(labels) + return addr, len(labels), reverseZoneSuffixV6, ok + default: + return netip.Addr{}, 0, "", false + } +} + +func reverseZoneAddrV4(labels []string) (netip.Addr, bool) { + if len(labels) == 0 || len(labels) > v4Octets { + return netip.Addr{}, false + } + + var octets [v4Octets]byte + for i, label := range labels { + octet, err := strconv.ParseUint(label, 10, 8) + if err != nil { + return netip.Addr{}, false + } + octets[len(labels)-1-i] = byte(octet) + } + + return netip.AddrFrom4(octets), true +} + +func reverseZoneAddrV6(labels []string) (netip.Addr, bool) { + if len(labels) == 0 || len(labels) > v6Nibbles { + return netip.Addr{}, false + } + + nibbles := make([]byte, 0, v6Nibbles) + for i := len(labels) - 1; i >= 0; i-- { + if len(labels[i]) != 1 || !isHexDigit(labels[i][0]) { + return netip.Addr{}, false + } + nibbles = append(nibbles, labels[i][0]) + } + for len(nibbles) < v6Nibbles { + nibbles = append(nibbles, '0') + } + + var groups []string + for i := 0; i < len(nibbles); i += 4 { + groups = append(groups, string(nibbles[i:i+4])) + } + + addr, err := netip.ParseAddr(strings.Join(groups, ":")) + if err != nil { + return netip.Addr{}, false + } + + return addr, true +} + +// reverseZoneName spells the first labelCount labels of addr backwards, the +// inverse of parseReverseZone, without the zone suffix. +func reverseZoneName(addr netip.Addr, labelCount int) string { + labels := make([]string, 0, labelCount) + + if addr.Is4() { + octets := addr.As4() + for i := labelCount - 1; i >= 0; i-- { + labels = append(labels, strconv.Itoa(int(octets[i]))) + } + return strings.Join(labels, ".") + } + + address := addr.As16() + nibbles := hex.EncodeToString(address[:]) + for i := labelCount - 1; i >= 0; i-- { + labels = append(labels, string(nibbles[i])) + } + + return strings.Join(labels, ".") +} + +func isHexDigit(c byte) bool { + return c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F' +} diff --git a/client/anonymize/reverse_zone_test.go b/client/anonymize/reverse_zone_test.go new file mode 100644 index 000000000..8c3b8954a --- /dev/null +++ b/client/anonymize/reverse_zone_test.go @@ -0,0 +1,171 @@ +package anonymize + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newLeveledAnonymizer(level Level) *Anonymizer { + a := NewAnonymizer(DefaultAddresses()) + a.SetLevel(level) + return a +} + +// TestAnonymizeDomainReverseZone covers reverse zones going through the address +// rules instead of the domain ones, so a zone stays a zone and an address that +// is preserved keeps the zone that names it. +func TestAnonymizeDomainReverseZone(t *testing.T) { + // 100.64.0.0/10 is the overlay range, which is CGNAT: preserved at the + // default level and replaced from the internal pool at the strict one + const overlayZone = "64.100.in-addr.arpa" + + t.Run("overlay zone preserved at the default level", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + assert.Equal(t, overlayZone, a.AnonymizeDomain(overlayZone), "should keep the zone of a preserved address") + }) + + t.Run("private zone preserved at the default level", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + assert.Equal(t, "168.192.in-addr.arpa", a.AnonymizeDomain("168.192.in-addr.arpa"), "should keep the zone of a private address") + }) + + t.Run("overlay zone replaced at the strict level", func(t *testing.T) { + a := newLeveledAnonymizer(LevelStrict) + + got := a.AnonymizeDomain(overlayZone) + require.True(t, strings.HasSuffix(got, reverseZoneSuffixV4), "should stay a reverse zone, got %q", got) + assert.NotEqual(t, overlayZone, got, "should replace the encoded prefix") + assert.Len(t, strings.Split(strings.TrimSuffix(got, reverseZoneSuffixV4), "."), 2, + "should keep the label count, got %q", got) + }) + + t.Run("public zone replaced at the default level", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + got := a.AnonymizeDomain("113.0.203.in-addr.arpa") + require.True(t, strings.HasSuffix(got, reverseZoneSuffixV4), "should stay a reverse zone, got %q", got) + assert.NotEqual(t, "113.0.203.in-addr.arpa", got, "should replace a public prefix") + }) + + t.Run("zone of an address keeps that address mapping", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + anonymizedAddr := a.AnonymizeIPString("203.0.113.7") + got := a.AnonymizeDomain("7.113.0.203.in-addr.arpa") + + octets := strings.Split(anonymizedAddr, ".") + want := octets[3] + "." + octets[2] + "." + octets[1] + "." + octets[0] + reverseZoneSuffixV4 + assert.Equal(t, want, got, "should name the same replacement as the address itself") + }) + + t.Run("ipv6 nibble labels stay single digits", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + zone := "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.0.0.0" + reverseZoneSuffixV6 + got := a.AnonymizeDomain(zone) + + require.True(t, strings.HasSuffix(got, reverseZoneSuffixV6), "should stay a reverse zone, got %q", got) + labels := strings.Split(strings.TrimSuffix(got, reverseZoneSuffixV6), ".") + assert.Len(t, labels, 28, "should keep every nibble label, got %q", got) + for _, label := range labels { + assert.Len(t, label, 1, "nibble label %q should stay a single digit", label) + } + }) + + t.Run("trailing dot is kept", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + assert.Equal(t, "64.100.in-addr.arpa.", a.AnonymizeDomain("64.100.in-addr.arpa."), "should keep the trailing dot") + }) + + t.Run("a domain that only looks like a zone is anonymized as a domain", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + got := a.AnonymizeDomain("not-a-zone.in-addr.arpa") + assert.NotContains(t, got, "in-addr.arpa", "should fall back to domain anonymization") + }) +} + +// TestAnonymizeStringReverseZone verifies that a zone inside free text, such as +// a DNS log line, is not chewed up by the address passes. The IPv4 pattern +// matches any run of dotted digits, which a reverse zone is made of. +func TestAnonymizeStringReverseZone(t *testing.T) { + t.Run("ipv6 zone survives the address passes", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + zone := "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.0.0.0" + reverseZoneSuffixV6 + got := a.AnonymizeString("question: domain=" + zone + " type=PTR") + + assert.Contains(t, got, "type=PTR", "should keep the rest of the line") + assert.NotContains(t, got, "198.51.100", "should not rewrite nibble labels as an address") + + labels := strings.Split(strings.TrimSuffix(strings.TrimPrefix(got, "question: domain="), reverseZoneSuffixV6+" type=PTR"), ".") + assert.Len(t, labels, 28, "should keep every nibble label, got %q", got) + }) + + t.Run("preserved ipv4 zone is untouched", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + line := "reverse zone 64.100.in-addr.arpa registered" + assert.Equal(t, line, a.AnonymizeString(line), "should keep the zone of a preserved address") + }) + + t.Run("public ipv4 zone is replaced consistently", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + got := a.AnonymizeString("zone 113.0.203.in-addr.arpa and address 203.0.113.7") + assert.NotContains(t, got, "113.0.203.in-addr.arpa", "should replace the zone") + assert.NotContains(t, got, "203.0.113.7", "should replace the address") + assert.Contains(t, got, reverseZoneSuffixV4, "should keep the zone suffix") + }) +} + +func TestParseReverseZone(t *testing.T) { + tests := []struct { + name string + zone string + addr string + labels int + }{ + {name: "v4 two labels", zone: "0.100" + reverseZoneSuffixV4, addr: "100.0.0.0", labels: 2}, + {name: "v4 three labels", zone: "1.168.192" + reverseZoneSuffixV4, addr: "192.168.1.0", labels: 3}, + {name: "v4 full address", zone: "7.113.0.203" + reverseZoneSuffixV4, addr: "203.0.113.7", labels: 4}, + { + name: "v6 prefix", + zone: "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.0.0.0" + reverseZoneSuffixV6, + addr: "2::", + labels: 28, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + addr, labels, suffix, ok := parseReverseZone(tc.zone) + require.True(t, ok, "should decode the reverse zone") + assert.Equal(t, tc.addr, addr.String(), "should decode to the encoded prefix") + assert.Equal(t, tc.labels, labels, "should count the labels") + assert.Equal(t, tc.zone, reverseZoneName(addr, labels)+suffix, "should re-encode to the original zone") + }) + } +} + +func TestParseReverseZoneRejectsNonZones(t *testing.T) { + tests := []string{ + "example.com", + "in-addr.arpa", + "x.100" + reverseZoneSuffixV4, + "256" + reverseZoneSuffixV4, + "1.2.3.4.5" + reverseZoneSuffixV4, + "ab" + reverseZoneSuffixV6, + "g" + reverseZoneSuffixV6, + } + + for _, zone := range tests { + t.Run(zone, func(t *testing.T) { + _, _, _, ok := parseReverseZone(zone) + assert.False(t, ok, "should reject %q", zone) + }) + } +} diff --git a/client/cmd/daemon_error.go b/client/cmd/daemon_error.go new file mode 100644 index 000000000..0d5b1307e --- /dev/null +++ b/client/cmd/daemon_error.go @@ -0,0 +1,66 @@ +package cmd + +import ( + "errors" + "fmt" + "strings" + + "google.golang.org/genproto/googleapis/rpc/errdetails" + gstatus "google.golang.org/grpc/status" + + "github.com/netbirdio/netbird/client/internal/ipcauth" +) + +// daemonCallError prepares a daemon error for display. A refusal the daemon +// raised because the operation needs root/administrator is already guidance +// written for the user, so it is surfaced on its own instead of buried under the +// gRPC envelope and the name of the RPC that hit it. Anything else is wrapped +// with context as usual. +func daemonCallError(context string, err error) error { + if guidance, ok := privilegeGuidance(err); ok { + return errors.New(guidance) + } + return fmt.Errorf("%s: %w", context, err) +} + +// privilegeGuidance renders the daemon's privilege refusal as a summary and the +// command that performs the operation with the privileges it needs. It reports +// false for any other error. +func privilegeGuidance(err error) (string, bool) { + info, ok := privilegeErrorInfo(err) + if !ok { + return "", false + } + + summary := info.GetMetadata()[ipcauth.ErrorMetaSummary] + command := info.GetMetadata()[ipcauth.ErrorMetaCommand] + if summary == "" { + // Detail without a summary: fall back to the status message, which + // carries the same text. + summary = strings.TrimSpace(gstatus.Convert(err).Message()) + } + if command == "" { + return summary, true + } + + return fmt.Sprintf("%s\n\n %s\n", summary, command), true +} + +// privilegeErrorInfo returns the daemon's privilege-refusal detail, if the error +// carries one. +func privilegeErrorInfo(err error) (*errdetails.ErrorInfo, bool) { + if err == nil { + return nil, false + } + + for _, detail := range gstatus.Convert(err).Details() { + info, ok := detail.(*errdetails.ErrorInfo) + if !ok { + continue + } + if info.GetReason() == ipcauth.ErrorReasonPrivilegeRequired && info.GetDomain() == ipcauth.ErrorDomain { + return info, true + } + } + return nil, false +} diff --git a/client/cmd/debug.go b/client/cmd/debug.go index 57e75f663..c4d5ad6d7 100644 --- a/client/cmd/debug.go +++ b/client/cmd/debug.go @@ -3,7 +3,6 @@ package cmd import ( "context" "fmt" - "os/user" "strings" "time" @@ -24,13 +23,17 @@ import ( "github.com/netbirdio/netbird/version" ) -const errCloseConnection = "Failed to close connection: %v" +const ( + errCloseConnection = "Failed to close connection: %v" + noUpDownFlag = "no-updown" +) var ( - logFileCount uint32 - systemInfoFlag bool - uploadBundleFlag bool - uploadBundleURLFlag string + logFileCount uint32 + systemInfoFlag bool + uploadBundleFlag bool + uploadBundleURLFlag string + uploadBundleInsecureFlag bool ) var debugCmd = &cobra.Command{ @@ -113,7 +116,7 @@ func debugConfigDump(cmd *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("get active profile: %v", err) } - currUser, err := user.Current() + currUser, err := profilemanager.InvokingUser() if err != nil { return fmt.Errorf("get current user: %v", err) } @@ -155,6 +158,11 @@ func debugConfigDump(cmd *cobra.Command, _ []string) error { // request. Returns an error if the RPC fails or if the daemon reports // an upload failure reason. func debugBundle(cmd *cobra.Command, _ []string) error { + anonymizeEnabled, anonymizeLevel, err := effectiveAnonymize() + if err != nil { + return err + } + conn, err := getClient(cmd) if err != nil { return err @@ -167,17 +175,19 @@ func debugBundle(cmd *cobra.Command, _ []string) error { client := proto.NewDaemonServiceClient(conn) request := &proto.DebugBundleRequest{ - Anonymize: anonymizeFlag, - SystemInfo: systemInfoFlag, - LogFileCount: logFileCount, - CliVersion: version.NetbirdVersion(), + Anonymize: anonymizeEnabled, + AnonymizeLevel: anonymizeLevel.String(), + SystemInfo: systemInfoFlag, + LogFileCount: logFileCount, + CliVersion: version.NetbirdVersion(), } if uploadBundleFlag { request.UploadURL = uploadBundleURLFlag + request.UploadInsecure = uploadBundleInsecureFlag } resp, err := client.DebugBundle(cmd.Context(), request) if err != nil { - return fmt.Errorf("failed to bundle debug: %v", status.Convert(err).Message()) + return daemonCallError("bundle debug", err) } cmd.Printf("Local file:\n%s\n", resp.GetPath()) @@ -227,6 +237,11 @@ func runForDuration(cmd *cobra.Command, args []string) error { return fmt.Errorf("invalid duration format: %v", err) } + anonymizeEnabled, anonymizeLevel, err := effectiveAnonymize() + if err != nil { + return err + } + conn, err := getClient(cmd) if err != nil { return err @@ -245,13 +260,14 @@ func runForDuration(cmd *cobra.Command, args []string) error { } stateWasDown := stat.Status != string(internal.StatusConnected) && stat.Status != string(internal.StatusConnecting) + noUpDown, _ := cmd.Flags().GetBool(noUpDownFlag) initialLogLevel, err := client.GetLogLevel(cmd.Context(), &proto.GetLogLevelRequest{}) if err != nil { return fmt.Errorf("failed to get log level: %v", status.Convert(err).Message()) } - if stateWasDown { + if stateWasDown && !noUpDown { if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil { cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message()) } else { @@ -272,34 +288,20 @@ func runForDuration(cmd *cobra.Command, args []string) error { } needsRestoreUp := false - if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil { - cmd.PrintErrf("Failed to bring service down: %v\n", status.Convert(err).Message()) + if noUpDown { + enableSyncResponsePersistence(cmd, client) } else { - needsRestoreUp = !stateWasDown - cmd.Println("netbird down") + needsRestoreUp = restartDaemon(cmd, client, stateWasDown) } - time.Sleep(1 * time.Second) - - // Enable sync response persistence before bringing the service up - if _, err := client.SetSyncResponsePersistence(cmd.Context(), &proto.SetSyncResponsePersistenceRequest{ - Enabled: true, - }); err != nil { - cmd.PrintErrf("Failed to enable sync response persistence: %v\n", status.Convert(err).Message()) - } - - if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil { - cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message()) - } else { - needsRestoreUp = false - cmd.Println("netbird up") - } - - time.Sleep(3 * time.Second) - cpuProfilingStarted := false if _, err := client.StartCPUProfile(cmd.Context(), &proto.StartCPUProfileRequest{}); err != nil { - cmd.PrintErrf("Failed to start CPU profiling: %v\n", err) + if msg := status.Convert(err).Message(); strings.Contains(msg, "already in progress") { + cmd.PrintErrln("CPU profiling is already running (started with `netbird debug cpu start`). " + + "It is left running and is included in a bundle created after `netbird debug cpu stop`.") + } else { + cmd.PrintErrf("Failed to start CPU profiling: %v\n", msg) + } } else { cpuProfilingStarted = true defer func() { @@ -366,17 +368,19 @@ func runForDuration(cmd *cobra.Command, args []string) error { cmd.Println("Creating debug bundle...") request := &proto.DebugBundleRequest{ - Anonymize: anonymizeFlag, - SystemInfo: systemInfoFlag, - LogFileCount: logFileCount, - CliVersion: version.NetbirdVersion(), + Anonymize: anonymizeEnabled, + AnonymizeLevel: anonymizeLevel.String(), + SystemInfo: systemInfoFlag, + LogFileCount: logFileCount, + CliVersion: version.NetbirdVersion(), } if uploadBundleFlag { request.UploadURL = uploadBundleURLFlag + request.UploadInsecure = uploadBundleInsecureFlag } resp, err := client.DebugBundle(cmd.Context(), request) if err != nil { - return fmt.Errorf("failed to bundle debug: %v", status.Convert(err).Message()) + return daemonCallError("bundle debug", err) } if needsRestoreUp { @@ -387,7 +391,7 @@ func runForDuration(cmd *cobra.Command, args []string) error { } } - if stateWasDown { + if stateWasDown && !noUpDown { if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil { cmd.PrintErrf("Failed to restore service down state: %v\n", status.Convert(err).Message()) } else { @@ -444,6 +448,47 @@ func setSyncResponsePersistence(cmd *cobra.Command, args []string) error { return nil } +// enableSyncResponsePersistence asks the daemon to keep the latest sync +// response so the bundle carries the network map. With a running daemon only +// syncs received after the call are kept. +func enableSyncResponsePersistence(cmd *cobra.Command, client proto.DaemonServiceClient) { + if _, err := client.SetSyncResponsePersistence(cmd.Context(), &proto.SetSyncResponsePersistenceRequest{ + Enabled: true, + }); err != nil { + cmd.PrintErrf("Failed to enable sync response persistence: %v\n", status.Convert(err).Message()) + } +} + +// restartDaemon cycles the daemon down and up with sync response persistence +// enabled so the bundle carries the network map. It reports whether the +// daemon was left down although it was running before, so the caller can +// bring it back up. +func restartDaemon(cmd *cobra.Command, client proto.DaemonServiceClient, stateWasDown bool) bool { + needsRestoreUp := false + if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil { + cmd.PrintErrf("Failed to bring service down: %v\n", status.Convert(err).Message()) + } else { + needsRestoreUp = !stateWasDown + cmd.Println("netbird down") + } + + time.Sleep(1 * time.Second) + + // Enable sync response persistence before bringing the service up + enableSyncResponsePersistence(cmd, client) + + if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil { + cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message()) + } else { + needsRestoreUp = false + cmd.Println("netbird up") + } + + time.Sleep(3 * time.Second) + + return needsRestoreUp +} + func waitForDurationOrCancel(ctx context.Context, duration time.Duration, cmd *cobra.Command) error { ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() @@ -524,10 +569,13 @@ func init() { debugBundleCmd.Flags().BoolVarP(&systemInfoFlag, "system-info", "S", true, "Adds system information to the debug bundle") debugBundleCmd.Flags().BoolVarP(&uploadBundleFlag, "upload-bundle", "U", false, "Uploads the debug bundle to a server") debugBundleCmd.Flags().StringVar(&uploadBundleURLFlag, "upload-bundle-url", types.DefaultBundleURL, "Service URL to get an URL to upload the debug bundle") + debugBundleCmd.Flags().BoolVar(&uploadBundleInsecureFlag, "upload-bundle-insecure", false, "Allow uploading to an http or untrusted-TLS upload server (self-hosted); requires root") forCmd.Flags().Uint32VarP(&logFileCount, "log-file-count", "C", 1, "Number of rotated log files to include in debug bundle") forCmd.Flags().BoolVarP(&systemInfoFlag, "system-info", "S", true, "Adds system information to the debug bundle") forCmd.Flags().BoolVarP(&uploadBundleFlag, "upload-bundle", "U", false, "Uploads the debug bundle to a server") forCmd.Flags().StringVar(&uploadBundleURLFlag, "upload-bundle-url", types.DefaultBundleURL, "Service URL to get an URL to upload the debug bundle") + forCmd.Flags().BoolVar(&uploadBundleInsecureFlag, "upload-bundle-insecure", false, "Allow uploading to an http or untrusted-TLS upload server (self-hosted); requires root") forCmd.Flags().Bool("capture", false, "Capture packets during the debug duration and include in bundle") + forCmd.Flags().Bool(noUpDownFlag, false, "Keep the daemon running instead of bringing it down and up before collecting. The bundle only includes the network map if a sync arrives during the run") } diff --git a/client/cmd/debug_cpu.go b/client/cmd/debug_cpu.go new file mode 100644 index 000000000..a01b845cf --- /dev/null +++ b/client/cmd/debug_cpu.go @@ -0,0 +1,83 @@ +package cmd + +import ( + "fmt" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "google.golang.org/grpc/status" + + "github.com/netbirdio/netbird/client/proto" +) + +var debugCPUCmd = &cobra.Command{ + Use: "cpu", + Short: "Profile the daemon's CPU usage", + Long: `Starts and stops CPU profiling in the running daemon without restarting it. +The profile is included in the next debug bundle as cpu.prof. + +Profiling is not time limited: it keeps running, and keeps costing CPU, until +"netbird debug cpu stop" is run.`, +} + +var debugCPUStartCmd = &cobra.Command{ + Use: "start", + Short: "Start CPU profiling in the daemon", + Example: " netbird debug cpu start", + Args: cobra.NoArgs, + RunE: debugCPUStart, +} + +var debugCPUStopCmd = &cobra.Command{ + Use: "stop", + Short: "Stop CPU profiling in the daemon", + Long: `Stops CPU profiling. The captured profile stays in the daemon until the next +debug bundle is created, which includes it as cpu.prof.`, + Example: " netbird debug cpu stop && netbird debug bundle", + Args: cobra.NoArgs, + RunE: debugCPUStop, +} + +func debugCPUStart(cmd *cobra.Command, _ []string) error { + conn, err := getClient(cmd) + if err != nil { + return err + } + defer func() { + if err := conn.Close(); err != nil { + log.Errorf(errCloseConnection, err) + } + }() + + if _, err := proto.NewDaemonServiceClient(conn).StartCPUProfile(cmd.Context(), &proto.StartCPUProfileRequest{}); err != nil { + return fmt.Errorf("start CPU profiling: %v", status.Convert(err).Message()) + } + + cmd.Println("CPU profiling started and runs until stopped. Run `netbird debug cpu stop` and then `netbird debug bundle` to collect it.") + return nil +} + +func debugCPUStop(cmd *cobra.Command, _ []string) error { + conn, err := getClient(cmd) + if err != nil { + return err + } + defer func() { + if err := conn.Close(); err != nil { + log.Errorf(errCloseConnection, err) + } + }() + + if _, err := proto.NewDaemonServiceClient(conn).StopCPUProfile(cmd.Context(), &proto.StopCPUProfileRequest{}); err != nil { + return fmt.Errorf("stop CPU profiling: %v", status.Convert(err).Message()) + } + + cmd.Println("CPU profiling stopped. Run `netbird debug bundle` to include cpu.prof.") + return nil +} + +func init() { + debugCPUCmd.AddCommand(debugCPUStartCmd) + debugCPUCmd.AddCommand(debugCPUStopCmd) + debugCmd.AddCommand(debugCPUCmd) +} diff --git a/client/cmd/debug_cpu_test.go b/client/cmd/debug_cpu_test.go new file mode 100644 index 000000000..85fffd462 --- /dev/null +++ b/client/cmd/debug_cpu_test.go @@ -0,0 +1,164 @@ +package cmd + +import ( + "bytes" + "context" + "os/user" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/internal" + "github.com/netbirdio/netbird/client/internal/profilemanager" +) + +// startDebugTestDaemon starts an in-process daemon with an isolated profile +// directory and returns the address the CLI should dial. +func startDebugTestDaemon(t *testing.T) string { + t.Helper() + + tempDir := t.TempDir() + origDefaultProfileDir := profilemanager.DefaultConfigPathDir + origActiveProfileStatePath := profilemanager.ActiveProfileStatePath + origConfigDirOverride := profilemanager.ConfigDirOverride + origDaemonAddr := daemonAddr + t.Cleanup(func() { + profilemanager.DefaultConfigPathDir = origDefaultProfileDir + profilemanager.ActiveProfileStatePath = origActiveProfileStatePath + profilemanager.ConfigDirOverride = origConfigDirOverride + daemonAddr = origDaemonAddr + }) + + profilemanager.DefaultConfigPathDir = tempDir + profilemanager.ActiveProfileStatePath = tempDir + "/active_profile.json" + profilemanager.ConfigDirOverride = tempDir + + currUser, err := user.Current() + require.NoError(t, err) + sm := profilemanager.ServiceManager{} + created, err := sm.AddProfile("test1", currUser.Username) + require.NoError(t, err) + require.NoError(t, sm.SetActiveProfileState(&profilemanager.ActiveProfileState{ + ID: created.ID, + Username: currUser.Username, + })) + + ctx, cancel := context.WithCancel(internal.CtxInitState(context.Background())) + srv, lis := startClientDaemon(t, ctx, "", tempDir+"/config.json") + t.Cleanup(func() { + cancel() + srv.Stop() + }) + + return "tcp://" + lis.Addr().String() +} + +// runDebugCmd runs `netbird debug ` against the daemon at addr and +// returns everything the command printed. +func runDebugCmd(addr string, args ...string) (string, error) { + daemonAddr = addr + var out bytes.Buffer + rootCmd.SetOut(&out) + rootCmd.SetErr(&out) + rootCmd.SetArgs(append(append([]string{"debug"}, args...), "--daemon-addr", addr, "--log-file", "")) + err := rootCmd.Execute() + rootCmd.SetOut(nil) + rootCmd.SetErr(nil) + rootCmd.SetArgs(nil) + resetFlags(rootCmd) + return out.String(), err +} + +// resetFlags puts every flag of the command and its subcommands back to its +// default so a value parsed in one run does not leak into the next in-process +// execution. +func resetFlags(cmd *cobra.Command) { + reset := func(f *pflag.Flag) { + // Set appends to a slice flag and would parse the "[a,b]" default + // text as elements, so slices are replaced instead. + if sv, ok := f.Value.(pflag.SliceValue); ok { + var def []string + if trimmed := strings.Trim(f.DefValue, "[]"); trimmed != "" { + def = strings.Split(trimmed, ",") + } + _ = sv.Replace(def) + } else { + _ = f.Value.Set(f.DefValue) + } + f.Changed = false + } + cmd.Flags().VisitAll(reset) + cmd.PersistentFlags().VisitAll(reset) + // Commands pin their writers to the buffer of the run that first used + // them, so a later run would print into the old buffer. + cmd.SetOut(nil) + cmd.SetErr(nil) + for _, sub := range cmd.Commands() { + resetFlags(sub) + } +} + +// TestResetFlagsSliceDefault guards against Set("[]") on slice flags, which +// stores a literal "[]" element instead of the empty default. +func TestResetFlagsSliceDefault(t *testing.T) { + cmd := &cobra.Command{Use: "x"} + var env, withDefault []string + cmd.Flags().StringSliceVar(&env, "env", nil, "") + cmd.Flags().StringSliceVar(&withDefault, "names", []string{"a", "b"}, "") + require.NoError(t, cmd.Flags().Parse([]string{"--env", "K=V", "--names", "c"})) + + resetFlags(cmd) + + assert.Empty(t, env, "slice flag with no default must reset to empty") + assert.Equal(t, []string{"a", "b"}, withDefault, "slice flag must reset to its default") +} + +func TestDebugCPUStartStop(t *testing.T) { + addr := startDebugTestDaemon(t) + + run := func(args ...string) error { + _, err := runDebugCmd(addr, append([]string{"cpu"}, args...)...) + return err + } + + require.Error(t, run("stop"), "stop without a running profile must fail") + require.NoError(t, run("start")) + assert.Error(t, run("start"), "second start must be rejected while profiling") + require.NoError(t, run("stop")) + assert.Error(t, run("stop"), "second stop must be rejected") + assert.NoError(t, run("start"), "profiling can be started again after a stop") + assert.NoError(t, run("stop")) +} + +// TestDebugForKeepsRunningCPUProfile covers `debug for` started while a +// profile from `debug cpu start` is running: it must say so, leave the +// profile alone, and still create the bundle. +func TestDebugForKeepsRunningCPUProfile(t *testing.T) { + addr := startDebugTestDaemon(t) + + _, err := runDebugCmd(addr, "cpu", "start") + require.NoError(t, err) + + out, err := runDebugCmd(addr, "for", "1s", "-S=false", "--no-updown") + require.NoError(t, err, "output: %s", out) + assert.Contains(t, out, "CPU profiling is already running", "the conflict must be explained") + assert.NotContains(t, out, "rpc error", "the raw RPC error must not reach the user") + assert.Contains(t, out, "Local file:", "the bundle must still be created") + + _, err = runDebugCmd(addr, "cpu", "stop") + assert.NoError(t, err, "the profile started by the user must still be running") +} + +func TestDebugForNoUpDown(t *testing.T) { + addr := startDebugTestDaemon(t) + + out, err := runDebugCmd(addr, "for", "1s", "-S=false", "--no-updown") + require.NoError(t, err, "output: %s", out) + assert.NotContains(t, out, "netbird down", "--no-updown must not bring the daemon down") + assert.NotContains(t, out, "netbird up", "--no-updown must not bring the daemon up") + assert.Contains(t, out, "Local file:", "the bundle must still be created") +} diff --git a/client/cmd/forwarding_rules.go b/client/cmd/forwarding_rules.go deleted file mode 100644 index b3052746a..000000000 --- a/client/cmd/forwarding_rules.go +++ /dev/null @@ -1,98 +0,0 @@ -package cmd - -import ( - "fmt" - "sort" - - "github.com/spf13/cobra" - "google.golang.org/grpc/status" - - "github.com/netbirdio/netbird/client/proto" -) - -var forwardingRulesCmd = &cobra.Command{ - Use: "forwarding", - Short: "List forwarding rules", - Long: `Commands to list forwarding rules.`, -} - -var forwardingRulesListCmd = &cobra.Command{ - Use: "list", - Aliases: []string{"ls"}, - Short: "List forwarding rules", - Example: " netbird forwarding list", - Long: "Commands to list forwarding rules.", - RunE: listForwardingRules, -} - -func listForwardingRules(cmd *cobra.Command, _ []string) error { - conn, err := getClient(cmd) - if err != nil { - return err - } - defer conn.Close() - - client := proto.NewDaemonServiceClient(conn) - resp, err := client.ForwardingRules(cmd.Context(), &proto.EmptyRequest{}) - if err != nil { - return fmt.Errorf("failed to list network: %v", status.Convert(err).Message()) - } - - if len(resp.GetRules()) == 0 { - cmd.Println("No forwarding rules available.") - return nil - } - - printForwardingRules(cmd, resp.GetRules()) - return nil -} - -func printForwardingRules(cmd *cobra.Command, rules []*proto.ForwardingRule) { - cmd.Println("Available forwarding rules:") - - // Sort rules by translated address - sort.Slice(rules, func(i, j int) bool { - if rules[i].GetTranslatedAddress() != rules[j].GetTranslatedAddress() { - return rules[i].GetTranslatedAddress() < rules[j].GetTranslatedAddress() - } - if rules[i].GetProtocol() != rules[j].GetProtocol() { - return rules[i].GetProtocol() < rules[j].GetProtocol() - } - - return getFirstPort(rules[i].GetDestinationPort()) < getFirstPort(rules[j].GetDestinationPort()) - }) - - var lastIP string - for _, rule := range rules { - dPort := portToString(rule.GetDestinationPort()) - tPort := portToString(rule.GetTranslatedPort()) - if lastIP != rule.GetTranslatedAddress() { - lastIP = rule.GetTranslatedAddress() - cmd.Printf("\nTranslated peer: %s\n", rule.GetTranslatedHostname()) - } - - cmd.Printf(" Local %s/%s to %s:%s\n", rule.GetProtocol(), dPort, rule.GetTranslatedAddress(), tPort) - } -} - -func getFirstPort(portInfo *proto.PortInfo) int { - switch v := portInfo.PortSelection.(type) { - case *proto.PortInfo_Port: - return int(v.Port) - case *proto.PortInfo_Range_: - return int(v.Range.GetStart()) - default: - return 0 - } -} - -func portToString(translatedPort *proto.PortInfo) string { - switch v := translatedPort.PortSelection.(type) { - case *proto.PortInfo_Port: - return fmt.Sprintf("%d", v.Port) - case *proto.PortInfo_Range_: - return fmt.Sprintf("%d-%d", v.Range.GetStart(), v.Range.GetEnd()) - default: - return "No port specified" - } -} diff --git a/client/cmd/jobs.go b/client/cmd/jobs.go new file mode 100644 index 000000000..36aab3570 --- /dev/null +++ b/client/cmd/jobs.go @@ -0,0 +1,13 @@ +package cmd + +// remoteJobsAllowedFlag opts this peer into running remote jobs (e.g. debug +// bundles) requested by the management server. It defaults to false: remote +// jobs are an explicit opt-in, and enabling it is a privileged change (see the +// daemon gate in client/server), mirroring the SSH server opt-in. +const remoteJobsAllowedFlag = "allow-remote-jobs" + +var remoteJobsAllowed bool + +func init() { + upCmd.PersistentFlags().BoolVar(&remoteJobsAllowed, remoteJobsAllowedFlag, false, "Allow the management server to run remote jobs (e.g. debug bundles) on this peer") +} diff --git a/client/cmd/login.go b/client/cmd/login.go index a7ee960b1..764fdd5b1 100644 --- a/client/cmd/login.go +++ b/client/cmd/login.go @@ -4,29 +4,36 @@ import ( "context" "fmt" "os" - "os/user" - "runtime" "strings" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/term" - "google.golang.org/grpc/codes" - gstatus "google.golang.org/grpc/status" "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/auth" "github.com/netbirdio/netbird/client/internal/profilemanager" + "github.com/netbirdio/netbird/client/mdm" + nbnet "github.com/netbirdio/netbird/client/net" "github.com/netbirdio/netbird/client/proto" + "github.com/netbirdio/netbird/client/server" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/util" ) +// extendSessionFlag drives the `netbird login --extend` flow: refresh the +// SSO session expiry on the management server without tearing down the +// tunnel. Mutually exclusive with setup-key login (a setup-key cannot +// refresh an SSO-tracked peer — see auth.errSetupKeyOnSSOExpiredPeer). +var extendSessionFlag bool + func init() { loginCmd.PersistentFlags().BoolVar(&noBrowser, noBrowserFlag, false, noBrowserDesc) loginCmd.PersistentFlags().BoolVar(&showQR, showQRFlag, false, showQRDesc) loginCmd.PersistentFlags().StringVar(&profileName, profileNameFlag, "", profileNameDesc) loginCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "(DEPRECATED) Netbird config file location") + loginCmd.PersistentFlags().BoolVar(&extendSessionFlag, "extend", false, + "refresh the SSO session expiry without tearing down the tunnel (requires an active connection)") } var loginCmd = &cobra.Command{ @@ -44,7 +51,7 @@ var loginCmd = &cobra.Command{ // nolint ctx = context.WithValue(ctx, system.DeviceNameCtxKey, hostName) } - username, err := user.Current() + username, err := profilemanager.InvokingUser() if err != nil { return fmt.Errorf("get current user: %v", err) } @@ -61,6 +68,16 @@ var loginCmd = &cobra.Command{ return err } + if extendSessionFlag { + if providedSetupKey != "" { + return fmt.Errorf("--extend cannot be combined with a setup key; setup keys can only enrol new peers") + } + if err := doExtendSession(ctx, cmd, activeProf); err != nil { + return fmt.Errorf("extend session failed: %v", err) + } + return nil + } + // workaround to run without service if util.FindFirstLogPath(logFiles) == "" { if err := doForegroundLogin(ctx, cmd, providedSetupKey, activeProf); err != nil { @@ -73,7 +90,7 @@ var loginCmd = &cobra.Command{ return fmt.Errorf("daemon login failed: %v", err) } - cmd.Println("Logging successfully") + cmd.Println("Login successful") return nil }, @@ -101,7 +118,7 @@ func doDaemonLogin(ctx context.Context, cmd *cobra.Command, providedSetupKey str loginRequest := proto.LoginRequest{ SetupKey: providedSetupKey, ManagementUrl: managementURL, - IsUnixDesktopClient: isUnixRunningDesktop(), + IsUnixDesktopClient: util.HasGraphicalSession(), Hostname: hostName, DnsLabels: dnsLabelsReq, ProfileName: &handle, @@ -126,10 +143,7 @@ func doDaemonLogin(ctx context.Context, cmd *cobra.Command, providedSetupKey str err = WithBackOff(func() error { var backOffErr error loginResp, backOffErr = client.Login(ctx, &loginRequest) - if s, ok := gstatus.FromError(backOffErr); ok && (s.Code() == codes.InvalidArgument || - s.Code() == codes.PermissionDenied || - s.Code() == codes.NotFound || - s.Code() == codes.Unimplemented) { + if terminalLoginError(backOffErr) { loginErr = backOffErr return nil } @@ -152,13 +166,73 @@ func doDaemonLogin(ctx context.Context, cmd *cobra.Command, providedSetupKey str return nil } +// doExtendSession drives the daemon's RequestExtendAuthSession / +// WaitExtendAuthSession pair. The user is sent through a regular SSO flow +// (browser + verification URL) and the resulting JWT is forwarded to the +// management server's ExtendAuthSession RPC. The tunnel stays up +// throughout — no Down/Up, no network-map resync. +func doExtendSession(ctx context.Context, cmd *cobra.Command, activeProf *profilemanager.Profile) error { + conn, err := DialClientGRPCServer(ctx, daemonAddr) + if err != nil { + //nolint + return fmt.Errorf("failed to connect to daemon error: %v\n"+ + "If the daemon is not running please run: "+ + "\nnetbird service install \nnetbird service start\n", err) + } + defer conn.Close() + + client := proto.NewDaemonServiceClient(conn) + + // the CLI runs in the user's session, the daemon does not: tell it what we can see + req := &proto.RequestExtendAuthSessionRequest{HasGraphicalSession: util.HasGraphicalSession()} + // Pre-fill the IdP login hint from the resolved profile so the user + // doesn't have to retype their email. Best-effort: we still proceed + // without a hint if the lookup fails. + pm := profilemanager.NewProfileManager() + if profState, perr := pm.GetProfileState(activeProf.ID); perr == nil && profState.Email != "" { + req.Hint = &profState.Email + } + + startResp, err := client.RequestExtendAuthSession(ctx, req) + if err != nil { + return fmt.Errorf("start extend session: %v", err) + } + + uri := startResp.GetVerificationURIComplete() + if uri == "" { + uri = startResp.GetVerificationURI() + } + openURL(cmd, uri, startResp.GetUserCode(), noBrowser, showQR) + + waitResp, err := client.WaitExtendAuthSession(ctx, &proto.WaitExtendAuthSessionRequest{ + DeviceCode: startResp.GetDeviceCode(), + UserCode: startResp.GetUserCode(), + }) + if err != nil { + return fmt.Errorf("wait for extend session: %v", err) + } + + if ts := waitResp.GetSessionExpiresAt(); ts.IsValid() && !ts.AsTime().IsZero() { + deadline := ts.AsTime().Local() + cmd.Printf("Session extended. New expiry: %s\n", deadline.Format("2006-01-02 15:04:05 MST")) + } else { + // Management reported the peer is not eligible (e.g. login + // expiration disabled on the account). Surface that fact + // instead of pretending the call succeeded. + cmd.Println("Session extension call completed, but the management server did not return a new deadline (peer may not be SSO-tracked or login expiration is disabled).") + } + return nil +} + func getActiveProfile(ctx context.Context, pm *profilemanager.ProfileManager, profileName string, username string) (*profilemanager.Profile, error) { // switch profile if provided if profileName != "" { - if err := switchProfileOnDaemon(ctx, pm, profileName, username); err != nil { + prof, err := switchProfileOnDaemon(ctx, pm, profileName, username) + if err != nil { return nil, fmt.Errorf("switch profile: %v", err) } + return prof, nil } activeProf, err := pm.GetActiveProfile() @@ -172,20 +246,19 @@ func getActiveProfile(ctx context.Context, pm *profilemanager.ProfileManager, pr return activeProf, nil } -func switchProfileOnDaemon(ctx context.Context, pm *profilemanager.ProfileManager, handle string, username string) error { +func switchProfileOnDaemon(ctx context.Context, pm *profilemanager.ProfileManager, handle string, username string) (*profilemanager.Profile, error) { resolvedID, err := switchProfile(ctx, handle, username) if err != nil { - return fmt.Errorf("switch profile on daemon: %v", err) + return nil, fmt.Errorf("switch profile on daemon: %v", err) } if err := pm.SwitchProfile(resolvedID); err != nil { - return fmt.Errorf("switch profile: %v", err) + return nil, fmt.Errorf("switch profile: %v", err) } conn, err := DialClientGRPCServer(ctx, daemonAddr) if err != nil { - log.Errorf("failed to connect to service CLI interface %v", err) - return err + return nil, fmt.Errorf("connect to service CLI interface: %w", err) } defer conn.Close() @@ -193,17 +266,17 @@ func switchProfileOnDaemon(ctx context.Context, pm *profilemanager.ProfileManage status, err := client.Status(ctx, &proto.StatusRequest{}) if err != nil { - return fmt.Errorf("unable to get daemon status: %v", err) + return nil, fmt.Errorf("unable to get daemon status: %v", err) } if status.Status == string(internal.StatusConnected) { if _, err := client.Down(ctx, &proto.DownRequest{}); err != nil { log.Errorf("call service down method: %v", err) - return err + return nil, err } } - return nil + return &profilemanager.Profile{ID: resolvedID}, nil } // switchProfile asks the daemon to switch to the profile identified by @@ -249,16 +322,47 @@ func doForegroundLogin(ctx context.Context, cmd *cobra.Command, setupKey string, } - config, err := profilemanager.ReadConfig(configFilePath) + config, err := profilemanager.ReadConfigOrDefault(configFilePath) if err != nil { return fmt.Errorf("read config file %s: %v", configFilePath, err) } + // Reading a config does not provision one: this login is about to dial + // management with the profile's identity, so mint the keys if the profile + // has none yet and put them on disk — a key that stayed in memory would + // come back different on the next run and register a second peer. + // + // Before the MDM overlay below, on purpose: the file must keep the + // profile's own values. The overlay is runtime-only and re-derived on + // every load, so persisting it would turn an enforced management URL or + // pre-shared key into one the user appears to own once the policy is + // withdrawn. + if generated, err := config.EnsureIdentity(); err != nil { + return fmt.Errorf("ensure profile identity: %v", err) + } else if generated { + if err := profilemanager.WriteOutConfig(configFilePath, config); err != nil { + return fmt.Errorf("write out config file %s: %v", configFilePath, err) + } + } + + // CLI standalone login: profilemanager no longer auto-applies MDM, + // so layer in the OS-native policy here. Desktop builds construct + // a Loader with no fetcher — the build-tagged loadPlatform reads + // the registry/plist directly. + config.ApplyMDMPolicy(mdm.NewLoader(nil).Load()) + + // Mirror runInForegroundMode: recover residual state (DNS, firewall, + // ssh config, legacy routing) from a previous unclean shutdown and + // enable advanced routing before dialing management. + if err := server.RestoreResidualState(ctx, profilemanager.NewServiceManager(configFilePath).GetStatePath()); err != nil { + log.Warnf("failed to restore residual state: %v", err) + } + nbnet.Init() err = foregroundLogin(ctx, cmd, config, setupKey, activeProf.ID) if err != nil { return fmt.Errorf("foreground login failed: %v", err) } - cmd.Println("Logging successfully") + cmd.Println("Login successful") return nil } @@ -321,11 +425,44 @@ func foregroundGetTokenInfo(ctx context.Context, cmd *cobra.Command, config *pro hint = profileState.Email } - oAuthFlow, err := auth.NewOAuthFlow(ctx, config, isUnixRunningDesktop(), false, hint) + oAuthFlow, err := auth.NewOAuthFlow(ctx, config, util.HasGraphicalSession(), false, hint, false) if err != nil { return nil, err } + tokenInfo, err := runInteractiveFlow(cmd, oAuthFlow) + if err != nil { + return nil, err + } + + if tokenInfo.MatchesAccount(hint) { + return tokenInfo, nil + } + + // The IdP answered from a session belonging to another account. Retrying is + // what makes this recoverable: on a peer already registered the server would + // reject the token, and on a fresh one it would silently register the peer + // under the wrong account and bind the profile to it. + cmd.Println("The login returned a different account than this profile uses. Asking to sign in again.") + retryFlow := auth.RetryFlowForAccount(oAuthFlow) + if retryFlow == nil { + return tokenInfo, nil + } + + retryToken, err := runInteractiveFlow(cmd, retryFlow) + if err != nil { + return nil, err + } + if !retryToken.MatchesAccount(hint) { + log.Warnf("login still returned a different account after the prompt, continuing with it") + } + + return retryToken, nil +} + +// runInteractiveFlow requests the authorization info, shows the URL to the user +// and blocks until the token comes back. +func runInteractiveFlow(cmd *cobra.Command, oAuthFlow auth.OAuthFlow) (*auth.TokenInfo, error) { flowInfo, err := oAuthFlow.RequestAuthInfo(context.TODO()) if err != nil { return nil, fmt.Errorf("getting a request OAuth flow info failed: %v", err) @@ -371,14 +508,6 @@ func openURL(cmd *cobra.Command, verificationURIComplete, userCode string, noBro } } -// isUnixRunningDesktop checks if a Linux OS is running desktop environment -func isUnixRunningDesktop() bool { - if runtime.GOOS != "linux" && runtime.GOOS != "freebsd" { - return false - } - return os.Getenv("DESKTOP_SESSION") != "" || os.Getenv("XDG_CURRENT_DESKTOP") != "" -} - func setEnvAndFlags(cmd *cobra.Command) error { SetFlagsFromEnvVars(rootCmd) diff --git a/client/cmd/logout.go b/client/cmd/logout.go index 1a5281acb..cf2a4e446 100644 --- a/client/cmd/logout.go +++ b/client/cmd/logout.go @@ -3,11 +3,11 @@ package cmd import ( "context" "fmt" - "os/user" "time" "github.com/spf13/cobra" + "github.com/netbirdio/netbird/client/internal/profilemanager" "github.com/netbirdio/netbird/client/proto" ) @@ -37,7 +37,7 @@ var logoutCmd = &cobra.Command{ if profileName != "" { req.ProfileName = &profileName - currUser, err := user.Current() + currUser, err := profilemanager.InvokingUser() if err != nil { return fmt.Errorf("get current user: %v", err) } @@ -46,7 +46,7 @@ var logoutCmd = &cobra.Command{ } if _, err := daemonClient.Logout(ctx, req); err != nil { - return fmt.Errorf("deregister: %v", err) + return daemonCallError("deregister", err) } cmd.Println("Deregistered successfully") diff --git a/client/cmd/posture.go b/client/cmd/posture.go new file mode 100644 index 000000000..b0870233d --- /dev/null +++ b/client/cmd/posture.go @@ -0,0 +1,30 @@ +package cmd + +import ( + "github.com/spf13/cobra" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/certproof" +) + +var postureCmd = &cobra.Command{ + Use: "posture", + Short: "Posture helpers invoked by the NetBird daemon", + Hidden: true, +} + +var postureCertProofCmd = &cobra.Command{ + Use: "cert-proof", + Short: "Answer certificate posture challenges from the calling user's keychain", + Long: "Reads a certificate challenge set as JSON on stdin and writes the proofs as JSON on stdout.\n" + + "The daemon launches this in the console user's desktop session, because a login keychain\n" + + "cannot be reached from a root daemon. Not intended to be run by hand.", + Hidden: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + // Proofs travel on stdout, so every log line has to go elsewhere. + log.SetOutput(cmd.ErrOrStderr()) + return certproof.RunHelper(cmd.Context(), cmd.InOrStdin(), cmd.OutOrStdout()) + }, +} diff --git a/client/cmd/profile.go b/client/cmd/profile.go index 268034e70..2d6653537 100644 --- a/client/cmd/profile.go +++ b/client/cmd/profile.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "os/user" "strings" "text/tabwriter" "time" @@ -97,7 +96,7 @@ func listProfilesFunc(cmd *cobra.Command, _ []string) error { } defer conn.Close() - currUser, err := user.Current() + currUser, err := profilemanager.InvokingUser() if err != nil { return fmt.Errorf("get current user: %w", err) } @@ -138,7 +137,7 @@ func addProfileFunc(cmd *cobra.Command, args []string) error { return err } - currUser, err := user.Current() + currUser, err := profilemanager.InvokingUser() if err != nil { return fmt.Errorf("get current user: %w", err) } @@ -179,7 +178,7 @@ func renameProfileFunc(cmd *cobra.Command, args []string) error { } defer conn.Close() - currUser, err := user.Current() + currUser, err := profilemanager.InvokingUser() if err != nil { return fmt.Errorf("get current user: %w", err) } @@ -233,7 +232,7 @@ func removeProfileFunc(cmd *cobra.Command, args []string) error { } defer conn.Close() - currUser, err := user.Current() + currUser, err := profilemanager.InvokingUser() if err != nil { return fmt.Errorf("get current user: %w", err) } @@ -261,7 +260,7 @@ func selectProfileFunc(cmd *cobra.Command, args []string) error { profileManager := profilemanager.NewProfileManager() handle := args[0] - currUser, err := user.Current() + currUser, err := profilemanager.InvokingUser() if err != nil { return fmt.Errorf("get current user: %w", err) } diff --git a/client/cmd/root.go b/client/cmd/root.go index f1ef32717..51179f77c 100644 --- a/client/cmd/root.go +++ b/client/cmd/root.go @@ -20,9 +20,12 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/codes" + gstatus "google.golang.org/grpc/status" + "github.com/netbirdio/netbird/client/anonymize" daddr "github.com/netbirdio/netbird/client/internal/daemonaddr" + "github.com/netbirdio/netbird/client/internal/localmetrics" "github.com/netbirdio/netbird/client/internal/profilemanager" ) @@ -31,6 +34,8 @@ const ( dnsResolverAddress = "dns-resolver-address" enableRosenpassFlag = "enable-rosenpass" rosenpassPermissiveFlag = "rosenpass-permissive" + enableLocalMetricsFlag = "enable-local-metrics" + localMetricsAddressFlag = "local-metrics-address" preSharedKeyFlag = "preshared-key" interfaceNameFlag = "interface-name" wireguardPortFlag = "wireguard-port" @@ -70,6 +75,7 @@ var ( autoConnectDisabled bool extraIFaceBlackList []string anonymizeFlag bool + anonymizeLevelFlag string dnsRouteInterval time.Duration // lazyConnEnabled is the parse target for the deprecated --enable-lazy-connection // flag. The flag is inert; the value is no longer read (use NB_LAZY_CONN instead). @@ -79,6 +85,8 @@ var ( updateSettingsDisabled bool captureEnabled bool networksDisabled bool + localMetricsEnabled bool + localMetricsAddr string rootCmd = &cobra.Command{ Use: "netbird", @@ -91,6 +99,7 @@ var ( // Don't resolve for service commands — they create the socket, not connect to it. if !isServiceCmd(cmd) { daemonAddr = daddr.ResolveUnixDaemonAddr(daemonAddr) + daemonAddr = daddr.ResolveDaemonAddr(daemonAddr) } return nil }, @@ -143,10 +152,10 @@ func init() { defaultDaemonAddr := "unix:///var/run/netbird.sock" if runtime.GOOS == "windows" { - defaultDaemonAddr = "tcp://127.0.0.1:41731" + defaultDaemonAddr = daddr.WindowsPipeAddr } - rootCmd.PersistentFlags().StringVar(&daemonAddr, "daemon-addr", defaultDaemonAddr, "Daemon service address to serve CLI requests [unix|tcp]://[path|host:port]") + rootCmd.PersistentFlags().StringVar(&daemonAddr, "daemon-addr", defaultDaemonAddr, "Daemon service address to serve CLI requests [unix|tcp|npipe]://[path|host:port|name]") rootCmd.PersistentFlags().StringVarP(&managementURL, "management-url", "m", "", fmt.Sprintf("Management Service URL [http|https]://[host]:[port] (default \"%s\")", profilemanager.DefaultManagementURL)) rootCmd.PersistentFlags().StringVar(&adminURL, "admin-url", "", fmt.Sprintf("Admin Panel URL [http|https]://[host]:[port] (default \"%s\")", profilemanager.DefaultAdminURL)) rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", "info", "sets NetBird log level") @@ -156,7 +165,8 @@ func init() { rootCmd.MarkFlagsMutuallyExclusive("setup-key", "setup-key-file") rootCmd.PersistentFlags().StringVar(&preSharedKey, preSharedKeyFlag, "", "Sets WireGuard PreSharedKey property. If set, then only peers that have the same key can communicate.") rootCmd.PersistentFlags().StringVarP(&hostName, "hostname", "n", "", "Sets a custom hostname for the device") - rootCmd.PersistentFlags().BoolVarP(&anonymizeFlag, "anonymize", "A", false, "anonymize IP addresses and non-netbird.io domains in logs and status output") + rootCmd.PersistentFlags().BoolVarP(&anonymizeFlag, "anonymize", "A", false, "anonymize public IP addresses, MAC addresses, and non-netbird.io domains in logs and status output; private, CGNAT, and link-local IP ranges are kept (see --anonymize-level strict)") + rootCmd.PersistentFlags().StringVar(&anonymizeLevelFlag, "anonymize-level", "", "anonymization level: \"default\" or \"strict\"; strict also anonymizes private, CGNAT, and link-local IP ranges, peer names, and WireGuard public keys. Setting this flag implies --anonymize") rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", profilemanager.DefaultConfigPath, "Overrides the default profile file location") rootCmd.AddCommand(upCmd) @@ -167,16 +177,16 @@ func init() { rootCmd.AddCommand(versionCmd) rootCmd.AddCommand(sshCmd) rootCmd.AddCommand(networksCMD) - rootCmd.AddCommand(forwardingRulesCmd) rootCmd.AddCommand(debugCmd) rootCmd.AddCommand(profileCmd) rootCmd.AddCommand(exposeCmd) + rootCmd.AddCommand(postureCmd) + + postureCmd.AddCommand(postureCertProofCmd) networksCMD.AddCommand(routesListCmd) networksCMD.AddCommand(routesSelectCmd, routesDeselectCmd) - forwardingRulesCmd.AddCommand(forwardingRulesListCmd) - debugCmd.AddCommand(debugBundleCmd) debugCmd.AddCommand(logCmd) logCmd.AddCommand(logLevelCmd) @@ -212,6 +222,8 @@ func init() { upCmd.PersistentFlags().BoolVar(&rosenpassEnabled, enableRosenpassFlag, false, "[Experimental] Enable Rosenpass feature. If enabled, the connection will be post-quantum secured via Rosenpass.") upCmd.PersistentFlags().BoolVar(&rosenpassPermissive, rosenpassPermissiveFlag, false, "[Experimental] Enable Rosenpass in permissive mode to allow this peer to accept WireGuard connections without requiring Rosenpass functionality from peers that do not have Rosenpass enabled.") upCmd.PersistentFlags().BoolVar(&autoConnectDisabled, disableAutoConnectFlag, false, "Disables auto-connect feature. If enabled, then the client won't connect automatically when the service starts.") + upCmd.PersistentFlags().BoolVar(&localMetricsEnabled, enableLocalMetricsFlag, false, "Enables a local Prometheus /metrics endpoint exposing connection state (peers, latency, P2P vs relay).") + upCmd.PersistentFlags().StringVar(&localMetricsAddr, localMetricsAddressFlag, localmetrics.DefaultListenAddress, "Listen address of the local Prometheus /metrics endpoint.") upCmd.PersistentFlags().BoolVar(&lazyConnEnabled, enableLazyConnectionFlag, false, "Deprecated: no longer used. Lazy connections are controlled by the server and the NB_LAZY_CONN environment variable.") _ = upCmd.PersistentFlags().MarkDeprecated(enableLazyConnectionFlag, "no longer used; lazy connections are controlled by the server and the NB_LAZY_CONN environment variable") @@ -269,12 +281,47 @@ func DialClientGRPCServer(ctx context.Context, addr string) (*grpc.ClientConn, e ctx, cancel := context.WithTimeout(ctx, time.Second*10) defer cancel() - return grpc.DialContext( - ctx, - strings.TrimPrefix(addr, "tcp://"), - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithBlock(), - ) + target, opts := daddr.DialTarget(addr) + opts = append(opts, grpc.WithBlock()) + + return grpc.DialContext(ctx, target, opts...) +} + +// terminalLoginError reports whether a Login failure is final, so the backoff +// cycle stops and the caller is told what the daemon said instead of "login +// backoff cycle failed" thirty seconds later. Retrying cannot change any of +// these answers: the request is malformed, the caller is not allowed, the +// target does not exist, a precondition on the daemon refuses it (the +// update-settings kill switch, an MDM-managed field), or the method is not +// implemented. +// +// Both `netbird up` and `netbird login` run Login through the backoff, and +// they each carried their own copy of this list — which is how one of them +// ended up retrying a refusal the other treated as final. +func terminalLoginError(err error) bool { + // A successful Login reaches here with a nil error, and that is not a + // terminal failure. Handled explicitly rather than left to + // gstatus.FromError, which answers (nil, true) for a nil error and leans on + // Status.Code tolerating a nil receiver to come back as codes.OK. + if err == nil { + return false + } + + s, ok := gstatus.FromError(err) + if !ok { + return false + } + + switch s.Code() { + case codes.InvalidArgument, + codes.PermissionDenied, + codes.NotFound, + codes.FailedPrecondition, + codes.Unimplemented: + return true + default: + return false + } } // WithBackOff execute function in backoff cycle. @@ -295,6 +342,19 @@ var CLIBackOffSettings = &backoff.ExponentialBackOff{ Clock: backoff.SystemClock, } +// effectiveAnonymize resolves the --anonymize and --anonymize-level flags: +// setting a level implies anonymization, and an invalid level is rejected. +func effectiveAnonymize() (bool, anonymize.Level, error) { + if anonymizeLevelFlag == "" { + return anonymizeFlag, anonymize.LevelDefault, nil + } + level := anonymize.ParseLevel(anonymizeLevelFlag) + if !strings.EqualFold(anonymizeLevelFlag, level.String()) { + return false, anonymize.LevelDefault, fmt.Errorf("invalid anonymize level %q: use %q or %q", anonymizeLevelFlag, anonymize.LevelDefault.String(), anonymize.LevelStrict.String()) + } + return true, level, nil +} + func getSetupKey() (string, error) { if setupKeyPath != "" && setupKey == "" { return getSetupKeyFromFile(setupKeyPath) diff --git a/client/cmd/service.go b/client/cmd/service.go index 56d8a8726..2a558e6d5 100644 --- a/client/cmd/service.go +++ b/client/cmd/service.go @@ -5,7 +5,9 @@ package cmd import ( "context" "fmt" + "net/http" "runtime" + "slices" "strings" "sync" @@ -22,15 +24,50 @@ var serviceCmd = &cobra.Command{ Short: "Manage the NetBird daemon service", } +const defaultJSONSocket = "unix:///var/run/netbird-http.sock" + +// forbiddenServiceEnvVars are the environment variables the service is never +// registered with, keyed in upper case since these are Windows names. Each one +// decides where the daemon resolves something it then uses with the privileges +// of the account it runs under — LocalSystem on Windows, root elsewhere: the +// executables it runs (PATH, PATHEXT, COMSPEC, SystemRoot, windir) or the +// directory it writes temporary files in (TEMP, TMP). The daemon needs none of +// them, and the utilities it shells out to are resolved by absolute path. +var forbiddenServiceEnvVars = map[string]struct{}{ + "PATH": {}, + "PATHEXT": {}, + "SYSTEMROOT": {}, + "WINDIR": {}, + "COMSPEC": {}, + "TEMP": {}, + "TMP": {}, +} + +// forbiddenServiceEnvPrefixes are the dynamic-loader families, refused whole +// rather than by name: LD_PRELOAD, DYLD_INSERT_LIBRARIES and their siblings all +// reach the loader of the process, the set differs per platform and libc, and +// new members arrive with new OS releases. Listing them one by one is a list +// that is wrong the moment it is written. +var forbiddenServiceEnvPrefixes = []string{"LD_", "DYLD_"} + var ( - serviceName string - serviceEnvVars []string + serviceName string + serviceEnvVars []string + jsonSocket string + enableJSONSocket bool ) type program struct { - ctx context.Context - cancel context.CancelFunc - serv *grpc.Server + ctx context.Context + cancel context.CancelFunc + serv *grpc.Server + jsonServ *http.Server + // jsonClient is the gateway's own connection to the daemon. It is held so + // shutting the gateway down also closes it: nothing else references it once + // the handlers are registered, so its transport goroutines would otherwise + // outlive the server. + jsonClient *grpc.ClientConn + jsonServMu sync.Mutex serverInstance *server.Server serverInstanceMu sync.Mutex } @@ -46,6 +83,8 @@ func init() { serviceCmd.PersistentFlags().BoolVar(&updateSettingsDisabled, "disable-update-settings", false, "Disables update settings feature. If enabled, the client will not be able to change or edit any settings. To persist this setting, use: netbird service install --disable-update-settings") serviceCmd.PersistentFlags().BoolVar(&captureEnabled, "enable-capture", false, "Enables packet capture via 'netbird debug capture'. To persist, use: netbird service install --enable-capture") serviceCmd.PersistentFlags().BoolVar(&networksDisabled, "disable-networks", false, "Disables network selection. If enabled, the client will not allow listing, selecting, or deselecting networks. To persist, use: netbird service install --disable-networks") + serviceCmd.PersistentFlags().BoolVar(&enableJSONSocket, "enable-json-socket", false, "Enables the HTTP/JSON API socket served by grpc-gateway. To persist, use: netbird service install --enable-json-socket") + serviceCmd.PersistentFlags().StringVar(&jsonSocket, "json-socket", defaultJSONSocket, "HTTP/JSON API socket address [unix|tcp]://[path|host:port]. Requires --enable-json-socket to serve. To persist, use: netbird service install --enable-json-socket --json-socket") rootCmd.PersistentFlags().StringVarP(&serviceName, "service", "s", defaultServiceName, "Netbird system service name") serviceEnvDesc := `Sets extra environment variables for the service. ` + @@ -113,8 +152,33 @@ func parseServiceEnvVars(envVars []string) (map[string]string, error) { return nil, fmt.Errorf("empty environment variable key in: %s", env) } + if isForbiddenServiceEnvVar(key) { + return nil, fmt.Errorf("environment variable %s cannot be set on the service: it decides where the service resolves the executables, libraries or temporary files it uses", key) + } + envMap[key] = value } return envMap, nil } + +// isForbiddenServiceEnvVar reports whether name is one the service must not be +// registered with. +// +// The names are matched case-insensitively only on Windows, where they are the +// same variable however they are spelled. Elsewhere the environment is +// case-sensitive, so Path and PATH are two different variables and only the +// exact spelling is the one the loader reads. +func isForbiddenServiceEnvVar(name string) bool { + if runtime.GOOS == "windows" { + name = strings.ToUpper(name) + } + + if _, forbidden := forbiddenServiceEnvVars[name]; forbidden { + return true + } + + return slices.ContainsFunc(forbiddenServiceEnvPrefixes, func(prefix string) bool { + return strings.HasPrefix(name, prefix) + }) +} diff --git a/client/cmd/service_controller.go b/client/cmd/service_controller.go index 8de147946..e9a0e055f 100644 --- a/client/cmd/service_controller.go +++ b/client/cmd/service_controller.go @@ -5,9 +5,7 @@ package cmd import ( "context" "fmt" - "net" - "os" - "strings" + "runtime" "time" "github.com/kardianos/service" @@ -16,69 +14,159 @@ import ( "github.com/spf13/cobra" "google.golang.org/grpc" + "github.com/netbirdio/netbird/client/internal/daemonaddr" + "github.com/netbirdio/netbird/client/internal/ipcauth" "github.com/netbirdio/netbird/client/proto" "github.com/netbirdio/netbird/client/server" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/util" ) +func validateJSONSocketFlags() error { + if serviceCmd.PersistentFlags().Changed("json-socket") && !enableJSONSocket { + return fmt.Errorf("--json-socket requires --enable-json-socket to configure the daemon JSON gateway") + } + return nil +} + +// daemonServerOptions installs the transport credentials that expose each +// caller's kernel-authenticated identity to the handlers, which is what lets +// the daemon require root/administrator for privileged operations. +// +// The handshake exchanges no bytes, so older CLI and UI binaries still +// interoperate. Callers on a TCP socket carry no identity at all: the daemon +// keeps serving them, and the privileged operations deny them, so a warning is +// logged to make the loss of functionality visible. +func daemonServerOptions(network string) []grpc.ServerOption { + if network == "tcp" { + log.Warnf("daemon is listening on TCP (%s): callers carry no verifiable identity over TCP, "+ + "so privileged operations (SSH root login, SSH auth, enabling the SSH server, management URL changes, "+ + "deregistration) will be denied, and the SSH JWT cache is neither filled nor served. "+ + "Use a unix socket, or npipe:// on Windows", daemonAddr) + return nil + } + + creds := ipcauth.NewTransportCredentials() //nolint:staticcheck + if creds == nil { //nolint:staticcheck // nil only on platforms without a peer-identity primitive + log.Warnf("daemon IPC has no peer-identity primitive on %s: privileged operations will be denied "+ + "and the SSH JWT cache is neither filled nor served", runtime.GOOS) + return nil + } + + return []grpc.ServerOption{grpc.Creds(creds)} +} + func (p *program) Start(svc service.Service) error { // Start should not block. Do the actual work async. log.Info("starting NetBird service") //nolint + if err := validateJSONSocketFlags(); err != nil { + return err + } + // Collect static system and platform information system.UpdateStaticInfoAsync() - // in any case, even if configuration does not exists we run daemon to serve CLI gRPC API. - p.serv = grpc.NewServer() - - split := strings.Split(daemonAddr, "://") - switch split[0] { - case "unix": - // cleanup failed close - stat, err := os.Stat(split[1]) - if err == nil && !stat.IsDir() { - if err := os.Remove(split[1]); err != nil { - log.Debugf("remove socket file: %v", err) - } - } - case "tcp": - default: - return fmt.Errorf("unsupported daemon address protocol: %v", split[0]) + // A daemon installed before named-pipe support has the loopback TCP address + // persisted. Move it to the named pipe so an upgraded daemon can identify + // its callers instead of silently serving an unauthenticated socket. + if migrated, ok := daemonaddr.MigrateLegacy(daemonAddr); ok { + log.Infof("daemon address %q predates named-pipe support, listening on %q so callers can be identified", daemonAddr, migrated) + daemonAddr = migrated } - listen, err := net.Listen(split[0], split[1]) + network, _, err := parseListenAddress(daemonAddr) if err != nil { - return fmt.Errorf("listen daemon interface: %w", err) + return fmt.Errorf("parse daemon address: %w", err) } + + // in any case, even if configuration does not exists we run daemon to serve CLI gRPC API. + p.serv = grpc.NewServer(daemonServerOptions(network)...) + + daemonListener, jsonListener, err := listenDaemonSockets() + if err != nil { + return err + } + go func() { - defer listen.Close() - - if split[0] == "unix" { - if err := os.Chmod(split[1], 0666); err != nil { - log.Errorf("failed setting daemon permissions: %v", split[1]) - return - } - } - - serverInstance := server.New(p.ctx, util.FindFirstLogPath(logFiles), configPath, profilesDisabled, updateSettingsDisabled, captureEnabled, networksDisabled) - if err := serverInstance.Start(); err != nil { - log.Fatalf("failed to start daemon: %v", err) - } - proto.RegisterDaemonServiceServer(p.serv, serverInstance) - - p.serverInstanceMu.Lock() - p.serverInstance = serverInstance - p.serverInstanceMu.Unlock() - - log.Printf("started daemon server: %v", split[1]) - if err := p.serv.Serve(listen); err != nil { - log.Errorf("failed to serve daemon requests: %v", err) + // Fatal here rather than inside serve, so serve's deferred listener + // closes run before the process exits. + if err := p.serve(daemonListener, jsonListener); err != nil { + log.Fatalf("failed to %v", err) } }() return nil } +// listenDaemonSockets opens the daemon control socket and, when it is enabled, the +// JSON gateway socket. The control socket is closed again if the second one fails, +// so a failed start leaves nothing listening. The returned JSON listener is nil +// when the socket is disabled. +func listenDaemonSockets() (*socketListener, *socketListener, error) { + daemonListener, err := listenOnAddress(daemonAddr) + if err != nil { + return nil, nil, fmt.Errorf("listen daemon interface: %w", err) + } + + if !enableJSONSocket { + removeStaleUnixSocketForAddress(jsonSocket) + return daemonListener, nil, nil + } + + jsonListener, err := listenOnAddress(jsonSocket) + if err != nil { + if cerr := daemonListener.Close(); cerr != nil { + log.Debugf("close daemon listener: %v", cerr) + } + return nil, nil, fmt.Errorf("listen daemon JSON interface: %w", err) + } + + return daemonListener, jsonListener, nil +} + +// serve brings up the daemon server on an already-open control socket and blocks +// until it stops. jsonListener is nil when the JSON socket is disabled. A returned +// error means the daemon cannot run at all and the caller is expected to exit; the +// failures it recovers from on its own are logged here. +func (p *program) serve(daemonListener, jsonListener *socketListener) error { + defer daemonListener.Close() + if jsonListener != nil { + defer jsonListener.Close() + } + + // chmodUnixSocket is a no-op for a nil listener and for a non-unix one. + if err := daemonListener.chmodUnixSocket("daemon"); err != nil { + log.Error(err) + return nil + } + if err := jsonListener.chmodUnixSocket("daemon JSON"); err != nil { + log.Error(err) + return nil + } + + serverInstance := server.New(p.ctx, util.FindFirstLogPath(logFiles), configPath, profilesDisabled, updateSettingsDisabled, captureEnabled, networksDisabled) + if err := serverInstance.Start(); err != nil { + return fmt.Errorf("start daemon: %w", err) + } + proto.RegisterDaemonServiceServer(p.serv, serverInstance) + + p.serverInstanceMu.Lock() + p.serverInstance = serverInstance + p.serverInstanceMu.Unlock() + + if jsonListener == nil { + log.Debug("daemon JSON socket disabled") + } else if err := p.startJSONGateway(jsonListener, daemonAddr); err != nil { + return fmt.Errorf("start daemon JSON server: %w", err) + } + + log.Printf("started daemon server: %v", daemonListener.address) + if err := p.serv.Serve(daemonListener.Listener); err != nil { + log.Errorf("failed to serve daemon requests: %v", err) + } + return nil +} + func (p *program) Stop(srv service.Service) error { p.serverInstanceMu.Lock() if p.serverInstance != nil { @@ -92,6 +180,25 @@ func (p *program) Stop(srv service.Service) error { p.cancel() + p.jsonServMu.Lock() + jsonServ, jsonClient := p.jsonServ, p.jsonClient + p.jsonServMu.Unlock() + if jsonClient != nil { + if err := jsonClient.Close(); err != nil { + log.Debugf("close daemon JSON gateway client: %v", err) + } + } + if jsonServ != nil { + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 2*time.Second) + if err := jsonServ.Shutdown(shutdownCtx); err != nil { + log.Errorf("failed to stop daemon JSON server gracefully: %v", err) + if err := jsonServ.Close(); err != nil { + log.Errorf("failed to close daemon JSON server: %v", err) + } + } + shutdownCancel() + } + if p.serv != nil { p.serv.Stop() } @@ -148,6 +255,9 @@ var runCmd = &cobra.Command{ if err != nil { return err } + if err := validateJSONSocketFlags(); err != nil { + return err + } return s.Run() }, @@ -162,6 +272,9 @@ var startCmd = &cobra.Command{ if err != nil { return err } + if err := validateJSONSocketFlags(); err != nil { + return err + } if err := s.Start(); err != nil { return fmt.Errorf("start service: %w", err) @@ -198,6 +311,9 @@ var restartCmd = &cobra.Command{ if err != nil { return err } + if err := validateJSONSocketFlags(); err != nil { + return err + } if err := s.Restart(); err != nil { return fmt.Errorf("restart service: %w", err) diff --git a/client/cmd/service_installer.go b/client/cmd/service_installer.go index 2d45fa063..ae2dfb9fa 100644 --- a/client/cmd/service_installer.go +++ b/client/cmd/service_installer.go @@ -67,6 +67,10 @@ func buildServiceArguments() []string { args = append(args, "--disable-networks") } + if enableJSONSocket { + args = append(args, "--enable-json-socket", "--json-socket", jsonSocket) + } + return args } @@ -106,6 +110,10 @@ func configurePlatformSpecificSettings(svcConfig *service.Config) error { // Create fully configured service config for install/reconfigure func createServiceConfigForInstall() (*service.Config, error) { + if err := validateJSONSocketFlags(); err != nil { + return nil, err + } + svcConfig, err := newSVCConfig() if err != nil { return nil, fmt.Errorf("create service config: %w", err) diff --git a/client/cmd/service_json_gateway.go b/client/cmd/service_json_gateway.go new file mode 100644 index 000000000..b6864f338 --- /dev/null +++ b/client/cmd/service_json_gateway.go @@ -0,0 +1,150 @@ +//go:build !ios && !android + +package cmd + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "sync" + "time" + + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + log "github.com/sirupsen/logrus" + "google.golang.org/grpc" + + "github.com/netbirdio/netbird/client/internal/daemonaddr" + "github.com/netbirdio/netbird/client/internal/ipcauth" + "github.com/netbirdio/netbird/client/proto" +) + +// jsonPeerIdentity is the context key under which the connecting HTTP client's +// identity is stashed for the lifetime of its connection. +type jsonPeerIdentity struct{} + +// jsonPeerIdentityValue pairs the identity with whether it could be read at +// all, so an unreadable identity is forwarded as "unknown" rather than omitted. +type jsonPeerIdentityValue struct { + id ipcauth.Identity + known bool +} + +// jsonConnContext reads the identity of the client connecting to the JSON +// socket and stashes it on the connection's context. The gateway re-dials the +// daemon in-process, so the daemon would otherwise see every JSON request as +// coming from the daemon itself. +func jsonConnContext(ctx context.Context, c net.Conn) context.Context { + value := jsonPeerIdentityValue{} + id, err := ipcauth.ConnIdentity(c) + if err != nil { + log.Warnf("json gateway: cannot read HTTP client identity, privileged operations will be denied for this connection: %v", err) + } else { + value.id = id + value.known = true + } + return context.WithValue(ctx, jsonPeerIdentity{}, value) +} + +// forwardIdentity stamps the HTTP client's identity onto every call the gateway +// makes to the daemon. +// +// It is an interceptor on the gateway's client connection rather than a +// runtime.WithMetadata annotator because grpc-gateway skips annotators when no +// request header maps to metadata, which an HTTP/1.0 request with no Host header +// over a unix socket achieves. The daemon would then receive no marker, see its own +// identity as the transport peer, and authorize the request as the daemon itself. +// An interceptor runs for every RPC whatever the request looked like. +func forwardIdentity(ctx context.Context) context.Context { + value, ok := ctx.Value(jsonPeerIdentity{}).(jsonPeerIdentityValue) + if !ok { + // No ConnContext ran for this request, so forward an unknown identity: + // the daemon must not mistake its own identity for the client's. + return ipcauth.WithForwardedIdentity(ctx, ipcauth.Identity{}, false) + } + return ipcauth.WithForwardedIdentity(ctx, value.id, value.known) +} + +func forwardIdentityUnary(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + return invoker(forwardIdentity(ctx), method, req, reply, cc, opts...) +} + +func forwardIdentityStream(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + return streamer(forwardIdentity(ctx), desc, cc, method, opts...) +} + +// reservedHeaderWarning limits the dropped-header warning to the first occurrence. +var reservedHeaderWarning sync.Once + +// jsonIncomingHeaderMatcher keeps an HTTP client from supplying the metadata the +// gateway uses to forward its identity. grpc-gateway turns "Grpc-Metadata-" +// headers into gRPC metadata and joins them ahead of what its annotators add, so +// without this filter a JSON client could send its own x-netbird-fwd-uid and the +// daemon would authorize that instead of the client's real identity. +func jsonIncomingHeaderMatcher(key string) (string, bool) { + mapped, ok := runtime.DefaultHeaderMatcher(key) + if !ok { + return "", false + } + if ipcauth.IsReservedForwardKey(mapped) { + // Warn once: any client can send these on every request, so warning each + // time hands it a way to fill the log. The rest are debug-level. + reservedHeaderWarning.Do(func() { + log.Warnf("json gateway: dropping reserved header %q from a request: only the gateway may set the caller's identity", key) + }) + log.Debugf("json gateway: dropping reserved header %q", key) + return "", false + } + return mapped, true +} + +func (p *program) startJSONGateway(jsonListener *socketListener, daemonEndpoint string) error { + if jsonListener.network == "tcp" { + log.Warnf("daemon JSON socket is listening on TCP (%s): callers carry no verifiable identity over TCP, "+ + "so privileged operations will be denied for JSON clients", jsonListener.address) + } + + mux := runtime.NewServeMux(runtime.WithIncomingHeaderMatcher(jsonIncomingHeaderMatcher)) + + // grpc.NewClient does not connect until the first request, so registering + // the handler here cannot block daemon startup. + target, opts := daemonaddr.DialTarget(daemonEndpoint) + opts = append(opts, + grpc.WithChainUnaryInterceptor(forwardIdentityUnary), + grpc.WithChainStreamInterceptor(forwardIdentityStream), + ) + conn, err := grpc.NewClient(target, opts...) + if err != nil { + return fmt.Errorf("create daemon client for JSON gateway: %w", err) + } + if err := proto.RegisterDaemonServiceHandler(p.ctx, mux, conn); err != nil { + if cerr := conn.Close(); cerr != nil { + log.Debugf("close daemon client after failed JSON gateway registration: %v", cerr) + } + return err + } + + jsonServer := &http.Server{ + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + BaseContext: func(net.Listener) context.Context { + return p.ctx + }, + ConnContext: jsonConnContext, + } + + p.jsonServMu.Lock() + p.jsonServ = jsonServer + p.jsonClient = conn + p.jsonServMu.Unlock() + + go func() { + log.Printf("started daemon JSON server: %v", jsonListener.address) + if err := jsonServer.Serve(jsonListener.Listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Errorf("failed to serve daemon JSON requests: %v", err) + } + }() + + return nil +} diff --git a/client/cmd/service_json_gateway_test.go b/client/cmd/service_json_gateway_test.go new file mode 100644 index 000000000..dfeef1c46 --- /dev/null +++ b/client/cmd/service_json_gateway_test.go @@ -0,0 +1,261 @@ +//go:build !windows && !ios && !android + +package cmd + +import ( + "context" + "net" + "net/http" + "path/filepath" + "testing" + "time" + + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + + "github.com/netbirdio/netbird/client/internal/ipcauth" +) + +// The JSON gateway runs inside the daemon and re-dials it locally, so every JSON +// request reaches a handler with the daemon's own identity as the transport peer. +// The gateway therefore forwards its HTTP client's identity as metadata, and the +// daemon authorizes that instead of itself. These tests drive the real wiring +// (jsonConnContext, forwardIdentity, jsonIncomingHeaderMatcher) and check the +// identity a handler would end up authorizing. + +// daemonSideCtx is what a handler sees for a gateway-relayed call. The transport +// peer must be this process's own identity: the gateway is the daemon, so the two +// cannot differ, and hardcoding root here instead would describe a state that +// never occurs. +func daemonSideCtx(t *testing.T, md metadata.MD) context.Context { + t.Helper() + self, err := ipcauth.CurrentProcessIdentity() + if err != nil { + t.Skipf("cannot read this process's identity: %v", err) + } + ctx := peer.NewContext(context.Background(), &peer.Peer{ + AuthInfo: ipcauth.AuthInfo{ + CommonAuthInfo: credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity}, + Identity: self, + }, + }) + return metadata.NewIncomingContext(ctx, md) +} + +// gatewayMetadata reproduces what the daemon receives for a JSON request: the +// mux annotates the context from the request's headers, then the interceptor on the +// gateway's client connection stamps the caller's identity. The order matters, +// since the interceptor must win over anything a header put there. +func gatewayMetadata(t *testing.T, req *http.Request, ctx context.Context) metadata.MD { + t.Helper() + mux := runtime.NewServeMux(runtime.WithIncomingHeaderMatcher(jsonIncomingHeaderMatcher)) + annotated, err := runtime.AnnotateContext(ctx, mux, req, + "/daemon.DaemonService/SetConfig", + runtime.WithHTTPPathPattern("/daemon.DaemonService/SetConfig")) + if err != nil { + t.Fatalf("annotate: %v", err) + } + + md, ok := metadata.FromOutgoingContext(forwardIdentity(annotated)) + if !ok { + t.Fatal("the interceptor produced no metadata") + } + return md +} + +// clientCtx is the connection context jsonConnContext would have produced for an +// HTTP client whose identity the gateway could read. +func clientCtx(id ipcauth.Identity, known bool) context.Context { + return context.WithValue(context.Background(), jsonPeerIdentity{}, + jsonPeerIdentityValue{id: id, known: known}) +} + +// An HTTP client must not be able to name its own identity. grpc-gateway turns +// Grpc-Metadata- headers into gRPC metadata, so without the header filter and +// the interceptor overwriting the reserved keys, this request would authorize as +// uid 0. +func TestJSONGateway_ForgedIdentityHeaderIsDropped(t *testing.T) { + req, err := http.NewRequest(http.MethodPost, "http://localhost/daemon.DaemonService/SetConfig", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Grpc-Metadata-X-Netbird-Fwd-Uid", "0") + req.Header.Set("Grpc-Metadata-X-Netbird-Fwd-Gid", "0") + req.Header.Set("Grpc-Metadata-X-Netbird-Fwd", "1") + req.Header.Set("Grpc-Metadata-X-Netbird-Fwd-Sid", "S-1-5-18") + + caller := ipcauth.Identity{UID: 31000, GID: 31000} + md := gatewayMetadata(t, req, clientCtx(caller, true)) + + id, ok := ipcauth.CallerIdentity(daemonSideCtx(t, md)) + if !ok { + t.Fatal("the forwarded identity should be usable") + } + if id.IsPrivileged() { + t.Errorf("forged header was believed: authorized as %v", id) + } + if id.UID != caller.UID { + t.Errorf("authorized as uid %d, want the real client %d", id.UID, caller.UID) + } +} + +// A request with no headers at all (HTTP/1.0 needs no Host, and a unix socket +// yields no host:port) makes grpc-gateway produce no metadata whatsoever and skip +// its annotators: "if len(pairs) == 0 { return ctx, nil, nil }" in +// runtime/context.go. That is why the identity is stamped by an interceptor +// instead. This is the case that previously reached the gate as the daemon itself. +func TestJSONGateway_HeaderlessRequestIsStillMarkedForwarded(t *testing.T) { + req, err := http.NewRequest(http.MethodPost, "http://localhost/daemon.DaemonService/SetConfig", nil) + if err != nil { + t.Fatal(err) + } + req.Header = http.Header{} + req.Host = "" + + caller := ipcauth.Identity{UID: 31000, GID: 31000} + ctx := clientCtx(caller, true) + + // Pin the skip path itself: if grpc-gateway ever produced a pair here, this + // test would still pass below while no longer covering what it was written for. + mux := runtime.NewServeMux(runtime.WithIncomingHeaderMatcher(jsonIncomingHeaderMatcher)) + annotated, err := runtime.AnnotateContext(ctx, mux, req, + "/daemon.DaemonService/SetConfig", + runtime.WithHTTPPathPattern("/daemon.DaemonService/SetConfig")) + if err != nil { + t.Fatalf("annotate: %v", err) + } + if md, ok := metadata.FromOutgoingContext(annotated); ok { + t.Fatalf("grpc-gateway produced metadata %v for a headerless request; "+ + "this test no longer covers the annotator-skip path", md) + } + + md := gatewayMetadata(t, req, ctx) + + id, ok := ipcauth.CallerIdentity(daemonSideCtx(t, md)) + if !ok { + t.Fatal("the forwarded identity should be usable") + } + if id.UID != caller.UID || id.IsPrivileged() { + t.Errorf("authorized as %v, want the real client uid %d", id, caller.UID) + } +} + +// When the gateway cannot read its client's identity (a TCP JSON socket, say) it +// forwards the marker alone. The daemon must then report "unidentified" so the +// privileged operations refuse, rather than falling back to the gateway's own +// identity. +func TestJSONGateway_UnreadableClientIdentityIsUnidentified(t *testing.T) { + req, err := http.NewRequest(http.MethodPost, "http://localhost/daemon.DaemonService/SetConfig", nil) + if err != nil { + t.Fatal(err) + } + + md := gatewayMetadata(t, req, clientCtx(ipcauth.Identity{}, false)) + + if id, ok := ipcauth.CallerIdentity(daemonSideCtx(t, md)); ok { + t.Errorf("a request with no client identity was authorized as %v", id) + } +} + +// A request that never passed through jsonConnContext (no stashed identity) must +// also come out unidentified rather than as the daemon. +func TestJSONGateway_MissingConnContextIsUnidentified(t *testing.T) { + req, err := http.NewRequest(http.MethodPost, "http://localhost/daemon.DaemonService/SetConfig", nil) + if err != nil { + t.Fatal(err) + } + + md := gatewayMetadata(t, req, context.Background()) + + if id, ok := ipcauth.CallerIdentity(daemonSideCtx(t, md)); ok { + t.Errorf("a request with no connection context was authorized as %v", id) + } +} + +// End to end over a real unix socket: the gateway reads the connecting client's +// identity from the socket itself, so a client cannot present anything else. +func TestJSONGateway_IdentityComesFromTheSocket(t *testing.T) { + mux := runtime.NewServeMux(runtime.WithIncomingHeaderMatcher(jsonIncomingHeaderMatcher)) + + type observed struct { + md metadata.MD + } + seen := make(chan observed, 1) + + srv := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, err := runtime.AnnotateContext(r.Context(), mux, r, + "/daemon.DaemonService/SetConfig", + runtime.WithHTTPPathPattern("/daemon.DaemonService/SetConfig")) + if err != nil { + t.Errorf("annotate: %v", err) + return + } + md, _ := metadata.FromOutgoingContext(forwardIdentity(ctx)) + seen <- observed{md: md} + w.WriteHeader(http.StatusOK) + }), + ReadHeaderTimeout: 5 * time.Second, + ConnContext: jsonConnContext, + } + + sock := filepath.Join(t.TempDir(), "http.sock") + ln, err := net.Listen("unix", sock) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := srv.Close(); err != nil { + t.Logf("close server: %v", err) + } + }) + go func() { + if err := srv.Serve(ln); err != nil && err != http.ErrServerClosed { + t.Logf("serve: %v", err) + } + }() + + conn, err := net.Dial("unix", sock) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := conn.Close(); err != nil { + t.Logf("close conn: %v", err) + } + }) + + // Forge the identity headers on the wire as well. + request := "POST /daemon.DaemonService/SetConfig HTTP/1.1\r\n" + + "Host: localhost\r\n" + + "Grpc-Metadata-X-Netbird-Fwd: 1\r\n" + + "Grpc-Metadata-X-Netbird-Fwd-Uid: 0\r\n" + + "Content-Length: 0\r\n\r\n" + if _, err := conn.Write([]byte(request)); err != nil { + t.Fatal(err) + } + + select { + case got := <-seen: + self, err := ipcauth.CurrentProcessIdentity() + if err != nil { + t.Skipf("cannot read this process's identity: %v", err) + } + // The socket peer is this test process, so that is the identity the + // gateway must forward, not the uid 0 the request asked for. + if uids := got.md.Get("x-netbird-fwd-uid"); len(uids) != 1 { + t.Fatalf("x-netbird-fwd-uid = %v, want exactly the gateway's own value", uids) + } + id, ok := ipcauth.CallerIdentity(daemonSideCtx(t, got.md)) + if !ok { + t.Fatal("the forwarded identity should be usable") + } + if id.UID != self.UID { + t.Errorf("authorized as uid %d, want the socket peer %d", id.UID, self.UID) + } + case <-time.After(5 * time.Second): + t.Fatal("the gateway never handled the request") + } +} diff --git a/client/cmd/service_json_socket_test.go b/client/cmd/service_json_socket_test.go new file mode 100644 index 000000000..4b39794d7 --- /dev/null +++ b/client/cmd/service_json_socket_test.go @@ -0,0 +1,176 @@ +//go:build !ios && !android + +package cmd + +import ( + "net" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func preserveJSONSocketTestState(t *testing.T) { + t.Helper() + + origJSONSocket := jsonSocket + origEnableJSONSocket := enableJSONSocket + origChanged := map[string]bool{} + serviceCmd.PersistentFlags().VisitAll(func(flag *pflag.Flag) { + origChanged[flag.Name] = flag.Changed + }) + + t.Cleanup(func() { + jsonSocket = origJSONSocket + enableJSONSocket = origEnableJSONSocket + serviceCmd.PersistentFlags().VisitAll(func(flag *pflag.Flag) { + flag.Changed = origChanged[flag.Name] + }) + }) +} + +func TestJSONSocketFlagsArePositiveEnableOnly(t *testing.T) { + assert.NotNil(t, serviceCmd.PersistentFlags().Lookup("enable-json-socket")) + assert.NotNil(t, serviceCmd.PersistentFlags().Lookup("json-socket")) + assert.Nil(t, serviceCmd.PersistentFlags().Lookup("disable-json-socket")) + assert.Equal(t, "false", serviceCmd.PersistentFlags().Lookup("enable-json-socket").DefValue) +} + +func TestBuildServiceArgumentsDefaultDisablesJSONSocket(t *testing.T) { + preserveJSONSocketTestState(t) + + enableJSONSocket = false + jsonSocket = "tcp://127.0.0.1:8080" + + args := buildServiceArguments() + + assert.NotContains(t, args, "--enable-json-socket") + assert.NotContains(t, args, "--json-socket") +} + +func TestBuildServiceArgumentsIncludesJSONSocketWhenEnabled(t *testing.T) { + preserveJSONSocketTestState(t) + + enableJSONSocket = true + jsonSocket = "tcp://127.0.0.1:8080" + + args := buildServiceArguments() + + enableIndex := indexOfArg(args, "--enable-json-socket") + jsonIndex := indexOfArg(args, "--json-socket") + require.NotEqual(t, -1, enableIndex) + require.NotEqual(t, -1, jsonIndex) + require.Less(t, enableIndex, jsonIndex) + require.Less(t, jsonIndex+1, len(args)) + assert.Equal(t, "tcp://127.0.0.1:8080", args[jsonIndex+1]) +} + +func TestJSONSocketWithoutEnableValidation(t *testing.T) { + preserveJSONSocketTestState(t) + + enableJSONSocket = false + require.NoError(t, serviceCmd.PersistentFlags().Set("json-socket", "tcp://127.0.0.1:8080")) + + err := validateJSONSocketFlags() + + require.Error(t, err) + assert.Contains(t, err.Error(), "--enable-json-socket") +} + +func TestJSONSocketWithEnableValidation(t *testing.T) { + preserveJSONSocketTestState(t) + + require.NoError(t, serviceCmd.PersistentFlags().Set("enable-json-socket", "true")) + require.NoError(t, serviceCmd.PersistentFlags().Set("json-socket", "tcp://127.0.0.1:8080")) + + assert.NoError(t, validateJSONSocketFlags()) +} + +func TestJSONSocketServiceParamsPersistEnableAndAddress(t *testing.T) { + preserveJSONSocketTestState(t) + serviceCmd.PersistentFlags().VisitAll(func(flag *pflag.Flag) { + flag.Changed = false + }) + + enableJSONSocket = true + jsonSocket = "tcp://127.0.0.1:8080" + + params := currentServiceParams() + require.True(t, params.EnableJSONSocket) + require.Equal(t, "tcp://127.0.0.1:8080", params.JSONSocket) + + enableJSONSocket = false + jsonSocket = defaultJSONSocket + applyServiceParams(testServiceEnvCommand(), params) + + assert.True(t, enableJSONSocket) + assert.Equal(t, "tcp://127.0.0.1:8080", jsonSocket) +} + +func TestRemoveStaleUnixSocketDoesNotRemoveRegularFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "netbird-http.sock") + require.NoError(t, os.WriteFile(path, []byte("not a socket"), 0600)) + + removeStaleUnixSocket(path) + + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, []byte("not a socket"), data) +} + +func TestRemoveStaleUnixSocketRemovesSocket(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("unix sockets are not available on Windows") + } + + path := filepath.Join(t.TempDir(), "netbird-http.sock") + addr := &net.UnixAddr{Name: path, Net: "unix"} + listener, err := net.ListenUnix("unix", addr) + require.NoError(t, err) + listener.SetUnlinkOnClose(false) + require.NoError(t, listener.Close()) + + _, err = os.Lstat(path) + require.NoError(t, err, "test setup must leave a stale Unix socket path") + + removeStaleUnixSocket(path) + + _, err = os.Lstat(path) + assert.True(t, os.IsNotExist(err), "expected stale Unix socket to be removed, got %v", err) +} + +func TestRemoveStaleUnixSocketDoesNotRemoveLiveSocket(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("unix sockets are not available on Windows") + } + + path := filepath.Join(t.TempDir(), "netbird-http.sock") + listener, err := net.Listen("unix", path) + require.NoError(t, err) + defer listener.Close() + + removeStaleUnixSocket(path) + + _, err = os.Lstat(path) + assert.NoError(t, err, "expected live Unix socket to be preserved") +} + +func testServiceEnvCommand() *cobra.Command { + cmd := &cobra.Command{} + cmd.Flags().StringSlice("service-env", nil, "") + return cmd +} + +func indexOfArg(args []string, arg string) int { + for i, candidate := range args { + if candidate == arg { + return i + } + } + return -1 +} diff --git a/client/cmd/service_params.go b/client/cmd/service_params.go index 192e0ac60..6e2dbec40 100644 --- a/client/cmd/service_params.go +++ b/client/cmd/service_params.go @@ -13,6 +13,8 @@ import ( "github.com/spf13/cobra" "github.com/netbirdio/netbird/client/configs" + "github.com/netbirdio/netbird/client/internal/daemonaddr" + "github.com/netbirdio/netbird/client/internal/elevate" "github.com/netbirdio/netbird/util" ) @@ -23,6 +25,7 @@ const serviceParamsFile = "service.json" type serviceParams struct { LogLevel string `json:"log_level"` DaemonAddr string `json:"daemon_addr"` + JSONSocket string `json:"json_socket"` ManagementURL string `json:"management_url,omitempty"` ConfigPath string `json:"config_path,omitempty"` LogFiles []string `json:"log_files,omitempty"` @@ -30,6 +33,7 @@ type serviceParams struct { DisableUpdateSettings bool `json:"disable_update_settings,omitempty"` EnableCapture bool `json:"enable_capture,omitempty"` DisableNetworks bool `json:"disable_networks,omitempty"` + EnableJSONSocket bool `json:"enable_json_socket,omitempty"` ServiceEnvVars map[string]string `json:"service_env_vars,omitempty"` } @@ -40,10 +44,33 @@ func serviceParamsPath() string { // loadServiceParams reads saved service parameters from disk. // Returns nil with no error if the file does not exist. +// +// The file is read by an elevated install and decides the arguments and the +// environment of the service it then registers, so it is used only when its +// ownership and permissions are the ones saveServiceParams leaves behind. That +// restricted ACL is applied when the file is written, which is not necessarily +// before it is first read, so this is checked rather than assumed. A file that +// fails the check is treated as absent, and the install proceeds with its +// defaults. func loadServiceParams() (*serviceParams, error) { path := serviceParamsPath() - data, err := os.ReadFile(path) + // Resolve links first so the checks apply to the file that is actually read. + // Since the check covers every directory above it as well, nobody who fails + // it can swap the file between here and the read below. + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil //nolint:nilnil + } + return nil, fmt.Errorf("resolve service params %s: %w", path, err) + } + + if err := elevate.CheckOnlyOwnerWritable(resolved); err != nil { + return nil, fmt.Errorf("refusing to read service params from %s: %w", resolved, err) + } + + data, err := os.ReadFile(resolved) if err != nil { if os.IsNotExist(err) { return nil, nil //nolint:nilnil @@ -75,6 +102,7 @@ func currentServiceParams() *serviceParams { params := &serviceParams{ LogLevel: logLevel, DaemonAddr: daemonAddr, + JSONSocket: jsonSocket, ManagementURL: managementURL, ConfigPath: configPath, LogFiles: logFiles, @@ -82,6 +110,7 @@ func currentServiceParams() *serviceParams { DisableUpdateSettings: updateSettingsDisabled, EnableCapture: captureEnabled, DisableNetworks: networksDisabled, + EnableJSONSocket: enableJSONSocket, } if len(serviceEnvVars) > 0 { @@ -113,15 +142,29 @@ func applyServiceParams(cmd *cobra.Command, params *serviceParams) { return } - // For fields with non-empty defaults (log-level, daemon-addr), keep the - // != "" guard so that an older service.json missing the field doesn't - // clobber the default with an empty string. + // For fields with non-empty defaults, keep the != "" guard so that an older + // service.json missing the field doesn't clobber the default with an empty string. if !rootCmd.PersistentFlags().Changed("log-level") && params.LogLevel != "" { logLevel = params.LogLevel } if !rootCmd.PersistentFlags().Changed("daemon-addr") && params.DaemonAddr != "" { daemonAddr = params.DaemonAddr + // An install that predates named-pipe support has the loopback TCP + // address saved. Callers carry no identity over TCP, so move it to the + // pipe instead of restoring a socket the daemon cannot authorize on. + if migrated, ok := daemonaddr.MigrateLegacy(daemonAddr); ok { + cmd.Printf("Moving the saved daemon address from %s to %s so the daemon can identify its callers\n", daemonAddr, migrated) + daemonAddr = migrated + } + } + + if !serviceCmd.PersistentFlags().Changed("json-socket") && params.JSONSocket != "" { + jsonSocket = params.JSONSocket + } + + if !serviceCmd.PersistentFlags().Changed("enable-json-socket") { + enableJSONSocket = params.EnableJSONSocket } // For optional fields where empty means "use default", always apply so @@ -163,10 +206,16 @@ func applyServiceParams(cmd *cobra.Command, params *serviceParams) { // If --service-env was explicitly set to empty, all saved env vars are cleared. // If --service-env was not set, saved env vars are used entirely. func applyServiceEnvParams(cmd *cobra.Command, params *serviceParams) { + // A forbidden name explicitly passed on the command line is an error the + // operator is told about, but one restored from a file written by an older + // version is dropped: an install that refuses to run would leave the host + // without a daemon over a variable nobody is asking for any more. + saved := dropForbiddenServiceEnvVars(cmd, params.ServiceEnvVars) + if !cmd.Flags().Changed("service-env") { - if len(params.ServiceEnvVars) > 0 { + if len(saved) > 0 { // No explicit env vars: rebuild serviceEnvVars from saved params. - serviceEnvVars = envMapToSlice(params.ServiceEnvVars) + serviceEnvVars = envMapToSlice(saved) } return } @@ -185,13 +234,13 @@ func applyServiceEnvParams(cmd *cobra.Command, params *serviceParams) { return } - if len(params.ServiceEnvVars) == 0 { + if len(saved) == 0 { return } // Merge saved values underneath explicit ones. - merged := make(map[string]string, len(params.ServiceEnvVars)+len(explicit)) - maps.Copy(merged, params.ServiceEnvVars) + merged := make(map[string]string, len(saved)+len(explicit)) + maps.Copy(merged, saved) maps.Copy(merged, explicit) // explicit wins on conflict serviceEnvVars = envMapToSlice(merged) } @@ -214,6 +263,20 @@ var resetParamsCmd = &cobra.Command{ }, } +// dropForbiddenServiceEnvVars returns the saved entries that may still be +// registered on the service, reporting every one it leaves behind. +func dropForbiddenServiceEnvVars(cmd *cobra.Command, saved map[string]string) map[string]string { + kept := make(map[string]string, len(saved)) + for key, value := range saved { + if isForbiddenServiceEnvVar(key) { + cmd.PrintErrf("Warning: ignoring saved service environment variable %s: it decides where the service resolves the executables, libraries or temporary files it uses\n", key) + continue + } + kept[key] = value + } + return kept +} + // envMapToSlice converts a map of env vars to a KEY=VALUE slice. func envMapToSlice(m map[string]string) []string { s := make([]string, 0, len(m)) diff --git a/client/cmd/service_params_test.go b/client/cmd/service_params_test.go index f338c12f4..1f83374cb 100644 --- a/client/cmd/service_params_test.go +++ b/client/cmd/service_params_test.go @@ -9,6 +9,7 @@ import ( "go/token" "os" "path/filepath" + "runtime" "strings" "testing" @@ -41,6 +42,8 @@ func TestSaveAndLoadServiceParams(t *testing.T) { params := &serviceParams{ LogLevel: "debug", DaemonAddr: "unix:///var/run/netbird.sock", + JSONSocket: "tcp://127.0.0.1:8080", + EnableJSONSocket: true, ManagementURL: "https://my.server.com", ConfigPath: "/etc/netbird/config.json", LogFiles: []string{"/var/log/netbird/client.log", "console"}, @@ -63,6 +66,8 @@ func TestSaveAndLoadServiceParams(t *testing.T) { assert.Equal(t, params.LogLevel, loaded.LogLevel) assert.Equal(t, params.DaemonAddr, loaded.DaemonAddr) + assert.Equal(t, params.JSONSocket, loaded.JSONSocket) + assert.Equal(t, params.EnableJSONSocket, loaded.EnableJSONSocket) assert.Equal(t, params.ManagementURL, loaded.ManagementURL) assert.Equal(t, params.ConfigPath, loaded.ConfigPath) assert.Equal(t, params.LogFiles, loaded.LogFiles) @@ -101,6 +106,8 @@ func TestLoadServiceParams_InvalidJSON(t *testing.T) { func TestCurrentServiceParams(t *testing.T) { origLogLevel := logLevel origDaemonAddr := daemonAddr + origJSONSocket := jsonSocket + origEnableJSONSocket := enableJSONSocket origManagementURL := managementURL origConfigPath := configPath origLogFiles := logFiles @@ -110,6 +117,8 @@ func TestCurrentServiceParams(t *testing.T) { t.Cleanup(func() { logLevel = origLogLevel daemonAddr = origDaemonAddr + jsonSocket = origJSONSocket + enableJSONSocket = origEnableJSONSocket managementURL = origManagementURL configPath = origConfigPath logFiles = origLogFiles @@ -120,6 +129,8 @@ func TestCurrentServiceParams(t *testing.T) { logLevel = "trace" daemonAddr = "tcp://127.0.0.1:9999" + jsonSocket = "tcp://127.0.0.1:8080" + enableJSONSocket = true managementURL = "https://mgmt.example.com" configPath = "/tmp/test-config.json" logFiles = []string{"/tmp/test.log"} @@ -131,6 +142,8 @@ func TestCurrentServiceParams(t *testing.T) { assert.Equal(t, "trace", params.LogLevel) assert.Equal(t, "tcp://127.0.0.1:9999", params.DaemonAddr) + assert.Equal(t, "tcp://127.0.0.1:8080", params.JSONSocket) + assert.True(t, params.EnableJSONSocket) assert.Equal(t, "https://mgmt.example.com", params.ManagementURL) assert.Equal(t, "/tmp/test-config.json", params.ConfigPath) assert.Equal(t, []string{"/tmp/test.log"}, params.LogFiles) @@ -142,6 +155,8 @@ func TestCurrentServiceParams(t *testing.T) { func TestApplyServiceParams_OnlyUnchangedFlags(t *testing.T) { origLogLevel := logLevel origDaemonAddr := daemonAddr + origJSONSocket := jsonSocket + origEnableJSONSocket := enableJSONSocket origManagementURL := managementURL origConfigPath := configPath origLogFiles := logFiles @@ -151,6 +166,8 @@ func TestApplyServiceParams_OnlyUnchangedFlags(t *testing.T) { t.Cleanup(func() { logLevel = origLogLevel daemonAddr = origDaemonAddr + jsonSocket = origJSONSocket + enableJSONSocket = origEnableJSONSocket managementURL = origManagementURL configPath = origConfigPath logFiles = origLogFiles @@ -162,6 +179,8 @@ func TestApplyServiceParams_OnlyUnchangedFlags(t *testing.T) { // Reset all flags to defaults. logLevel = "info" daemonAddr = "unix:///var/run/netbird.sock" + jsonSocket = defaultJSONSocket + enableJSONSocket = false managementURL = "" configPath = "/etc/netbird/config.json" logFiles = []string{"/var/log/netbird/client.log"} @@ -184,6 +203,8 @@ func TestApplyServiceParams_OnlyUnchangedFlags(t *testing.T) { saved := &serviceParams{ LogLevel: "debug", DaemonAddr: "tcp://127.0.0.1:5555", + JSONSocket: "tcp://127.0.0.1:8080", + EnableJSONSocket: true, ManagementURL: "https://saved.example.com", ConfigPath: "/saved/config.json", LogFiles: []string{"/saved/client.log"}, @@ -201,6 +222,8 @@ func TestApplyServiceParams_OnlyUnchangedFlags(t *testing.T) { // All other fields were not Changed, so they should use saved values. assert.Equal(t, "tcp://127.0.0.1:5555", daemonAddr) + assert.Equal(t, "tcp://127.0.0.1:8080", jsonSocket) + assert.True(t, enableJSONSocket) assert.Equal(t, "https://saved.example.com", managementURL) assert.Equal(t, "/saved/config.json", configPath) assert.Equal(t, []string{"/saved/client.log"}, logFiles) @@ -212,14 +235,17 @@ func TestApplyServiceParams_OnlyUnchangedFlags(t *testing.T) { func TestApplyServiceParams_BooleanRevertToFalse(t *testing.T) { origProfilesDisabled := profilesDisabled origUpdateSettingsDisabled := updateSettingsDisabled + origEnableJSONSocket := enableJSONSocket t.Cleanup(func() { profilesDisabled = origProfilesDisabled updateSettingsDisabled = origUpdateSettingsDisabled + enableJSONSocket = origEnableJSONSocket }) // Simulate current state where booleans are true (e.g. set by previous install). profilesDisabled = true updateSettingsDisabled = true + enableJSONSocket = true // Reset Changed state so flags appear unset. serviceCmd.PersistentFlags().VisitAll(func(f *pflag.Flag) { @@ -238,6 +264,7 @@ func TestApplyServiceParams_BooleanRevertToFalse(t *testing.T) { assert.False(t, profilesDisabled, "saved false should override current true") assert.False(t, updateSettingsDisabled, "saved false should override current true") + assert.False(t, enableJSONSocket, "saved false should override current true") } func TestApplyServiceParams_ClearManagementURL(t *testing.T) { @@ -327,6 +354,59 @@ func TestApplyServiceEnvParams_NotChanged(t *testing.T) { assert.Equal(t, map[string]string{"FROM_SAVED": "val"}, result) } +func TestParseServiceEnvVars_RejectsForbiddenNames(t *testing.T) { + for _, env := range []string{"PATH=C:\\somewhere", "LD_PRELOAD=/tmp/lib.so", "DYLD_FALLBACK_LIBRARY_PATH=/tmp"} { + _, err := parseServiceEnvVars([]string{"KEEP=me", env}) + require.Errorf(t, err, "%s selects what the service resolves and must be refused", env) + } +} + +func TestIsForbiddenServiceEnvVar(t *testing.T) { + // The loader families are matched by prefix, so a name nobody has heard of + // yet is refused too. + for _, name := range []string{ + "PATH", "PATHEXT", "COMSPEC", "SYSTEMROOT", "WINDIR", "TEMP", "TMP", + "LD_PRELOAD", "LD_AUDIT", "DYLD_INSERT_LIBRARIES", "DYLD_FALLBACK_FRAMEWORK_PATH", + } { + assert.Truef(t, isForbiddenServiceEnvVar(name), "%s must be refused", name) + } + + // The prefix must not swallow names that merely start with the same letters. + for _, name := range []string{"NB_LOG_LEVEL", "NB_WG_DEBUG", "HTTPS_PROXY", "LDAP_URL", "DYLDX"} { + assert.Falsef(t, isForbiddenServiceEnvVar(name), "%s has no reason to be refused", name) + } + + // On Windows a variable is the same one however it is spelled; elsewhere + // Path and PATH are two variables and only the exact one is read. + if runtime.GOOS == "windows" { + assert.True(t, isForbiddenServiceEnvVar("Path")) + assert.True(t, isForbiddenServiceEnvVar("ld_preload")) + } else { + assert.False(t, isForbiddenServiceEnvVar("Path")) + assert.False(t, isForbiddenServiceEnvVar("ld_preload")) + } +} + +func TestApplyServiceEnvParams_DropsForbiddenSavedNames(t *testing.T) { + origServiceEnvVars := serviceEnvVars + t.Cleanup(func() { serviceEnvVars = origServiceEnvVars }) + + serviceEnvVars = nil + + cmd := &cobra.Command{} + cmd.Flags().StringSlice("service-env", nil, "") + + saved := &serviceParams{ + ServiceEnvVars: map[string]string{"PATH": "C:\\attacker", "NB_LOG_FORMAT": "json"}, + } + + applyServiceEnvParams(cmd, saved) + + result, err := parseServiceEnvVars(serviceEnvVars) + require.NoError(t, err, "a saved PATH must be dropped rather than fail the install") + assert.Equal(t, map[string]string{"NB_LOG_FORMAT": "json"}, result) +} + func TestApplyServiceEnvParams_ExplicitEmptyClears(t *testing.T) { origServiceEnvVars := serviceEnvVars t.Cleanup(func() { serviceEnvVars = origServiceEnvVars }) @@ -530,6 +610,7 @@ func fieldToGlobalVar(field string) string { m := map[string]string{ "LogLevel": "logLevel", "DaemonAddr": "daemonAddr", + "JSONSocket": "jsonSocket", "ManagementURL": "managementURL", "ConfigPath": "configPath", "LogFiles": "logFiles", @@ -537,6 +618,7 @@ func fieldToGlobalVar(field string) string { "DisableUpdateSettings": "updateSettingsDisabled", "EnableCapture": "captureEnabled", "DisableNetworks": "networksDisabled", + "EnableJSONSocket": "enableJSONSocket", "ServiceEnvVars": "serviceEnvVars", } if v, ok := m[field]; ok { diff --git a/client/cmd/service_params_trust_test.go b/client/cmd/service_params_trust_test.go new file mode 100644 index 000000000..1cf564445 --- /dev/null +++ b/client/cmd/service_params_trust_test.go @@ -0,0 +1,57 @@ +//go:build !windows && !ios && !android + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/configs" +) + +// The Windows equivalent of this is the ACL check in +// elevate.CheckOnlyOwnerWritable, covered by that package's own tests; here the +// point is that loadServiceParams asks the question at all. +func TestLoadServiceParams_RefusesWorldWritableFile(t *testing.T) { + tmpDir := t.TempDir() + + original := configs.StateDir + t.Cleanup(func() { configs.StateDir = original }) + configs.StateDir = tmpDir + + path := filepath.Join(tmpDir, serviceParamsFile) + require.NoError(t, os.WriteFile(path, []byte(`{"log_level":"debug"}`), 0o666)) + // WriteFile is subject to the umask, so set the bits that matter explicitly. + require.NoError(t, os.Chmod(path, 0o666)) + + params, err := loadServiceParams() + require.Error(t, err, "a service.json anyone can rewrite must not be trusted") + assert.Nil(t, params) + + require.NoError(t, os.Chmod(path, 0o600)) + params, err = loadServiceParams() + require.NoError(t, err) + require.NotNil(t, params) + assert.Equal(t, "debug", params.LogLevel) +} + +func TestLoadServiceParams_RefusesWorldWritableDirectory(t *testing.T) { + tmpDir := t.TempDir() + stateDir := filepath.Join(tmpDir, "state") + require.NoError(t, os.Mkdir(stateDir, 0o777)) + require.NoError(t, os.Chmod(stateDir, 0o777)) + + original := configs.StateDir + t.Cleanup(func() { configs.StateDir = original }) + configs.StateDir = stateDir + + require.NoError(t, os.WriteFile(filepath.Join(stateDir, serviceParamsFile), []byte(`{}`), 0o600)) + + params, err := loadServiceParams() + require.Error(t, err, "a service.json in a directory anyone can replace entries in must not be trusted") + assert.Nil(t, params) +} diff --git a/client/cmd/service_pipe_other.go b/client/cmd/service_pipe_other.go new file mode 100644 index 000000000..c7cc72469 --- /dev/null +++ b/client/cmd/service_pipe_other.go @@ -0,0 +1,14 @@ +//go:build !windows + +package cmd + +import ( + "fmt" + "net" +) + +// listenNamedPipe is Windows-only: no other platform serves the daemon on a +// named pipe. +func listenNamedPipe(string) (net.Listener, string, error) { + return nil, "", fmt.Errorf("named pipes are only supported on Windows") +} diff --git a/client/cmd/service_pipe_windows.go b/client/cmd/service_pipe_windows.go new file mode 100644 index 000000000..b6e860f51 --- /dev/null +++ b/client/cmd/service_pipe_windows.go @@ -0,0 +1,41 @@ +//go:build windows + +package cmd + +import ( + "errors" + "fmt" + "net" + + "github.com/Microsoft/go-winio" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/daemonaddr" + "github.com/netbirdio/netbird/client/internal/ipcauth" +) + +// listenNamedPipe creates the daemon control pipe and reports the path it ended +// up on. The security descriptor lets any local caller connect, as a Unix socket +// at 0666 does, and the privileged operations are authorized separately from the +// caller's token. +// +// The protected name comes first so that an unprivileged process cannot take the +// name before the service does. Creating it requires being an administrator or +// LocalSystem, so a daemon an ordinary user runs themselves, as in netstack mode, +// falls back to the plain name; clients try both and check who serves them. +func listenNamedPipe(name string) (net.Listener, string, error) { + var errs []error + for _, path := range daemonaddr.PipePaths(name) { + listener, err := winio.ListenPipe(path, &winio.PipeConfig{ + SecurityDescriptor: ipcauth.DefaultPipeSDDL(), + }) + if err != nil { + log.Debugf("not serving the daemon on %s: %v", path, err) + errs = append(errs, fmt.Errorf("%s: %w", path, err)) + continue + } + return listener, path, nil + } + + return nil, "", errors.Join(errs...) +} diff --git a/client/cmd/service_socket.go b/client/cmd/service_socket.go new file mode 100644 index 000000000..bf3122f7c --- /dev/null +++ b/client/cmd/service_socket.go @@ -0,0 +1,119 @@ +//go:build !ios && !android + +package cmd + +import ( + "errors" + "fmt" + "net" + "os" + "strings" + "syscall" + "time" + + log "github.com/sirupsen/logrus" +) + +type socketListener struct { + net.Listener + network string + address string +} + +func listenOnAddress(addr string) (*socketListener, error) { + network, address, err := parseListenAddress(addr) + if err != nil { + return nil, err + } + + if network == "npipe" { + listener, path, err := listenNamedPipe(address) //nolint:staticcheck + if err != nil { //nolint:staticcheck // always errors on non-Windows builds + return nil, err + } + return &socketListener{Listener: listener, network: network, address: path}, nil + } + + if network == "unix" { + removeStaleUnixSocket(address) + } + + listener, err := net.Listen(network, address) + if err != nil { + return nil, err + } + + return &socketListener{Listener: listener, network: network, address: address}, nil +} + +func parseListenAddress(addr string) (string, string, error) { + network, address, ok := strings.Cut(addr, "://") + if !ok || network == "" || address == "" { + return "", "", fmt.Errorf("address must be in [unix|tcp|npipe]://[path|host:port|name] format: %q", addr) + } + + switch network { + case "unix", "tcp", "npipe": + return network, address, nil + default: + return "", "", fmt.Errorf("unsupported daemon address protocol: %v", network) + } +} + +func removeStaleUnixSocket(path string) { + stat, err := os.Lstat(path) + if err != nil { + if !os.IsNotExist(err) { + log.Debugf("stat socket file: %v", err) + } + return + } + + if stat.Mode()&os.ModeSocket == 0 { + return + } + + if !isStaleUnixSocket(path) { + return + } + + if err := os.Remove(path); err != nil { + log.Debugf("remove socket file: %v", err) + } +} + +func isStaleUnixSocket(path string) bool { + conn, err := net.DialTimeout("unix", path, 100*time.Millisecond) + if err == nil { + if closeErr := conn.Close(); closeErr != nil { + log.Debugf("close unix socket probe: %v", closeErr) + } + return false + } + + if os.IsNotExist(err) || os.IsPermission(err) || os.IsTimeout(err) { + log.Debugf("not removing unix socket %s after probe error: %v", path, err) + return false + } + + return errors.Is(err, syscall.ECONNREFUSED) +} + +func removeStaleUnixSocketForAddress(addr string) { + network, address, err := parseListenAddress(addr) + if err != nil || network != "unix" { + return + } + removeStaleUnixSocket(address) +} + +func (l *socketListener) chmodUnixSocket(description string) error { + if l == nil || l.network != "unix" { + return nil + } + + if err := os.Chmod(l.address, 0666); err != nil { + return fmt.Errorf("failed setting %s permissions for %s: %w", description, l.address, err) + } + return nil +} diff --git a/client/cmd/status.go b/client/cmd/status.go index 5a7559cf1..f2e5bcc66 100644 --- a/client/cmd/status.go +++ b/client/cmd/status.go @@ -6,6 +6,7 @@ import ( "net" "net/netip" "strings" + "time" "github.com/spf13/cobra" "google.golang.org/grpc/status" @@ -115,8 +116,19 @@ func statusFunc(cmd *cobra.Command, args []string) error { // manager only knows the active profile ID, not its display name. profName := getActiveProfileName(ctx) + var sessionExpiresAt time.Time + if ts := resp.GetSessionExpiresAt(); ts.IsValid() { + sessionExpiresAt = ts.AsTime().UTC() + } + + anonymizeEnabled, anonymizeLevel, err := effectiveAnonymize() + if err != nil { + return err + } + var outputInformationHolder = nbstatus.ConvertToStatusOutputOverview(resp.GetFullStatus(), nbstatus.ConvertOptions{ - Anonymize: anonymizeFlag, + Anonymize: anonymizeEnabled, + AnonymizeLevel: anonymizeLevel, DaemonVersion: resp.GetDaemonVersion(), DaemonStatus: nbstatus.ParseDaemonStatus(status), StatusFilter: statusFilter, @@ -125,6 +137,7 @@ func statusFunc(cmd *cobra.Command, args []string) error { IPsFilter: ipsFilterMap, ConnectionTypeFilter: connectionTypeFilter, ProfileName: profName, + SessionExpiresAt: sessionExpiresAt, }) var statusOutputString string switch { diff --git a/client/cmd/testutil_test.go b/client/cmd/testutil_test.go index 205327ef5..46bf31837 100644 --- a/client/cmd/testutil_test.go +++ b/client/cmd/testutil_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" + "go.uber.org/mock/gomock" "google.golang.org/grpc" "github.com/netbirdio/netbird/management/server/integrations/integrated_validator/validator" @@ -28,7 +28,6 @@ import ( mgmt "github.com/netbirdio/netbird/management/server" "github.com/netbirdio/netbird/management/server/activity" "github.com/netbirdio/netbird/management/server/groups" - "github.com/netbirdio/netbird/management/server/integrations/port_forwarding" "github.com/netbirdio/netbird/management/server/permissions" "github.com/netbirdio/netbird/management/server/settings" "github.com/netbirdio/netbird/management/server/store" @@ -124,9 +123,9 @@ func startManagement(t *testing.T, config *config.Config, testFile string) (*grp updateManager := update_channel.NewPeersUpdateManager(metrics) requestBuffer := mgmt.NewAccountRequestBuffer(ctx, store) - networkMapController := controller.NewController(ctx, store, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.cloud", port_forwarding.NewControllerMock(), manager.NewEphemeralManager(store, peersmanager), config) + networkMapController := controller.NewController(ctx, store, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.cloud", manager.NewEphemeralManager(store, peersmanager), config, nil) - accountManager, err := mgmt.BuildManager(ctx, config, store, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManagerMock, false, cacheStore) + accountManager, err := mgmt.BuildManager(ctx, config, store, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, settingsMockManager, permissionsManagerMock, false, cacheStore) if err != nil { t.Fatal(err) } diff --git a/client/cmd/up.go b/client/cmd/up.go index 8b3de3c66..e336ba257 100644 --- a/client/cmd/up.go +++ b/client/cmd/up.go @@ -2,10 +2,9 @@ package cmd import ( "context" + "errors" "fmt" - "net" "net/netip" - "os/user" "runtime" "strings" "time" @@ -21,18 +20,16 @@ import ( "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/profilemanager" + "github.com/netbirdio/netbird/client/mdm" + nbnet "github.com/netbirdio/netbird/client/net" "github.com/netbirdio/netbird/client/proto" + "github.com/netbirdio/netbird/client/server" "github.com/netbirdio/netbird/client/system" + "github.com/netbirdio/netbird/shared/externalipmap" "github.com/netbirdio/netbird/shared/management/domain" "github.com/netbirdio/netbird/util" ) -const ( - invalidInputType int = iota - ipInputType - interfaceInputType -) - const ( dnsLabelsFlag = "extra-dns-labels" @@ -46,6 +43,8 @@ const ( profileNameDesc = "profile name to use for the login. If not specified, the last used profile will be used." ) +var errDaemonActiveProfileUnsupported = errors.New("daemon does not support active profile lookup") + var ( foregroundMode bool dnsLabels []string @@ -101,7 +100,7 @@ func upFunc(cmd *cobra.Command, args []string) error { return fmt.Errorf("failed initializing log %v", err) } - err = validateNATExternalIPs(natExternalIPs) + err = externalipmap.Validate(natExternalIPs) if err != nil { return err } @@ -120,23 +119,25 @@ func upFunc(cmd *cobra.Command, args []string) error { pm := profilemanager.NewProfileManager() - username, err := user.Current() + username, err := profilemanager.InvokingUser() if err != nil { return fmt.Errorf("get current user: %v", err) } + var activeProf *profilemanager.Profile var profileSwitched bool // switch profile if provided if profileName != "" { - if err := switchOrCreateProfile(cmd.Context(), pm, profileName, username.Username); err != nil { + activeProf, err = switchOrCreateProfile(cmd.Context(), pm, profileName, username.Username) + if err != nil { return fmt.Errorf("switch profile: %v", err) } profileSwitched = true - } - - activeProf, err := pm.GetActiveProfile() - if err != nil { - return fmt.Errorf("get active profile: %v", err) + } else { + activeProf, err = pm.GetActiveProfile() + if err != nil { + return fmt.Errorf("get active profile: %v", err) + } } if foregroundMode { @@ -148,13 +149,15 @@ func upFunc(cmd *cobra.Command, args []string) error { // switchOrCreateProfile switches the active profile to the one identified by // handle, creating it first when it does not exist yet. This restores the // pre-0.73 behaviour where `netbird up --profile ` auto-creates a -// missing profile instead of failing. -func switchOrCreateProfile(ctx context.Context, pm *profilemanager.ProfileManager, handle, username string) error { +// missing profile instead of failing. Returns the daemon-resolved profile so +// callers act on it directly instead of re-reading the local state, which is +// not updated under sudo. +func switchOrCreateProfile(ctx context.Context, pm *profilemanager.ProfileManager, handle, username string) (*profilemanager.Profile, error) { resolvedID, err := switchProfile(ctx, handle, username) if err != nil { st, ok := gstatus.FromError(err) if !ok || st.Code() != codes.NotFound { - return err + return nil, err } // Don't fail immediately on a create error: a concurrent run may // have created the profile between the NotFound above and this @@ -163,16 +166,16 @@ func switchOrCreateProfile(ctx context.Context, pm *profilemanager.ProfileManage _, createErr := createProfile(ctx, handle, username) if resolvedID, err = switchProfile(ctx, handle, username); err != nil { if createErr != nil { - return fmt.Errorf("create profile: %w", createErr) + return nil, fmt.Errorf("create profile: %w", createErr) } - return err + return nil, err } } if err := pm.SwitchProfile(resolvedID); err != nil { - return err + return nil, err } - return nil + return &profilemanager.Profile{ID: resolvedID}, nil } // createProfile dials the daemon and creates a new profile with the given @@ -226,9 +229,31 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command, activeProf *pr if err != nil { return fmt.Errorf("get config file: %v", err) } + // CLI foreground path runs without the daemon Server: layer in the + // active MDM policy explicitly so a forced ManagementURL / PSK / + // other managed key actually takes effect on this run. + config.ApplyMDMPolicy(mdm.NewLoader(nil).Load()) _, _ = profilemanager.UpdateOldManagementURL(ctx, config, configFilePath) + // Restore residual state left by a previous run that did not shut down + // cleanly, mirroring what the daemon does before connecting: it recovers + // DNS config (a stale resolv.conf takeover can make the management + // hostname unresolvable), firewall rules, ssh config and legacy routing. + // Route cleanup itself happens at engine start; nbnet.Init() below lets + // the management dial bypass a leftover fwmark rule until then. + // Foreground mode is particularly exposed in containers: a crashed + // container restarts inside the same (pod) network namespace, so stale + // state survives while the process does not. + if err := server.RestoreResidualState(ctx, profilemanager.NewServiceManager(configPath).GetStatePath()); err != nil { + log.Warnf("failed to restore residual state: %v", err) + } + + // Enable advanced routing (as the daemon does on startup) so the + // management dial bypasses a leftover fwmark rule instead of being + // shunted into a stale routing table. + nbnet.Init() + err = foregroundLogin(ctx, cmd, config, providedSetupKey, activeProf.ID) if err != nil { return fmt.Errorf("foreground login failed: %v", err) @@ -282,6 +307,30 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command, pm *profilemanager return fmt.Errorf("unable to get daemon status: %v", err) } + // Under sudo the invoking user's local active-profile mirror is never + // written (the SwitchProfile write is a no-op), and plain root has no + // invoking user at all — so the mirror read into activeProf above is stale + // or defaulted and must not drive the daemon. With no --profile to make the + // choice explicit, take the profile the daemon already holds for this user + // instead: it stays on the user's current profile rather than silently + // switching to the mirror's default, and refuses when the daemon is on + // another user's profile. + if profileName == "" && !profilemanager.MirrorIsAuthoritative() { + u, err := profilemanager.InvokingUser() + if err != nil { + return fmt.Errorf("get current user: %v", err) + } + resolved, err := daemonActiveProfileForUser(ctx, client, u.Username) + switch { + case errors.Is(err, errDaemonActiveProfileUnsupported): + log.Warnf("keeping the locally resolved profile: %v", err) + case err != nil: + return err + default: + activeProf = resolved + } + } + if status.Status == string(internal.StatusConnected) { if !profileSwitched { cmd.Println("Already connected") @@ -294,7 +343,7 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command, pm *profilemanager } } - username, err := user.Current() + username, err := profilemanager.InvokingUser() if err != nil { return fmt.Errorf("get current user: %v", err) } @@ -302,10 +351,18 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command, pm *profilemanager // set the new config req := setupSetConfigReq(customDNSAddressConverted, cmd, activeProf.ID.String(), username.Username) if _, err := client.SetConfig(ctx, req); err != nil { - if st, ok := gstatus.FromError(err); ok && st.Code() == codes.Unavailable { - log.Warnf("setConfig method is not available in the daemon: %s", st.Message()) - } else { - return fmt.Errorf("call service setConfig method: %v", err) + switch reason, refused := refusedSettingsUpdate(err); { + case refused: + // Failing here is the point: carrying on would connect while + // silently dropping the settings the caller asked for, since + // nothing further down the line applies them. + return fmt.Errorf("the daemon refused the settings update: %s", reason) + case gstatus.Code(err) == codes.Unavailable: + // The daemon cannot serve the method at all, which is what this + // code means; an older daemon without it lands here. + log.Warnf("the daemon did not apply the settings update: %s", gstatus.Convert(err).Message()) + default: + return daemonCallError("call service setConfig method", err) } } @@ -345,10 +402,7 @@ func doDaemonUp(ctx context.Context, cmd *cobra.Command, client proto.DaemonServ err = WithBackOff(func() error { var backOffErr error loginResp, backOffErr = client.Login(ctx, loginRequest) - if s, ok := gstatus.FromError(backOffErr); ok && (s.Code() == codes.InvalidArgument || - s.Code() == codes.PermissionDenied || - s.Code() == codes.NotFound || - s.Code() == codes.Unimplemented) { + if terminalLoginError(backOffErr) { loginErr = backOffErr return nil } @@ -359,7 +413,7 @@ func doDaemonUp(ctx context.Context, cmd *cobra.Command, client proto.DaemonServ } if loginErr != nil { - return fmt.Errorf("login failed: %v", loginErr) + return daemonCallError("login failed", loginErr) } if loginResp.NeedsSSOLogin { @@ -372,32 +426,27 @@ func doDaemonUp(ctx context.Context, cmd *cobra.Command, client proto.DaemonServ ProfileName: &profileID, Username: &username, }); err != nil { - return fmt.Errorf("call service up method: %v", err) + return daemonCallError("call service up method", err) } return nil } -func setupSetConfigReq(customDNSAddressConverted []byte, cmd *cobra.Command, profileName, username string) *proto.SetConfigRequest { - var req proto.SetConfigRequest - req.ProfileName = profileName - req.Username = username - - req.ManagementUrl = managementURL - req.AdminURL = adminURL - req.NatExternalIPs = natExternalIPs - req.CustomDNSAddress = customDNSAddressConverted - req.ExtraIFaceBlacklist = extraIFaceBlackList - req.DnsLabels = dnsLabelsValidated.ToPunycodeList() - req.CleanDNSLabels = dnsLabels != nil && len(dnsLabels) == 0 - req.CleanNATExternalIPs = natExternalIPs != nil && len(natExternalIPs) == 0 - - if cmd.Flag(enableRosenpassFlag).Changed { - req.RosenpassEnabled = &rosenpassEnabled - } - if cmd.Flag(rosenpassPermissiveFlag).Changed { - req.RosenpassPermissive = &rosenpassPermissive +// setBoolPtrIfChanged points dst at a copy of val when the named bool flag was +// explicitly set on cmd. It collapses the repeated +// "if cmd.Flag(x).Changed { field = &val }" pattern in the request builders into +// a single call, keeping their cognitive complexity within bounds. +func setBoolPtrIfChanged(cmd *cobra.Command, name string, dst **bool, val bool) { + if cmd.Flag(name).Changed { + dst2 := val + *dst = &dst2 } +} + +// setSSHSetConfigFields copies the SSH server flags the user actually +// passed into req, leaving the rest unset so the daemon keeps the +// persisted values. +func setSSHSetConfigFields(req *proto.SetConfigRequest, cmd *cobra.Command) { if cmd.Flag(serverSSHAllowedFlag).Changed { req.ServerSSHAllowed = &serverSSHAllowed } @@ -420,6 +469,47 @@ func setupSetConfigReq(customDNSAddressConverted []byte, cmd *cobra.Command, pro sshJWTCacheTTL32 := int32(sshJWTCacheTTL) req.SshJWTCacheTTL = &sshJWTCacheTTL32 } +} + +// refusedSettingsUpdate reports whether err is the daemon refusing the settings +// a request carried — the update-settings kill switch, or a field an MDM policy +// manages — and returns the reason it gave. +// +// The distinction that matters is against codes.Unavailable, which means the +// daemon cannot serve the call: that one is worth a warning, because an older +// daemon without the method lands there and the rest of `netbird up` still +// works. A refusal is not, because the settings would be silently dropped. +func refusedSettingsUpdate(err error) (string, bool) { + st, ok := gstatus.FromError(err) + if !ok || st.Code() != codes.FailedPrecondition { + return "", false + } + return st.Message(), true +} + +func setupSetConfigReq(customDNSAddressConverted []byte, cmd *cobra.Command, profileName, username string) *proto.SetConfigRequest { + var req proto.SetConfigRequest + req.ProfileName = profileName + req.Username = username + + req.ManagementUrl = managementURL + req.AdminURL = adminURL + req.NatExternalIPs = natExternalIPs + req.CustomDNSAddress = customDNSAddressConverted + req.ExtraIFaceBlacklist = extraIFaceBlackList + req.DnsLabels = dnsLabelsValidated.ToPunycodeList() + req.CleanDNSLabels = dnsLabels != nil && len(dnsLabels) == 0 + req.CleanNATExternalIPs = natExternalIPs != nil && len(natExternalIPs) == 0 + + if cmd.Flag(enableRosenpassFlag).Changed { + req.RosenpassEnabled = &rosenpassEnabled + } + if cmd.Flag(rosenpassPermissiveFlag).Changed { + req.RosenpassPermissive = &rosenpassPermissive + } + setSSHSetConfigFields(&req, cmd) + setBoolPtrIfChanged(cmd, remoteJobsAllowedFlag, &req.RemoteJobsAllowed, remoteJobsAllowed) + if cmd.Flag(interfaceNameFlag).Changed { if err := parseInterfaceName(interfaceName); err != nil { log.Errorf("parse interface name: %v", err) @@ -479,6 +569,13 @@ func setupSetConfigReq(customDNSAddressConverted []byte, cmd *cobra.Command, pro req.DisableIpv6 = &disableIPv6 } + if cmd.Flag(enableLocalMetricsFlag).Changed { + req.EnableLocalMetrics = &localMetricsEnabled + } + if cmd.Flag(localMetricsAddressFlag).Changed { + req.LocalMetricsAddress = &localMetricsAddr + } + return &req } @@ -503,6 +600,7 @@ func setupConfig(customDNSAddressConverted []byte, cmd *cobra.Command, configFil if cmd.Flag(serverSSHAllowedFlag).Changed { ic.ServerSSHAllowed = &serverSSHAllowed } + setBoolPtrIfChanged(cmd, remoteJobsAllowedFlag, &ic.RemoteJobsAllowed, remoteJobsAllowed) if cmd.Flag(enableSSHRootFlag).Changed { ic.EnableSSHRoot = &enableSSHRoot @@ -596,9 +694,45 @@ func setupConfig(customDNSAddressConverted []byte, cmd *cobra.Command, configFil ic.DisableIPv6 = &disableIPv6 } + if cmd.Flag(enableLocalMetricsFlag).Changed { + ic.LocalMetricsEnabled = &localMetricsEnabled + } + + if cmd.Flag(localMetricsAddressFlag).Changed { + ic.LocalMetricsAddress = &localMetricsAddr + } + return &ic, nil } +// setSSHLoginFields copies the SSH server flags the user actually passed +// into req, leaving the rest unset so the daemon keeps the persisted +// values. +func setSSHLoginFields(req *proto.LoginRequest, cmd *cobra.Command) { + if cmd.Flag(serverSSHAllowedFlag).Changed { + req.ServerSSHAllowed = &serverSSHAllowed + } + if cmd.Flag(enableSSHRootFlag).Changed { + req.EnableSSHRoot = &enableSSHRoot + } + if cmd.Flag(enableSSHSFTPFlag).Changed { + req.EnableSSHSFTP = &enableSSHSFTP + } + if cmd.Flag(enableSSHLocalPortForwardFlag).Changed { + req.EnableSSHLocalPortForwarding = &enableSSHLocalPortForward + } + if cmd.Flag(enableSSHRemotePortForwardFlag).Changed { + req.EnableSSHRemotePortForwarding = &enableSSHRemotePortForward + } + if cmd.Flag(disableSSHAuthFlag).Changed { + req.DisableSSHAuth = &disableSSHAuth + } + if cmd.Flag(sshJWTCacheTTLFlag).Changed { + sshJWTCacheTTL32 := int32(sshJWTCacheTTL) + req.SshJWTCacheTTL = &sshJWTCacheTTL32 + } +} + func setupLoginRequest(providedSetupKey string, customDNSAddressConverted []byte, cmd *cobra.Command) (*proto.LoginRequest, error) { loginRequest := proto.LoginRequest{ SetupKey: providedSetupKey, @@ -606,7 +740,7 @@ func setupLoginRequest(providedSetupKey string, customDNSAddressConverted []byte NatExternalIPs: natExternalIPs, CleanNATExternalIPs: natExternalIPs != nil && len(natExternalIPs) == 0, CustomDNSAddress: customDNSAddressConverted, - IsUnixDesktopClient: isUnixRunningDesktop(), + IsUnixDesktopClient: util.HasGraphicalSession(), Hostname: hostName, ExtraIFaceBlacklist: extraIFaceBlackList, DnsLabels: dnsLabels, @@ -625,39 +759,21 @@ func setupLoginRequest(providedSetupKey string, customDNSAddressConverted []byte loginRequest.RosenpassPermissive = &rosenpassPermissive } - if cmd.Flag(serverSSHAllowedFlag).Changed { - loginRequest.ServerSSHAllowed = &serverSSHAllowed - } - - if cmd.Flag(enableSSHRootFlag).Changed { - loginRequest.EnableSSHRoot = &enableSSHRoot - } - - if cmd.Flag(enableSSHSFTPFlag).Changed { - loginRequest.EnableSSHSFTP = &enableSSHSFTP - } - - if cmd.Flag(enableSSHLocalPortForwardFlag).Changed { - loginRequest.EnableSSHLocalPortForwarding = &enableSSHLocalPortForward - } - - if cmd.Flag(enableSSHRemotePortForwardFlag).Changed { - loginRequest.EnableSSHRemotePortForwarding = &enableSSHRemotePortForward - } - - if cmd.Flag(disableSSHAuthFlag).Changed { - loginRequest.DisableSSHAuth = &disableSSHAuth - } - - if cmd.Flag(sshJWTCacheTTLFlag).Changed { - sshJWTCacheTTL32 := int32(sshJWTCacheTTL) - loginRequest.SshJWTCacheTTL = &sshJWTCacheTTL32 - } + setSSHLoginFields(&loginRequest, cmd) + setBoolPtrIfChanged(cmd, remoteJobsAllowedFlag, &loginRequest.RemoteJobsAllowed, remoteJobsAllowed) if cmd.Flag(disableAutoConnectFlag).Changed { loginRequest.DisableAutoConnect = &autoConnectDisabled } + if cmd.Flag(enableLocalMetricsFlag).Changed { + loginRequest.EnableLocalMetrics = &localMetricsEnabled + } + + if cmd.Flag(localMetricsAddressFlag).Changed { + loginRequest.LocalMetricsAddress = &localMetricsAddr + } + if cmd.Flag(interfaceNameFlag).Changed { if err := parseInterfaceName(interfaceName); err != nil { return nil, err @@ -714,36 +830,6 @@ func setupLoginRequest(providedSetupKey string, customDNSAddressConverted []byte return &loginRequest, nil } -func validateNATExternalIPs(list []string) error { - for _, element := range list { - if element == "" { - return fmt.Errorf("empty string is not a valid input for %s", externalIPMapFlag) - } - - subElements := strings.Split(element, "/") - if len(subElements) > 2 { - return fmt.Errorf("%s is not a valid input for %s. it should be formatted as \"String\" or \"String/String\"", element, externalIPMapFlag) - } - - if len(subElements) == 1 && !isValidIP(subElements[0]) { - return fmt.Errorf("%s is not a valid input for %s. it should be formatted as \"IP\" or \"IP/IP\", or \"IP/Interface Name\"", element, externalIPMapFlag) - } - - last := 0 - for _, singleElement := range subElements { - inputType, err := validateElement(singleElement) - if err != nil { - return fmt.Errorf("%s is not a valid input for %s. it should be an IP string or a network name", singleElement, externalIPMapFlag) - } - if last == interfaceInputType && inputType == interfaceInputType { - return fmt.Errorf("%s is not a valid input for %s. it should not contain two interface names", element, externalIPMapFlag) - } - last = inputType - } - } - return nil -} - func parseInterfaceName(name string) error { if runtime.GOOS != "darwin" { return nil @@ -756,39 +842,6 @@ func parseInterfaceName(name string) error { return fmt.Errorf("invalid interface name %s. Please use the prefix utun followed by a number on MacOS. e.g., utun1 or utun199", name) } -func validateElement(element string) (int, error) { - if isValidIP(element) { - return ipInputType, nil - } - validIface, err := isValidInterface(element) - if err != nil { - return invalidInputType, fmt.Errorf("unable to validate the network interface name, error: %s", err) - } - - if validIface { - return interfaceInputType, nil - } - - return interfaceInputType, fmt.Errorf("invalid IP or network interface name not found") -} - -func isValidIP(ip string) bool { - return net.ParseIP(ip) != nil -} - -func isValidInterface(name string) (bool, error) { - netInterfaces, err := net.Interfaces() - if err != nil { - return false, err - } - for _, iface := range netInterfaces { - if iface.Name == name { - return true, nil - } - } - return false, nil -} - func parseCustomDNSAddress(modified bool) ([]byte, error) { var parsed []byte if modified { @@ -829,3 +882,31 @@ func isValidAddrPort(input string) bool { _, err := netip.ParseAddrPort(input) return err == nil } + +// daemonActiveProfileForUser returns the profile the daemon currently holds for +// username, for the no --profile case where the local mirror is not +// authoritative (sudo or plain root). It returns that profile when the daemon +// owns it for this user or when the profile is unowned (empty username, as on a +// fresh install), so the caller acts on the daemon's real state instead of the +// stale mirror. It denies with a --profile hint when the daemon is on another +// user's profile, when the lookup fails, or when the daemon reports no active +// profile. Returns errDaemonActiveProfileUnsupported when the daemon predates +// the RPC; the caller keeps the mirror-derived profile in that case. +func daemonActiveProfileForUser(ctx context.Context, client proto.DaemonServiceClient, username string) (*profilemanager.Profile, error) { + active, err := client.GetActiveProfile(ctx, &proto.GetActiveProfileRequest{}) + if err != nil { + if st, ok := gstatus.FromError(err); ok && st.Code() == codes.Unimplemented { + return nil, fmt.Errorf("%w: %v", errDaemonActiveProfileUnsupported, err) + } + return nil, fmt.Errorf("pass --profile to choose the profile explicitly: the daemon's active profile could not be verified: %v", err) + } + if active.GetId() == "" { + return nil, fmt.Errorf("pass --profile to choose the profile explicitly: the daemon reported no active profile") + } + if active.GetUsername() != "" && active.GetUsername() != username { + return nil, fmt.Errorf( + "pass --profile to choose the profile explicitly: the daemon's active profile is %q (user %q) but this invocation runs for %q", + active.GetProfileName(), active.GetUsername(), username) + } + return &profilemanager.Profile{ID: profilemanager.ID(active.GetId())}, nil +} diff --git a/client/cmd/up_setconfig_refusal_test.go b/client/cmd/up_setconfig_refusal_test.go new file mode 100644 index 000000000..fdf580102 --- /dev/null +++ b/client/cmd/up_setconfig_refusal_test.go @@ -0,0 +1,85 @@ +package cmd + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + gstatus "google.golang.org/grpc/status" +) + +// A refused settings update has to fail `netbird up`, or a caller that asked +// for a setting the daemon will not apply connects as if it had been applied. +// The daemon being unable to serve the call is the case that stays a warning. +func TestRefusedSettingsUpdate(t *testing.T) { + tests := []struct { + name string + err error + wantRefused bool + }{ + { + name: "the kill switch refused the change", + err: gstatus.Errorf(codes.FailedPrecondition, "update settings are disabled, you cannot use this feature without update settings enabled"), + wantRefused: true, + }, + { + name: "an MDM policy manages the field", + err: gstatus.Errorf(codes.FailedPrecondition, "fields managed by MDM policy: managementURL"), + wantRefused: true, + }, + { + name: "the daemon cannot serve the call", + err: gstatus.Errorf(codes.Unavailable, "connection refused"), + wantRefused: false, + }, + { + name: "any other RPC failure", + err: gstatus.Errorf(codes.Internal, "boom"), + wantRefused: false, + }, + { + name: "not a status error at all", + err: errors.New("boom"), + wantRefused: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reason, refused := refusedSettingsUpdate(tt.err) + require.Equal(t, tt.wantRefused, refused) + if tt.wantRefused { + require.Equal(t, gstatus.Convert(tt.err).Message(), reason, "the daemon's reason must reach the caller") + } + }) + } +} + +// Both `netbird up` and `netbird login` drive Login through the backoff cycle, +// and a final answer has to stop it: retrying a refusal only replaces the +// daemon's reason with "login backoff cycle failed" thirty seconds later. +func TestTerminalLoginError(t *testing.T) { + tests := []struct { + name string + err error + wantTerminal bool + }{ + {name: "settings refused by the kill switch", err: gstatus.Errorf(codes.FailedPrecondition, "update settings are disabled"), wantTerminal: true}, + {name: "field managed by MDM", err: gstatus.Errorf(codes.FailedPrecondition, "fields managed by MDM policy: managementURL"), wantTerminal: true}, + {name: "caller not allowed", err: gstatus.Errorf(codes.PermissionDenied, "nope"), wantTerminal: true}, + {name: "malformed request", err: gstatus.Errorf(codes.InvalidArgument, "nope"), wantTerminal: true}, + {name: "profile not found", err: gstatus.Errorf(codes.NotFound, "nope"), wantTerminal: true}, + {name: "method missing on an older daemon", err: gstatus.Errorf(codes.Unimplemented, "nope"), wantTerminal: true}, + {name: "daemon unreachable, worth retrying", err: gstatus.Errorf(codes.Unavailable, "connection refused"), wantTerminal: false}, + {name: "transient internal failure", err: gstatus.Errorf(codes.Internal, "boom"), wantTerminal: false}, + {name: "not a status error", err: errors.New("boom"), wantTerminal: false}, + {name: "no error at all, the login succeeded", err: nil, wantTerminal: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.wantTerminal, terminalLoginError(tt.err)) + }) + } +} diff --git a/client/cmd/up_test.go b/client/cmd/up_test.go new file mode 100644 index 000000000..9b5f9fbea --- /dev/null +++ b/client/cmd/up_test.go @@ -0,0 +1,88 @@ +package cmd + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + gstatus "google.golang.org/grpc/status" + + "github.com/netbirdio/netbird/client/internal/profilemanager" + "github.com/netbirdio/netbird/client/proto" +) + +type fakeActiveProfileClient struct { + proto.DaemonServiceClient + resp *proto.GetActiveProfileResponse + err error +} + +func (f *fakeActiveProfileClient) GetActiveProfile(_ context.Context, _ *proto.GetActiveProfileRequest, _ ...grpc.CallOption) (*proto.GetActiveProfileResponse, error) { + return f.resp, f.err +} + +func TestDaemonActiveProfileForUserReturnsOwnProfile(t *testing.T) { + client := &fakeActiveProfileClient{resp: &proto.GetActiveProfileResponse{Id: "default", ProfileName: "default", Username: "root"}} + prof, err := daemonActiveProfileForUser(context.Background(), client, "root") + require.NoError(t, err) + require.NotNil(t, prof) + assert.Equal(t, profilemanager.ID("default"), prof.ID) +} + +func TestDaemonActiveProfileForUserReturnsUnownedProfile(t *testing.T) { + client := &fakeActiveProfileClient{resp: &proto.GetActiveProfileResponse{Id: "default", ProfileName: "default", Username: ""}} + prof, err := daemonActiveProfileForUser(context.Background(), client, "root") + require.NoError(t, err) + require.NotNil(t, prof) + assert.Equal(t, profilemanager.ID("default"), prof.ID) +} + +func TestDaemonActiveProfileForUserKeepsDaemonProfileOverStaleMirror(t *testing.T) { + client := &fakeActiveProfileClient{resp: &proto.GetActiveProfileResponse{Id: "ab12", ProfileName: "work", Username: "misha"}} + prof, err := daemonActiveProfileForUser(context.Background(), client, "misha") + require.NoError(t, err) + require.NotNil(t, prof) + assert.Equal(t, profilemanager.ID("ab12"), prof.ID) +} + +func TestDaemonActiveProfileForUserRejectsOtherUsersProfile(t *testing.T) { + client := &fakeActiveProfileClient{resp: &proto.GetActiveProfileResponse{Id: "ab12", ProfileName: "work", Username: "misha"}} + prof, err := daemonActiveProfileForUser(context.Background(), client, "root") + require.Error(t, err) + assert.Nil(t, prof) + assert.Contains(t, err.Error(), "--profile") +} + +func TestDaemonActiveProfileForUserRejectsOtherUsersDefaultProfile(t *testing.T) { + client := &fakeActiveProfileClient{resp: &proto.GetActiveProfileResponse{Id: "default", ProfileName: "default", Username: "misha"}} + prof, err := daemonActiveProfileForUser(context.Background(), client, "root") + require.Error(t, err) + assert.Nil(t, prof) + assert.Contains(t, err.Error(), "--profile") +} + +func TestDaemonActiveProfileForUserRejectsLookupError(t *testing.T) { + client := &fakeActiveProfileClient{err: gstatus.Error(codes.Internal, "boom")} + prof, err := daemonActiveProfileForUser(context.Background(), client, "root") + require.Error(t, err) + assert.Nil(t, prof) + assert.Contains(t, err.Error(), "--profile") +} + +func TestDaemonActiveProfileForUserRejectsEmptyResponse(t *testing.T) { + client := &fakeActiveProfileClient{resp: &proto.GetActiveProfileResponse{}} + prof, err := daemonActiveProfileForUser(context.Background(), client, "root") + require.Error(t, err) + assert.Nil(t, prof) + assert.Contains(t, err.Error(), "--profile") +} + +func TestDaemonActiveProfileForUserKeepsMirrorWhenDaemonWithoutRPC(t *testing.T) { + client := &fakeActiveProfileClient{err: gstatus.Error(codes.Unimplemented, "unknown method")} + prof, err := daemonActiveProfileForUser(context.Background(), client, "root") + require.ErrorIs(t, err, errDaemonActiveProfileUnsupported) + assert.Nil(t, prof) +} diff --git a/client/configs/configs.go b/client/configs/configs.go index 8f9c3ba28..a1ecf0feb 100644 --- a/client/configs/configs.go +++ b/client/configs/configs.go @@ -6,6 +6,11 @@ import ( "runtime" ) +// UILogFile is the file name the desktop UI writes its log to. It is defined +// here so the UI (writer), the daemon's RegisterUILog validation, and the debug +// bundle collector all share one definition. +const UILogFile = "gui-client.log" + var StateDir string func init() { diff --git a/client/embed/embed.go b/client/embed/embed.go index d0d88b177..208e71817 100644 --- a/client/embed/embed.go +++ b/client/embed/embed.go @@ -21,7 +21,8 @@ import ( "github.com/netbirdio/netbird/client/internal/auth" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/profilemanager" - sshcommon "github.com/netbirdio/netbird/client/ssh" + "github.com/netbirdio/netbird/client/mdm" + nbssh "github.com/netbirdio/netbird/client/ssh" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/shared/management/domain" mgmProto "github.com/netbirdio/netbird/shared/management/proto" @@ -53,6 +54,9 @@ type Client struct { jwtToken string connect *internal.ConnectClient recorder *peer.Status + // inMemoryState is set when no StatePath was given in netstack mode, + // where the client changes nothing on the host that needs restoring. + inMemoryState bool } // Options configures a new Client. @@ -77,7 +81,8 @@ type Options struct { NoUserspace bool // ConfigPath is the path to the netbird config file. If empty, the config will be stored in memory and not persisted. ConfigPath string - // StatePath is the path to the netbird state file + // StatePath is the path to the netbird state file. When empty in netstack + // mode, state is kept in memory; with NoUserspace the default path is used. StatePath string // DisableClientRoutes disables the client routes DisableClientRoutes bool @@ -85,12 +90,24 @@ type Options struct { DisableIPv6 bool // BlockInbound blocks all inbound connections from peers BlockInbound bool + // EnableRosenpass enables the Rosenpass post-quantum key exchange. + EnableRosenpass bool + // RosenpassPermissive lets a Rosenpass-enabled peer still connect to peers + // that do not run Rosenpass (falling back to the plain WireGuard PSK). + RosenpassPermissive bool // BlockLANAccess blocks the embedded peer from reaching the host's // LAN (RFC 1918, link-local, loopback) when it's used as a routing // peer. Mirrors profilemanager.ConfigInput.BlockLANAccess. Useful // when the embedded client must never act as a stepping stone into // the host's local network (e.g. the proxy's overlay peer). BlockLANAccess bool + // LazyConnectionEnabled is a tri-state local override for lazy connections, + // mirroring the NB_LAZY_CONN env var. Nil defers to the management feature + // flag; a set value overrides it in both directions. A short-lived client + // that reaches only a few known peers can set this to false, so its peers + // connect eagerly and the first request does not wait for the connection to + // be established. + LazyConnectionEnabled *bool // WireguardPort is the port for the tunnel interface. Use 0 for a random port. WireguardPort *int // MTU is the MTU for the tunnel interface. @@ -101,6 +118,9 @@ type Options struct { MTU *uint16 // DNSLabels defines additional DNS labels configured in the peer. DNSLabels []string + // NATExternalIPs maps external IPs to internal IPs or interfaces for ICE + // candidates, same format as the agent's --external-ip-map flag. + NATExternalIPs []string // Performance configures the tunnel's buffer pool cap and batch size. Performance Performance } @@ -180,7 +200,6 @@ func New(opts Options) (*Client, error) { } if opts.StatePath != "" { - // TODO: Disable state if path not provided if err := os.Setenv("NB_DNS_STATE_FILE", opts.StatePath); err != nil { return nil, fmt.Errorf("setenv: %w", err) } @@ -203,9 +222,12 @@ func New(opts Options) (*Client, error) { DisableIPv6: &opts.DisableIPv6, BlockInbound: &opts.BlockInbound, BlockLANAccess: &opts.BlockLANAccess, + RosenpassEnabled: &opts.EnableRosenpass, + RosenpassPermissive: &opts.RosenpassPermissive, WireguardPort: opts.WireguardPort, MTU: opts.MTU, DNSLabels: parsedLabels, + NATExternalIPs: opts.NATExternalIPs, } if opts.ConfigPath != "" { config, err = profilemanager.UpdateOrCreateConfig(input) @@ -215,11 +237,24 @@ func New(opts Options) (*Client, error) { if err != nil { return nil, fmt.Errorf("create config: %w", err) } + // Embedded path runs without the daemon Server: apply the active + // MDM policy explicitly so a forced ManagementURL / PSK / other + // managed key takes effect on this embedded engine instance. + config.ApplyMDMPolicy(mdm.NewLoader(nil).Load()) if opts.PrivateKey != "" { config.PrivateKey = opts.PrivateKey } + if opts.LazyConnectionEnabled != nil { + // Runtime-only override, read back through lazyconn.ParseState; a set value + // wins over the management feature flag in both directions. + config.LazyConnection = "off" + if *opts.LazyConnectionEnabled { + config.LazyConnection = "on" + } + } + if opts.Performance.PreallocatedBuffersPerPool != nil { wgdevice.SetPreallocatedBuffersPerPool(*opts.Performance.PreallocatedBuffersPerPool) } @@ -233,6 +268,9 @@ func New(opts Options) (*Client, error) { jwtToken: opts.JWTToken, config: config, recorder: peer.NewRecorder(config.ManagementURL.String()), + // Without a path the default would be the installed agent's state + // file, which this client must neither read nor overwrite. + inMemoryState: opts.StatePath == "" && !opts.NoUserspace, }, nil } @@ -264,7 +302,11 @@ func (c *Client) Start(startCtx context.Context) error { if err, _ := authClient.Login(ctx, c.setupKey, c.jwtToken); err != nil { return fmt.Errorf("login: %w", err) } - client := internal.NewConnectClient(ctx, c.config, c.recorder) + var connectOpts []internal.ConnectClientOption + if c.inMemoryState { + connectOpts = append(connectOpts, internal.WithInMemoryState()) + } + client := internal.NewConnectClient(ctx, c.config, c.recorder, connectOpts...) client.SetSyncResponsePersistence(true) // either startup error (permanent backoff err) or nil err (successful engine up) @@ -470,7 +512,7 @@ func (c *Client) Status() (peer.FullStatus, error) { if connect != nil { engine := connect.Engine() if engine != nil { - _ = engine.RunHealthProbes(false) + _ = engine.RunHealthProbes(context.Background(), false) } } @@ -521,12 +563,7 @@ func (c *Client) VerifySSHHostKey(peerAddress string, key []byte) error { return err } - storedKey, found := engine.GetPeerSSHKey(peerAddress) - if !found { - return sshcommon.ErrPeerNotFound - } - - return sshcommon.VerifyHostKey(storedKey, key, peerAddress) + return nbssh.PeerKeyLookup(engine.GetPeerSSHKey).VerifySSHHostKey(peerAddress, key) } // SetPerformance retunes a running Client. Only PreallocatedBuffersPerPool diff --git a/client/embed/embed_test.go b/client/embed/embed_test.go index a2f438975..a818af055 100644 --- a/client/embed/embed_test.go +++ b/client/embed/embed_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "google.golang.org/grpc" "github.com/netbirdio/netbird/management/internals/controllers/network_map/controller" @@ -21,7 +21,6 @@ import ( nbcache "github.com/netbirdio/netbird/management/server/cache" "github.com/netbirdio/netbird/management/server/groups" "github.com/netbirdio/netbird/management/server/integrations/integrated_validator/validator" - "github.com/netbirdio/netbird/management/server/integrations/port_forwarding" "github.com/netbirdio/netbird/management/server/job" "github.com/netbirdio/netbird/management/server/permissions" "github.com/netbirdio/netbird/management/server/settings" @@ -146,8 +145,8 @@ func startManagement(t *testing.T, signalAddr string) string { updateManager := update_channel.NewPeersUpdateManager(metrics) requestBuffer := mgmt.NewAccountRequestBuffer(context.Background(), testStore) - networkMapController := controller.NewController(context.Background(), testStore, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.selfhosted", port_forwarding.NewControllerMock(), manager.NewEphemeralManager(testStore, peersManager), cfg) - accountManager, err := mgmt.BuildManager(context.Background(), cfg, testStore, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false, cacheStore) + networkMapController := controller.NewController(context.Background(), testStore, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.selfhosted", manager.NewEphemeralManager(testStore, peersManager), cfg, nil) + accountManager, err := mgmt.BuildManager(context.Background(), cfg, testStore, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, settingsMockManager, permissionsManager, false, cacheStore) require.NoError(t, err) secretsManager, err := nbgrpc.NewTimeBasedAuthSecretsManager(updateManager, cfg.TURNConfig, cfg.Relay, settingsMockManager, groupsManager) diff --git a/client/firewall/allower_other.go b/client/firewall/allower_other.go new file mode 100644 index 000000000..4d2ec9094 --- /dev/null +++ b/client/firewall/allower_other.go @@ -0,0 +1,11 @@ +//go:build android || (!linux && !windows) + +package firewall + +import "github.com/netbirdio/netbird/client/firewall/uspfilter" + +// interfaceAllower returns no allower: these platforms have no host firewall to +// open for the interface. +func interfaceAllower(IFaceMapper, uint16) uspfilter.InterfaceAllower { + return nil +} diff --git a/client/firewall/allower_windows.go b/client/firewall/allower_windows.go new file mode 100644 index 000000000..b9efa18a4 --- /dev/null +++ b/client/firewall/allower_windows.go @@ -0,0 +1,10 @@ +//go:build windows + +package firewall + +import "github.com/netbirdio/netbird/client/firewall/uspfilter" + +// interfaceAllower returns the Windows netsh-based interface allower. +func interfaceAllower(iface IFaceMapper, _ uint16) uspfilter.InterfaceAllower { + return uspfilter.NewWindowsInterfaceAllower(iface) +} diff --git a/client/firewall/create.go b/client/firewall/create.go index 24f12bc6d..cb68a0d04 100644 --- a/client/firewall/create.go +++ b/client/firewall/create.go @@ -6,8 +6,6 @@ import ( "fmt" "runtime" - log "github.com/sirupsen/logrus" - firewall "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/firewall/uspfilter" nftypes "github.com/netbirdio/netbird/client/internal/netflow/types" @@ -21,13 +19,11 @@ func NewFirewall(iface IFaceMapper, _ *statemanager.Manager, flowLogger nftypes. } // use userspace packet filtering firewall - fm, err := uspfilter.Create(iface, disableServerRoutes, flowLogger, mtu) - if err != nil { - return nil, err - } - err = fm.AllowNetbird() - if err != nil { - log.Warnf("failed to allow netbird interface traffic: %v", err) - } - return fm, nil + return uspfilter.Create(uspfilter.Config{ + IFace: iface, + DisableServerRoutes: disableServerRoutes, + FlowLogger: flowLogger, + MTU: mtu, + InterfaceAllower: interfaceAllower(iface, mtu), + }) } diff --git a/client/firewall/create_linux.go b/client/firewall/create_linux.go index d916ebad4..d585e85d7 100644 --- a/client/firewall/create_linux.go +++ b/client/firewall/create_linux.go @@ -16,6 +16,7 @@ import ( firewall "github.com/netbirdio/netbird/client/firewall/manager" nbnftables "github.com/netbirdio/netbird/client/firewall/nftables" "github.com/netbirdio/netbird/client/firewall/uspfilter" + "github.com/netbirdio/netbird/client/iface/netstack" nftypes "github.com/netbirdio/netbird/client/internal/netflow/types" "github.com/netbirdio/netbird/client/internal/statemanager" ) @@ -29,47 +30,107 @@ const ( NFTABLES ) -// SKIP_NFTABLES_ENV is the environment variable to skip nftables check -const SKIP_NFTABLES_ENV = "NB_SKIP_NFTABLES_CHECK" +// SkipNftablesEnv is the environment variable to skip nftables check +const SkipNftablesEnv = "NB_SKIP_NFTABLES_CHECK" + +// errNoFirewallManager indicates no kernel firewall backend is present, +// as opposed to a backend that exists but failed to create or initialize. +var errNoFirewallManager = errors.New("no firewall manager found") // FWType is the type for the firewall type type FWType int func NewFirewall(iface IFaceMapper, stateManager *statemanager.Manager, flowLogger nftypes.FlowLogger, disableServerRoutes bool, mtu uint16) (firewall.Manager, error) { - // We run in userspace mode and force userspace firewall was requested. We don't attempt native firewall. - if iface.IsUserspaceBind() && forceUserspaceFirewall() { - log.Info("forcing userspace firewall") - return createUserspaceFirewall(iface, nil, disableServerRoutes, flowLogger, mtu) + // Userspace firewall without a native counterpart: routing is handled + // entirely in userspace. The interface is opened in the kernel's foreign + // filter chains via a table-less allower, except in netstack mode where no + // kernel interface exists. + if netstack.IsEnabled() || (iface.IsUserspaceBind() && forceUserspaceFirewall()) { + if netstack.IsEnabled() { + log.Info("netstack mode, using userspace firewall") + } else { + log.Info("forcing userspace firewall") + } + cfg := uspfilter.Config{ + IFace: iface, + DisableServerRoutes: disableServerRoutes, + FlowLogger: flowLogger, + MTU: mtu, + InterfaceAllower: interfaceAllower(iface, mtu), + } + + return uspfilter.Create(cfg) } // Use native firewall for either kernel or userspace, the interface appears identical to netfilter - fm, err := createNativeFirewall(iface, stateManager, disableServerRoutes, mtu) - - // Kernel cannot fall back to anything else, need to return error - if !iface.IsUserspaceBind() { - return fm, err - } - - // Fall back to the userspace packet filter if native is unavailable - if err != nil { - log.Warnf("failed to create native firewall: %v. Proceeding with userspace", err) - return createUserspaceFirewall(iface, nil, disableServerRoutes, flowLogger, mtu) - } - - // Native firewall handles packet filtering, but the userspace WireGuard bind - // needs a device filter for DNS interception hooks. Install a minimal - // hooks-only filter that passes all traffic through to the kernel firewall. - if err := iface.SetFilter(&uspfilter.HooksFilter{}); err != nil { - log.Warnf("failed to set hooks filter, DNS via memory hooks will not work: %v", err) + fm, err := createNativeFirewall(iface, stateManager, mtu) + switch { + case err == nil && !iface.IsUserspaceBind(): + // Nothing to do, fall through + case err == nil && iface.IsUserspaceBind(): + // Native firewall handles packet filtering, but the userspace WireGuard bind + // needs a device filter for DNS interception hooks. Install a minimal + // hooks-only filter that passes all traffic through to the kernel firewall. + if err := iface.SetFilter(&uspfilter.HooksFilter{}); err != nil { + log.Warnf("failed to set hooks filter, DNS via memory hooks will not work: %v", err) + } + case err != nil && !iface.IsUserspaceBind(): + // Kernel cannot fall back to anything else, need to return error + return nil, err + case err != nil && iface.IsUserspaceBind(): + // Fall back to the userspace packet filter if native is unavailable + logNativeFirewallUnavailable(err) + return uspfilter.Create(uspfilter.Config{ + IFace: iface, + DisableServerRoutes: disableServerRoutes, + FlowLogger: flowLogger, + MTU: mtu, + InterfaceAllower: interfaceAllower(iface, mtu), + }) } return fm, nil } -func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager, routes bool, mtu uint16) (firewall.Manager, error) { +// interfaceAllower selects how the userspace firewall opens the interface in +// foreign kernel chains: nftables when available (which also opens foreign nft +// tables), else iptables (the legacy fallback, filter INPUT only), else nil. +// firewalld trust is applied separately by the manager. Netstack has no kernel +// interface to open. +func interfaceAllower(iface IFaceMapper, mtu uint16) uspfilter.InterfaceAllower { + if netstack.IsEnabled() { + return nil + } + + nftAllower, err := nbnftables.NewInterfaceAllower(iface, mtu) + if err == nil { + return nftAllower + } + log.Infof("no nftables interface allower: %v", err) + + iptAllower, err := nbiptables.NewInterfaceAllower(iface) + if err == nil { + return iptAllower + } + log.Infof("no iptables interface allower: %v", err) + + return nil +} + +// logNativeFirewallUnavailable logs the fallback to userspace at info level +// when no kernel firewall backend exists, and at warn level otherwise. +func logNativeFirewallUnavailable(err error) { + if errors.Is(err, errNoFirewallManager) { + log.Infof("no native firewall backend available: %v. Proceeding with userspace", err) + } else { + log.Warnf("failed to create native firewall: %v. Proceeding with userspace", err) + } +} + +func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager, mtu uint16) (firewall.Manager, error) { fm, err := createFW(iface, mtu) if err != nil { - return nil, fmt.Errorf("create firewall: %s", err) + return nil, fmt.Errorf("create firewall: %w", err) } if err = fm.Init(stateManager); err != nil { @@ -88,29 +149,10 @@ func createFW(iface IFaceMapper, mtu uint16) (firewall.Manager, error) { log.Info("creating an nftables firewall manager") return nbnftables.Create(iface, mtu) default: - log.Info("no firewall manager found, trying to use userspace packet filtering firewall") - return nil, errors.New("no firewall manager found") + return nil, errNoFirewallManager } } -func createUserspaceFirewall(iface IFaceMapper, fm firewall.Manager, disableServerRoutes bool, flowLogger nftypes.FlowLogger, mtu uint16) (firewall.Manager, error) { - var errUsp error - if fm != nil { - fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm, disableServerRoutes, flowLogger, mtu) - } else { - fm, errUsp = uspfilter.Create(iface, disableServerRoutes, flowLogger, mtu) - } - - if errUsp != nil { - return nil, fmt.Errorf("create userspace firewall: %s", errUsp) - } - - if err := fm.AllowNetbird(); err != nil { - log.Errorf("failed to allow netbird interface traffic: %v", err) - } - return fm, nil -} - // check returns the firewall type based on common lib checks. It returns UNKNOWN if no firewall is found. func check() FWType { useIPTABLES := false @@ -132,35 +174,38 @@ func check() FWType { } } - nf := nftables.Conn{} - if chains, err := nf.ListChains(); err == nil && os.Getenv(SKIP_NFTABLES_ENV) != "true" { - if !useIPTABLES { - return NFTABLES - } - - // search for chains where table is filter - // if we find one, we assume that nftables manager can be used with iptables - for _, chain := range chains { - if chain.Table.Name == "filter" { + // Honor the skip env before probing nftables at all. + if os.Getenv(SkipNftablesEnv) != "true" { + nf := nftables.Conn{} + if chains, err := nf.ListChains(); err == nil { + if !useIPTABLES { return NFTABLES } - } - // check tables for the following constraints: - // 1. there is no chain in nftables for the filter table and there is at least one chain in iptables, we assume that nftables manager can not be used - // 2. there is no tables or more than one table, we assume that nftables manager can be used - // 3. there is only one table and its name is filter, we assume that nftables manager can not be used, since there was no chain in it - // 4. if we find an error we log and continue with iptables check - nbTablesList, err := nf.ListTables() - switch { - case err == nil && len(iptablesChains) > 0: - return IPTABLES - case err == nil && len(nbTablesList) != 1: - return NFTABLES - case err == nil && len(nbTablesList) == 1 && nbTablesList[0].Name == "filter": - return IPTABLES - case err != nil: - log.Errorf("failed to list nftables tables on fw manager discovery: %s", err) + // search for chains where table is filter + // if we find one, we assume that nftables manager can be used with iptables + for _, chain := range chains { + if chain.Table.Name == "filter" { + return NFTABLES + } + } + + // check tables for the following constraints: + // 1. there is no chain in nftables for the filter table and there is at least one chain in iptables, we assume that nftables manager can not be used + // 2. there is no tables or more than one table, we assume that nftables manager can be used + // 3. there is only one table and its name is filter, we assume that nftables manager can not be used, since there was no chain in it + // 4. if we find an error we log and continue with iptables check + nbTablesList, err := nf.ListTables() + switch { + case err == nil && len(iptablesChains) > 0: + return IPTABLES + case err == nil && len(nbTablesList) != 1: + return NFTABLES + case err == nil && len(nbTablesList) == 1 && nbTablesList[0].Name == "filter": + return IPTABLES + case err != nil: + log.Errorf("failed to list nftables tables on fw manager discovery: %s", err) + } } } @@ -176,15 +221,21 @@ func isIptablesClientAvailable(client *iptables.IPTables) bool { return err == nil } +// forceUserspaceFirewall reports whether the userspace firewall is forced. +// NB_FORCE_USERSPACE_ROUTER is an alias: forcing userspace routing implies the +// userspace firewall, since the two are no longer separable. func forceUserspaceFirewall() bool { - val := os.Getenv(EnvForceUserspaceFirewall) + return envForceBool(EnvForceUserspaceFirewall) || envForceBool(uspfilter.EnvForceUserspaceRouter) +} + +func envForceBool(name string) bool { + val := os.Getenv(name) if val == "" { return false } - force, err := strconv.ParseBool(val) if err != nil { - log.Warnf("failed to parse %s: %v", EnvForceUserspaceFirewall, err) + log.Warnf("failed to parse %s: %v", name, err) return false } return force diff --git a/client/firewall/firewalld/firewalld.go b/client/firewall/firewalld/firewalld.go index 188ea61dd..38a4efdbc 100644 --- a/client/firewall/firewalld/firewalld.go +++ b/client/firewall/firewalld/firewalld.go @@ -2,8 +2,8 @@ // its wg interface into firewalld's "trusted" zone. This is required because // firewalld's nftables chains are created with NFT_CHAIN_OWNER on recent // versions, which returns EPERM to any other process that tries to insert -// rules into them. The workaround mirrors what Tailscale does: let firewalld -// itself add the accept rules to its own chains by trusting the interface. +// rules into them. Trusting the interface makes firewalld itself add the +// accept rules to its own chains instead. package firewalld // TrustedZone is the firewalld zone name used for interfaces whose traffic diff --git a/client/firewall/iptables/acl_linux.go b/client/firewall/iptables/acl_linux.go deleted file mode 100644 index 4b4cebf9c..000000000 --- a/client/firewall/iptables/acl_linux.go +++ /dev/null @@ -1,560 +0,0 @@ -package iptables - -import ( - "errors" - "fmt" - "maps" - "net" - "slices" - - "github.com/coreos/go-iptables/iptables" - "github.com/google/uuid" - ipset "github.com/lrh3321/ipset-go" - log "github.com/sirupsen/logrus" - - firewall "github.com/netbirdio/netbird/client/firewall/manager" - "github.com/netbirdio/netbird/client/internal/statemanager" - nbnet "github.com/netbirdio/netbird/client/net" -) - -const ( - tableName = "filter" - - // rules chains contains the effective ACL rules - chainNameInputRules = "NETBIRD-ACL-INPUT" - - // mangleFwdKey is the entries map key for mangle FORWARD guard rules that prevent - // external DNAT from bypassing ACL rules. - mangleFwdKey = "MANGLE-FORWARD" -) - -type aclEntries map[string][][]string - -type entry struct { - spec []string - position int -} - -type aclManager struct { - iptablesClient *iptables.IPTables - wgIface iFaceMapper - entries aclEntries - optionalEntries map[string][]entry - ipsetStore *ipsetStore - v6 bool - - stateManager *statemanager.Manager -} - -func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*aclManager, error) { - return &aclManager{ - iptablesClient: iptablesClient, - wgIface: wgIface, - entries: make(map[string][][]string), - optionalEntries: make(map[string][]entry), - ipsetStore: newIpsetStore(), - v6: iptablesClient.Proto() == iptables.ProtocolIPv6, - }, nil -} - -func (m *aclManager) init(stateManager *statemanager.Manager) error { - m.stateManager = stateManager - - m.seedInitialEntries() - m.seedInitialOptionalEntries() - - if err := m.cleanChains(); err != nil { - return fmt.Errorf("clean chains: %w", err) - } - - if err := m.createDefaultChains(); err != nil { - return fmt.Errorf("create default chains: %w", err) - } - - m.updateState() - - return nil -} - -func (m *aclManager) AddPeerFiltering( - id []byte, - ip net.IP, - protocol firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, - ipsetName string, -) ([]firewall.Rule, error) { - chain := chainNameInputRules - - ipsetName = transformIPsetName(ipsetName, sPort, dPort, action) - if m.v6 && ipsetName != "" { - ipsetName += "-v6" - } - proto := protoForFamily(protocol, m.v6) - specs := filterRuleSpecs(ip, proto, sPort, dPort, action, ipsetName) - - mangleSpecs := slices.Clone(specs) - mangleSpecs = append(mangleSpecs, - "-i", m.wgIface.Name(), - "-m", "addrtype", "--dst-type", "LOCAL", - "-j", "MARK", "--set-xmark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), - ) - - specs = append(specs, "-j", actionToStr(action)) - if ipsetName != "" { - if ipList, ipsetExists := m.ipsetStore.ipset(ipsetName); ipsetExists { - if err := m.addToIPSet(ipsetName, ip); err != nil { - return nil, fmt.Errorf("add IP to ipset: %w", err) - } - // if ruleset already exists it means we already have the firewall rule - // so we need to update IPs in the ruleset and return new fw.Rule object for ACL manager. - ipList.addIP(ip.String()) - return []firewall.Rule{&Rule{ - ruleID: uuid.New().String(), - ipsetName: ipsetName, - ip: ip.String(), - chain: chain, - specs: specs, - v6: m.v6, - }}, nil - } - - if err := m.flushIPSet(ipsetName); err != nil { - if errors.Is(err, ipset.ErrSetNotExist) { - log.Debugf("flush ipset %s before use: %v", ipsetName, err) - } else { - log.Errorf("flush ipset %s before use: %v", ipsetName, err) - } - } - if err := m.createIPSet(ipsetName); err != nil { - return nil, fmt.Errorf("create ipset: %w", err) - } - if err := m.addToIPSet(ipsetName, ip); err != nil { - return nil, fmt.Errorf("add IP to ipset: %w", err) - } - - ipList := newIpList(ip.String()) - m.ipsetStore.addIpList(ipsetName, ipList) - } - - ok, err := m.iptablesClient.Exists(tableFilter, chain, specs...) - if err != nil { - return nil, fmt.Errorf("failed to check rule: %w", err) - } - if ok { - return nil, fmt.Errorf("rule already exists") - } - - // Insert DROP rules at the beginning, append ACCEPT rules at the end - if action == firewall.ActionDrop { - // Insert at the beginning of the chain (position 1) - err = m.iptablesClient.Insert(tableFilter, chain, 1, specs...) - } else { - err = m.iptablesClient.Append(tableFilter, chain, specs...) - } - if err != nil { - return nil, err - } - - if err := m.iptablesClient.Append(tableMangle, chainRTPRE, mangleSpecs...); err != nil { - log.Errorf("failed to add mangle rule: %v", err) - mangleSpecs = nil - } - - rule := &Rule{ - ruleID: uuid.New().String(), - specs: specs, - mangleSpecs: mangleSpecs, - ipsetName: ipsetName, - ip: ip.String(), - chain: chain, - v6: m.v6, - } - - m.updateState() - - return []firewall.Rule{rule}, nil -} - -// DeletePeerRule from the firewall by rule definition -func (m *aclManager) DeletePeerRule(rule firewall.Rule) error { - r, ok := rule.(*Rule) - if !ok { - return fmt.Errorf("invalid rule type") - } - - shouldDestroyIpset := false - if ipsetList, ok := m.ipsetStore.ipset(r.ipsetName); ok { - // delete IP from ruleset IPs list and ipset - if _, ok := ipsetList.ips[r.ip]; ok { - ip := net.ParseIP(r.ip) - if ip == nil { - return fmt.Errorf("parse IP %s", r.ip) - } - if err := m.delFromIPSet(r.ipsetName, ip); err != nil { - return fmt.Errorf("delete ip from ipset: %w", err) - } - delete(ipsetList.ips, r.ip) - } - - // if after delete, set still contains other IPs, - // no need to delete firewall rule and we should exit here - if len(ipsetList.ips) != 0 { - return nil - } - - // we delete last IP from the set, that means we need to delete - // set itself and associated firewall rule too - m.ipsetStore.deleteIpset(r.ipsetName) - shouldDestroyIpset = true - } - - if err := m.iptablesClient.Delete(tableName, r.chain, r.specs...); err != nil { - return fmt.Errorf("failed to delete rule: %s, %v: %w", r.chain, r.specs, err) - } - - if r.mangleSpecs != nil { - if err := m.iptablesClient.Delete(tableMangle, chainRTPRE, r.mangleSpecs...); err != nil { - log.Errorf("failed to delete mangle rule: %v", err) - } - } - - if shouldDestroyIpset { - if err := m.destroyIPSet(r.ipsetName); err != nil { - if errors.Is(err, ipset.ErrBusy) || errors.Is(err, ipset.ErrSetNotExist) { - log.Debugf("destroy empty ipset: %v", err) - } else { - log.Errorf("destroy empty ipset: %v", err) - } - } - } - - m.updateState() - - return nil -} - -func (m *aclManager) Reset() error { - if err := m.cleanChains(); err != nil { - return fmt.Errorf("clean chains: %w", err) - } - - m.updateState() - - return nil -} - -// todo write less destructive cleanup mechanism -func (m *aclManager) cleanChains() error { - ok, err := m.iptablesClient.ChainExists(tableName, chainNameInputRules) - if err != nil { - log.Debugf("failed to list chains: %s", err) - return err - } - if ok { - for _, rule := range m.entries["INPUT"] { - err := m.iptablesClient.DeleteIfExists(tableName, "INPUT", rule...) - if err != nil { - log.Errorf("failed to delete rule: %v, %s", rule, err) - } - } - - for _, rule := range m.entries["FORWARD"] { - err := m.iptablesClient.DeleteIfExists(tableName, "FORWARD", rule...) - if err != nil { - log.Errorf("failed to delete rule: %v, %s", rule, err) - } - } - - err = m.iptablesClient.ClearAndDeleteChain(tableName, chainNameInputRules) - if err != nil { - log.Debugf("failed to clear and delete %s chain: %s", chainNameInputRules, err) - return err - } - } - - ok, err = m.iptablesClient.ChainExists("mangle", "PREROUTING") - if err != nil { - return fmt.Errorf("list chains: %w", err) - } - if ok { - for _, rule := range m.entries["PREROUTING"] { - err := m.iptablesClient.DeleteIfExists("mangle", "PREROUTING", rule...) - if err != nil { - log.Errorf("failed to delete rule: %v, %s", rule, err) - } - } - } - - for _, rule := range m.entries[mangleFwdKey] { - if err := m.iptablesClient.DeleteIfExists(tableMangle, chainFORWARD, rule...); err != nil { - log.Errorf("failed to delete mangle FORWARD guard rule: %v, %s", rule, err) - } - } - - for _, ipsetName := range m.ipsetStore.ipsetNames() { - if err := m.flushIPSet(ipsetName); err != nil { - if errors.Is(err, ipset.ErrSetNotExist) { - log.Debugf("flush ipset %q during reset: %v", ipsetName, err) - } else { - log.Errorf("flush ipset %q during reset: %v", ipsetName, err) - } - } - if err := m.destroyIPSet(ipsetName); err != nil { - if errors.Is(err, ipset.ErrBusy) || errors.Is(err, ipset.ErrSetNotExist) { - log.Debugf("destroy ipset %q during reset: %v", ipsetName, err) - } else { - log.Errorf("destroy ipset %q during reset: %v", ipsetName, err) - } - } - m.ipsetStore.deleteIpset(ipsetName) - } - - return nil -} - -func (m *aclManager) createDefaultChains() error { - // chain netbird-acl-input-rules - if err := m.iptablesClient.NewChain(tableName, chainNameInputRules); err != nil { - log.Debugf("failed to create '%s' chain: %s", chainNameInputRules, err) - return err - } - - for chainName, rules := range m.entries { - // mangle FORWARD guard rules are handled separately below - if chainName == mangleFwdKey { - continue - } - for _, rule := range rules { - if err := m.iptablesClient.InsertUnique(tableName, chainName, 1, rule...); err != nil { - log.Debugf("failed to create input chain jump rule: %s", err) - return err - } - } - } - - for chainName, entries := range m.optionalEntries { - for _, entry := range entries { - if err := m.iptablesClient.InsertUnique(tableName, chainName, entry.position, entry.spec...); err != nil { - log.Errorf("failed to insert optional entry %v: %v", entry.spec, err) - continue - } - m.entries[chainName] = append(m.entries[chainName], entry.spec) - } - } - clear(m.optionalEntries) - - // Insert mangle FORWARD guard rules to prevent external DNAT bypass. - for _, rule := range m.entries[mangleFwdKey] { - if err := m.iptablesClient.AppendUnique(tableMangle, chainFORWARD, rule...); err != nil { - log.Errorf("failed to add mangle FORWARD guard rule: %v", err) - } - } - - return nil -} - -// seedInitialEntries adds default rules to the entries map, rules are inserted on pos 1, hence the order is reversed. -// We want to make sure our traffic is not dropped by existing rules. - -// The existing FORWARD rules/policies decide outbound traffic towards our interface. -// In case the FORWARD policy is set to "drop", we add an established/related rule to allow return traffic for the inbound rule. -func (m *aclManager) seedInitialEntries() { - established := getConntrackEstablished() - - m.appendToEntries("INPUT", []string{"-i", m.wgIface.Name(), "-j", "DROP"}) - m.appendToEntries("INPUT", []string{"-i", m.wgIface.Name(), "-j", chainNameInputRules}) - m.appendToEntries("INPUT", append([]string{"-i", m.wgIface.Name()}, established...)) - - // Inbound is handled by our ACLs, the rest is dropped. - // For outbound we respect the FORWARD policy. However, we need to allow established/related traffic for inbound rules. - m.appendToEntries("FORWARD", []string{"-i", m.wgIface.Name(), "-j", "DROP"}) - - m.appendToEntries("FORWARD", []string{"-o", m.wgIface.Name(), "-j", chainRTFWDOUT}) - m.appendToEntries("FORWARD", []string{"-i", m.wgIface.Name(), "-j", chainRTFWDIN}) - - // Mangle FORWARD guard: when external DNAT redirects traffic from the wg interface, it - // traverses FORWARD instead of INPUT, bypassing ACL rules. ACCEPT rules in filter FORWARD - // can be inserted above ours. Mangle runs before filter, so these guard rules enforce the - // ACL mark check where it cannot be overridden. - m.appendToEntries(mangleFwdKey, []string{ - "-i", m.wgIface.Name(), - "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", - "-j", "ACCEPT", - }) - m.appendToEntries(mangleFwdKey, []string{ - "-i", m.wgIface.Name(), - "-m", "conntrack", "--ctstate", "DNAT", - "-m", "mark", "!", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), - "-j", "DROP", - }) -} - -func (m *aclManager) seedInitialOptionalEntries() { - m.optionalEntries["FORWARD"] = []entry{ - { - spec: []string{"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), "-j", "ACCEPT"}, - position: 2, - }, - } -} - -func (m *aclManager) appendToEntries(chainName string, spec []string) { - m.entries[chainName] = append(m.entries[chainName], spec) -} - -func (m *aclManager) updateState() { - if m.stateManager == nil { - return - } - - var currentState *ShutdownState - if existing := m.stateManager.GetState(currentState); existing != nil { - if existingState, ok := existing.(*ShutdownState); ok { - currentState = existingState - } - } - if currentState == nil { - currentState = &ShutdownState{} - } - - currentState.Lock() - defer currentState.Unlock() - - // Clone the maps so the persisted state holds a private snapshot. The - // live maps keep being mutated by subsequent rule operations while the - // state manager marshals the state from its periodic-save goroutine. - // Sharing them by reference races the two and aborts the process with a - // concurrent map iteration and write. - if m.v6 { - currentState.ACLEntries6 = maps.Clone(m.entries) - currentState.ACLIPsetStore6 = m.ipsetStore.clone() - } else { - currentState.ACLEntries = maps.Clone(m.entries) - currentState.ACLIPsetStore = m.ipsetStore.clone() - } - - if err := m.stateManager.UpdateState(currentState); err != nil { - log.Errorf("failed to update state: %v", err) - } -} - -// filterRuleSpecs returns the specs of a filtering rule -// protoForFamily translates ICMP to ICMPv6 for ip6tables. -// ip6tables requires "ipv6-icmp" (or "icmpv6") instead of "icmp". -func protoForFamily(protocol firewall.Protocol, v6 bool) string { - if v6 && protocol == firewall.ProtocolICMP { - return "ipv6-icmp" - } - return string(protocol) -} - -func filterRuleSpecs(ip net.IP, protocol string, sPort, dPort *firewall.Port, action firewall.Action, ipsetName string) (specs []string) { - // don't use IP matching if IP is 0.0.0.0 - matchByIP := !ip.IsUnspecified() - - if matchByIP { - if ipsetName != "" { - specs = append(specs, "-m", "set", "--match-set", ipsetName, "src") - } else { - specs = append(specs, "-s", ip.String()) - } - } - if protocol != "all" { - specs = append(specs, "-p", protocol) - } - specs = append(specs, applyPort("--sport", sPort)...) - specs = append(specs, applyPort("--dport", dPort)...) - return specs -} - -func actionToStr(action firewall.Action) string { - if action == firewall.ActionAccept { - return "ACCEPT" - } - return "DROP" -} - -func transformIPsetName(ipsetName string, sPort, dPort *firewall.Port, action firewall.Action) string { - if ipsetName == "" { - return "" - } - - actionSuffix := "" - if action == firewall.ActionDrop { - actionSuffix = "-drop" - } - - switch { - case sPort != nil && dPort != nil: - return ipsetName + "-sport-dport" + actionSuffix - case sPort != nil: - return ipsetName + "-sport" + actionSuffix - case dPort != nil: - return ipsetName + "-dport" + actionSuffix - default: - return ipsetName + actionSuffix - } -} - -func (m *aclManager) createIPSet(name string) error { - opts := ipset.CreateOptions{ - Replace: true, - } - if m.v6 { - opts.Family = ipset.FamilyIPV6 - } - - if err := ipset.Create(name, ipset.TypeHashNet, opts); err != nil { - return fmt.Errorf("create ipset %s: %w", name, err) - } - - log.Debugf("created ipset %s with type hash:net", name) - return nil -} - -func (m *aclManager) addToIPSet(name string, ip net.IP) error { - cidr := uint8(32) - if ip.To4() == nil { - cidr = 128 - } - - entry := &ipset.Entry{ - IP: ip, - CIDR: cidr, - Replace: true, - } - - if err := ipset.Add(name, entry); err != nil { - return fmt.Errorf("add IP to ipset %s: %w", name, err) - } - - return nil -} - -func (m *aclManager) delFromIPSet(name string, ip net.IP) error { - cidr := uint8(32) - if ip.To4() == nil { - cidr = 128 - } - - entry := &ipset.Entry{ - IP: ip, - CIDR: cidr, - } - - if err := ipset.Del(name, entry); err != nil { - return fmt.Errorf("delete IP from ipset %s: %w", name, err) - } - - return nil -} - -func (m *aclManager) flushIPSet(name string) error { - return ipset.Flush(name) -} - -func (m *aclManager) destroyIPSet(name string) error { - return ipset.Destroy(name) -} diff --git a/client/firewall/iptables/chains_linux.go b/client/firewall/iptables/chains_linux.go new file mode 100644 index 000000000..58bfa8c6a --- /dev/null +++ b/client/firewall/iptables/chains_linux.go @@ -0,0 +1,346 @@ +//go:build !android + +package iptables + +import ( + "fmt" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbnet "github.com/netbirdio/netbird/client/net" +) + +func (r *family) createContainers() error { + for _, chainInfo := range []struct { + chain string + table string + }{ + {chainRTFwdIn, tableFilter}, + {chainRTFwdOut, tableFilter}, + {chainRTPre, tableMangle}, + {chainRTNAT, tableNat}, + {chainRTRdr, tableNat}, + {chainRTMSSClamp, tableMangle}, + } { + // Fallback: clear chains that survived an unclean shutdown. + if ok, _ := r.iptablesClient.ChainExists(chainInfo.table, chainInfo.chain); ok { + if err := r.iptablesClient.ClearAndDeleteChain(chainInfo.table, chainInfo.chain); err != nil { + log.Warnf("clear stale chain %s in %s: %v", chainInfo.chain, chainInfo.table, err) + } + } + if err := r.iptablesClient.NewChain(chainInfo.table, chainInfo.chain); err != nil { + return fmt.Errorf("create chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err) + } + } + + if err := r.insertEstablishedRule(chainRTFwdIn); err != nil { + return fmt.Errorf("insert established rule: %w", err) + } + + if err := r.insertEstablishedRule(chainRTFwdOut); err != nil { + return fmt.Errorf("insert established rule: %w", err) + } + + if err := r.addPostroutingRules(); err != nil { + return fmt.Errorf("add static nat rules: %w", err) + } + + if err := r.addJumpRules(); err != nil { + return fmt.Errorf("add jump rules: %w", err) + } + + if err := r.addMSSClampingRules(); err != nil { + log.Errorf("failed to add MSS clamping rules: %s", err) + } + + return nil +} + +func (r *family) addJumpRules() error { + // Jump to nat chain + natRule := jumpRuleSpec(chainRTNAT) + if err := r.iptablesClient.Insert(tableNat, chainPostrouting, 1, natRule...); err != nil { + return fmt.Errorf("add nat postrouting jump rule: %w", err) + } + r.rules[jumpNATPost] = natRule + + // Jump to mangle prerouting chain + preRule := jumpRuleSpec(chainRTPre) + if err := r.iptablesClient.Insert(tableMangle, chainPrerouting, 1, preRule...); err != nil { + return fmt.Errorf("add mangle prerouting jump rule: %w", err) + } + r.rules[jumpManglePre] = preRule + + // Jump to nat prerouting chain + rdrRule := jumpRuleSpec(chainRTRdr) + if err := r.iptablesClient.Insert(tableNat, chainPrerouting, 1, rdrRule...); err != nil { + return fmt.Errorf("add nat prerouting jump rule: %w", err) + } + r.rules[jumpNATPre] = rdrRule + + return nil +} + +func (r *family) setupDataPlaneMark() error { + var merr *multierror.Error + preRule := []string{ + "-i", r.wgIface.Name(), + "-m", "conntrack", "--ctstate", "NEW", + "-j", "CONNMARK", "--set-mark", fmt.Sprintf("%#x", nbnet.DataPlaneMarkIn), + } + + if err := r.iptablesClient.AppendUnique(tableMangle, chainPrerouting, preRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add mangle prerouting rule: %w", err)) + } else { + r.rules[markManglePre] = preRule + } + + postRule := []string{ + "-o", r.wgIface.Name(), + "-m", "conntrack", "--ctstate", "NEW", + "-j", "CONNMARK", "--set-mark", fmt.Sprintf("%#x", nbnet.DataPlaneMarkOut), + } + + if err := r.iptablesClient.AppendUnique(tableMangle, chainPostrouting, postRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add mangle postrouting rule: %w", err)) + } else { + r.rules[markManglePost] = postRule + } + + return nberrors.FormatErrorOrNil(merr) +} + +// seedInitialEntries adds default rules to the entries map. Rules are +// inserted at position 1, so the order here is reversed. +// +// Existing FORWARD policy decides outbound traffic towards our +// interface. If FORWARD policy is "drop", we add an +// established/related rule to allow return traffic for inbound rules. +func (r *family) seedInitialEntries() { + established := getConntrackEstablished() + + r.appendToEntries(chainInput, []string{"-i", r.wgIface.Name(), "-j", "DROP"}) + r.appendToEntries(chainInput, []string{"-i", r.wgIface.Name(), "-j", chainACLInput}) + r.appendToEntries(chainInput, append([]string{"-i", r.wgIface.Name()}, established...)) + + r.appendToEntries(chainForward, []string{"-i", r.wgIface.Name(), "-j", "DROP"}) + r.appendToEntries(chainForward, []string{"-o", r.wgIface.Name(), "-j", chainRTFwdOut}) + r.appendToEntries(chainForward, []string{"-i", r.wgIface.Name(), "-j", chainRTFwdIn}) + + // Mangle FORWARD guard: when external DNAT redirects traffic from + // the wg interface, it traverses FORWARD instead of INPUT, + // bypassing ACL rules. ACCEPT rules in filter FORWARD can be + // inserted above ours. Mangle runs before filter, so these guard + // rules enforce the ACL mark check where it cannot be overridden. + r.appendToEntries(mangleForwardKey, []string{ + "-i", r.wgIface.Name(), + "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", + "-j", "ACCEPT", + }) + r.appendToEntries(mangleForwardKey, []string{ + "-i", r.wgIface.Name(), + "-m", "conntrack", "--ctstate", "DNAT", + "-m", "mark", "!", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), + "-j", "DROP", + }) +} + +func (r *family) seedInitialOptionalEntries() { + r.optionalEntries[chainForward] = []entry{ + { + spec: []string{"-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), "-j", "ACCEPT"}, + position: 2, + }, + } +} + +func (r *family) appendToEntries(chain chainKey, spec ruleSpec) { + r.entries[chain] = append(r.entries[chain], spec) +} + +func (r *family) createDefaultChains() error { + if err := r.iptablesClient.NewChain(tableFilter, chainACLInput); err != nil { + return fmt.Errorf("create %s chain: %w", chainACLInput, err) + } + + for chain, rules := range r.entries { + // mangle FORWARD guard rules are handled separately below + if chain == mangleForwardKey { + continue + } + for _, rule := range rules { + if err := r.iptablesClient.InsertUnique(tableFilter, string(chain), 1, rule...); err != nil { + return fmt.Errorf("insert jump rule into %s: %w", chain, err) + } + } + } + + for chain, entries := range r.optionalEntries { + for _, entry := range entries { + if err := r.iptablesClient.InsertUnique(tableFilter, string(chain), entry.position, entry.spec...); err != nil { + log.Errorf("failed to insert optional entry %v: %v", entry.spec, err) + continue + } + r.entries[chain] = append(r.entries[chain], entry.spec) + } + } + clear(r.optionalEntries) + + // Insert mangle FORWARD guard rules to prevent external DNAT bypass. + for _, rule := range r.entries[mangleForwardKey] { + if err := r.iptablesClient.AppendUnique(tableMangle, chainForward, rule...); err != nil { + log.Errorf("failed to add mangle FORWARD guard rule: %v", err) + } + } + + return nil +} + +func (r *family) cleanUpDefaultForwardRules() error { + var merr *multierror.Error + + // cleanJumpRules removes the OUTPUT jump to NETBIRD-NAT-OUTPUT among + // the others, so the chain below deletes cleanly instead of failing + // with "device or resource busy". + if err := r.cleanJumpRules(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("clean jump rules: %w", err)) + } + + for _, chainInfo := range []struct { + chain string + table string + }{ + {chainRTFwdIn, tableFilter}, + {chainRTFwdOut, tableFilter}, + {chainRTPre, tableMangle}, + {chainRTNAT, tableNat}, + {chainRTRdr, tableNat}, + {chainNATOutput, tableNat}, + {chainRTMSSClamp, tableMangle}, + } { + ok, err := r.iptablesClient.ChainExists(chainInfo.table, chainInfo.chain) + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("check chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)) + continue + } + if ok { + if err := r.iptablesClient.ClearAndDeleteChain(chainInfo.table, chainInfo.chain); err != nil { + merr = multierror.Append(merr, fmt.Errorf("clear and delete chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)) + } + } + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) cleanJumpRules() error { + // locations maps each jump rule to the built-in table and chain it + // was inserted into, plus the netbird chain it targets. + locations := map[firewall.RuleID]struct{ table, chain, target string }{ + jumpNATPost: {tableNat, chainPostrouting, chainRTNAT}, + jumpManglePre: {tableMangle, chainPrerouting, chainRTPre}, + jumpNATPre: {tableNat, chainPrerouting, chainRTRdr}, + jumpMSSClamp: {tableMangle, chainForward, chainRTMSSClamp}, + jumpNATOutput: {tableNat, chainOutput, chainNATOutput}, + } + + var merr *multierror.Error + for ruleID, loc := range locations { + rule, exists := r.rules[ruleID] + if !exists { + // Untracked (e.g. fresh start after an unclean shutdown with no + // restored state): if the target chain survived, remove the stale + // jump to it so the chain can be deleted. + ok, err := r.iptablesClient.ChainExists(loc.table, loc.target) + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("check chain %s in table %s: %w", loc.target, loc.table, err)) + continue + } + if !ok { + continue + } + rule = jumpRuleSpec(loc.target) + } + if err := r.iptablesClient.DeleteIfExists(loc.table, loc.chain, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete rule from chain %s in table %s: %w", loc.chain, loc.table, err)) + continue + } + delete(r.rules, ruleID) + } + return nberrors.FormatErrorOrNil(merr) +} + +// jumpRuleSpec builds the iptables rule spec that jumps to target. Create +// and cleanup sites share it so the installed and deleted specs cannot drift. +func jumpRuleSpec(target string) []string { + return []string{"-j", target} +} + +func (r *family) cleanAclChains() error { + var merr *multierror.Error + + if err := r.cleanInputAclChain(); err != nil { + merr = multierror.Append(merr, err) + } + + for _, rule := range r.entries[mangleForwardKey] { + if err := r.iptablesClient.DeleteIfExists(tableMangle, chainForward, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete mangle %s guard rule %v: %w", chainForward, rule, err)) + } + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) cleanInputAclChain() error { + ok, err := r.iptablesClient.ChainExists(tableFilter, chainACLInput) + if err != nil { + return fmt.Errorf("check chain %s: %w", chainACLInput, err) + } + if !ok { + return nil + } + + var merr *multierror.Error + for _, rule := range r.entries[chainInput] { + if err := r.iptablesClient.DeleteIfExists(tableFilter, chainInput, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete %s rule %v: %w", chainInput, rule, err)) + } + } + + for _, rule := range r.entries[chainForward] { + if err := r.iptablesClient.DeleteIfExists(tableFilter, chainForward, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete %s rule %v: %w", chainForward, rule, err)) + } + } + + if err := r.iptablesClient.ClearAndDeleteChain(tableFilter, chainACLInput); err != nil { + merr = multierror.Append(merr, fmt.Errorf("clear and delete %s chain: %w", chainACLInput, err)) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) cleanupDataPlaneMark() error { + var merr *multierror.Error + if preRule, exists := r.rules[markManglePre]; exists { + if err := r.iptablesClient.DeleteIfExists(tableMangle, chainPrerouting, preRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove mangle prerouting rule: %w", err)) + } else { + delete(r.rules, markManglePre) + } + } + + if postRule, exists := r.rules[markManglePost]; exists { + if err := r.iptablesClient.DeleteIfExists(tableMangle, chainPostrouting, postRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove mangle postrouting rule: %w", err)) + } else { + delete(r.rules, markManglePost) + } + } + + return nberrors.FormatErrorOrNil(merr) +} diff --git a/client/firewall/iptables/dnat_linux.go b/client/firewall/iptables/dnat_linux.go new file mode 100644 index 000000000..f118c9dfe --- /dev/null +++ b/client/firewall/iptables/dnat_linux.go @@ -0,0 +1,136 @@ +//go:build !android + +package iptables + +import ( + "fmt" + "net/netip" + "strconv" + "strings" + + log "github.com/sirupsen/logrus" + + firewall "github.com/netbirdio/netbird/client/firewall/manager" +) + +func (r *family) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + if _, exists := r.rules[ruleID]; exists { + return nil + } + + dnatRule := []string{ + "-i", r.wgIface.Name(), + "-p", strings.ToLower(protoForFamily(protocol, r.v6)), + "--dport", strconv.Itoa(int(originalPort)), + "-d", localAddr.String(), + "-m", "addrtype", "--dst-type", "LOCAL", + "-j", "DNAT", + "--to-destination", ":" + strconv.Itoa(int(translatedPort)), + } + + info := ruleInfo{ + table: tableNat, + chain: chainRTRdr, + rule: dnatRule, + } + + if err := r.iptablesClient.Append(info.table, info.chain, info.rule...); err != nil { + return fmt.Errorf("add inbound DNAT rule: %w", err) + } + r.rules[ruleID] = info.rule + + r.updateState() + return nil +} + +// RemoveInboundDNAT removes an inbound DNAT rule. +func (r *family) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + if dnatRule, exists := r.rules[ruleID]; exists { + if err := r.iptablesClient.Delete(tableNat, chainRTRdr, dnatRule...); err != nil { + return fmt.Errorf("delete inbound DNAT rule: %w", err) + } + delete(r.rules, ruleID) + } + + r.updateState() + return nil +} + +// ensureNATOutputChain lazily creates the OUTPUT NAT chain and jump rule on first use. +func (r *family) ensureNATOutputChain() error { + if _, exists := r.rules[jumpNATOutput]; exists { + return nil + } + + chainExists, err := r.iptablesClient.ChainExists(tableNat, chainNATOutput) + if err != nil { + return fmt.Errorf("check chain %s: %w", chainNATOutput, err) + } + if !chainExists { + if err := r.iptablesClient.NewChain(tableNat, chainNATOutput); err != nil { + return fmt.Errorf("create chain %s: %w", chainNATOutput, err) + } + } + + jumpRule := jumpRuleSpec(chainNATOutput) + if err := r.iptablesClient.Insert(tableNat, chainOutput, 1, jumpRule...); err != nil { + if !chainExists { + if delErr := r.iptablesClient.ClearAndDeleteChain(tableNat, chainNATOutput); delErr != nil { + log.Warnf("failed to rollback chain %s: %v", chainNATOutput, delErr) + } + } + return fmt.Errorf("add OUTPUT jump rule: %w", err) + } + r.rules[jumpNATOutput] = jumpRule + + r.updateState() + return nil +} + +// AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic. +func (r *family) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + ruleID := firewall.RuleID(fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + if _, exists := r.rules[ruleID]; exists { + return nil + } + + if err := r.ensureNATOutputChain(); err != nil { + return err + } + + dnatRule := []string{ + "-p", strings.ToLower(protoForFamily(protocol, localAddr.Is6())), + "--dport", strconv.Itoa(int(originalPort)), + "-d", localAddr.String(), + "-j", "DNAT", + "--to-destination", ":" + strconv.Itoa(int(translatedPort)), + } + + if err := r.iptablesClient.Append(tableNat, chainNATOutput, dnatRule...); err != nil { + return fmt.Errorf("add output DNAT rule: %w", err) + } + r.rules[ruleID] = dnatRule + + r.updateState() + return nil +} + +// RemoveOutputDNAT removes an OUTPUT chain DNAT rule. +func (r *family) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + ruleID := firewall.RuleID(fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + if dnatRule, exists := r.rules[ruleID]; exists { + if err := r.iptablesClient.Delete(tableNat, chainNATOutput, dnatRule...); err != nil { + return fmt.Errorf("delete output DNAT rule: %w", err) + } + delete(r.rules, ruleID) + } + + r.updateState() + return nil +} diff --git a/client/firewall/iptables/family_linux.go b/client/firewall/iptables/family_linux.go new file mode 100644 index 000000000..2ac860a0a --- /dev/null +++ b/client/firewall/iptables/family_linux.go @@ -0,0 +1,256 @@ +//go:build !android + +package iptables + +import ( + "fmt" + "maps" + "net/netip" + + "github.com/coreos/go-iptables/iptables" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbid "github.com/netbirdio/netbird/client/internal/acl/id" + "github.com/netbirdio/netbird/client/internal/routemanager/ipfwdstate" + "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" + "github.com/netbirdio/netbird/client/internal/statemanager" +) + +// constants needed to manage and create iptable rules +const ( + tableFilter = "filter" + tableNat = "nat" + tableMangle = "mangle" + tableRaw = "raw" + + // chainACLInput is the peer ACL chain that holds installed + // peer-filtering rules. + chainACLInput = "NETBIRD-ACL-INPUT" + + // mangleForwardKey is the entries map key for mangle FORWARD guard + // rules that prevent external DNAT from bypassing ACL rules. + mangleForwardKey chainKey = "MANGLE-FORWARD" + + chainInput = "INPUT" + chainOutput = "OUTPUT" + chainPostrouting = "POSTROUTING" + chainPrerouting = "PREROUTING" + chainForward = "FORWARD" + chainRTNAT = "NETBIRD-RT-NAT" + chainRTFwdIn = "NETBIRD-RT-FWD-IN" + chainRTFwdOut = "NETBIRD-RT-FWD-OUT" + chainRTPre = "NETBIRD-RT-PRE" + chainRTRdr = "NETBIRD-RT-RDR" + chainNATOutput = "NETBIRD-NAT-OUTPUT" + chainRTMSSClamp = "NETBIRD-RT-MSSCLAMP" + + jumpManglePre = "jump-mangle-pre" + jumpNATPre = "jump-nat-pre" + jumpNATPost = "jump-nat-post" + jumpNATOutput = "jump-nat-output" + jumpMSSClamp = "jump-mss-clamp" + markManglePre = "mark-mangle-pre" + markManglePost = "mark-mangle-post" + matchSet = "--match-set" + + // ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation. + ipv4TCPHeaderSize = 40 + // ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation. + ipv6TCPHeaderSize = 60 +) + +type ruleInfo struct { + chain string + table string + rule []string +} + +type routeRules map[firewall.RuleID][]string + +// ruleSpec is a single iptables rule expressed as its argument list +// (e.g. {"-i", "wg0", "-j", "DROP"}). +type ruleSpec []string + +// chainKey identifies the chain a seeded entry belongs to. It holds +// built-in chain names ("INPUT", "FORWARD", "PREROUTING") plus the +// synthetic mangleForwardKey bucket for the mangle FORWARD guard rules. +type chainKey string + +// aclEntries maps a chain to the rules seeded into it to jump into or +// guard the netbird ACL chains. +type aclEntries map[chainKey][]ruleSpec + +type entry struct { + spec ruleSpec + position int +} + +// ipsetCounter is the shared hash:net refcounter used by peer and +// route ACLs alike. The ipset library does not support comments, so +// the key is just the set name (string). +type ipsetCounter = refcounter.Counter[string, []netip.Prefix, struct{}] + +// family holds the per-address-family iptables state. One instance +// handles route ACLs, peer ACLs, NAT, DNAT, and MSS clamping for a +// single family; the top-level Manager owns one for v4 and another +// for v6. +type family struct { + iptablesClient *iptables.IPTables + wgIface iFaceMapper + v6 bool + + // Peer ACL chain bookkeeping. + entries aclEntries + optionalEntries map[chainKey][]entry + + // filters holds peer + route filter rules keyed by content hash. + // AddFilterRule writes here; DeleteFilterRule looks up by id. + filters map[nbid.RuleID]*Rule + ipsetCounter *ipsetCounter + // ipsetSupported records whether the kernel can create the hash:net + // sets the source matches rely on; probed once at init. When false, + // multi-source rules expand to one rule per source prefix. + ipsetSupported bool + + // rules holds NAT, jump, and MSS-clamping rules (auxiliary + // plumbing that isn't a filter rule). + rules routeRules + + // Routing / NAT. + legacyManagement bool + mtu uint16 + ipFwdState *ipfwdstate.IPForwardingState + + stateManager *statemanager.Manager +} + +func newFamily(iptablesClient *iptables.IPTables, wgIface iFaceMapper, mtu uint16) (*family, error) { + r := &family{ + iptablesClient: iptablesClient, + wgIface: wgIface, + v6: iptablesClient.Proto() == iptables.ProtocolIPv6, + entries: make(aclEntries), + optionalEntries: make(map[chainKey][]entry), + filters: make(map[nbid.RuleID]*Rule), + rules: make(routeRules), + mtu: mtu, + ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()), + } + + r.ipsetCounter = refcounter.New( + func(name string, sources []netip.Prefix) (struct{}, error) { + return struct{}{}, r.createIpSet(name, sources) + }, + func(name string, _ struct{}) error { + return r.deleteIpSet(name) + }, + ) + + return r, nil +} + +// init wires the family to the state manager and installs both the +// route ACL containers and the peer ACL chain skeleton. +func (r *family) init(stateManager *statemanager.Manager) error { + r.stateManager = stateManager + + r.ipsetSupported = r.probeIPSetSupport() + + if err := r.cleanUpDefaultForwardRules(); err != nil { + log.Errorf("failed to clean up rules from FORWARD chain: %s", err) + } + + if err := r.createContainers(); err != nil { + return fmt.Errorf("create containers: %w", err) + } + + if err := r.setupDataPlaneMark(); err != nil { + log.Errorf("failed to set up data plane mark: %v", err) + } + + r.seedInitialEntries() + r.seedInitialOptionalEntries() + + if err := r.cleanAclChains(); err != nil { + return fmt.Errorf("clean acl chains: %w", err) + } + if err := r.createDefaultChains(); err != nil { + return fmt.Errorf("create default chains: %w", err) + } + + r.updateState() + + return nil +} + +// Reset tears down all firewall state owned by this family. ACL +// chain cleanup runs before route-chain cleanup because the route +// chains are still referenced by FORWARD jumps installed during +// seedInitialEntries; deleting them first would trip EBUSY. +func (r *family) Reset() error { + var merr *multierror.Error + + if err := r.cleanAclChains(); err != nil { + merr = multierror.Append(merr, err) + } + + if err := r.cleanUpDefaultForwardRules(); err != nil { + merr = multierror.Append(merr, err) + } + + if err := r.ipsetCounter.Flush(); err != nil { + merr = multierror.Append(merr, err) + } + + if err := r.cleanupDataPlaneMark(); err != nil { + merr = multierror.Append(merr, err) + } + + clear(r.rules) + clear(r.filters) + r.updateState() + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) updateState() { + if r.stateManager == nil { + return + } + + var currentState *ShutdownState + if existing := r.stateManager.GetState(currentState); existing != nil { + if existingState, ok := existing.(*ShutdownState); ok { + currentState = existingState + } + } + if currentState == nil { + currentState = &ShutdownState{} + } + + currentState.Lock() + defer currentState.Unlock() + + // Clone the rule maps so the persisted state holds a private snapshot. + // The live maps keep being mutated by subsequent rule operations while + // the state manager marshals the state from its periodic-save goroutine. + // Sharing the maps by reference races the two and aborts the process with + // a concurrent map iteration and write. The ipset counter guards itself + // during marshaling, so it can be shared directly. + if r.v6 { + currentState.RouteRules6 = maps.Clone(r.rules) + currentState.RouteIPsetCounter6 = r.ipsetCounter + currentState.ACLEntries6 = maps.Clone(r.entries) + } else { + currentState.RouteRules = maps.Clone(r.rules) + currentState.RouteIPsetCounter = r.ipsetCounter + currentState.ACLEntries = maps.Clone(r.entries) + } + + if err := r.stateManager.UpdateState(currentState); err != nil { + log.Errorf("failed to update state: %v", err) + } +} diff --git a/client/firewall/iptables/filter_linux.go b/client/firewall/iptables/filter_linux.go new file mode 100644 index 000000000..30cd81018 --- /dev/null +++ b/client/firewall/iptables/filter_linux.go @@ -0,0 +1,421 @@ +//go:build !android + +package iptables + +import ( + "fmt" + "net/netip" + "slices" + "strconv" + "strings" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbid "github.com/netbirdio/netbird/client/internal/acl/id" + nbnet "github.com/netbirdio/netbird/client/net" +) + +// AddFilterRule installs a packet-filtering rule. With destination +// empty, the rule goes to the peer ACL input chain plus a paired +// mangle PREROUTING rule for the redirect mark. With destination set +// (prefix or named set), it goes to the route ACL forward chain. +// Multi-source rules collapse to one iptables rule via the shared +// hash:net ipset. +func (r *family) AddFilterRule( + id []byte, + sources []netip.Prefix, + destination firewall.Network, + proto firewall.Protocol, + sPort *firewall.Port, + dPort *firewall.Port, + action firewall.Action, +) (firewall.Rule, error) { + ruleID := nbid.GenerateRuleID(sources, destination, proto, sPort, dPort, action) + if existing, ok := r.filters[ruleID]; ok { + return existing, nil + } + + rule, err := r.installFilterRules(ruleID, sources, destination, proto, sPort, dPort, action, r.ipsetSupported) + if err != nil { + return nil, err + } + + r.filters[ruleID] = rule + r.updateState() + return rule, nil +} + +// installFilterRules resolves the source matches and installs one +// iptables rule per match. It is more than one rule only when useIPSet +// is false and a multi-source rule has to be expanded per prefix. +func (r *family) installFilterRules( + ruleID nbid.RuleID, + sources []netip.Prefix, + destination firewall.Network, + proto firewall.Protocol, + sPort *firewall.Port, + dPort *firewall.Port, + action firewall.Action, + useIPSet bool, +) (*Rule, error) { + srcMatches, err := r.applySourceMatches(sources, useIPSet) + if err != nil { + return nil, fmt.Errorf("apply source match: %w", err) + } + + rule, err := r.installFilterRule(ruleID, srcMatches, destination, proto, sPort, dPort, action) + if err != nil { + for _, srcMatch := range srcMatches { + r.dropSourceMatch(srcMatch) + } + return nil, err + } + return rule, nil +} + +func (r *family) hasRule(id nbid.RuleID) bool { + _, ok := r.filters[id] + return ok +} + +// DeleteFilterRule removes a previously installed filter rule. The +// rule's stored chain/table identify where to delete from; source set +// references are recovered from the spec via findSets and dropped +// from the shared ipset counter. +func (r *family) DeleteFilterRule(rule firewall.Rule) error { + ruleID := rule.ID() + pr, ok := r.filters[ruleID] + if !ok { + log.Debugf("filter rule %s not found", ruleID) + return nil + } + + // DeleteIfExists keeps the deletes idempotent so a retry after a + // partial failure does not error on the parts already removed. + var merr *multierror.Error + for _, fs := range pr.allSpecs() { + if err := r.iptablesClient.DeleteIfExists(tableFilter, pr.chain, fs.specs...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete rule from %s: %w", pr.chain, err)) + } + if fs.mangleSpecs != nil { + if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPre, fs.mangleSpecs...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete mangle rule: %w", err)) + } + } + } + if merr != nil { + // Leave the rule tracked so the caller retries the remaining part. + return nberrors.FormatErrorOrNil(merr) + } + + // The rule is gone from iptables, so untrack it regardless of how the + // refcount decrement goes, but surface decrement failures so callers + // see the ipset desync. Only the primary spec can reference sets: the + // per-prefix expansion never uses them. + delete(r.filters, ruleID) + r.updateState() + if err := r.decrementSetCounter(pr.specs); err != nil { + return fmt.Errorf("drop source set references: %w", err) + } + return nil +} + +// findSets scans an iptables rule spec for "-m set --match-set +// " fragments and returns the named sets in occurrence order. +// Used at delete time to drop ipsetCounter references. +func findSets(rule []string) []string { + var sets []string + for i, arg := range rule { + if arg == "-m" && i+3 < len(rule) && rule[i+1] == "set" && rule[i+2] == matchSet { + sets = append(sets, rule[i+3]) + } + } + return sets +} + +// sourceNetwork classifies a source-prefix list into the firewall.Network +// shape the rest of the spec-builder consumes: empty for match-any, a +// single prefix inline, or an ipset for multiple sources. +func sourceNetwork(sources []netip.Prefix) firewall.Network { + switch { + case len(sources) == 0: + return firewall.Network{} + case len(sources) == 1 && sources[0].Bits() == 0: + return firewall.Network{} + case len(sources) == 1: + return firewall.Network{Prefix: sources[0]} + default: + return firewall.Network{Set: firewall.NewPrefixSet(sources)} + } +} + +// applySourceMatches returns one source match fragment per iptables +// rule needed for the sources: normally a single fragment (a set match, +// a direct -s match, or nil for match-any), and one -s fragment per +// prefix when a multi-source rule cannot use ipset. Per-prefix rules +// are the only form a kernel without the ipset modules can express. +func (r *family) applySourceMatches(sources []netip.Prefix, useIPSet bool) ([][]string, error) { + network := sourceNetwork(sources) + if !network.IsSet() || useIPSet { + match, err := r.applySourceMatch(network, sources) + if err != nil { + return nil, err + } + return [][]string{match}, nil + } + + matches := make([][]string, 0, len(sources)) + for _, source := range sources { + matches = append(matches, []string{"-s", source.String()}) + } + return matches, nil +} + +// applySourceMatch returns the iptables match fragment for the rule's +// source. For a Set it increments the shared ipset's refcount; for a +// Prefix it emits a direct -s match; for the wildcard it returns nil. +func (r *family) applySourceMatch(network firewall.Network, prefixes []netip.Prefix) ([]string, error) { + switch { + case network.IsSet(): + if r.ipsetCounter == nil { + return nil, fmt.Errorf("multi-source peer rule requires shared ipset counter") + } + name := r.ipsetName(network.Set.HashedName()) + if _, err := r.ipsetCounter.Increment(name, prefixes); err != nil { + return nil, fmt.Errorf("ipset increment %s: %w", name, err) + } + return []string{"-m", "set", matchSet, name, "src"}, nil + case network.IsPrefix(): + return []string{"-s", network.Prefix.String()}, nil + default: + return nil, nil + } +} + +// dropSourceMatch undoes whatever applySourceMatch reserved when +// installing a rule fails. Safe to call when the spec is empty or holds +// only inline matchers. Decrement errors are logged but not returned: +// the install error is what the caller needs to see. +func (r *family) dropSourceMatch(srcMatch []string) { + if r.ipsetCounter == nil { + return + } + for _, name := range findSets(srcMatch) { + if _, err := r.ipsetCounter.Decrement(name); err != nil { + log.Errorf("rollback ipset decrement %s: %v", name, err) + } + } +} + +// decrementSetCounter drops ipset references owned by a raw rule spec +// stored in r.rules (NAT / legacy route entries). It returns an error +// aggregate so the caller surfaces decrement failures. +func (r *family) decrementSetCounter(rule []string) error { + if r.ipsetCounter == nil { + return nil + } + var merr *multierror.Error + for _, name := range findSets(rule) { + if _, err := r.ipsetCounter.Decrement(name); err != nil { + merr = multierror.Append(merr, fmt.Errorf("decrement counter: %w", err)) + } + } + return nberrors.FormatErrorOrNil(merr) +} + +// installFilterRule assembles and writes the iptables filter-chain +// rules for one filter rule, one per source match fragment. With +// destination empty the rules land in the peer ACL input chain and each +// gets a paired mangle PREROUTING rule for the redirect mark. With +// destination set the rules land in the route ACL forward chain and +// there is no mangle pairing. +func (r *family) installFilterRule( + ruleID nbid.RuleID, + srcMatches [][]string, + destination firewall.Network, + protocol firewall.Protocol, + sPort, dPort *firewall.Port, + action firewall.Action, +) (*Rule, error) { + isRoute := !destination.IsZero() + + proto := protoForFamily(protocol, r.v6) + + var destExp []string + if isRoute { + var err error + destExp, err = r.applyNetwork("-d", destination, nil) + if err != nil { + return nil, fmt.Errorf("apply network -d: %w", err) + } + } + matchSpecs := filterMatchSpecs(proto, sPort, dPort) + + chain := chainACLInput + if isRoute { + chain = chainRTFwdIn + } + + var installed []filterSpecs + for _, srcMatch := range srcMatches { + specs := slices.Clone(srcMatch) + specs = append(specs, destExp...) + specs = append(specs, matchSpecs...) + + var mangleSpecs []string + if !isRoute { + mangleSpecs = slices.Clone(specs) + mangleSpecs = append(mangleSpecs, + "-i", r.wgIface.Name(), + "-m", "addrtype", "--dst-type", "LOCAL", + "-j", "MARK", "--set-xmark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkRedirected), + ) + } + + specs = append(specs, "-j", actionToStr(action)) + + if err := r.insertFilterRule(chain, action, specs); err != nil { + // Leave nothing half-installed: the caller sees an error, so a + // partial rule would silently keep matching without being tracked. + r.removeFilterSpecs(chain, installed) + r.dropSourceMatch(destExp) + return nil, fmt.Errorf("install filter rule on %s: %w", chain, err) + } + + // The mangle redirect-mark rule is best effort: the filter rule itself + // is what enforces the ACL, so a mangle failure must not undo it. Drop + // the spec so teardown does not try to remove a rule that was not added. + if mangleSpecs != nil { + if err := r.iptablesClient.Append(tableMangle, chainRTPre, mangleSpecs...); err != nil { + log.Errorf("add mangle rule: %v", err) + mangleSpecs = nil + } + } + + installed = append(installed, filterSpecs{specs: specs, mangleSpecs: mangleSpecs}) + } + + return &Rule{ + id: ruleID, + specs: installed[0].specs, + mangleSpecs: installed[0].mangleSpecs, + extraRules: installed[1:], + chain: chain, + v6: r.v6, + }, nil +} + +// insertFilterRule writes one assembled rule spec into the given ACL +// chain. Peer ACL drops are inserted at position 1 so they precede the +// chain's catch-all; route ACL drops are inserted at position 2 to sit +// immediately after the established/related accept rule. +func (r *family) insertFilterRule(chain string, action firewall.Action, specs []string) error { + if action == firewall.ActionDrop { + pos := 1 + if chain == chainRTFwdIn { + pos = 2 + } + return r.iptablesClient.Insert(tableFilter, chain, pos, specs...) + } + return r.iptablesClient.Append(tableFilter, chain, specs...) +} + +// removeFilterSpecs deletes the already-installed rules of a partially +// applied filter rule. +func (r *family) removeFilterSpecs(chain string, installed []filterSpecs) { + for _, fs := range installed { + if err := r.iptablesClient.DeleteIfExists(tableFilter, chain, fs.specs...); err != nil { + log.Debugf("delete partial filter rule: %v", err) + } + if fs.mangleSpecs != nil { + if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPre, fs.mangleSpecs...); err != nil { + log.Debugf("delete partial mangle rule: %v", err) + } + } + } +} + +// applyNetwork resolves a firewall.Network into the iptables match +// fragment for the given direction flag (-s or -d). Set networks +// increment the shared ipset refcount; prefixes emit a direct match; +// an empty network returns no spec ("match any"). +func (r *family) applyNetwork(flag string, network firewall.Network, prefixes []netip.Prefix) ([]string, error) { + direction := "src" + if flag == "-d" { + direction = "dst" + } + + if network.IsSet() { + // A destination set is populated later from DNS results, so unlike a + // source set it cannot be expanded into per-prefix rules. Without + // ipset such a rule is not expressible; report it instead of + // installing something broader than the policy allows. + if flag == "-d" && !r.ipsetSupported { + return nil, fmt.Errorf("destination set %s requires ipset (ip_set_hash_net and xt_set)", network.Set.HashedName()) + } + + name := r.ipsetName(network.Set.HashedName()) + if _, err := r.ipsetCounter.Increment(name, prefixes); err != nil { + return nil, fmt.Errorf("create or get ipset: %w", err) + } + + return []string{"-m", "set", matchSet, name, direction}, nil + } + if network.IsPrefix() { + return []string{flag, network.Prefix.String()}, nil + } + + // nolint:nilnil + return nil, nil +} + +// protoForFamily translates ICMP to ICMPv6 for ip6tables. +// ip6tables requires "ipv6-icmp" (or "icmpv6") instead of "icmp". +func protoForFamily(protocol firewall.Protocol, v6 bool) string { + if v6 && protocol == firewall.ProtocolICMP { + return "ipv6-icmp" + } + return string(protocol) +} + +// filterMatchSpecs returns the proto/port match fragment for a +// filtering rule. The source match (-s or -m set) is built by the +// caller and prepended. +func filterMatchSpecs(protocol string, sPort, dPort *firewall.Port) (specs []string) { + if protocol != "all" { + specs = append(specs, "-p", protocol) + } + specs = append(specs, applyPort("--sport", sPort)...) + specs = append(specs, applyPort("--dport", dPort)...) + return specs +} + +func actionToStr(action firewall.Action) string { + if action == firewall.ActionAccept { + return "ACCEPT" + } + return "DROP" +} + +func applyPort(flag string, port *firewall.Port) []string { + if port == nil { + return nil + } + + if port.IsRange && len(port.Values) == 2 { + return []string{flag, fmt.Sprintf("%d:%d", port.Values[0], port.Values[1])} + } + + if len(port.Values) > 1 { + portList := make([]string, len(port.Values)) + for i, p := range port.Values { + portList[i] = strconv.Itoa(int(p)) + } + return []string{"-m", "multiport", flag, strings.Join(portList, ",")} + } + + return []string{flag, strconv.Itoa(int(port.Values[0]))} +} diff --git a/client/firewall/iptables/interface_allower_linux.go b/client/firewall/iptables/interface_allower_linux.go new file mode 100644 index 000000000..40e9728e2 --- /dev/null +++ b/client/firewall/iptables/interface_allower_linux.go @@ -0,0 +1,93 @@ +package iptables + +import ( + "fmt" + + "github.com/coreos/go-iptables/iptables" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" +) + +// InterfaceAllower opens the NetBird interface on the iptables filter INPUT +// chain so the host firewall doesn't drop traffic the userspace firewall +// handles. It is the fallback used when nftables is unavailable (an +// iptables-legacy host). +// +// It opens INPUT only: the userspace router never forwards in the kernel. +// firewalld trust is handled by the uspfilter manager, not here. +type InterfaceAllower struct { + ifaceName string + ipt4 *iptables.IPTables + // ipt6 is nil when the interface has no IPv6 overlay address. + ipt6 *iptables.IPTables +} + +// NewInterfaceAllower builds an iptables allower for the interface. It returns +// an error when iptables is unavailable, so the caller can fall back to +// firewalld trust. +func NewInterfaceAllower(wgIface iFaceMapper) (*InterfaceAllower, error) { + ipt4, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + if err != nil { + return nil, fmt.Errorf("iptables not available: %w", err) + } + if _, err := ipt4.ListChains(tableFilter); err != nil { + return nil, fmt.Errorf("iptables filter table not available: %w", err) + } + + a := &InterfaceAllower{ifaceName: wgIface.Name(), ipt4: ipt4} + + // Missing v6 must not break the v4 path: open v4 only and continue. + if wgIface.Address().HasIPv6() { + ipt6, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) + if err != nil { + log.Warnf("ip6tables not available, opening interface on v4 only: %v", err) + } else if _, err := ipt6.ListChains(tableFilter); err != nil { + log.Warnf("ip6tables filter table not available, opening interface on v4 only: %v", err) + } else { + a.ipt6 = ipt6 + } + } + + return a, nil +} + +// Apply inserts the interface accept rule on the filter INPUT chain. It removes +// any stale rule first so an unclean exit (e.g. SIGKILL, where Close never ran) +// is recovered deterministically rather than accumulating duplicates. +func (a *InterfaceAllower) Apply() error { + var merr *multierror.Error + for _, ipt := range a.clients() { + if err := ipt.DeleteIfExists(tableFilter, chainInput, a.inputRule()...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("clean stale interface accept rule: %w", err)) + } + if err := ipt.Insert(tableFilter, chainInput, 1, a.inputRule()...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add interface accept rule: %w", err)) + } + } + return nberrors.FormatErrorOrNil(merr) +} + +// Close removes the interface accept rule. +func (a *InterfaceAllower) Close() error { + var merr *multierror.Error + for _, ipt := range a.clients() { + if err := ipt.DeleteIfExists(tableFilter, chainInput, a.inputRule()...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove interface accept rule: %w", err)) + } + } + return nberrors.FormatErrorOrNil(merr) +} + +func (a *InterfaceAllower) inputRule() []string { + return []string{"-i", a.ifaceName, "-j", "ACCEPT"} +} + +func (a *InterfaceAllower) clients() []*iptables.IPTables { + clients := []*iptables.IPTables{a.ipt4} + if a.ipt6 != nil { + clients = append(clients, a.ipt6) + } + return clients +} diff --git a/client/firewall/iptables/ipset_linux.go b/client/firewall/iptables/ipset_linux.go new file mode 100644 index 000000000..2a3685af7 --- /dev/null +++ b/client/firewall/iptables/ipset_linux.go @@ -0,0 +1,131 @@ +//go:build !android + +package iptables + +import ( + "fmt" + "net/netip" + + "github.com/google/uuid" + "github.com/hashicorp/go-multierror" + "github.com/lrh3321/ipset-go" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" +) + +// probeIPSetSupport checks whether the kernel can create the ipset type +// used for source and destination matches. On kernels lacking the +// required ipset hash module, set creation fails (e.g. "invalid +// argument"), which would otherwise fail every multi-source rule and +// leave traffic the policy permits blocked by the catch-all drop. When +// unsupported, multi-source rules fall back to one rule per prefix. +func (r *family) probeIPSetSupport() bool { + // Use a unique name so concurrent processes don't collide and we only ever + // destroy the set we created ourselves. ipset names are limited to 31 chars, + // so use a short random suffix. + probeName := "nb-probe-" + uuid.New().String()[:8] + + if err := r.createIPSet(probeName); err != nil { + log.Warnf("ipset is not available (failed to create probe set: %v); "+ + "falling back to per-IP iptables ACL rules. Ensure the kernel provides "+ + "the ipset hash:net module (ip_set_hash_net) for better performance with large rule sets", err) + return false + } + + if err := r.destroyIPSet(probeName); err != nil { + log.Debugf("destroy ipset probe set %q: %v", probeName, err) + } + + return true +} + +func (r *family) createIpSet(setName string, sources []netip.Prefix) error { + if err := r.createIPSet(setName); err != nil { + return fmt.Errorf("create set %s: %w", setName, err) + } + + for _, prefix := range sources { + if err := r.addPrefixToIPSet(setName, prefix); err != nil { + // The refcounter records nothing when this callback errors, + // so destroy the set or it leaks in the kernel. A partial + // source set would also fail-open for deny rules, so the + // rule must fail rather than install with a missing source. + if derr := r.destroyIPSet(setName); derr != nil { + log.Warnf("rollback ipset %s after add failure: %v", setName, derr) + } + return fmt.Errorf("add element to set %s: %w", setName, err) + } + } + + return nil +} + +func (r *family) deleteIpSet(setName string) error { + if err := r.destroyIPSet(setName); err != nil { + return fmt.Errorf("destroy set %s: %w", setName, err) + } + + log.Debugf("deleted unused ipset %s", setName) + return nil +} + +func (r *family) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { + name := r.ipsetName(set.HashedName()) + var merr *multierror.Error + for _, prefix := range prefixes { + if err := r.addPrefixToIPSet(name, prefix); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add prefix to ipset: %w", err)) + } + } + if merr == nil { + log.Debugf("updated set %s with prefixes %v", name, prefixes) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) ipsetName(name string) string { + if r.v6 { + return name + "-v6" + } + return name +} + +func (r *family) createIPSet(name string) error { + opts := ipset.CreateOptions{ + Replace: true, + } + if r.v6 { + opts.Family = ipset.FamilyIPV6 + } + + if err := ipset.Create(name, ipset.TypeHashNet, opts); err != nil { + return fmt.Errorf("create ipset %s: %w", name, err) + } + + log.Debugf("created ipset %s with type hash:net", name) + return nil +} + +func (r *family) addPrefixToIPSet(name string, prefix netip.Prefix) error { + addr := prefix.Addr() + ip := addr.AsSlice() + + entry := &ipset.Entry{ + IP: ip, + CIDR: uint8(prefix.Bits()), + Replace: true, + } + + if err := ipset.Add(name, entry); err != nil { + return fmt.Errorf("add prefix to ipset %s: %w", name, err) + } + + return nil +} + +func (r *family) destroyIPSet(name string) error { + return ipset.Destroy(name) +} diff --git a/client/firewall/iptables/manager_linux.go b/client/firewall/iptables/manager_linux.go index 696537dd8..a566909c8 100644 --- a/client/firewall/iptables/manager_linux.go +++ b/client/firewall/iptables/manager_linux.go @@ -3,7 +3,6 @@ package iptables import ( "context" "fmt" - "net" "net/netip" "sync" @@ -18,25 +17,20 @@ import ( "github.com/netbirdio/netbird/client/internal/statemanager" ) -type resetter interface { - Reset() error -} - -// Manager of iptables firewall +// Manager of iptables firewall. Per-family state (peer ACLs, route +// ACLs, NAT, DNAT, MSS clamping) lives on family; Manager dispatches +// by family and provides the public firewall.Manager surface. type Manager struct { mutex sync.Mutex wgIface iFaceMapper - ipv4Client *iptables.IPTables - aclMgr *aclManager - router *router - rawSupported bool + ipv4Client *iptables.IPTables + family4 *family // IPv6 counterparts, nil when no v6 overlay ipv6Client *iptables.IPTables - aclMgr6 *aclManager - router6 *router + family6 *family } // iFaceMapper defines subset methods of interface required for manager @@ -57,14 +51,9 @@ func Create(wgIface iFaceMapper, mtu uint16) (*Manager, error) { ipv4Client: iptablesClient, } - m.router, err = newRouter(iptablesClient, wgIface, mtu) + m.family4, err = newFamily(iptablesClient, wgIface, mtu) if err != nil { - return nil, fmt.Errorf("create router: %w", err) - } - - m.aclMgr, err = newAclManager(iptablesClient, wgIface) - if err != nil { - return nil, fmt.Errorf("create acl manager: %w", err) + return nil, fmt.Errorf("create family: %w", err) } if wgIface.Address().HasIPv6() { @@ -81,21 +70,18 @@ func (m *Manager) createIPv6Components(wgIface iFaceMapper, mtu uint16) error { if err != nil { return fmt.Errorf("init ip6tables: %w", err) } + + family6, err := newFamily(ip6Client, wgIface, mtu) + if err != nil { + return fmt.Errorf("create v6 family: %w", err) + } + + // Share the same IP forwarding state with the v4 family, since the + // forwarding refcounter is per-family but shared between both families. + family6.ipFwdState = m.family4.ipFwdState + m.ipv6Client = ip6Client - - m.router6, err = newRouter(ip6Client, wgIface, mtu) - if err != nil { - return fmt.Errorf("create v6 router: %w", err) - } - - // Share the same IP forwarding state with the v4 router, since - // EnableIPForwarding controls both v4 and v6 sysctls. - m.router6.ipFwdState = m.router.ipFwdState - - m.aclMgr6, err = newAclManager(ip6Client, wgIface) - if err != nil { - return fmt.Errorf("create v6 acl manager: %w", err) - } + m.family6 = family6 return nil } @@ -109,7 +95,7 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error { InterfaceState: &InterfaceState{ NameStr: m.wgIface.Name(), WGAddress: m.wgIface.Address(), - MTU: m.router.mtu, + MTU: m.family4.mtu, }, } stateManager.RegisterState(state) @@ -121,10 +107,6 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error { return err } - if err := m.initNoTrackChain(); err != nil { - log.Warnf("raw table not available, notrack rules will be disabled: %v", err) - } - // Trust after all fatal init steps so a later failure doesn't leave the // interface in firewalld's trusted zone without a corresponding Close. if err := firewalld.TrustInterface(m.wgIface.Name()); err != nil { @@ -141,31 +123,24 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error { return nil } -// initChains initializes router and ACL chains for both address families, -// rolling back on failure. +// initChains initializes the per-family firewall state for both +// address families, rolling back on failure. func (m *Manager) initChains(stateManager *statemanager.Manager) error { type initStep struct { name string - init func(*statemanager.Manager) error - mgr resetter + r *family } - steps := []initStep{ - {"router", m.router.init, m.router}, - {"acl manager", m.aclMgr.init, m.aclMgr}, - } + steps := []initStep{{"v4", m.family4}} if m.hasIPv6() { - steps = append(steps, - initStep{"v6 router", m.router6.init, m.router6}, - initStep{"v6 acl manager", m.aclMgr6.init, m.aclMgr6}, - ) + steps = append(steps, initStep{"v6", m.family6}) } var initialized []initStep for _, s := range steps { - if err := s.init(stateManager); err != nil { + if err := s.r.init(stateManager); err != nil { for i := len(initialized) - 1; i >= 0; i-- { - if rerr := initialized[i].mgr.Reset(); rerr != nil { + if rerr := initialized[i].r.Reset(); rerr != nil { log.Warnf("rollback %s: %v", initialized[i].name, rerr) } } @@ -176,84 +151,50 @@ func (m *Manager) initChains(stateManager *statemanager.Manager) error { return nil } -// AddPeerFiltering adds a rule to the firewall -// -// Comment will be ignored because some system this feature is not supported -func (m *Manager) AddPeerFiltering( - id []byte, - ip net.IP, - proto firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, - ipsetName string, -) ([]firewall.Rule, error) { - m.mutex.Lock() - defer m.mutex.Unlock() - - if ip.To4() != nil { - return m.aclMgr.AddPeerFiltering(id, ip, proto, sPort, dPort, action, ipsetName) - } - if !m.hasIPv6() { - return nil, fmt.Errorf("add peer filtering for %s: %w", ip, firewall.ErrIPv6NotInitialized) - } - return m.aclMgr6.AddPeerFiltering(id, ip, proto, sPort, dPort, action, ipsetName) -} - -func (m *Manager) AddRouteFiltering( +// AddFilterRule installs a packet-filtering rule. See firewall.Manager +// docs for destination semantics. Sources are a single address family; +// the rule is dispatched to the matching v4 / v6 backend. +func (m *Manager) AddFilterRule( id []byte, sources []netip.Prefix, destination firewall.Network, proto firewall.Protocol, - sPort, dPort *firewall.Port, + sPort *firewall.Port, + dPort *firewall.Port, action firewall.Action, ) (firewall.Rule, error) { + if len(sources) == 0 { + return nil, firewall.ErrNoSources + } + m.mutex.Lock() defer m.mutex.Unlock() - if isIPv6RouteRule(sources, destination) { + fam := m.family4 + if isIPv6Rule(sources, destination) { if !m.hasIPv6() { - return nil, fmt.Errorf("add route filtering: %w", firewall.ErrIPv6NotInitialized) + return nil, fmt.Errorf("add filtering: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddRouteFiltering(id, sources, destination, proto, sPort, dPort, action) + fam = m.family6 } - - return m.router.AddRouteFiltering(id, sources, destination, proto, sPort, dPort, action) + return fam.AddFilterRule(id, sources, destination, proto, sPort, dPort, action) } -func isIPv6RouteRule(sources []netip.Prefix, destination firewall.Network) bool { - if destination.IsPrefix() { - return destination.Prefix.Addr().Is6() - } - return len(sources) > 0 && sources[0].Addr().Is6() -} - -// DeletePeerRule from the firewall by rule definition -func (m *Manager) DeletePeerRule(rule firewall.Rule) error { +// DeleteFilterRule removes a rule previously added via AddFilterRule. +// The rule is looked up by id in each family's filter cache. +func (m *Manager) DeleteFilterRule(rule firewall.Rule) error { m.mutex.Lock() defer m.mutex.Unlock() - if m.hasIPv6() && isIPv6IptRule(rule) { - return m.aclMgr6.DeletePeerRule(rule) + id := rule.ID() + if m.family4.hasRule(id) { + return m.family4.DeleteFilterRule(rule) } - return m.aclMgr.DeletePeerRule(rule) -} - -func isIPv6IptRule(rule firewall.Rule) bool { - r, ok := rule.(*Rule) - return ok && r.v6 -} - -// DeleteRouteRule deletes a routing rule. -// Route rules are keyed by content hash. Check v4 first, try v6 if not found. -func (m *Manager) DeleteRouteRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if m.hasIPv6() && !m.router.hasRule(rule.ID()) { - return m.router6.DeleteRouteRule(rule) + if m.hasIPv6() && m.family6.hasRule(id) { + return m.family6.DeleteFilterRule(rule) } - return m.router.DeleteRouteRule(rule) + log.Debugf("filter rule %s not found in any family", id) + return nil } func (m *Manager) IsServerRouteSupported() bool { @@ -272,10 +213,10 @@ func (m *Manager) AddNatRule(pair firewall.RouterPair) error { if !m.hasIPv6() { return fmt.Errorf("add NAT rule: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddNatRule(pair) + return m.family6.AddNatRule(pair) } - if err := m.router.AddNatRule(pair); err != nil { + if err := m.family4.AddNatRule(pair); err != nil { return err } @@ -284,7 +225,7 @@ func (m *Manager) AddNatRule(pair firewall.RouterPair) error { // wildcard 0.0.0.0/0 destination where the client resolves DNS. if m.hasIPv6() && pair.Dynamic { v6Pair := firewall.ToV6NatPair(pair) - if err := m.router6.AddNatRule(v6Pair); err != nil { + if err := m.family6.AddNatRule(v6Pair); err != nil { return fmt.Errorf("add v6 NAT rule: %w", err) } } @@ -300,18 +241,18 @@ func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error { if !m.hasIPv6() { return nil } - return m.router6.RemoveNatRule(pair) + return m.family6.RemoveNatRule(pair) } var merr *multierror.Error - if err := m.router.RemoveNatRule(pair); err != nil { + if err := m.family4.RemoveNatRule(pair); err != nil { merr = multierror.Append(merr, fmt.Errorf("remove v4 NAT rule: %w", err)) } if m.hasIPv6() && pair.Dynamic { v6Pair := firewall.ToV6NatPair(pair) - if err := m.router6.RemoveNatRule(v6Pair); err != nil { + if err := m.family6.RemoveNatRule(v6Pair); err != nil { merr = multierror.Append(merr, fmt.Errorf("remove v6 NAT rule: %w", err)) } } @@ -320,11 +261,14 @@ func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error { } func (m *Manager) SetLegacyManagement(isLegacy bool) error { - if err := firewall.SetLegacyManagement(m.router, isLegacy); err != nil { + m.mutex.Lock() + defer m.mutex.Unlock() + + if err := firewall.SetLegacyManagement(m.family4, isLegacy); err != nil { return err } if m.hasIPv6() { - return firewall.SetLegacyManagement(m.router6, isLegacy) + return firewall.SetLegacyManagement(m.family6, isLegacy) } return nil } @@ -336,24 +280,14 @@ func (m *Manager) Close(stateManager *statemanager.Manager) error { var merr *multierror.Error - if err := m.cleanupNoTrackChain(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("cleanup notrack chain: %w", err)) - } - if m.hasIPv6() { - if err := m.aclMgr6.Reset(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("reset v6 acl manager: %w", err)) - } - if err := m.router6.Reset(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("reset v6 router: %w", err)) + if err := m.family6.Reset(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("reset v6 family: %w", err)) } } - if err := m.aclMgr.Reset(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("reset acl manager: %w", err)) - } - if err := m.router.Reset(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("reset router: %w", err)) + if err := m.family4.Reset(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("reset family: %w", err)) } // Appending to merr intentionally blocks DeleteState below so ShutdownState @@ -372,27 +306,6 @@ func (m *Manager) Close(stateManager *statemanager.Manager) error { return nberrors.FormatErrorOrNil(merr) } -// AllowNetbird allows netbird interface traffic. -// This is called when USPFilter wraps the native firewall, adding blanket accept -// rules so that packet filtering is handled in userspace instead of by netfilter. -func (m *Manager) AllowNetbird() error { - var merr *multierror.Error - if _, err := m.AddPeerFiltering(nil, net.IP{0, 0, 0, 0}, firewall.ProtocolALL, nil, nil, firewall.ActionAccept, ""); err != nil { - merr = multierror.Append(merr, fmt.Errorf("allow netbird v4 interface traffic: %w", err)) - } - if m.hasIPv6() { - if _, err := m.AddPeerFiltering(nil, net.IPv6zero, firewall.ProtocolALL, nil, nil, firewall.ActionAccept, ""); err != nil { - merr = multierror.Append(merr, fmt.Errorf("allow netbird v6 interface traffic: %w", err)) - } - } - - if err := firewalld.TrustInterface(m.wgIface.Name()); err != nil { - log.Warnf("failed to trust interface in firewalld: %v", err) - } - - return nberrors.FormatErrorOrNil(merr) -} - // Flush doesn't need to be implemented for this manager func (m *Manager) Flush() error { return nil } @@ -402,42 +315,12 @@ func (m *Manager) SetLogLevel(log.Level) { } func (m *Manager) EnableRouting() error { - if err := m.router.ipFwdState.RequestForwarding(); err != nil { - return fmt.Errorf("enable IP forwarding: %w", err) - } - return nil + // v6 only when the overlay actually has v6. + return m.family4.ipFwdState.RequestRouting(m.hasIPv6()) } func (m *Manager) DisableRouting() error { - if err := m.router.ipFwdState.ReleaseForwarding(); err != nil { - return fmt.Errorf("disable IP forwarding: %w", err) - } - return nil -} - -// AddDNATRule adds a DNAT rule -func (m *Manager) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - m.mutex.Lock() - defer m.mutex.Unlock() - - if rule.TranslatedAddress.Is6() { - if !m.hasIPv6() { - return nil, fmt.Errorf("add DNAT rule: %w", firewall.ErrIPv6NotInitialized) - } - return m.router6.AddDNATRule(rule) - } - return m.router.AddDNATRule(rule) -} - -// DeleteDNATRule deletes a DNAT rule -func (m *Manager) DeleteDNATRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if m.hasIPv6() && !m.router.hasRule(rule.ID()+dnatSuffix) { - return m.router6.DeleteDNATRule(rule) - } - return m.router.DeleteDNATRule(rule) + return m.family4.ipFwdState.ReleaseRouting() } // UpdateSet updates the set with the given prefixes @@ -454,12 +337,12 @@ func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { } } - if err := m.router.UpdateSet(set, v4Prefixes); err != nil { + if err := m.family4.UpdateSet(set, v4Prefixes); err != nil { return err } if m.hasIPv6() && len(v6Prefixes) > 0 { - if err := m.router6.UpdateSet(set, v6Prefixes); err != nil { + if err := m.family6.UpdateSet(set, v6Prefixes); err != nil { return fmt.Errorf("update v6 set: %w", err) } } @@ -476,9 +359,9 @@ func (m *Manager) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protoco if !m.hasIPv6() { return fmt.Errorf("add inbound DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) } // RemoveInboundDNAT removes an inbound DNAT rule. @@ -490,9 +373,9 @@ func (m *Manager) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Prot if !m.hasIPv6() { return fmt.Errorf("remove inbound DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) } // AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic. @@ -504,9 +387,9 @@ func (m *Manager) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol if !m.hasIPv6() { return fmt.Errorf("add output DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) } // RemoveOutputDNAT removes an OUTPUT chain DNAT rule. @@ -518,139 +401,21 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto if !m.hasIPv6() { return fmt.Errorf("remove output DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) -} - -const ( - chainNameRaw = "NETBIRD-RAW" - chainOUTPUT = "OUTPUT" - tableRaw = "raw" -) - -// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic. -// This prevents conntrack from tracking WireGuard proxy traffic on loopback, which -// can interfere with MASQUERADE rules (e.g., from container runtimes like Podman/netavark). -// -// Traffic flows that need NOTRACK: -// -// 1. Egress: WireGuard -> fake endpoint (before eBPF rewrite) -// src=127.0.0.1:wgPort -> dst=127.0.0.1:fakePort -// Matched by: sport=wgPort -// -// 2. Egress: Proxy -> WireGuard (via raw socket) -// src=127.0.0.1:fakePort -> dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 3. Ingress: Packets to WireGuard -// dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 4. Ingress: Packets to proxy (after eBPF rewrite) -// dst=127.0.0.1:proxyPort -// Matched by: dport=proxyPort -// -// Rules are cleaned up when the firewall manager is closed. -func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if !m.rawSupported { - return fmt.Errorf("raw table not available") - } - - wgPortStr := fmt.Sprintf("%d", wgPort) - proxyPortStr := fmt.Sprintf("%d", proxyPort) - - // Egress rules: match outgoing loopback UDP packets - outputRuleSport := []string{"-o", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--sport", wgPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, outputRuleSport...); err != nil { - return fmt.Errorf("add output sport notrack rule: %w", err) - } - - outputRuleDport := []string{"-o", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, outputRuleDport...); err != nil { - return fmt.Errorf("add output dport notrack rule: %w", err) - } - - // Ingress rules: match incoming loopback UDP packets - preroutingRuleWg := []string{"-i", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, preroutingRuleWg...); err != nil { - return fmt.Errorf("add prerouting wg notrack rule: %w", err) - } - - preroutingRuleProxy := []string{"-i", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", proxyPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, preroutingRuleProxy...); err != nil { - return fmt.Errorf("add prerouting proxy notrack rule: %w", err) - } - - log.Debugf("set up ebpf proxy notrack rules for ports %d,%d", proxyPort, wgPort) - return nil -} - -func (m *Manager) initNoTrackChain() error { - if err := m.cleanupNoTrackChain(); err != nil { - log.Debugf("cleanup notrack chain: %v", err) - } - - if err := m.ipv4Client.NewChain(tableRaw, chainNameRaw); err != nil { - return fmt.Errorf("create chain: %w", err) - } - - jumpRule := []string{"-j", chainNameRaw} - - if err := m.ipv4Client.InsertUnique(tableRaw, chainOUTPUT, 1, jumpRule...); err != nil { - if delErr := m.ipv4Client.DeleteChain(tableRaw, chainNameRaw); delErr != nil { - log.Debugf("delete orphan chain: %v", delErr) - } - return fmt.Errorf("add output jump rule: %w", err) - } - - if err := m.ipv4Client.InsertUnique(tableRaw, chainPREROUTING, 1, jumpRule...); err != nil { - if delErr := m.ipv4Client.DeleteIfExists(tableRaw, chainOUTPUT, jumpRule...); delErr != nil { - log.Debugf("delete output jump rule: %v", delErr) - } - if delErr := m.ipv4Client.DeleteChain(tableRaw, chainNameRaw); delErr != nil { - log.Debugf("delete orphan chain: %v", delErr) - } - return fmt.Errorf("add prerouting jump rule: %w", err) - } - - m.rawSupported = true - return nil -} - -func (m *Manager) cleanupNoTrackChain() error { - exists, err := m.ipv4Client.ChainExists(tableRaw, chainNameRaw) - if err != nil { - if !m.rawSupported { - return nil - } - return fmt.Errorf("check chain exists: %w", err) - } - if !exists { - return nil - } - - jumpRule := []string{"-j", chainNameRaw} - - if err := m.ipv4Client.DeleteIfExists(tableRaw, chainOUTPUT, jumpRule...); err != nil { - return fmt.Errorf("remove output jump rule: %w", err) - } - - if err := m.ipv4Client.DeleteIfExists(tableRaw, chainPREROUTING, jumpRule...); err != nil { - return fmt.Errorf("remove prerouting jump rule: %w", err) - } - - if err := m.ipv4Client.ClearAndDeleteChain(tableRaw, chainNameRaw); err != nil { - return fmt.Errorf("clear and delete chain: %w", err) - } - - m.rawSupported = false - return nil + return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } func getConntrackEstablished() []string { return []string{"-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"} } + +// isIPv6Rule reports whether the rule belongs to the IPv6 family, from +// the destination prefix when set, otherwise from the (single-family) +// sources. +func isIPv6Rule(sources []netip.Prefix, destination firewall.Network) bool { + if destination.IsPrefix() { + return destination.Prefix.Addr().Is6() + } + return len(sources) > 0 && sources[0].Addr().Is6() +} diff --git a/client/firewall/iptables/manager_linux_test.go b/client/firewall/iptables/manager_linux_test.go index 7b0989f6c..8435bf6a5 100644 --- a/client/firewall/iptables/manager_linux_test.go +++ b/client/firewall/iptables/manager_linux_test.go @@ -5,16 +5,19 @@ package iptables import ( "fmt" "net/netip" + "slices" "strings" "testing" "time" "github.com/coreos/go-iptables/iptables" + "github.com/lrh3321/ipset-go" "github.com/stretchr/testify/require" fw "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/iface" "github.com/netbirdio/netbird/client/iface/wgaddr" + "github.com/netbirdio/netbird/shared/management/domain" ) var ifaceMock = &iFaceMock{ @@ -67,47 +70,37 @@ func TestIptablesManager(t *testing.T) { time.Sleep(time.Second) }() - var rule2 []fw.Rule + var rule2 fw.Rule t.Run("add second rule", func(t *testing.T) { ip := netip.MustParseAddr("10.20.0.3") port := &fw.Port{ IsRange: true, Values: []uint16{8043, 8046}, } - rule2, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", port, nil, fw.ActionAccept, "") + rule2, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", port, nil, fw.ActionAccept) require.NoError(t, err, "failed to add rule") - for _, r := range rule2 { - rr := r.(*Rule) - checkRuleSpecs(t, ipv4Client, rr.chain, true, rr.specs...) - } + rr := rule2.(*Rule) + checkRuleSpecs(t, ipv4Client, rr.chain, true, rr.specs...) }) t.Run("delete second rule", func(t *testing.T) { - for _, r := range rule2 { - err := manager.DeletePeerRule(r) - require.NoError(t, err, "failed to delete rule") - } - - require.Empty(t, manager.aclMgr.ipsetStore.ipsets, "rulesets index after removed second rule must be empty") + require.NoError(t, manager.DeleteFilterRule(rule2), "failed to delete rule") }) t.Run("reset check", func(t *testing.T) { // add second rule ip := netip.MustParseAddr("10.20.0.3") port := &fw.Port{Values: []uint16{5353}} - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "udp", nil, port, fw.ActionAccept, "") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "udp", nil, port, fw.ActionAccept) require.NoError(t, err, "failed to add rule") err = manager.Close(nil) require.NoError(t, err, "failed to reset") - ok, err := ipv4Client.ChainExists("filter", chainNameInputRules) + ok, err := ipv4Client.ChainExists("filter", chainACLInput) require.NoError(t, err, "failed check chain exists") - - if ok { - require.NoErrorf(t, err, "chain '%v' still exists after Close", chainNameInputRules) - } + require.Falsef(t, ok, "chain %q still exists after Close", chainACLInput) }) } @@ -128,15 +121,13 @@ func TestIptablesManagerDenyRules(t *testing.T) { ip := netip.MustParseAddr("10.20.0.3") port := &fw.Port{Values: []uint16{22}} - rule, err := manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionDrop, "deny-ssh") + rule, err := manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", nil, port, fw.ActionDrop) require.NoError(t, err, "failed to add deny rule") - require.NotEmpty(t, rule, "deny rule should not be empty") + require.NotNil(t, rule, "deny rule should not be nil") // Verify the rule was added by checking iptables - for _, r := range rule { - rr := r.(*Rule) - checkRuleSpecs(t, ipv4Client, rr.chain, true, rr.specs...) - } + rr := rule.(*Rule) + checkRuleSpecs(t, ipv4Client, rr.chain, true, rr.specs...) }) t.Run("deny rule precedence test", func(t *testing.T) { @@ -144,36 +135,40 @@ func TestIptablesManagerDenyRules(t *testing.T) { port := &fw.Port{Values: []uint16{80}} // Add accept rule first - _, err := manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "accept-http") + _, err := manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", nil, port, fw.ActionAccept) require.NoError(t, err, "failed to add accept rule") // Add deny rule second for same IP/port - this should take precedence - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionDrop, "deny-http") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", nil, port, fw.ActionDrop) require.NoError(t, err, "failed to add deny rule") // Inspect the actual iptables rules to verify deny rule comes before accept rule - rules, err := ipv4Client.List("filter", chainNameInputRules) + rules, err := ipv4Client.List("filter", chainACLInput) require.NoError(t, err, "failed to list iptables rules") // Debug: print all rules - t.Logf("All iptables rules in chain %s:", chainNameInputRules) + t.Logf("All iptables rules in chain %s:", chainACLInput) for i, rule := range rules { t.Logf(" [%d] %s", i, rule) } + // Single-source rules emit a direct `-s /32 ... --dport 80` + // match. Match on that shape instead of the legacy + // per-(action,port) ipset names ("deny-http"/"accept-http") + // that this test predates. + srcMatch := fmt.Sprintf("-s %s/32", ip) var denyRuleIndex, acceptRuleIndex = -1, -1 for i, rule := range rules { - if strings.Contains(rule, "DROP") { - t.Logf("Found DROP rule at index %d: %s", i, rule) - if strings.Contains(rule, "deny-http") && strings.Contains(rule, "80") { - denyRuleIndex = i - } + if !strings.Contains(rule, srcMatch) || !strings.Contains(rule, "--dport 80") { + continue } - if strings.Contains(rule, "ACCEPT") { + if strings.Contains(rule, "-j DROP") { + t.Logf("Found DROP rule at index %d: %s", i, rule) + denyRuleIndex = i + } + if strings.Contains(rule, "-j ACCEPT") { t.Logf("Found ACCEPT rule at index %d: %s", i, rule) - if strings.Contains(rule, "accept-http") && strings.Contains(rule, "80") { - acceptRuleIndex = i - } + acceptRuleIndex = i } } @@ -198,7 +193,6 @@ func TestIptablesManagerIPSet(t *testing.T) { }, } - // just check on the local interface manager, err := Create(mock, iface.DefaultMTU) require.NoError(t, err) require.NoError(t, manager.Init(nil)) @@ -212,27 +206,39 @@ func TestIptablesManagerIPSet(t *testing.T) { time.Sleep(time.Second) }() - var rule2 []fw.Rule - t.Run("add second rule", func(t *testing.T) { + var rule2 fw.Rule + t.Run("single source uses direct -s match (no ipset)", func(t *testing.T) { ip := netip.MustParseAddr("10.20.0.3") port := &fw.Port{ Values: []uint16{443}, } - rule2, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", port, nil, fw.ActionAccept, "default") - for _, r := range rule2 { - require.NoError(t, err, "failed to add rule") - require.Equal(t, r.(*Rule).ipsetName, "default-sport", "ipset name must be set") - require.Equal(t, r.(*Rule).ip, "10.20.0.3", "ipset IP must be set") - } + rule2, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", port, nil, fw.ActionAccept) + require.NoError(t, err, "failed to add rule") + require.NotNil(t, rule2) + require.Contains(t, rule2.(*Rule).specs, "-s", + "single-source rule should use direct -s match, not an ipset") + require.Empty(t, findSets(rule2.(*Rule).specs), + "single-source rule should not allocate a shared ipset") }) - t.Run("delete second rule", func(t *testing.T) { - for _, r := range rule2 { - err := manager.DeletePeerRule(r) - require.NoError(t, err, "failed to delete rule") + t.Run("delete single-source rule", func(t *testing.T) { + require.NoError(t, manager.DeleteFilterRule(rule2), "failed to delete rule") + }) - require.Empty(t, manager.aclMgr.ipsetStore.ipsets, "rulesets index after removed second rule must be empty") + t.Run("multi-source uses shared ipset", func(t *testing.T) { + sources := []netip.Prefix{ + netip.PrefixFrom(netip.MustParseAddr("10.20.0.3"), 32), + netip.PrefixFrom(netip.MustParseAddr("10.20.0.4"), 32), + netip.PrefixFrom(netip.MustParseAddr("10.20.0.5"), 32), } + port := &fw.Port{Values: []uint16{8080}} + multi, err := manager.AddFilterRule(nil, sources, fw.Network{}, "tcp", nil, port, fw.ActionAccept) + require.NoError(t, err, "failed to add multi-source rule") + require.NotNil(t, multi, "multi-source rule must produce one iptables rule") + sets := findSets(multi.(*Rule).specs) + require.Len(t, sets, 1, "multi-source rule must reference exactly one ipset") + + require.NoError(t, manager.DeleteFilterRule(multi)) }) t.Run("reset check", func(t *testing.T) { @@ -241,9 +247,314 @@ func TestIptablesManagerIPSet(t *testing.T) { }) } +// TestIptablesFilterIPSetFallback verifies that when the kernel lacks +// ipset support, a multi-source rule falls back to one iptables rule +// per source prefix instead of silently leaving the chain empty. See +// discussion #6125. +func TestIptablesFilterIPSetFallback(t *testing.T) { + ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + require.NoError(t, err) + + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + defer func() { + require.NoError(t, manager.Close(nil)) + }() + + // Simulate a kernel without the ipset hash module. + manager.family4.ipsetSupported = false + + sources := []netip.Prefix{ + netip.MustParsePrefix("10.20.0.42/32"), + netip.MustParsePrefix("10.20.0.43/32"), + } + port := &fw.Port{Values: []uint16{22}} + + rule, err := manager.AddFilterRule(nil, sources, fw.Network{}, "tcp", nil, port, fw.ActionAccept) + require.NoError(t, err, "AddFilterRule should succeed via fallback") + + rr := rule.(*Rule) + all := rr.allSpecs() + require.Len(t, all, len(sources), "each source prefix needs its own rule") + for i, fs := range all { + joined := strings.Join(fs.specs, " ") + require.Contains(t, joined, "-s "+sources[i].String(), "fallback rule must match by source prefix") + require.NotContains(t, joined, matchSet, "fallback rule must not use ipset matching") + + // The rule must actually be present in the ACL chain (not silently dropped). + checkRuleSpecs(t, ipv4Client, rr.chain, true, fs.specs...) + + // Every expanded peer rule keeps its own redirect-mark pairing. + require.NotNil(t, fs.mangleSpecs, "peer rule must carry a mangle pairing") + checkTableRuleSpecs(t, ipv4Client, tableMangle, chainRTPre, true, fs.mangleSpecs...) + } + + require.NoError(t, manager.DeleteFilterRule(rule), "failed to delete fallback rule") + for _, fs := range all { + checkRuleSpecs(t, ipv4Client, rr.chain, false, fs.specs...) + checkTableRuleSpecs(t, ipv4Client, tableMangle, chainRTPre, false, fs.mangleSpecs...) + } +} + +// TestIptablesFilterDestinationSetRequiresIPSet documents that a dynamic +// (domain) destination cannot be expressed without ipset: its prefixes are only +// known after DNS resolution, so there is nothing to expand into per-prefix +// rules. The call must report that rather than install a broader rule than the +// policy allows. +func TestIptablesFilterDestinationSetRequiresIPSet(t *testing.T) { + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + defer func() { + require.NoError(t, manager.Close(nil)) + }() + + manager.family4.ipsetSupported = false + + destination := fw.Network{Set: fw.NewDomainSet(domain.List{"example.com"})} + + _, err = manager.AddFilterRule(nil, []netip.Prefix{netip.MustParsePrefix("172.16.0.0/16")}, + destination, fw.ProtocolALL, nil, nil, fw.ActionAccept) + require.Error(t, err, "a domain destination is not expressible without ipset") + require.ErrorContains(t, err, "requires ipset") +} + +// TestIptablesNatRuleDropsSourceSetOnDestinationFailure covers a marking rule +// whose source set is created but whose destination set is not: the source +// reference has to go back, or the set it created stays in the kernel with a +// count nothing will ever drop. +func TestIptablesNatRuleDropsSourceSetOnDestinationFailure(t *testing.T) { + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + defer func() { + require.NoError(t, manager.Close(nil)) + }() + + sourceSet := fw.NewPrefixSet([]netip.Prefix{ + netip.MustParsePrefix("100.0.0.0/16"), + netip.MustParsePrefix("10.10.0.0/16"), + }) + destSet := fw.NewDomainSet(domain.List{"example.org"}) + + // Poison the destination set's name so its hash:net creation fails after + // the source set has already been created. + poisoned := manager.family4.ipsetName(destSet.HashedName()) + require.NoError(t, ipset.Create(poisoned, ipset.TypeHashIP, ipset.CreateOptions{})) + t.Cleanup(func() { + if err := ipset.Destroy(poisoned); err != nil { + t.Logf("destroy poisoned set %s: %v", poisoned, err) + } + }) + + pair := fw.RouterPair{ + ID: "nat-source-set-test", + Source: fw.Network{Set: sourceSet}, + Destination: fw.Network{Set: destSet}, + Masquerade: true, + Dynamic: true, + } + + require.Error(t, manager.AddNatRule(pair), "the destination set must fail to be created") + + _, ok := manager.family4.ipsetCounter.Get(manager.family4.ipsetName(sourceSet.HashedName())) + require.False(t, ok, "the source set reference must be released") +} + +// TestIptablesNatRuleReAddKeepsSetReferences re-adds the same NAT rule the way +// a repeated network-map update does. The marking rule's set references must not +// grow, or RemoveNatRule can never drop the count to zero and the set stays in +// the kernel for the rest of the process lifetime. +func TestIptablesNatRuleReAddKeepsSetReferences(t *testing.T) { + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + defer func() { + require.NoError(t, manager.Close(nil)) + }() + + set := fw.NewDomainSet(domain.List{"example.com"}) + pair := fw.RouterPair{ + ID: "nat-reference-test", + Source: fw.Network{Prefix: netip.MustParsePrefix("100.0.0.0/16")}, + Destination: fw.Network{Set: set}, + Masquerade: true, + Dynamic: true, + } + + require.NoError(t, manager.AddNatRule(pair), "add nat rule") + name := manager.family4.ipsetName(set.HashedName()) + first, ok := manager.family4.ipsetCounter.Get(name) + require.True(t, ok, "the marking rule must hold a reference to its set") + + require.NoError(t, manager.AddNatRule(pair), "re-add nat rule") + second, ok := manager.family4.ipsetCounter.Get(name) + require.True(t, ok, "the set must still be referenced") + require.Equal(t, first.Count, second.Count, "re-adding the same rule must not add references") + + require.NoError(t, manager.RemoveNatRule(pair), "remove nat rule") + _, ok = manager.family4.ipsetCounter.Get(name) + require.False(t, ok, "removing the rule must drop the last reference") +} + +// TestIptablesRouteFilterIPSetFallback covers the route ACL side of the +// fallback: with a destination set, the expanded per-source rules land +// in the route forward chain and are all removed on delete. +func TestIptablesRouteFilterIPSetFallback(t *testing.T) { + ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + require.NoError(t, err) + + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + defer func() { + require.NoError(t, manager.Close(nil)) + }() + + manager.family4.ipsetSupported = false + + sources := []netip.Prefix{ + netip.MustParsePrefix("172.16.0.0/16"), + netip.MustParsePrefix("192.168.0.0/16"), + } + destination := fw.Network{Prefix: netip.MustParsePrefix("10.0.0.0/8")} + port := &fw.Port{Values: []uint16{443}} + + rule, err := manager.AddFilterRule(nil, sources, destination, "tcp", nil, port, fw.ActionAccept) + require.NoError(t, err, "route ACL must install without ipset") + + rr := rule.(*Rule) + require.Equal(t, chainRTFwdIn, rr.chain, "route rule must land in the forward chain") + + all := rr.allSpecs() + require.Len(t, all, len(sources), "each source prefix needs its own rule") + for i, fs := range all { + joined := strings.Join(fs.specs, " ") + require.Contains(t, joined, "-s "+sources[i].String(), "fallback rule must match by source prefix") + require.NotContains(t, joined, matchSet, "fallback rule must not use ipset matching") + require.Nil(t, fs.mangleSpecs, "route rules have no mangle pairing") + + checkRuleSpecs(t, ipv4Client, rr.chain, true, fs.specs...) + } + + require.NoError(t, manager.DeleteFilterRule(rule), "failed to delete fallback rule") + for _, fs := range all { + checkRuleSpecs(t, ipv4Client, rr.chain, false, fs.specs...) + } +} + +// TestIptablesCloseRemovesAllState exercises a spread of rule kinds and then +// asserts Close puts every table it touches back exactly as it found it. A +// leaked chain, jump, or ipset survives the daemon and nothing can remove it +// afterwards, since the tracking that knew about it is gone. +func TestIptablesCloseRemovesAllState(t *testing.T) { + ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + require.NoError(t, err) + + before := snapshotIptables(t, ipv4Client) + + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + // A failed assertion below returns before the Close under test, which would + // leave this test's chains and sets in the kernel for the next one. + t.Cleanup(func() { + if err := manager.Close(nil); err != nil { + t.Logf("close after failure: %v", err) + } + }) + + sources := []netip.Prefix{ + netip.MustParsePrefix("10.20.0.42/32"), + netip.MustParsePrefix("10.20.0.43/32"), + } + + // A multi-source peer rule: shared ipset plus the mangle redirect pairing. + _, err = manager.AddFilterRule(nil, sources, fw.Network{}, "tcp", + nil, &fw.Port{Values: []uint16{22}}, fw.ActionAccept) + require.NoError(t, err, "add peer rule") + + // A route rule with a dynamic destination: a second set, in the forward chain. + _, err = manager.AddFilterRule(nil, sources, + fw.Network{Set: fw.NewDomainSet(domain.List{"example.com"})}, + fw.ProtocolALL, nil, nil, fw.ActionDrop) + require.NoError(t, err, "add route rule") + + // NAT marking for a routed destination, both directions. + pair := fw.RouterPair{ + ID: "cleanup-test", + Source: fw.Network{Prefix: netip.MustParsePrefix("100.0.0.0/16")}, + Destination: fw.Network{Prefix: netip.MustParsePrefix("192.168.55.0/24")}, + Masquerade: true, + } + require.NoError(t, manager.AddNatRule(pair), "add nat rule") + require.NoError(t, manager.EnableRouting(), "enable routing") + + require.NotEqual(t, before, snapshotIptables(t, ipv4Client), "the manager must have installed state") + + // Everything above stays in place, so Close is what has to remove it. + require.NoError(t, manager.Close(nil), "close") + + after := snapshotIptables(t, ipv4Client) + require.Equal(t, before.chains, after.chains, "Close must remove every chain it created") + require.Equal(t, before.rules, after.rules, "Close must remove every rule it created") + require.Equal(t, before.sets, after.sets, "Close must destroy every ipset it created") +} + +// iptablesState is a snapshot of the tables the manager writes to, used to +// compare the kernel before and after a manager lifetime. +type iptablesState struct { + chains map[string][]string + rules map[string][]string + sets []string +} + +func snapshotIptables(t *testing.T, client *iptables.IPTables) iptablesState { + t.Helper() + + state := iptablesState{ + chains: map[string][]string{}, + rules: map[string][]string{}, + } + + for _, table := range []string{tableFilter, tableNat, tableMangle, tableRaw} { + chains, err := client.ListChains(table) + require.NoErrorf(t, err, "list chains in %s", table) + slices.Sort(chains) + state.chains[table] = chains + + for _, chain := range chains { + rules, err := client.List(table, chain) + require.NoErrorf(t, err, "list rules in %s/%s", table, chain) + state.rules[table+"/"+chain] = rules + } + } + + sets, err := ipset.ListAll() + require.NoError(t, err, "list ipsets") + for _, set := range sets { + state.sets = append(state.sets, set.SetName) + } + slices.Sort(state.sets) + + return state +} + func checkRuleSpecs(t *testing.T, ipv4Client *iptables.IPTables, chainName string, mustExists bool, rulespec ...string) { t.Helper() - exists, err := ipv4Client.Exists("filter", chainName, rulespec...) + checkTableRuleSpecs(t, ipv4Client, tableFilter, chainName, mustExists, rulespec...) +} + +func checkTableRuleSpecs(t *testing.T, ipv4Client *iptables.IPTables, table, chainName string, mustExists bool, rulespec ...string) { + t.Helper() + exists, err := ipv4Client.Exists(table, chainName, rulespec...) require.NoError(t, err, "failed to check rule") require.Falsef(t, !exists && mustExists, "rule '%v' does not exist", rulespec) require.Falsef(t, exists && !mustExists, "rule '%v' exist", rulespec) @@ -283,7 +594,7 @@ func TestIptablesCreatePerformance(t *testing.T) { start := time.Now() for i := 0; i < testMax; i++ { port := &fw.Port{Values: []uint16{uint16(1000 + i)}} - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", nil, port, fw.ActionAccept) require.NoError(t, err, "failed to add rule") } diff --git a/client/firewall/iptables/router_linux.go b/client/firewall/iptables/router_linux.go deleted file mode 100644 index 42d305f5c..000000000 --- a/client/firewall/iptables/router_linux.go +++ /dev/null @@ -1,1154 +0,0 @@ -//go:build !android - -package iptables - -import ( - "fmt" - "maps" - "net/netip" - "strconv" - "strings" - - "github.com/coreos/go-iptables/iptables" - "github.com/hashicorp/go-multierror" - ipset "github.com/lrh3321/ipset-go" - log "github.com/sirupsen/logrus" - - nberrors "github.com/netbirdio/netbird/client/errors" - firewall "github.com/netbirdio/netbird/client/firewall/manager" - nbid "github.com/netbirdio/netbird/client/internal/acl/id" - "github.com/netbirdio/netbird/client/internal/routemanager/ipfwdstate" - "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" - "github.com/netbirdio/netbird/client/internal/statemanager" - nbnet "github.com/netbirdio/netbird/client/net" -) - -// constants needed to manage and create iptable rules -const ( - tableFilter = "filter" - tableNat = "nat" - tableMangle = "mangle" - - chainPOSTROUTING = "POSTROUTING" - chainPREROUTING = "PREROUTING" - chainFORWARD = "FORWARD" - chainRTNAT = "NETBIRD-RT-NAT" - chainRTFWDIN = "NETBIRD-RT-FWD-IN" - chainRTFWDOUT = "NETBIRD-RT-FWD-OUT" - chainRTPRE = "NETBIRD-RT-PRE" - chainRTRDR = "NETBIRD-RT-RDR" - chainNATOutput = "NETBIRD-NAT-OUTPUT" - chainRTMSSCLAMP = "NETBIRD-RT-MSSCLAMP" - routingFinalForwardJump = "ACCEPT" - routingFinalNatJump = "MASQUERADE" - - jumpManglePre = "jump-mangle-pre" - jumpNatPre = "jump-nat-pre" - jumpNatPost = "jump-nat-post" - jumpNatOutput = "jump-nat-output" - jumpMSSClamp = "jump-mss-clamp" - markManglePre = "mark-mangle-pre" - markManglePost = "mark-mangle-post" - matchSet = "--match-set" - - dnatSuffix = "_dnat" - snatSuffix = "_snat" - fwdSuffix = "_fwd" - - // ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation. - ipv4TCPHeaderSize = 40 - // ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation. - ipv6TCPHeaderSize = 60 -) - -type ruleInfo struct { - chain string - table string - rule []string -} - -type routeFilteringRuleParams struct { - Source firewall.Network - Destination firewall.Network - Proto firewall.Protocol - SPort *firewall.Port - DPort *firewall.Port - Direction firewall.RuleDirection - Action firewall.Action -} - -type routeRules map[string][]string - -// the ipset library currently does not support comments, so we use the name only (string) -type ipsetCounter = refcounter.Counter[string, []netip.Prefix, struct{}] - -type router struct { - iptablesClient *iptables.IPTables - rules routeRules - ipsetCounter *ipsetCounter - wgIface iFaceMapper - legacyManagement bool - mtu uint16 - v6 bool - - stateManager *statemanager.Manager - ipFwdState *ipfwdstate.IPForwardingState -} - -func newRouter(iptablesClient *iptables.IPTables, wgIface iFaceMapper, mtu uint16) (*router, error) { - r := &router{ - iptablesClient: iptablesClient, - rules: make(map[string][]string), - wgIface: wgIface, - mtu: mtu, - v6: iptablesClient.Proto() == iptables.ProtocolIPv6, - ipFwdState: ipfwdstate.NewIPForwardingState(), - } - - r.ipsetCounter = refcounter.New( - func(name string, sources []netip.Prefix) (struct{}, error) { - return struct{}{}, r.createIpSet(name, sources) - }, - func(name string, _ struct{}) error { - return r.deleteIpSet(name) - }, - ) - - return r, nil -} - -func (r *router) init(stateManager *statemanager.Manager) error { - r.stateManager = stateManager - - if err := r.cleanUpDefaultForwardRules(); err != nil { - log.Errorf("failed to clean up rules from FORWARD chain: %s", err) - } - - if err := r.createContainers(); err != nil { - return fmt.Errorf("create containers: %w", err) - } - - if err := r.setupDataPlaneMark(); err != nil { - log.Errorf("failed to set up data plane mark: %v", err) - } - - r.updateState() - - return nil -} - -func (r *router) AddRouteFiltering( - id []byte, - sources []netip.Prefix, - destination firewall.Network, - proto firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, -) (firewall.Rule, error) { - ruleKey := nbid.GenerateRouteRuleKey(sources, destination, proto, sPort, dPort, action) - if _, ok := r.rules[string(ruleKey)]; ok { - return ruleKey, nil - } - - var source firewall.Network - if len(sources) > 1 { - source.Set = firewall.NewPrefixSet(sources) - } else if len(sources) > 0 { - source.Prefix = sources[0] - } - - params := routeFilteringRuleParams{ - Source: source, - Destination: destination, - Proto: proto, - SPort: sPort, - DPort: dPort, - Action: action, - } - - rule, err := r.genRouteRuleSpec(params, sources) - if err != nil { - return nil, fmt.Errorf("generate route rule spec: %w", err) - } - - // Insert DROP rules at the beginning, append ACCEPT rules at the end - if action == firewall.ActionDrop { - // after the established rule - err = r.iptablesClient.Insert(tableFilter, chainRTFWDIN, 2, rule...) - } else { - err = r.iptablesClient.Append(tableFilter, chainRTFWDIN, rule...) - } - - if err != nil { - return nil, fmt.Errorf("add route rule: %v", err) - } - - r.rules[string(ruleKey)] = rule - - r.updateState() - - return ruleKey, nil -} - -func (r *router) hasRule(id string) bool { - _, ok := r.rules[id] - return ok -} - -func (r *router) DeleteRouteRule(rule firewall.Rule) error { - ruleKey := rule.ID() - - if rule, exists := r.rules[ruleKey]; exists { - if err := r.iptablesClient.Delete(tableFilter, chainRTFWDIN, rule...); err != nil { - return fmt.Errorf("delete route rule: %v", err) - } - delete(r.rules, ruleKey) - - if err := r.decrementSetCounter(rule); err != nil { - return fmt.Errorf("decrement ipset counter: %w", err) - } - } else { - log.Debugf("route rule %s not found", ruleKey) - } - - r.updateState() - - return nil -} - -func (r *router) decrementSetCounter(rule []string) error { - sets := r.findSets(rule) - var merr *multierror.Error - for _, setName := range sets { - if _, err := r.ipsetCounter.Decrement(setName); err != nil { - merr = multierror.Append(merr, fmt.Errorf("decrement counter: %w", err)) - } - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) findSets(rule []string) []string { - var sets []string - for i, arg := range rule { - if arg == "-m" && i+3 < len(rule) && rule[i+1] == "set" && rule[i+2] == matchSet { - sets = append(sets, rule[i+3]) - } - } - return sets -} - -func (r *router) createIpSet(setName string, sources []netip.Prefix) error { - if err := r.createIPSet(setName); err != nil { - return fmt.Errorf("create set %s: %w", setName, err) - } - - for _, prefix := range sources { - if err := r.addPrefixToIPSet(setName, prefix); err != nil { - return fmt.Errorf("add element to set %s: %w", setName, err) - } - } - - return nil -} - -func (r *router) deleteIpSet(setName string) error { - if err := r.destroyIPSet(setName); err != nil { - return fmt.Errorf("destroy set %s: %w", setName, err) - } - - log.Debugf("Deleted unused ipset %s", setName) - return nil -} - -// AddNatRule inserts an iptables rule pair into the nat chain -func (r *router) AddNatRule(pair firewall.RouterPair) error { - if r.legacyManagement { - log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination) - if err := r.addLegacyRouteRule(pair); err != nil { - return fmt.Errorf("add legacy routing rule: %w", err) - } - } - - if !pair.Masquerade { - return nil - } - - if err := r.addNatRule(pair); err != nil { - return fmt.Errorf("add nat rule: %w", err) - } - - if err := r.addNatRule(firewall.GetInversePair(pair)); err != nil { - return fmt.Errorf("add inverse nat rule: %w", err) - } - - r.updateState() - - return nil -} - -// RemoveNatRule removes an iptables rule pair from forwarding and nat chains -func (r *router) RemoveNatRule(pair firewall.RouterPair) error { - if pair.Masquerade { - if err := r.removeNatRule(pair); err != nil { - return fmt.Errorf("remove nat rule: %w", err) - } - - if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil { - return fmt.Errorf("remove inverse nat rule: %w", err) - } - } - - if err := r.removeLegacyRouteRule(pair); err != nil { - return fmt.Errorf("remove legacy routing rule: %w", err) - } - - r.updateState() - - return nil -} - -// addLegacyRouteRule adds a legacy routing rule for mgmt servers pre route acls -func (r *router) addLegacyRouteRule(pair firewall.RouterPair) error { - ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair) - - if err := r.removeLegacyRouteRule(pair); err != nil { - return err - } - - rule := []string{"-s", pair.Source.String(), "-d", pair.Destination.String(), "-j", routingFinalForwardJump} - if err := r.iptablesClient.Append(tableFilter, chainRTFWDIN, rule...); err != nil { - return fmt.Errorf("add legacy forwarding rule %s -> %s: %v", pair.Source, pair.Destination, err) - } - - r.rules[ruleKey] = rule - - return nil -} - -func (r *router) removeLegacyRouteRule(pair firewall.RouterPair) error { - ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair) - - if rule, exists := r.rules[ruleKey]; exists { - if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFWDIN, rule...); err != nil { - return fmt.Errorf("remove legacy forwarding rule %s -> %s: %v", pair.Source, pair.Destination, err) - } - delete(r.rules, ruleKey) - - if err := r.decrementSetCounter(rule); err != nil { - return fmt.Errorf("decrement ipset counter: %w", err) - } - } - - return nil -} - -// GetLegacyManagement returns the current legacy management mode -func (r *router) GetLegacyManagement() bool { - return r.legacyManagement -} - -// SetLegacyManagement sets the route manager to use legacy management mode -func (r *router) SetLegacyManagement(isLegacy bool) { - r.legacyManagement = isLegacy -} - -// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls -func (r *router) RemoveAllLegacyRouteRules() error { - var merr *multierror.Error - for k, rule := range r.rules { - if !strings.HasPrefix(k, firewall.ForwardingFormatPrefix) { - continue - } - if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFWDIN, rule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %v", err)) - } else { - delete(r.rules, k) - } - } - - r.updateState() - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) Reset() error { - var merr *multierror.Error - if err := r.cleanUpDefaultForwardRules(); err != nil { - merr = multierror.Append(merr, err) - } - - if err := r.ipsetCounter.Flush(); err != nil { - merr = multierror.Append(merr, err) - } - - if err := r.cleanupDataPlaneMark(); err != nil { - merr = multierror.Append(merr, err) - } - - r.rules = make(map[string][]string) - r.updateState() - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) cleanUpDefaultForwardRules() error { - if err := r.cleanJumpRules(); err != nil { - return fmt.Errorf("clean jump rules: %w", err) - } - - log.Debug("flushing routing related tables") - - // Remove jump rules from built-in chains before deleting custom chains, - // otherwise the chain deletion fails with "device or resource busy". - if ok, err := r.iptablesClient.ChainExists(tableNat, chainNATOutput); err != nil { - return fmt.Errorf("check chain %s: %w", chainNATOutput, err) - } else if ok { - jumpRule := []string{"-j", chainNATOutput} - if err := r.iptablesClient.Delete(tableNat, "OUTPUT", jumpRule...); err != nil { - log.Debugf("clean OUTPUT jump rule: %v", err) - } - } - - for _, chainInfo := range []struct { - chain string - table string - }{ - {chainRTFWDIN, tableFilter}, - {chainRTFWDOUT, tableFilter}, - {chainRTPRE, tableMangle}, - {chainRTNAT, tableNat}, - {chainRTRDR, tableNat}, - {chainNATOutput, tableNat}, - {chainRTMSSCLAMP, tableMangle}, - } { - ok, err := r.iptablesClient.ChainExists(chainInfo.table, chainInfo.chain) - if err != nil { - return fmt.Errorf("check chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err) - } else if ok { - if err = r.iptablesClient.ClearAndDeleteChain(chainInfo.table, chainInfo.chain); err != nil { - return fmt.Errorf("clear and delete chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err) - } - } - } - - return nil -} - -func (r *router) createContainers() error { - for _, chainInfo := range []struct { - chain string - table string - }{ - {chainRTFWDIN, tableFilter}, - {chainRTFWDOUT, tableFilter}, - {chainRTPRE, tableMangle}, - {chainRTNAT, tableNat}, - {chainRTRDR, tableNat}, - {chainRTMSSCLAMP, tableMangle}, - } { - // Fallback: clear chains that survived an unclean shutdown. - if ok, _ := r.iptablesClient.ChainExists(chainInfo.table, chainInfo.chain); ok { - if err := r.iptablesClient.ClearAndDeleteChain(chainInfo.table, chainInfo.chain); err != nil { - log.Warnf("clear stale chain %s in %s: %v", chainInfo.chain, chainInfo.table, err) - } - } - if err := r.iptablesClient.NewChain(chainInfo.table, chainInfo.chain); err != nil { - return fmt.Errorf("create chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err) - } - } - - if err := r.insertEstablishedRule(chainRTFWDIN); err != nil { - return fmt.Errorf("insert established rule: %w", err) - } - - if err := r.insertEstablishedRule(chainRTFWDOUT); err != nil { - return fmt.Errorf("insert established rule: %w", err) - } - - if err := r.addPostroutingRules(); err != nil { - return fmt.Errorf("add static nat rules: %w", err) - } - - if err := r.addJumpRules(); err != nil { - return fmt.Errorf("add jump rules: %w", err) - } - - if err := r.addMSSClampingRules(); err != nil { - log.Errorf("failed to add MSS clamping rules: %s", err) - } - - return nil -} - -// setupDataPlaneMark configures the fwmark for the data plane -func (r *router) setupDataPlaneMark() error { - var merr *multierror.Error - preRule := []string{ - "-i", r.wgIface.Name(), - "-m", "conntrack", "--ctstate", "NEW", - "-j", "CONNMARK", "--set-mark", fmt.Sprintf("%#x", nbnet.DataPlaneMarkIn), - } - - if err := r.iptablesClient.AppendUnique(tableMangle, chainPREROUTING, preRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("add mangle prerouting rule: %w", err)) - } else { - r.rules[markManglePre] = preRule - } - - postRule := []string{ - "-o", r.wgIface.Name(), - "-m", "conntrack", "--ctstate", "NEW", - "-j", "CONNMARK", "--set-mark", fmt.Sprintf("%#x", nbnet.DataPlaneMarkOut), - } - - if err := r.iptablesClient.AppendUnique(tableMangle, chainPOSTROUTING, postRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("add mangle postrouting rule: %w", err)) - } else { - r.rules[markManglePost] = postRule - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) cleanupDataPlaneMark() error { - var merr *multierror.Error - if preRule, exists := r.rules[markManglePre]; exists { - if err := r.iptablesClient.DeleteIfExists(tableMangle, chainPREROUTING, preRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove mangle prerouting rule: %w", err)) - } else { - delete(r.rules, markManglePre) - } - } - - if postRule, exists := r.rules[markManglePost]; exists { - if err := r.iptablesClient.DeleteIfExists(tableMangle, chainPOSTROUTING, postRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove mangle postrouting rule: %w", err)) - } else { - delete(r.rules, markManglePost) - } - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) addPostroutingRules() error { - // First rule for outbound masquerade - rule1 := []string{ - "-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade), - "!", "-o", "lo", - "-j", routingFinalNatJump, - } - if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule1...); err != nil { - return fmt.Errorf("add outbound masquerade rule: %v", err) - } - r.rules["static-nat-outbound"] = rule1 - - // Second rule for return traffic masquerade - rule2 := []string{ - "-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn), - "-o", r.wgIface.Name(), - "-j", routingFinalNatJump, - } - if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule2...); err != nil { - return fmt.Errorf("add return masquerade rule: %v", err) - } - r.rules["static-nat-return"] = rule2 - - return nil -} - -// addMSSClampingRules adds MSS clamping rules to prevent fragmentation for forwarded traffic. -func (r *router) addMSSClampingRules() error { - overhead := uint16(ipv4TCPHeaderSize) - if r.v6 { - overhead = ipv6TCPHeaderSize - } - mss := r.mtu - overhead - - // Add jump rule from FORWARD chain in mangle table to our custom chain - jumpRule := []string{ - "-j", chainRTMSSCLAMP, - } - if err := r.iptablesClient.Insert(tableMangle, chainFORWARD, 1, jumpRule...); err != nil { - return fmt.Errorf("add jump to MSS clamp chain: %w", err) - } - r.rules[jumpMSSClamp] = jumpRule - - ruleOut := []string{ - "-o", r.wgIface.Name(), - "-p", "tcp", - "--tcp-flags", "SYN,RST", "SYN", - "-j", "TCPMSS", - "--set-mss", fmt.Sprintf("%d", mss), - } - if err := r.iptablesClient.Append(tableMangle, chainRTMSSCLAMP, ruleOut...); err != nil { - return fmt.Errorf("add outbound MSS clamp rule: %w", err) - } - r.rules["mss-clamp-out"] = ruleOut - - return nil -} - -func (r *router) insertEstablishedRule(chain string) error { - establishedRule := getConntrackEstablished() - - err := r.iptablesClient.Insert(tableFilter, chain, 1, establishedRule...) - if err != nil { - return fmt.Errorf("failed to insert established rule: %v", err) - } - - ruleKey := "established-" + chain - r.rules[ruleKey] = establishedRule - - return nil -} - -func (r *router) addJumpRules() error { - // Jump to nat chain - natRule := []string{"-j", chainRTNAT} - if err := r.iptablesClient.Insert(tableNat, chainPOSTROUTING, 1, natRule...); err != nil { - return fmt.Errorf("add nat postrouting jump rule: %v", err) - } - r.rules[jumpNatPost] = natRule - - // Jump to mangle prerouting chain - preRule := []string{"-j", chainRTPRE} - if err := r.iptablesClient.Insert(tableMangle, chainPREROUTING, 1, preRule...); err != nil { - return fmt.Errorf("add mangle prerouting jump rule: %v", err) - } - r.rules[jumpManglePre] = preRule - - // Jump to nat prerouting chain - rdrRule := []string{"-j", chainRTRDR} - if err := r.iptablesClient.Insert(tableNat, chainPREROUTING, 1, rdrRule...); err != nil { - return fmt.Errorf("add nat prerouting jump rule: %v", err) - } - r.rules[jumpNatPre] = rdrRule - - return nil -} - -func (r *router) cleanJumpRules() error { - for _, ruleKey := range []string{jumpNatPost, jumpManglePre, jumpNatPre, jumpMSSClamp} { - if rule, exists := r.rules[ruleKey]; exists { - var table, chain string - switch ruleKey { - case jumpNatPost: - table = tableNat - chain = chainPOSTROUTING - case jumpManglePre: - table = tableMangle - chain = chainPREROUTING - case jumpNatPre: - table = tableNat - chain = chainPREROUTING - case jumpMSSClamp: - table = tableMangle - chain = chainFORWARD - default: - return fmt.Errorf("unknown jump rule: %s", ruleKey) - } - - if err := r.iptablesClient.DeleteIfExists(table, chain, rule...); err != nil { - return fmt.Errorf("delete rule from chain %s in table %s, err: %v", chain, table, err) - } - delete(r.rules, ruleKey) - } - } - return nil -} - -func (r *router) addNatRule(pair firewall.RouterPair) error { - ruleKey := firewall.GenKey(firewall.NatFormat, pair) - - if rule, exists := r.rules[ruleKey]; exists { - if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPRE, rule...); err != nil { - return fmt.Errorf("error while removing existing marking rule for %s: %v", pair.Destination, err) - } - delete(r.rules, ruleKey) - } - - markValue := nbnet.PreroutingFwmarkMasquerade - if pair.Inverse { - markValue = nbnet.PreroutingFwmarkMasqueradeReturn - } - - rule := []string{"-i", r.wgIface.Name()} - if pair.Inverse { - rule = []string{"!", "-i", r.wgIface.Name()} - } - - rule = append(rule, - "-m", "conntrack", - "--ctstate", "NEW", - ) - sourceExp, err := r.applyNetwork("-s", pair.Source, nil) - if err != nil { - return fmt.Errorf("apply network -s: %w", err) - } - destExp, err := r.applyNetwork("-d", pair.Destination, nil) - if err != nil { - return fmt.Errorf("apply network -d: %w", err) - } - - rule = append(rule, sourceExp...) - rule = append(rule, destExp...) - rule = append(rule, - "-j", "MARK", "--set-mark", fmt.Sprintf("%#x", markValue), - ) - - // Ensure nat rules come first, so the mark can be overwritten. - // Currently overwritten by the dst-type LOCAL rules for redirected traffic. - if err := r.iptablesClient.Insert(tableMangle, chainRTPRE, 1, rule...); err != nil { - // TODO: rollback ipset counter - return fmt.Errorf("error while adding marking rule for %s: %v", pair.Destination, err) - } - - r.rules[ruleKey] = rule - - r.updateState() - return nil -} - -func (r *router) removeNatRule(pair firewall.RouterPair) error { - ruleKey := firewall.GenKey(firewall.NatFormat, pair) - - if rule, exists := r.rules[ruleKey]; exists { - if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPRE, rule...); err != nil { - return fmt.Errorf("error while removing marking rule for %s: %v", pair.Destination, err) - } - delete(r.rules, ruleKey) - - if err := r.decrementSetCounter(rule); err != nil { - return fmt.Errorf("decrement ipset counter: %w", err) - } - } else { - log.Debugf("marking rule %s not found", ruleKey) - } - - r.updateState() - return nil -} - -func (r *router) updateState() { - if r.stateManager == nil { - return - } - - var currentState *ShutdownState - if existing := r.stateManager.GetState(currentState); existing != nil { - if existingState, ok := existing.(*ShutdownState); ok { - currentState = existingState - } - } - if currentState == nil { - currentState = &ShutdownState{} - } - - currentState.Lock() - defer currentState.Unlock() - - // Clone the rule map so the persisted state holds a private snapshot. The - // live map keeps being mutated by subsequent rule operations while the - // state manager marshals the state from its periodic-save goroutine. - // Sharing it by reference races the two and aborts the process with a - // concurrent map iteration and write. The ipset counter guards itself - // during marshaling, so it can be shared directly. - if r.v6 { - currentState.RouteRules6 = maps.Clone(r.rules) - currentState.RouteIPsetCounter6 = r.ipsetCounter - } else { - currentState.RouteRules = maps.Clone(r.rules) - currentState.RouteIPsetCounter = r.ipsetCounter - } - - if err := r.stateManager.UpdateState(currentState); err != nil { - log.Errorf("failed to update state: %v", err) - } -} - -func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - if err := r.ipFwdState.RequestForwarding(); err != nil { - return nil, err - } - - ruleKey := rule.ID() - if _, exists := r.rules[ruleKey+dnatSuffix]; exists { - return rule, nil - } - - toDestination := rule.TranslatedAddress.String() - switch { - case len(rule.TranslatedPort.Values) == 0: - // no translated port, use original port - case len(rule.TranslatedPort.Values) == 1: - toDestination += fmt.Sprintf(":%d", rule.TranslatedPort.Values[0]) - case rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2: - // need the "/originalport" suffix to avoid dnat port randomization - toDestination += fmt.Sprintf(":%d-%d/%d", rule.TranslatedPort.Values[0], rule.TranslatedPort.Values[1], rule.DestinationPort.Values[0]) - default: - return nil, fmt.Errorf("invalid translated port: %v", rule.TranslatedPort) - } - - proto := strings.ToLower(string(rule.Protocol)) - - rules := make(map[string]ruleInfo, 3) - - // DNAT rule - dnatRule := []string{ - "!", "-i", r.wgIface.Name(), - "-p", proto, - "-j", "DNAT", - "--to-destination", toDestination, - } - dnatRule = append(dnatRule, applyPort("--dport", &rule.DestinationPort)...) - rules[ruleKey+dnatSuffix] = ruleInfo{ - table: tableNat, - chain: chainRTRDR, - rule: dnatRule, - } - - // SNAT rule - snatRule := []string{ - "-o", r.wgIface.Name(), - "-p", proto, - "-d", rule.TranslatedAddress.String(), - "-j", "MASQUERADE", - } - snatRule = append(snatRule, applyPort("--dport", &rule.TranslatedPort)...) - rules[ruleKey+snatSuffix] = ruleInfo{ - table: tableNat, - chain: chainRTNAT, - rule: snatRule, - } - - // Forward filtering rule, if fwd policy is DROP - forwardRule := []string{ - "-o", r.wgIface.Name(), - "-p", proto, - "-d", rule.TranslatedAddress.String(), - "-j", "ACCEPT", - } - forwardRule = append(forwardRule, applyPort("--dport", &rule.TranslatedPort)...) - rules[ruleKey+fwdSuffix] = ruleInfo{ - table: tableFilter, - chain: chainRTFWDOUT, - rule: forwardRule, - } - - for key, ruleInfo := range rules { - if err := r.iptablesClient.Append(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { - if rollbackErr := r.rollbackRules(rules); rollbackErr != nil { - log.Errorf("rollback failed: %v", rollbackErr) - } - return nil, fmt.Errorf("add rule %s: %w", key, err) - } - r.rules[key] = ruleInfo.rule - } - - r.updateState() - return rule, nil -} - -func (r *router) rollbackRules(rules map[string]ruleInfo) error { - var merr *multierror.Error - for key, ruleInfo := range rules { - if err := r.iptablesClient.DeleteIfExists(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("rollback rule %s: %w", key, err)) - // On rollback error, add to rules map for next cleanup - r.rules[key] = ruleInfo.rule - } - } - if merr != nil { - r.updateState() - } - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) DeleteDNATRule(rule firewall.Rule) error { - if err := r.ipFwdState.ReleaseForwarding(); err != nil { - log.Errorf("%v", err) - } - - ruleKey := rule.ID() - - var merr *multierror.Error - if dnatRule, exists := r.rules[ruleKey+dnatSuffix]; exists { - if err := r.iptablesClient.Delete(tableNat, chainRTRDR, dnatRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete DNAT rule: %w", err)) - } - delete(r.rules, ruleKey+dnatSuffix) - } - - if snatRule, exists := r.rules[ruleKey+snatSuffix]; exists { - if err := r.iptablesClient.Delete(tableNat, chainRTNAT, snatRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete SNAT rule: %w", err)) - } - delete(r.rules, ruleKey+snatSuffix) - } - - if fwdRule, exists := r.rules[ruleKey+fwdSuffix]; exists { - if err := r.iptablesClient.Delete(tableFilter, chainRTFWDOUT, fwdRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete forward rule: %w", err)) - } - delete(r.rules, ruleKey+fwdSuffix) - } - - r.updateState() - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) genRouteRuleSpec(params routeFilteringRuleParams, sources []netip.Prefix) ([]string, error) { - var rule []string - - sourceExp, err := r.applyNetwork("-s", params.Source, sources) - if err != nil { - return nil, fmt.Errorf("apply network -s: %w", err) - - } - destExp, err := r.applyNetwork("-d", params.Destination, nil) - if err != nil { - return nil, fmt.Errorf("apply network -d: %w", err) - } - - rule = append(rule, sourceExp...) - rule = append(rule, destExp...) - - if params.Proto != firewall.ProtocolALL { - rule = append(rule, "-p", strings.ToLower(protoForFamily(params.Proto, r.v6))) - rule = append(rule, applyPort("--sport", params.SPort)...) - rule = append(rule, applyPort("--dport", params.DPort)...) - } - - rule = append(rule, "-j", actionToStr(params.Action)) - - return rule, nil -} - -func (r *router) applyNetwork(flag string, network firewall.Network, prefixes []netip.Prefix) ([]string, error) { - direction := "src" - if flag == "-d" { - direction = "dst" - } - - if network.IsSet() { - name := r.ipsetName(network.Set.HashedName()) - if _, err := r.ipsetCounter.Increment(name, prefixes); err != nil { - return nil, fmt.Errorf("create or get ipset: %w", err) - } - - return []string{"-m", "set", matchSet, name, direction}, nil - } - if network.IsPrefix() { - return []string{flag, network.Prefix.String()}, nil - } - - // nolint:nilnil - return nil, nil -} - -func (r *router) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { - name := r.ipsetName(set.HashedName()) - var merr *multierror.Error - for _, prefix := range prefixes { - if err := r.addPrefixToIPSet(name, prefix); err != nil { - merr = multierror.Append(merr, fmt.Errorf("add prefix to ipset: %w", err)) - } - } - if merr == nil { - log.Debugf("updated set %s with prefixes %v", name, prefixes) - } - - return nberrors.FormatErrorOrNil(merr) -} - -// AddInboundDNAT adds an inbound DNAT rule redirecting traffic from NetBird peers to local services. -func (r *router) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - ruleID := fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - if _, exists := r.rules[ruleID]; exists { - return nil - } - - dnatRule := []string{ - "-i", r.wgIface.Name(), - "-p", strings.ToLower(protoForFamily(protocol, r.v6)), - "--dport", strconv.Itoa(int(originalPort)), - "-d", localAddr.String(), - "-m", "addrtype", "--dst-type", "LOCAL", - "-j", "DNAT", - "--to-destination", ":" + strconv.Itoa(int(translatedPort)), - } - - ruleInfo := ruleInfo{ - table: tableNat, - chain: chainRTRDR, - rule: dnatRule, - } - - if err := r.iptablesClient.Append(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { - return fmt.Errorf("add inbound DNAT rule: %w", err) - } - r.rules[ruleID] = ruleInfo.rule - - r.updateState() - return nil -} - -// RemoveInboundDNAT removes an inbound DNAT rule. -func (r *router) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - ruleID := fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - if dnatRule, exists := r.rules[ruleID]; exists { - if err := r.iptablesClient.Delete(tableNat, chainRTRDR, dnatRule...); err != nil { - return fmt.Errorf("delete inbound DNAT rule: %w", err) - } - delete(r.rules, ruleID) - } - - r.updateState() - return nil -} - -// ensureNATOutputChain lazily creates the OUTPUT NAT chain and jump rule on first use. -func (r *router) ensureNATOutputChain() error { - if _, exists := r.rules[jumpNatOutput]; exists { - return nil - } - - chainExists, err := r.iptablesClient.ChainExists(tableNat, chainNATOutput) - if err != nil { - return fmt.Errorf("check chain %s: %w", chainNATOutput, err) - } - if !chainExists { - if err := r.iptablesClient.NewChain(tableNat, chainNATOutput); err != nil { - return fmt.Errorf("create chain %s: %w", chainNATOutput, err) - } - } - - jumpRule := []string{"-j", chainNATOutput} - if err := r.iptablesClient.Insert(tableNat, "OUTPUT", 1, jumpRule...); err != nil { - if !chainExists { - if delErr := r.iptablesClient.ClearAndDeleteChain(tableNat, chainNATOutput); delErr != nil { - log.Warnf("failed to rollback chain %s: %v", chainNATOutput, delErr) - } - } - return fmt.Errorf("add OUTPUT jump rule: %w", err) - } - r.rules[jumpNatOutput] = jumpRule - - r.updateState() - return nil -} - -// AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic. -func (r *router) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - ruleID := fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - if _, exists := r.rules[ruleID]; exists { - return nil - } - - if err := r.ensureNATOutputChain(); err != nil { - return err - } - - dnatRule := []string{ - "-p", strings.ToLower(protoForFamily(protocol, localAddr.Is6())), - "--dport", strconv.Itoa(int(originalPort)), - "-d", localAddr.String(), - "-j", "DNAT", - "--to-destination", ":" + strconv.Itoa(int(translatedPort)), - } - - if err := r.iptablesClient.Append(tableNat, chainNATOutput, dnatRule...); err != nil { - return fmt.Errorf("add output DNAT rule: %w", err) - } - r.rules[ruleID] = dnatRule - - r.updateState() - return nil -} - -// RemoveOutputDNAT removes an OUTPUT chain DNAT rule. -func (r *router) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - ruleID := fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - if dnatRule, exists := r.rules[ruleID]; exists { - if err := r.iptablesClient.Delete(tableNat, chainNATOutput, dnatRule...); err != nil { - return fmt.Errorf("delete output DNAT rule: %w", err) - } - delete(r.rules, ruleID) - } - - r.updateState() - return nil -} - -func applyPort(flag string, port *firewall.Port) []string { - if port == nil { - return nil - } - - if port.IsRange && len(port.Values) == 2 { - return []string{flag, fmt.Sprintf("%d:%d", port.Values[0], port.Values[1])} - } - - if len(port.Values) > 1 { - portList := make([]string, len(port.Values)) - for i, p := range port.Values { - portList[i] = strconv.Itoa(int(p)) - } - return []string{"-m", "multiport", flag, strings.Join(portList, ",")} - } - - return []string{flag, strconv.Itoa(int(port.Values[0]))} -} - -// ipsetName returns the ipset name, suffixed with "-v6" for the v6 router -// to avoid collisions since ipsets are global in the kernel. -func (r *router) ipsetName(name string) string { - if r.v6 { - return name + "-v6" - } - return name -} - -func (r *router) createIPSet(name string) error { - opts := ipset.CreateOptions{ - Replace: true, - } - if r.v6 { - opts.Family = ipset.FamilyIPV6 - } - - if err := ipset.Create(name, ipset.TypeHashNet, opts); err != nil { - return fmt.Errorf("create ipset %s: %w", name, err) - } - - log.Debugf("created ipset %s with type hash:net", name) - return nil -} - -func (r *router) addPrefixToIPSet(name string, prefix netip.Prefix) error { - addr := prefix.Addr() - ip := addr.AsSlice() - - entry := &ipset.Entry{ - IP: ip, - CIDR: uint8(prefix.Bits()), - Replace: true, - } - - if err := ipset.Add(name, entry); err != nil { - return fmt.Errorf("add prefix to ipset %s: %w", name, err) - } - - return nil -} - -func (r *router) destroyIPSet(name string) error { - return ipset.Destroy(name) -} diff --git a/client/firewall/iptables/router_linux_test.go b/client/firewall/iptables/router_linux_test.go index 9ca6b9f7e..6c4ae9425 100644 --- a/client/firewall/iptables/router_linux_test.go +++ b/client/firewall/iptables/router_linux_test.go @@ -31,7 +31,7 @@ func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) { iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) require.NoError(t, err, "failed to init iptables client") - manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU) + manager, err := newFamily(iptablesClient, ifaceMock, iface.DefaultMTU) require.NoError(t, err, "should return a valid iptables manager") require.NoError(t, manager.init(nil)) @@ -52,12 +52,12 @@ func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) { // 11. MSS clamping rule for outbound traffic require.Len(t, manager.rules, 11, "should have created rules map") - exists, err := manager.iptablesClient.Exists(tableNat, chainPOSTROUTING, "-j", chainRTNAT) - require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainPOSTROUTING) + exists, err := manager.iptablesClient.Exists(tableNat, chainPostrouting, "-j", chainRTNAT) + require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainPostrouting) require.True(t, exists, "postrouting jump rule should exist") - exists, err = manager.iptablesClient.Exists(tableMangle, chainPREROUTING, "-j", chainRTPRE) - require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainPREROUTING) + exists, err = manager.iptablesClient.Exists(tableMangle, chainPrerouting, "-j", chainRTPre) + require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainPrerouting) require.True(t, exists, "prerouting jump rule should exist") pair := firewall.RouterPair{ @@ -84,7 +84,7 @@ func TestIptablesManager_AddNatRule(t *testing.T) { iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) require.NoError(t, err, "failed to init iptables client") - manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU) + manager, err := newFamily(iptablesClient, ifaceMock, iface.DefaultMTU) require.NoError(t, err, "shouldn't return error") require.NoError(t, manager.init(nil)) @@ -95,7 +95,7 @@ func TestIptablesManager_AddNatRule(t *testing.T) { err = manager.AddNatRule(testCase.InputPair) require.NoError(t, err, "marking rule should be inserted") - natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair) + natRuleKey := testCase.InputPair.GenKey(firewall.NatFormat) markingRule := []string{ "-i", ifaceMock.Name(), "-m", "conntrack", @@ -106,8 +106,8 @@ func TestIptablesManager_AddNatRule(t *testing.T) { fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade), } - exists, err := iptablesClient.Exists(tableMangle, chainRTPRE, markingRule...) - require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE) + exists, err := iptablesClient.Exists(tableMangle, chainRTPre, markingRule...) + require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPre) if testCase.InputPair.Masquerade { require.True(t, exists, "marking rule should be created") foundRule, found := manager.rules[natRuleKey] @@ -121,7 +121,7 @@ func TestIptablesManager_AddNatRule(t *testing.T) { // Check inverse rule inversePair := firewall.GetInversePair(testCase.InputPair) - inverseRuleKey := firewall.GenKey(firewall.NatFormat, inversePair) + inverseRuleKey := inversePair.GenKey(firewall.NatFormat) inverseMarkingRule := []string{ "!", "-i", ifaceMock.Name(), "-m", "conntrack", @@ -132,8 +132,8 @@ func TestIptablesManager_AddNatRule(t *testing.T) { fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn), } - exists, err = iptablesClient.Exists(tableMangle, chainRTPRE, inverseMarkingRule...) - require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE) + exists, err = iptablesClient.Exists(tableMangle, chainRTPre, inverseMarkingRule...) + require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPre) if testCase.InputPair.Masquerade { require.True(t, exists, "inverse marking rule should be created") foundRule, found := manager.rules[inverseRuleKey] @@ -157,7 +157,7 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) { t.Run(testCase.Name, func(t *testing.T) { iptablesClient, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4) - manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU) + manager, err := newFamily(iptablesClient, ifaceMock, iface.DefaultMTU) require.NoError(t, err, "shouldn't return error") require.NoError(t, manager.init(nil)) defer func() { @@ -170,7 +170,7 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) { err = manager.RemoveNatRule(testCase.InputPair) require.NoError(t, err, "shouldn't return error") - natRuleKey := firewall.GenKey(firewall.NatFormat, testCase.InputPair) + natRuleKey := testCase.InputPair.GenKey(firewall.NatFormat) markingRule := []string{ "-i", ifaceMock.Name(), "-m", "conntrack", @@ -181,8 +181,8 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) { fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade), } - exists, err := iptablesClient.Exists(tableMangle, chainRTPRE, markingRule...) - require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE) + exists, err := iptablesClient.Exists(tableMangle, chainRTPre, markingRule...) + require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPre) require.False(t, exists, "marking rule should not exist") _, found := manager.rules[natRuleKey] @@ -190,7 +190,7 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) { // Check inverse rule removal inversePair := firewall.GetInversePair(testCase.InputPair) - inverseRuleKey := firewall.GenKey(firewall.NatFormat, inversePair) + inverseRuleKey := inversePair.GenKey(firewall.NatFormat) inverseMarkingRule := []string{ "!", "-i", ifaceMock.Name(), "-m", "conntrack", @@ -201,8 +201,8 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) { fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn), } - exists, err = iptablesClient.Exists(tableMangle, chainRTPRE, inverseMarkingRule...) - require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPRE) + exists, err = iptablesClient.Exists(tableMangle, chainRTPre, inverseMarkingRule...) + require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableMangle, chainRTPre) require.False(t, exists, "inverse marking rule should not exist") _, found = manager.rules[inverseRuleKey] @@ -219,13 +219,13 @@ func TestRouter_AddRouteFiltering(t *testing.T) { iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) require.NoError(t, err, "Failed to create iptables client") - r, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU) - require.NoError(t, err, "Failed to create router manager") + r, err := newFamily(iptablesClient, ifaceMock, iface.DefaultMTU) + require.NoError(t, err, "Failed to create family manager") require.NoError(t, r.init(nil)) defer func() { err := r.Reset() - require.NoError(t, err, "Failed to reset router") + require.NoError(t, err, "Failed to reset family") }() tests := []struct { @@ -334,62 +334,30 @@ func TestRouter_AddRouteFiltering(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ruleKey, err := r.AddRouteFiltering(nil, tt.sources, firewall.Network{Prefix: tt.destination}, tt.proto, tt.sPort, tt.dPort, tt.action) - require.NoError(t, err, "AddRouteFiltering failed") + ruleKey, err := r.AddFilterRule(nil, tt.sources, firewall.Network{Prefix: tt.destination}, tt.proto, tt.sPort, tt.dPort, tt.action) + require.NoError(t, err, "AddFilterRule failed") - // Check if the rule is in the internal map - rule, ok := r.rules[ruleKey.ID()] - assert.True(t, ok, "Rule not found in internal map") + stored, ok := r.filters[ruleKey.ID()] + require.True(t, ok, "rule not stored in filters") + t.Logf("Internal rule: %v", stored.specs) - // Log the internal rule - t.Logf("Internal rule: %v", rule) - - // Check if the rule exists in iptables - exists, err := iptablesClient.Exists(tableFilter, chainRTFWDIN, rule...) + exists, err := iptablesClient.Exists(tableFilter, chainRTFwdIn, stored.specs...) assert.NoError(t, err, "Failed to check rule existence") assert.True(t, exists, "Rule not found in iptables") - var source firewall.Network - if len(tt.sources) > 1 { - source.Set = firewall.NewPrefixSet(tt.sources) - } else if len(tt.sources) > 0 { - source.Prefix = tt.sources[0] - } - // Verify rule content - params := routeFilteringRuleParams{ - Source: source, - Destination: firewall.Network{Prefix: tt.destination}, - Proto: tt.proto, - SPort: tt.sPort, - DPort: tt.dPort, - Action: tt.action, - } - - expectedRule, err := r.genRouteRuleSpec(params, nil) - require.NoError(t, err, "Failed to generate expected rule spec") - if tt.expectSet { setName := firewall.NewPrefixSet(tt.sources).HashedName() - expectedRule, err = r.genRouteRuleSpec(params, nil) - require.NoError(t, err, "Failed to generate expected rule spec with set") - - // Check if the set was created _, exists := r.ipsetCounter.Get(setName) assert.True(t, exists, "IPSet not created") + assert.NotEmpty(t, findSets(stored.specs), "Rule should reference an ipset") } - assert.Equal(t, expectedRule, rule, "Rule content mismatch") - - // Clean up - err = r.DeleteRouteRule(ruleKey) - require.NoError(t, err, "Failed to delete rule") + require.NoError(t, r.DeleteFilterRule(ruleKey), "Failed to delete rule") }) } } func TestFindSetNameInRule(t *testing.T) { - r := &router{} - testCases := []struct { name string rule []string @@ -430,7 +398,7 @@ func TestFindSetNameInRule(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - result := r.findSets(tc.rule) + result := findSets(tc.rule) if len(result) != len(tc.expected) { t.Errorf("Expected %d sets, got %d. Sets found: %v", len(tc.expected), len(result), result) diff --git a/client/firewall/iptables/routing_linux.go b/client/firewall/iptables/routing_linux.go new file mode 100644 index 000000000..c63be4e43 --- /dev/null +++ b/client/firewall/iptables/routing_linux.go @@ -0,0 +1,273 @@ +//go:build !android + +package iptables + +import ( + "fmt" + "strings" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbnet "github.com/netbirdio/netbird/client/net" +) + +func (r *family) AddNatRule(pair firewall.RouterPair) error { + if r.legacyManagement { + log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination) + if err := r.addLegacyRouteRule(pair); err != nil { + return fmt.Errorf("add legacy routing rule: %w", err) + } + } + + if pair.Masquerade { + if err := r.addNatRule(pair); err != nil { + return fmt.Errorf("add nat rule: %w", err) + } + + if err := r.addNatRule(firewall.GetInversePair(pair)); err != nil { + return fmt.Errorf("add inverse nat rule: %w", err) + } + } + + r.updateState() + + return nil +} + +// RemoveNatRule removes an iptables rule pair from forwarding and nat chains +func (r *family) RemoveNatRule(pair firewall.RouterPair) error { + if pair.Masquerade { + if err := r.removeNatRule(pair); err != nil { + return fmt.Errorf("remove nat rule: %w", err) + } + + if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil { + return fmt.Errorf("remove inverse nat rule: %w", err) + } + } + + if err := r.removeLegacyRouteRule(pair); err != nil { + return fmt.Errorf("remove legacy routing rule: %w", err) + } + + r.updateState() + + return nil +} + +// addLegacyRouteRule adds a legacy routing rule for mgmt servers pre route acls +func (r *family) addLegacyRouteRule(pair firewall.RouterPair) error { + ruleID := pair.GenKey(firewall.ForwardingFormat) + + if err := r.removeLegacyRouteRule(pair); err != nil { + return err + } + + rule := []string{"-s", pair.Source.String(), "-d", pair.Destination.String(), "-j", "ACCEPT"} + if err := r.iptablesClient.Append(tableFilter, chainRTFwdIn, rule...); err != nil { + return fmt.Errorf("add legacy forwarding rule %s -> %s: %w", pair.Source, pair.Destination, err) + } + + r.rules[ruleID] = rule + + return nil +} + +func (r *family) removeLegacyRouteRule(pair firewall.RouterPair) error { + ruleID := pair.GenKey(firewall.ForwardingFormat) + + if rule, exists := r.rules[ruleID]; exists { + if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFwdIn, rule...); err != nil { + return fmt.Errorf("remove legacy forwarding rule %s -> %s: %w", pair.Source, pair.Destination, err) + } + delete(r.rules, ruleID) + + if err := r.decrementSetCounter(rule); err != nil { + return fmt.Errorf("decrement ipset counter: %w", err) + } + } + + return nil +} + +// GetLegacyManagement returns the current legacy management mode +func (r *family) GetLegacyManagement() bool { + return r.legacyManagement +} + +// SetLegacyManagement sets the route manager to use legacy management mode +func (r *family) SetLegacyManagement(isLegacy bool) { + r.legacyManagement = isLegacy +} + +// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls +func (r *family) RemoveAllLegacyRouteRules() error { + var merr *multierror.Error + for k, rule := range r.rules { + if !strings.HasPrefix(string(k), firewall.ForwardingFormatPrefix) { + continue + } + if err := r.iptablesClient.DeleteIfExists(tableFilter, chainRTFwdIn, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %w", err)) + } else { + delete(r.rules, k) + } + } + + r.updateState() + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) addPostroutingRules() error { + // First rule for outbound masquerade + rule1 := []string{ + "-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasquerade), + "!", "-o", "lo", + "-j", "MASQUERADE", + } + if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule1...); err != nil { + return fmt.Errorf("add outbound masquerade rule: %w", err) + } + r.rules["static-nat-outbound"] = rule1 + + // Second rule for return traffic masquerade + rule2 := []string{ + "-m", "mark", "--mark", fmt.Sprintf("%#x", nbnet.PreroutingFwmarkMasqueradeReturn), + "-o", r.wgIface.Name(), + "-j", "MASQUERADE", + } + if err := r.iptablesClient.Append(tableNat, chainRTNAT, rule2...); err != nil { + return fmt.Errorf("add return masquerade rule: %w", err) + } + r.rules["static-nat-return"] = rule2 + + return nil +} + +// addMSSClampingRules adds MSS clamping rules to prevent fragmentation for forwarded traffic. +func (r *family) addMSSClampingRules() error { + overhead := uint16(ipv4TCPHeaderSize) + if r.v6 { + overhead = ipv6TCPHeaderSize + } + mss := r.mtu - overhead + + // Add jump rule from FORWARD chain in mangle table to our custom chain + jumpRule := jumpRuleSpec(chainRTMSSClamp) + if err := r.iptablesClient.Insert(tableMangle, chainForward, 1, jumpRule...); err != nil { + return fmt.Errorf("add jump to MSS clamp chain: %w", err) + } + r.rules[jumpMSSClamp] = jumpRule + + ruleOut := []string{ + "-o", r.wgIface.Name(), + "-p", "tcp", + "--tcp-flags", "SYN,RST", "SYN", + "-j", "TCPMSS", + "--set-mss", fmt.Sprintf("%d", mss), + } + if err := r.iptablesClient.Append(tableMangle, chainRTMSSClamp, ruleOut...); err != nil { + return fmt.Errorf("add outbound MSS clamp rule: %w", err) + } + r.rules["mss-clamp-out"] = ruleOut + + return nil +} + +func (r *family) insertEstablishedRule(chain string) error { + establishedRule := getConntrackEstablished() + + err := r.iptablesClient.Insert(tableFilter, chain, 1, establishedRule...) + if err != nil { + return fmt.Errorf("insert established rule: %w", err) + } + + ruleID := firewall.RuleID("established-" + chain) + r.rules[ruleID] = establishedRule + + return nil +} + +func (r *family) addNatRule(pair firewall.RouterPair) (err error) { + ruleID := pair.GenKey(firewall.NatFormat) + + if rule, exists := r.rules[ruleID]; exists { + if derr := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPre, rule...); derr != nil { + return fmt.Errorf("remove existing marking rule for %s: %w", pair.Destination, derr) + } + delete(r.rules, ruleID) + + // Drop the replaced spec's set references only once the new spec has + // taken its own, so a set both specs share is not destroyed and + // recreated, which would lose the prefixes UpdateSet put in it. + defer func() { + if derr := r.decrementSetCounter(rule); derr != nil && err == nil { + err = fmt.Errorf("decrement ipset counter: %w", derr) + } + }() + } + + markValue := nbnet.PreroutingFwmarkMasquerade + if pair.Inverse { + markValue = nbnet.PreroutingFwmarkMasqueradeReturn + } + + rule := []string{"-i", r.wgIface.Name()} + if pair.Inverse { + rule = []string{"!", "-i", r.wgIface.Name()} + } + + rule = append(rule, + "-m", "conntrack", + "--ctstate", "NEW", + ) + sourceExp, err := r.applyNetwork("-s", pair.Source, nil) + if err != nil { + return fmt.Errorf("apply network -s: %w", err) + } + destExp, err := r.applyNetwork("-d", pair.Destination, nil) + if err != nil { + r.dropSourceMatch(sourceExp) + return fmt.Errorf("apply network -d: %w", err) + } + + rule = append(rule, sourceExp...) + rule = append(rule, destExp...) + rule = append(rule, + "-j", "MARK", "--set-mark", fmt.Sprintf("%#x", markValue), + ) + + // Ensure nat rules come first, so the mark can be overwritten. + // Currently overwritten by the dst-type LOCAL rules for redirected traffic. + if err := r.iptablesClient.Insert(tableMangle, chainRTPre, 1, rule...); err != nil { + r.dropSourceMatch(rule) + return fmt.Errorf("add marking rule for %s: %w", pair.Destination, err) + } + + r.rules[ruleID] = rule + + return nil +} + +func (r *family) removeNatRule(pair firewall.RouterPair) error { + ruleID := pair.GenKey(firewall.NatFormat) + + if rule, exists := r.rules[ruleID]; exists { + if err := r.iptablesClient.DeleteIfExists(tableMangle, chainRTPre, rule...); err != nil { + return fmt.Errorf("remove marking rule for %s: %w", pair.Destination, err) + } + delete(r.rules, ruleID) + + if err := r.decrementSetCounter(rule); err != nil { + return fmt.Errorf("decrement ipset counter: %w", err) + } + } else { + log.Debugf("marking rule %s not found", ruleID) + } + + return nil +} diff --git a/client/firewall/iptables/rule.go b/client/firewall/iptables/rule.go index 4f4eab167..33bcbd1d2 100644 --- a/client/firewall/iptables/rule.go +++ b/client/firewall/iptables/rule.go @@ -1,18 +1,37 @@ package iptables -// Rule to handle management of rules -type Rule struct { - ruleID string - ipsetName string +import "github.com/netbirdio/netbird/client/firewall/manager" +// Rule to handle management of rules. Source set membership (when the +// rule was built against a shared hash:net ipset) is encoded in specs; +// DeleteFilterRule recovers it via findSets so the refcounter can drop +// the right reference. +type Rule struct { + id manager.RuleID specs []string mangleSpecs []string - ip string - chain string - v6 bool + // extraRules holds the rules beyond the first when the ipset + // fallback expands a multi-source rule into one rule per prefix. + extraRules []filterSpecs + chain string + v6 bool } -// GetRuleID returns the rule id -func (r *Rule) ID() string { - return r.ruleID +// filterSpecs is one installed iptables rule: its filter-table spec and +// the paired mangle redirect-mark spec (nil for route rules or when the +// mangle rule could not be added). +type filterSpecs struct { + specs []string + mangleSpecs []string +} + +// allSpecs returns the spec pairs of every iptables rule backing this +// Rule, the primary one first. +func (r *Rule) allSpecs() []filterSpecs { + return append([]filterSpecs{{specs: r.specs, mangleSpecs: r.mangleSpecs}}, r.extraRules...) +} + +// ID returns the rule id +func (r *Rule) ID() manager.RuleID { + return r.id } diff --git a/client/firewall/iptables/rulestore_linux.go b/client/firewall/iptables/rulestore_linux.go deleted file mode 100644 index a6d36540e..000000000 --- a/client/firewall/iptables/rulestore_linux.go +++ /dev/null @@ -1,127 +0,0 @@ -package iptables - -import ( - "encoding/json" - "maps" -) - -type ipList struct { - ips map[string]struct{} -} - -func newIpList(ip string) *ipList { - ips := make(map[string]struct{}) - ips[ip] = struct{}{} - - return &ipList{ - ips: ips, - } -} - -func (s *ipList) addIP(ip string) { - s.ips[ip] = struct{}{} -} - -// clone returns a deep copy of the ipList with its own ips map. -func (s *ipList) clone() *ipList { - if s == nil { - return nil - } - return &ipList{ips: maps.Clone(s.ips)} -} - -// MarshalJSON implements json.Marshaler -func (s *ipList) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - IPs map[string]struct{} `json:"ips"` - }{ - IPs: s.ips, - }) -} - -// UnmarshalJSON implements json.Unmarshaler -func (s *ipList) UnmarshalJSON(data []byte) error { - temp := struct { - IPs map[string]struct{} `json:"ips"` - }{} - if err := json.Unmarshal(data, &temp); err != nil { - return err - } - s.ips = temp.IPs - - if temp.IPs == nil { - temp.IPs = make(map[string]struct{}) - } - - return nil -} - -type ipsetStore struct { - ipsets map[string]*ipList -} - -func newIpsetStore() *ipsetStore { - return &ipsetStore{ - ipsets: make(map[string]*ipList), - } -} - -// clone returns a deep copy of the ipsetStore with its own ipsets map and -// independent ipList entries. -func (s *ipsetStore) clone() *ipsetStore { - if s == nil { - return nil - } - cloned := &ipsetStore{ipsets: make(map[string]*ipList, len(s.ipsets))} - for name, list := range s.ipsets { - cloned.ipsets[name] = list.clone() - } - return cloned -} - -func (s *ipsetStore) ipset(ipsetName string) (*ipList, bool) { - r, ok := s.ipsets[ipsetName] - return r, ok -} - -func (s *ipsetStore) addIpList(ipsetName string, list *ipList) { - s.ipsets[ipsetName] = list -} - -func (s *ipsetStore) deleteIpset(ipsetName string) { - delete(s.ipsets, ipsetName) -} - -func (s *ipsetStore) ipsetNames() []string { - names := make([]string, 0, len(s.ipsets)) - for name := range s.ipsets { - names = append(names, name) - } - return names -} - -// MarshalJSON implements json.Marshaler -func (s *ipsetStore) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - IPSets map[string]*ipList `json:"ipsets"` - }{ - IPSets: s.ipsets, - }) -} - -// UnmarshalJSON implements json.Unmarshaler -func (s *ipsetStore) UnmarshalJSON(data []byte) error { - temp := struct { - IPSets map[string]*ipList `json:"ipsets"` - }{} - if err := json.Unmarshal(data, &temp); err != nil { - return err - } - s.ipsets = temp.IPSets - - if temp.IPSets == nil { - temp.IPSets = make(map[string]*ipList) - } - - return nil -} diff --git a/client/firewall/iptables/state_linux.go b/client/firewall/iptables/state_linux.go index f4be37d01..00bf1cebd 100644 --- a/client/firewall/iptables/state_linux.go +++ b/client/firewall/iptables/state_linux.go @@ -29,17 +29,13 @@ type ShutdownState struct { InterfaceState *InterfaceState `json:"interface_state,omitempty"` - RouteRules routeRules `json:"route_rules,omitempty"` - RouteIPsetCounter *ipsetCounter `json:"route_ipset_counter,omitempty"` - - ACLEntries aclEntries `json:"acl_entries,omitempty"` - ACLIPsetStore *ipsetStore `json:"acl_ipset_store,omitempty"` - - // IPv6 counterparts + RouteRules routeRules `json:"route_rules,omitempty"` RouteRules6 routeRules `json:"route_rules_v6,omitempty"` + RouteIPsetCounter *ipsetCounter `json:"route_ipset_counter,omitempty"` RouteIPsetCounter6 *ipsetCounter `json:"route_ipset_counter_v6,omitempty"` - ACLEntries6 aclEntries `json:"acl_entries_v6,omitempty"` - ACLIPsetStore6 *ipsetStore `json:"acl_ipset_store_v6,omitempty"` + + ACLEntries aclEntries `json:"acl_entries,omitempty"` + ACLEntries6 aclEntries `json:"acl_entries_v6,omitempty"` } func (s *ShutdownState) Name() string { @@ -57,17 +53,14 @@ func (s *ShutdownState) Cleanup() error { } if s.RouteRules != nil { - ipt.router.rules = s.RouteRules + ipt.family4.rules = s.RouteRules } if s.RouteIPsetCounter != nil { - ipt.router.ipsetCounter.LoadData(s.RouteIPsetCounter) + ipt.family4.ipsetCounter.LoadData(s.RouteIPsetCounter) } if s.ACLEntries != nil { - ipt.aclMgr.entries = s.ACLEntries - } - if s.ACLIPsetStore != nil { - ipt.aclMgr.ipsetStore = s.ACLIPsetStore + ipt.family4.entries = s.ACLEntries } // Clean up v6 state even if the current run has no IPv6. @@ -79,16 +72,13 @@ func (s *ShutdownState) Cleanup() error { } if ipt.hasIPv6() { if s.RouteRules6 != nil { - ipt.router6.rules = s.RouteRules6 + ipt.family6.rules = s.RouteRules6 } if s.RouteIPsetCounter6 != nil { - ipt.router6.ipsetCounter.LoadData(s.RouteIPsetCounter6) + ipt.family6.ipsetCounter.LoadData(s.RouteIPsetCounter6) } if s.ACLEntries6 != nil { - ipt.aclMgr6.entries = s.ACLEntries6 - } - if s.ACLIPsetStore6 != nil { - ipt.aclMgr6.ipsetStore = s.ACLIPsetStore6 + ipt.family6.entries = s.ACLEntries6 } } diff --git a/client/firewall/iptables/testhelpers_linux_test.go b/client/firewall/iptables/testhelpers_linux_test.go new file mode 100644 index 000000000..fe44f7cc3 --- /dev/null +++ b/client/firewall/iptables/testhelpers_linux_test.go @@ -0,0 +1,27 @@ +//go:build privileged + +package iptables + +import ( + "fmt" + "net" + "net/netip" +) + +func pfx(ip net.IP) []netip.Prefix { + if ip == nil { + return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + } + if ip.IsUnspecified() { + if ip.To4() != nil { + return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + } + return []netip.Prefix{netip.PrefixFrom(netip.IPv6Unspecified(), 0)} + } + a, ok := netip.AddrFromSlice(ip) + if !ok { + panic(fmt.Sprintf("invalid IP length: %d", len(ip))) + } + a = a.Unmap() + return []netip.Prefix{netip.PrefixFrom(a, a.BitLen())} +} diff --git a/client/firewall/manager/firewall.go b/client/firewall/manager/firewall.go index 149c6db83..f8de1e2b5 100644 --- a/client/firewall/manager/firewall.go +++ b/client/firewall/manager/firewall.go @@ -3,7 +3,6 @@ package manager import ( "errors" "fmt" - "net" "net/netip" "sort" @@ -16,6 +15,12 @@ import ( // method but the IPv6 firewall components were not initialized. var ErrIPv6NotInitialized = errors.New("IPv6 firewall not initialized") +// ErrNoSources is returned when AddFilterRule is called with an empty +// source list. "Match any source" must be expressed explicitly with a +// /0 prefix; an empty list is a caller error and is rejected rather +// than silently widening the rule to every source. +var ErrNoSources = errors.New("rule has no sources") + const ( ForwardingFormatPrefix = "netbird-fwd-" ForwardingFormat = "netbird-fwd-%s-%t" @@ -23,13 +28,18 @@ const ( NatFormat = "netbird-nat-%s-%t" ) +// RuleID identifies a firewall rule. It is a typed string so the +// compiler catches accidental mixing with arbitrary string keys. It is +// only an identifier and does not implement Rule. +type RuleID string + // Rule abstraction should be implemented by each firewall manager // // Each firewall type for different OS can use different type // of the properties to hold data of the created rule type Rule interface { // ID returns the rule id - ID() string + ID() RuleID } // RuleDirection is the traffic direction which a rule is applied @@ -91,6 +101,13 @@ func (d Network) IsPrefix() bool { return d.Prefix.IsValid() } +// IsZero returns true if the network designates no destination, i.e. it +// is the zero value. A zero Network is the peer-rule sentinel; a non-zero +// one carries a prefix or set destination. +func (d Network) IsZero() bool { + return !d.IsPrefix() && !d.IsSet() +} + // Manager is the high level abstraction of a firewall manager // // It declares methods which handle actions required by the @@ -98,46 +115,42 @@ func (d Network) IsPrefix() bool { type Manager interface { Init(stateManager *statemanager.Manager) error - // AllowNetbird allows netbird interface traffic - AllowNetbird() error - - // AddPeerFiltering adds a rule to the firewall + // AddFilterRule adds a packet-filtering rule to the firewall. // - // If comment argument is empty firewall manager should set - // rule ID as comment for the rule + // If destination is the zero Network, the rule applies to traffic + // inbound to this node, i.e. peer ACL semantics, installed in + // the kernel's input chain. If destination is set (prefix or + // set), the rule applies to forwarded traffic with that + // destination, route ACL semantics, installed in the forward + // chain. // - // Note: Callers should call Flush() after adding rules to ensure - // they are applied to the kernel and rule handles are refreshed. - AddPeerFiltering( + // sources must be a single address family; the caller splits mixed + // families and calls once per family. "Match any source" must be + // expressed with an explicit /0 prefix; an empty sources list is + // rejected with ErrNoSources so a zeroed list can never widen a + // rule to every source. + // + // Note: callers should call Flush() after adding rules. + AddFilterRule( id []byte, - ip net.IP, + sources []netip.Prefix, + destination Network, proto Protocol, sPort *Port, dPort *Port, action Action, - ipsetName string, - ) ([]Rule, error) + ) (Rule, error) - // DeletePeerRule from the firewall by rule definition - DeletePeerRule(rule Rule) error + // DeleteFilterRule removes a filtering rule previously added via + // AddFilterRule. The rule's own type identifies whether it lives + // in the peer (input) or route (forward) path. + DeleteFilterRule(rule Rule) error // IsServerRouteSupported returns true if the firewall supports server side routing operations IsServerRouteSupported() bool IsStateful() bool - AddRouteFiltering( - id []byte, - sources []netip.Prefix, - destination Network, - proto Protocol, - sPort, dPort *Port, - action Action, - ) (Rule, error) - - // DeleteRouteRule deletes a routing rule - DeleteRouteRule(rule Rule) error - // AddNatRule inserts a routing NAT rule AddNatRule(pair RouterPair) error @@ -159,12 +172,6 @@ type Manager interface { DisableRouting() error - // AddDNATRule adds outbound DNAT rule for forwarding external traffic to the NetBird network. - AddDNATRule(ForwardRule) (Rule, error) - - // DeleteDNATRule deletes the outbound DNAT rule. - DeleteDNATRule(Rule) error - // UpdateSet updates the set with the given prefixes UpdateSet(hash Set, prefixes []netip.Prefix) error @@ -179,14 +186,11 @@ type Manager interface { // RemoveOutputDNAT removes an OUTPUT chain DNAT rule. RemoveOutputDNAT(localAddr netip.Addr, protocol Protocol, originalPort, translatedPort uint16) error - - // SetupEBPFProxyNoTrack creates static notrack rules for eBPF proxy loopback traffic. - // This prevents conntrack from interfering with WireGuard proxy communication. - SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error } -func GenKey(format string, pair RouterPair) string { - return fmt.Sprintf(format, pair.ID, pair.Inverse) +// GenKey builds the rule id for this pair from the given format. +func (p RouterPair) GenKey(format string) RuleID { + return RuleID(fmt.Sprintf(format, p.ID, p.Inverse)) } // LegacyManager defines the interface for legacy management operations @@ -242,6 +246,20 @@ func MergeIPRanges(prefixes []netip.Prefix) []netip.Prefix { return merged } +// UnmapPrefix normalizes a v4-mapped v6 prefix (::ffff:a.b.c.d) to its +// plain v4 form, shifting the prefix length out of the 96-bit mapped +// range. Other prefixes are returned unchanged. Keeping prefixes +// unmapped ensures v4 rules match consistently and the match builders +// read the correct address length. +func UnmapPrefix(p netip.Prefix) netip.Prefix { + addr := p.Addr() + if !addr.Is4In6() { + return p + } + bits := max(p.Bits()-96, 0) + return netip.PrefixFrom(addr.Unmap(), bits) +} + // SortPrefixes sorts the given slice of netip.Prefix in place. // It sorts first by IP address, then by prefix length (most specific to least specific). func SortPrefixes(prefixes []netip.Prefix) { diff --git a/client/firewall/manager/forward_rule.go b/client/firewall/manager/forward_rule.go deleted file mode 100644 index 21a43520e..000000000 --- a/client/firewall/manager/forward_rule.go +++ /dev/null @@ -1,27 +0,0 @@ -package manager - -import ( - "fmt" - "net/netip" -) - -// ForwardRule todo figure out better place to this to avoid circular imports -type ForwardRule struct { - Protocol Protocol - DestinationPort Port - TranslatedAddress netip.Addr - TranslatedPort Port -} - -func (r ForwardRule) ID() string { - id := fmt.Sprintf("%s;%s;%s;%s", - r.Protocol, - r.DestinationPort.String(), - r.TranslatedAddress.String(), - r.TranslatedPort.String()) - return id -} - -func (r ForwardRule) String() string { - return fmt.Sprintf("protocol: %s, destinationPort: %s, translatedAddress: %s, translatedPort: %s", r.Protocol, r.DestinationPort.String(), r.TranslatedAddress.String(), r.TranslatedPort.String()) -} diff --git a/client/firewall/manager/set.go b/client/firewall/manager/set.go index dda93bf47..fa55471ea 100644 --- a/client/firewall/manager/set.go +++ b/client/firewall/manager/set.go @@ -40,7 +40,7 @@ func (h Set) Comment() string { // NewPrefixSet generates a unique name for an ipset based on the given prefixes. func NewPrefixSet(prefixes []netip.Prefix) Set { - // sort for consistent naming + prefixes = slices.Clone(prefixes) SortPrefixes(prefixes) hash := sha256.New() diff --git a/client/firewall/nftables/acl_linux.go b/client/firewall/nftables/acl_linux.go deleted file mode 100644 index 9d2ea7264..000000000 --- a/client/firewall/nftables/acl_linux.go +++ /dev/null @@ -1,713 +0,0 @@ -package nftables - -import ( - "bytes" - "fmt" - "net" - "slices" - "strconv" - "strings" - "time" - - "github.com/google/nftables" - "github.com/google/nftables/binaryutil" - "github.com/google/nftables/expr" - log "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" - - firewall "github.com/netbirdio/netbird/client/firewall/manager" - nbnet "github.com/netbirdio/netbird/client/net" -) - -const ( - - // rules chains contains the effective ACL rules - chainNameInputRules = "netbird-acl-input-rules" - - // filter chains contains the rules that jump to the rules chains - chainNameInputFilter = "netbird-acl-input-filter" - chainNameForwardFilter = "netbird-acl-forward-filter" - chainNameManglePrerouting = "netbird-mangle-prerouting" - chainNameManglePostrouting = "netbird-mangle-postrouting" -) - -const flushError = "flush: %w" - -type AclManager struct { - rConn *nftables.Conn - sConn *nftables.Conn - wgIface iFaceMapper - routingFwChainName string - af addrFamily - - workTable *nftables.Table - chainInputRules *nftables.Chain - chainPrerouting *nftables.Chain - - ipsetStore *ipsetStore - rules map[string]*Rule -} - -func newAclManager(table *nftables.Table, wgIface iFaceMapper, routingFwChainName string) (*AclManager, error) { - // sConn is used for creating sets and adding/removing elements from them - // it's differ then rConn (which does create new conn for each flush operation) - // and is permanent. Using same connection for both type of operations - // overloads netlink with high amount of rules ( > 10000) - sConn, err := nftables.New(nftables.AsLasting()) - if err != nil { - return nil, fmt.Errorf("create nf conn: %w", err) - } - - return &AclManager{ - rConn: &nftables.Conn{}, - sConn: sConn, - wgIface: wgIface, - workTable: table, - routingFwChainName: routingFwChainName, - af: familyForAddr(table.Family == nftables.TableFamilyIPv4), - - ipsetStore: newIpsetStore(), - rules: make(map[string]*Rule), - }, nil -} - -func (m *AclManager) init(workTable *nftables.Table) error { - m.workTable = workTable - return m.createDefaultChains() -} - -// AddPeerFiltering rule to the firewall -// -// If comment argument is empty firewall manager should set -// rule ID as comment for the rule -func (m *AclManager) AddPeerFiltering( - id []byte, - ip net.IP, - proto firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, - ipsetName string, -) ([]firewall.Rule, error) { - var ipset *nftables.Set - if ipsetName != "" { - var err error - ipset, err = m.addIpToSet(ipsetName, ip) - if err != nil { - return nil, err - } - } - - newRules := make([]firewall.Rule, 0, 2) - ioRule, err := m.addIOFiltering(ip, proto, sPort, dPort, action, ipset) - if err != nil { - return nil, err - } - - newRules = append(newRules, ioRule) - return newRules, nil -} - -// DeletePeerRule from the firewall by rule definition -func (m *AclManager) DeletePeerRule(rule firewall.Rule) error { - r, ok := rule.(*Rule) - if !ok { - return fmt.Errorf("invalid rule type") - } - - if r.nftSet == nil { - if err := m.rConn.DelRule(r.nftRule); err != nil { - log.Errorf("failed to delete rule: %v", err) - } - if r.mangleRule != nil { - if err := m.rConn.DelRule(r.mangleRule); err != nil { - log.Errorf("failed to delete mangle rule: %v", err) - } - } - delete(m.rules, r.ID()) - return m.rConn.Flush() - } - - ips, ok := m.ipsetStore.ips(r.nftSet.Name) - if !ok { - if err := m.rConn.DelRule(r.nftRule); err != nil { - log.Errorf("failed to delete rule: %v", err) - } - if r.mangleRule != nil { - if err := m.rConn.DelRule(r.mangleRule); err != nil { - log.Errorf("failed to delete mangle rule: %v", err) - } - } - delete(m.rules, r.ID()) - return m.rConn.Flush() - } - - if _, ok := ips[r.ip.String()]; ok { - err := m.sConn.SetDeleteElements(r.nftSet, []nftables.SetElement{{Key: ipToBytes(r.ip, m.af)}}) - if err != nil { - log.Errorf("delete elements for set %q: %v", r.nftSet.Name, err) - } - if err := m.sConn.Flush(); err != nil { - log.Debugf("flush error of set delete element, %s", r.nftSet.Name) - return err - } - m.ipsetStore.DeleteIpFromSet(r.nftSet.Name, r.ip) - } - - // if after delete, set still contains other IPs, - // no need to delete firewall rule and we should exit here - if len(ips) > 0 { - return nil - } - - if err := m.rConn.DelRule(r.nftRule); err != nil { - log.Errorf("failed to delete rule: %v", err) - } - if r.mangleRule != nil { - if err := m.rConn.DelRule(r.mangleRule); err != nil { - log.Errorf("failed to delete mangle rule: %v", err) - } - } - - if err := m.rConn.Flush(); err != nil { - return err - } - - delete(m.rules, r.ID()) - m.ipsetStore.DeleteReferenceFromIpSet(r.nftSet.Name) - - if m.ipsetStore.HasReferenceToSet(r.nftSet.Name) { - return nil - } - - // we delete last IP from the set, that means we need to delete - // set itself and associated firewall rule too - m.rConn.FlushSet(r.nftSet) - m.rConn.DelSet(r.nftSet) - m.ipsetStore.deleteIpset(r.nftSet.Name) - return nil -} - -// createDefaultAllowRules creates default allow rules for the input and output chains -func (m *AclManager) createDefaultAllowRules() error { - expIn := []expr.Any{ - &expr.Verdict{ - Kind: expr.VerdictAccept, - }, - } - - _ = m.rConn.InsertRule(&nftables.Rule{ - Table: m.workTable, - Chain: m.chainInputRules, - Position: 0, - Exprs: expIn, - }) - - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf(flushError, err) - } - return nil -} - -// Flush rule/chain/set operations from the buffer -// -// Method also get all rules after flush and refreshes handle values in the rulesets -func (m *AclManager) Flush() error { - if err := m.flushWithBackoff(); err != nil { - return err - } - - if err := m.refreshRuleHandles(m.chainInputRules, false); err != nil { - log.Errorf("failed to refresh rule handles ipv4 input chain: %v", err) - } - if err := m.refreshRuleHandles(m.chainPrerouting, true); err != nil { - log.Errorf("failed to refresh rule handles prerouting chain: %v", err) - } - - return nil -} - -func (m *AclManager) addIOFiltering( - ip net.IP, - proto firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, - ipset *nftables.Set, -) (*Rule, error) { - ruleId := generatePeerRuleId(ip, proto, sPort, dPort, action, ipset) - if r, ok := m.rules[ruleId]; ok { - return &Rule{ - nftRule: r.nftRule, - mangleRule: r.mangleRule, - nftSet: r.nftSet, - ruleID: r.ruleID, - ip: ip, - }, nil - } - - var expressions []expr.Any - - if proto != firewall.ProtocolALL { - expressions = append(expressions, &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: m.af.protoOffset, - Len: uint32(1), - }) - - protoData, err := m.af.protoNum(proto) - if err != nil { - return nil, fmt.Errorf("convert protocol to number: %v", err) - } - - expressions = append(expressions, &expr.Cmp{ - Register: 1, - Op: expr.CmpOpEq, - Data: []byte{protoData}, - }) - } - - rawIP := ipToBytes(ip, m.af) - // check if rawIP contains zeroed IPv4 0.0.0.0 value - // in that case not add IP match expression into the rule definition - if slices.ContainsFunc(rawIP, func(v byte) bool { return v != 0 }) { - expressions = append(expressions, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: m.af.srcAddrOffset, - Len: m.af.addrLen, - }, - ) - // add individual IP for match if no ipset defined - if ipset == nil { - expressions = append(expressions, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: rawIP, - }, - ) - } else { - expressions = append(expressions, - &expr.Lookup{ - SourceRegister: 1, - SetName: ipset.Name, - SetID: ipset.ID, - }, - ) - } - } - - expressions = append(expressions, applyPort(sPort, true)...) - expressions = append(expressions, applyPort(dPort, false)...) - - mainExpressions := slices.Clone(expressions) - - switch action { - case firewall.ActionAccept: - mainExpressions = append(mainExpressions, &expr.Verdict{Kind: expr.VerdictAccept}) - case firewall.ActionDrop: - mainExpressions = append(mainExpressions, &expr.Verdict{Kind: expr.VerdictDrop}) - } - - userData := []byte(ruleId) - - chain := m.chainInputRules - rule := &nftables.Rule{ - Table: m.workTable, - Chain: chain, - Exprs: mainExpressions, - UserData: userData, - } - - // Insert DROP rules at the beginning, append ACCEPT rules at the end - var nftRule *nftables.Rule - if action == firewall.ActionDrop { - nftRule = m.rConn.InsertRule(rule) - } else { - nftRule = m.rConn.AddRule(rule) - } - - if err := m.rConn.Flush(); err != nil { - return nil, fmt.Errorf("flush input rule %s: %v", ruleId, err) - } - - ruleStruct := &Rule{ - nftRule: nftRule, - // best effort mangle rule - mangleRule: m.createPreroutingRule(expressions, userData), - nftSet: ipset, - ruleID: ruleId, - ip: ip, - } - m.rules[ruleId] = ruleStruct - if ipset != nil { - m.ipsetStore.AddReferenceToIpset(ipset.Name) - } - - return ruleStruct, nil -} - -func (m *AclManager) createPreroutingRule(expressions []expr.Any, userData []byte) *nftables.Rule { - if m.chainPrerouting == nil { - log.Warn("prerouting chain is not created") - return nil - } - - preroutingExprs := slices.Clone(expressions) - - // interface - preroutingExprs = append([]expr.Any{ - &expr.Meta{ - Key: expr.MetaKeyIIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(m.wgIface.Name()), - }, - }, preroutingExprs...) - - // local destination and mark - preroutingExprs = append(preroutingExprs, - &expr.Fib{ - Register: 1, - ResultADDRTYPE: true, - FlagDADDR: true, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(unix.RTN_LOCAL), - }, - - &expr.Immediate{ - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected), - }, - &expr.Meta{ - Key: expr.MetaKeyMARK, - Register: 1, - SourceRegister: true, - }, - ) - - nfRule := m.rConn.AddRule(&nftables.Rule{ - Table: m.workTable, - Chain: m.chainPrerouting, - Exprs: preroutingExprs, - UserData: userData, - }) - - if err := m.rConn.Flush(); err != nil { - log.Errorf("failed to flush mangle rule %s: %v", string(userData), err) - return nil - } - - return nfRule -} - -func (m *AclManager) createDefaultChains() (err error) { - // chainNameInputRules - chain := m.createChain(chainNameInputRules) - err = m.rConn.Flush() - if err != nil { - log.Debugf("failed to create chain (%s): %s", chain.Name, err) - return fmt.Errorf(flushError, err) - } - m.chainInputRules = chain - - // netbird-acl-input-filter - // type filter hook input priority filter; policy accept; - chain = m.createFilterChainWithHook(chainNameInputFilter, nftables.ChainHookInput) - m.addJumpRule(chain, m.chainInputRules.Name, expr.MetaKeyIIFNAME) // to netbird-acl-input-rules - m.addDropExpressions(chain, expr.MetaKeyIIFNAME) - err = m.rConn.Flush() - if err != nil { - log.Debugf("failed to create chain (%s): %s", chain.Name, err) - return err - } - - // netbird-acl-forward-filter - chainFwFilter := m.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward) - m.addJumpRulesToRtForward(chainFwFilter) // to netbird-rt-fwd - m.addDropExpressions(chainFwFilter, expr.MetaKeyIIFNAME) - - err = m.rConn.Flush() - if err != nil { - log.Debugf("failed to create chain (%s): %s", chainNameForwardFilter, err) - return fmt.Errorf(flushError, err) - } - - if err := m.allowRedirectedTraffic(chainFwFilter); err != nil { - log.Errorf("failed to allow redirected traffic: %s", err) - } - - return nil -} - -// Makes redirected traffic originally destined for the host itself (now subject to the forward filter) -// go through the input filter as well. This will enable e.g. Docker services to keep working by accessing the -// netbird peer IP. -func (m *AclManager) allowRedirectedTraffic(chainFwFilter *nftables.Chain) error { - // Chain is created by route manager - // TODO: move creation to a common place - m.chainPrerouting = &nftables.Chain{ - Name: chainNameManglePrerouting, - Table: m.workTable, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityMangle, - } - - m.addFwmarkToForward(chainFwFilter) - - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf(flushError, err) - } - - return nil -} - -func (m *AclManager) addFwmarkToForward(chainFwFilter *nftables.Chain) { - m.rConn.InsertRule(&nftables.Rule{ - Table: m.workTable, - Chain: chainFwFilter, - Exprs: []expr.Any{ - &expr.Meta{ - Key: expr.MetaKeyMARK, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected), - }, - &expr.Verdict{ - Kind: expr.VerdictAccept, - }, - }, - }) -} - -func (m *AclManager) addJumpRulesToRtForward(chainFwFilter *nftables.Chain) { - expressions := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(m.wgIface.Name()), - }, - &expr.Verdict{ - Kind: expr.VerdictJump, - Chain: m.routingFwChainName, - }, - } - - _ = m.rConn.AddRule(&nftables.Rule{ - Table: m.workTable, - Chain: chainFwFilter, - Exprs: expressions, - }) -} - -func (m *AclManager) createChain(name string) *nftables.Chain { - chain := &nftables.Chain{ - Name: name, - Table: m.workTable, - } - - chain = m.rConn.AddChain(chain) - - insertReturnTrafficRule(m.rConn, m.workTable, chain) - - return chain -} - -func (m *AclManager) createFilterChainWithHook(name string, hookNum *nftables.ChainHook) *nftables.Chain { - polAccept := nftables.ChainPolicyAccept - chain := &nftables.Chain{ - Name: name, - Table: m.workTable, - Hooknum: hookNum, - Priority: nftables.ChainPriorityFilter, - Type: nftables.ChainTypeFilter, - Policy: &polAccept, - } - - return m.rConn.AddChain(chain) -} - -func (m *AclManager) addDropExpressions(chain *nftables.Chain, ifaceKey expr.MetaKey) []expr.Any { - expressions := []expr.Any{ - &expr.Meta{Key: ifaceKey, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(m.wgIface.Name()), - }, - &expr.Verdict{Kind: expr.VerdictDrop}, - } - _ = m.rConn.AddRule(&nftables.Rule{ - Table: m.workTable, - Chain: chain, - Exprs: expressions, - }) - return nil -} - -func (m *AclManager) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr.MetaKey) { - expressions := []expr.Any{ - &expr.Meta{Key: ifaceKey, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(m.wgIface.Name()), - }, - &expr.Verdict{ - Kind: expr.VerdictJump, - Chain: to, - }, - } - - _ = m.rConn.AddRule(&nftables.Rule{ - Table: chain.Table, - Chain: chain, - Exprs: expressions, - }) -} - -func (m *AclManager) addIpToSet(ipsetName string, ip net.IP) (*nftables.Set, error) { - ipset, err := m.rConn.GetSetByName(m.workTable, ipsetName) - rawIP := ipToBytes(ip, m.af) - if err != nil { - if ipset, err = m.createSet(m.workTable, ipsetName); err != nil { - return nil, fmt.Errorf("get set name: %v", err) - } - - m.ipsetStore.newIpset(ipset.Name) - } - - if m.ipsetStore.IsIpInSet(ipset.Name, ip) { - return ipset, nil - } - - if err := m.sConn.SetAddElements(ipset, []nftables.SetElement{{Key: rawIP}}); err != nil { - return nil, fmt.Errorf("add set element for the first time: %v", err) - } - - m.ipsetStore.AddIpToSet(ipset.Name, ip) - - if err := m.sConn.Flush(); err != nil { - return nil, fmt.Errorf("flush add elements: %v", err) - } - - return ipset, nil -} - -// createSet in given table by name -func (m *AclManager) createSet(table *nftables.Table, name string) (*nftables.Set, error) { - ipset := &nftables.Set{ - Name: name, - Table: table, - Dynamic: true, - KeyType: m.af.setKeyType, - } - - if err := m.rConn.AddSet(ipset, nil); err != nil { - return nil, fmt.Errorf("create set: %v", err) - } - - if err := m.rConn.Flush(); err != nil { - return nil, fmt.Errorf("flush created set: %v", err) - } - - return ipset, nil -} - -func (m *AclManager) flushWithBackoff() (err error) { - backoff := 4 - backoffTime := 1000 * time.Millisecond - for i := 0; ; i++ { - err = m.rConn.Flush() - if err != nil { - log.Debugf("failed to flush nftables: %v", err) - if !strings.Contains(err.Error(), "busy") { - return - } - log.Error("failed to flush nftables, retrying...") - if i == backoff-1 { - return err - } - time.Sleep(backoffTime) - backoffTime *= 2 - continue - } - break - } - return -} - -func (m *AclManager) refreshRuleHandles(chain *nftables.Chain, mangle bool) error { - if m.workTable == nil || chain == nil { - return nil - } - - list, err := m.rConn.GetRules(m.workTable, chain) - if err != nil { - return err - } - - for _, rule := range list { - if len(rule.UserData) == 0 { - continue - } - split := bytes.Split(rule.UserData, []byte(" ")) - r, ok := m.rules[string(split[0])] - if ok { - if mangle { - *r.mangleRule = *rule - } else { - *r.nftRule = *rule - } - } - } - - return nil -} - -func generatePeerRuleId(ip net.IP, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action, ipset *nftables.Set) string { - rulesetID := ":" + string(proto) + ":" - if sPort != nil { - rulesetID += sPort.String() - } - rulesetID += ":" - if dPort != nil { - rulesetID += dPort.String() - } - rulesetID += ":" - rulesetID += strconv.Itoa(int(action)) - if ipset == nil { - return "ip:" + ip.String() + rulesetID - } - return "set:" + ipset.Name + rulesetID -} - -func ifname(n string) []byte { - b := make([]byte, 16) - copy(b, n+"\x00") - return b -} - - -// ipToBytes converts net.IP to the correct byte length for the address family. -func ipToBytes(ip net.IP, af addrFamily) []byte { - if af.addrLen == 4 { - return ip.To4() - } - return ip.To16() -} - diff --git a/client/firewall/nftables/chains_linux.go b/client/firewall/nftables/chains_linux.go new file mode 100644 index 000000000..71f0c60f2 --- /dev/null +++ b/client/firewall/nftables/chains_linux.go @@ -0,0 +1,880 @@ +//go:build !android + +package nftables + +import ( + "bytes" + "errors" + "fmt" + "slices" + "strings" + "time" + + "github.com/coreos/go-iptables/iptables" + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" + + nberrors "github.com/netbirdio/netbird/client/errors" + "github.com/netbirdio/netbird/client/firewall/firewalld" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbnet "github.com/netbirdio/netbird/client/net" +) + +func (r *family) createContainers() error { + r.chains[chainNameRoutingFw] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameRoutingFw, + Table: r.workTable, + }) + + prio := *nftables.ChainPriorityNATSource - 1 + r.chains[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameRoutingNat, + Table: r.workTable, + Hooknum: nftables.ChainHookPostrouting, + Priority: &prio, + Type: nftables.ChainTypeNAT, + }) + + r.chains[chainNameRoutingRdr] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameRoutingRdr, + Table: r.workTable, + Hooknum: nftables.ChainHookPrerouting, + Priority: nftables.ChainPriorityNATDest, + Type: nftables.ChainTypeNAT, + }) + + r.chains[chainNameManglePostrouting] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameManglePostrouting, + Table: r.workTable, + Hooknum: nftables.ChainHookPostrouting, + Priority: nftables.ChainPriorityMangle, + Type: nftables.ChainTypeFilter, + }) + + r.chains[chainNameManglePrerouting] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameManglePrerouting, + Table: r.workTable, + Hooknum: nftables.ChainHookPrerouting, + Priority: nftables.ChainPriorityMangle, + Type: nftables.ChainTypeFilter, + }) + + r.chains[chainNameMangleForward] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameMangleForward, + Table: r.workTable, + Hooknum: nftables.ChainHookForward, + Priority: nftables.ChainPriorityMangle, + Type: nftables.ChainTypeFilter, + }) + + insertReturnTrafficRule(r.conn, r.workTable, r.chains[chainNameRoutingFw]) + + r.addPostroutingRules() + + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("initialize tables: %v", err) + } + + if err := r.addMSSClampingRules(); err != nil { + log.Errorf("failed to add MSS clamping rules: %s", err) + } + + // Kernel routing opens both INPUT and FORWARD. + if err := r.openInterface(true); err != nil { + log.Errorf("failed to open interface in foreign chains: %s", err) + } + + if err := firewalld.TrustInterface(r.wgIface.Name()); err != nil { + log.Warnf("failed to trust interface in firewalld: %v", err) + } + + if err := r.refreshRulesMap(); err != nil { + log.Errorf("failed to refresh rules: %s", err) + } + + return nil +} + +// setupDataPlaneMark configures the fwmark for the data plane +func (r *family) setupDataPlaneMark() error { + if r.chains[chainNameManglePrerouting] == nil || r.chains[chainNameManglePostrouting] == nil { + return errors.New("no mangle chains found") + } + + ctNew := getCtNewExprs() + preExprs := []expr.Any{ + &expr.Meta{ + Key: expr.MetaKeyIIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + } + preExprs = append(preExprs, ctNew...) + preExprs = append(preExprs, + &expr.Immediate{ + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(nbnet.DataPlaneMarkIn), + }, + &expr.Ct{ + Key: expr.CtKeyMARK, + Register: 1, + SourceRegister: true, + }, + ) + + preNftRule := &nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameManglePrerouting], + Exprs: preExprs, + } + r.conn.AddRule(preNftRule) + + postExprs := []expr.Any{ + &expr.Meta{ + Key: expr.MetaKeyOIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + } + postExprs = append(postExprs, ctNew...) + postExprs = append(postExprs, + &expr.Immediate{ + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(nbnet.DataPlaneMarkOut), + }, + &expr.Ct{ + Key: expr.CtKeyMARK, + Register: 1, + SourceRegister: true, + }, + ) + + postNftRule := &nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameManglePostrouting], + Exprs: postExprs, + } + r.conn.AddRule(postNftRule) + + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush: %w", err) + } + + return nil +} + +// openInterface adds passthrough accept rules for the NetBird interface to the +// kernel's filter table and external chains so they don't drop our traffic. +// includeForward also opens the FORWARD chains (kernel routing); when false only +// INPUT is opened, which is all the userspace router needs since it never +// forwards in the kernel. +func (r *family) openInterface(includeForward bool) error { + var merr *multierror.Error + + if err := r.acceptFilterTableRules(includeForward); err != nil { + merr = multierror.Append(merr, err) + } + + if err := r.acceptExternalChainsRules(includeForward); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add accept rules to external chains: %w", err)) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) acceptFilterTableRules(includeForward bool) error { + if r.filterTable == nil { + return nil + } + + fw := "iptables" + + defer func() { + log.Debugf("Used %s to add accept input/forward rules", fw) + }() + + // Try iptables first and fallback to nftables if iptables is not available. + // Use the correct protocol (iptables vs ip6tables) for the address family. + ipt, err := iptables.NewWithProtocol(r.iptablesProto()) + if err != nil { + log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err) + + fw = "nftables" + return r.acceptFilterRulesNftables(r.filterTable, includeForward) + } + + if err := r.acceptFilterRulesIptables(ipt, includeForward); err != nil { + log.Warnf("iptables failed (table may be incompatible), falling back to nftables: %v", err) + fw = "nftables" + return r.acceptFilterRulesNftables(r.filterTable, includeForward) + } + return nil +} + +func (r *family) acceptFilterRulesIptables(ipt *iptables.IPTables, includeForward bool) error { + var merr *multierror.Error + + if includeForward { + for _, rule := range r.getAcceptForwardRules() { + if err := ipt.Insert("filter", chainNameForward, 1, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add iptables forward rule: %v", err)) + } else { + log.Debugf("added iptables forward rule: %v", rule) + } + } + } + + inputRule := r.getAcceptInputRule() + if err := ipt.Insert("filter", chainNameInput, 1, inputRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("add iptables input rule: %v", err)) + } else { + log.Debugf("added iptables input rule: %v", inputRule) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) getAcceptForwardRules() [][]string { + intf := r.wgIface.Name() + return [][]string{ + {"-i", intf, "-j", "ACCEPT"}, + {"-o", intf, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}, + } +} + +func (r *family) getAcceptInputRule() []string { + return []string{"-i", r.wgIface.Name(), "-j", "ACCEPT"} +} + +// acceptFilterRulesNftables adds accept rules to the ip filter table using nftables. +// This is used when iptables is not available. +func (r *family) acceptFilterRulesNftables(table *nftables.Table, includeForward bool) error { + intf := ifname(r.wgIface.Name()) + + if includeForward { + forwardChain := &nftables.Chain{ + Name: chainNameForward, + Table: table, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookForward, + Priority: nftables.ChainPriorityFilter, + } + r.insertForwardAcceptRules(forwardChain, intf) + } + + inputChain := &nftables.Chain{ + Name: chainNameInput, + Table: table, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookInput, + Priority: nftables.ChainPriorityFilter, + } + r.insertInputAcceptRule(inputChain, intf) + + return r.conn.Flush() +} + +// acceptExternalChainsRules adds accept rules to external chains (non-netbird, non-iptables tables). +// It dynamically finds chains at call time to handle chains that may have been created after startup. +func (r *family) acceptExternalChainsRules(includeForward bool) error { + chains := r.findExternalChains() + if len(chains) == 0 { + return nil + } + + intf := ifname(r.wgIface.Name()) + for _, chain := range chains { + r.applyExternalChainAccept(chain, intf, includeForward) + } + + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush external chain rules: %w", err) + } + return nil +} + +func (r *family) applyExternalChainAccept(chain *nftables.Chain, intf []byte, includeForward bool) { + if chain.Hooknum == nil { + log.Debugf("skipping external chain %s/%s: hooknum is nil", chain.Table.Name, chain.Name) + return + } + + log.Debugf("adding accept rules to external %s chain: %s %s/%s", + hookName(chain.Hooknum), familyName(chain.Table.Family), chain.Table.Name, chain.Name) + + switch *chain.Hooknum { + case *nftables.ChainHookForward: + if includeForward { + r.insertForwardAcceptRules(chain, intf) + } + case *nftables.ChainHookInput: + r.insertInputAcceptRule(chain, intf) + } +} + +func (r *family) insertForwardAcceptRules(chain *nftables.Chain, intf []byte) { + existing, err := r.existingNetbirdRulesInChain(chain) + if err != nil { + log.Warnf("skip forward accept rules in %s/%s: %v", chain.Table.Name, chain.Name, err) + return + } + r.insertForwardIifRule(chain, intf, existing) + r.insertForwardOifEstablishedRule(chain, intf, existing) +} + +func (r *family) insertForwardIifRule(chain *nftables.Chain, intf []byte, existing map[string]bool) { + if existing[userDataAcceptForwardRuleIif] { + return + } + r.conn.InsertRule(&nftables.Rule{ + Table: chain.Table, + Chain: chain, + Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf}, + &expr.Counter{}, + &expr.Verdict{Kind: expr.VerdictAccept}, + }, + UserData: []byte(userDataAcceptForwardRuleIif), + }) +} + +func (r *family) insertForwardOifEstablishedRule(chain *nftables.Chain, intf []byte, existing map[string]bool) { + if existing[userDataAcceptForwardRuleOif] { + return + } + exprs := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf}, + } + r.conn.InsertRule(&nftables.Rule{ + Table: chain.Table, + Chain: chain, + Exprs: append(exprs, getEstablishedExprs(2)...), + UserData: []byte(userDataAcceptForwardRuleOif), + }) +} + +func (r *family) insertInputAcceptRule(chain *nftables.Chain, intf []byte) { + existing, err := r.existingNetbirdRulesInChain(chain) + if err != nil { + log.Warnf("skip input accept rule in %s/%s: %v", chain.Table.Name, chain.Name, err) + return + } + if existing[userDataAcceptInputRule] { + return + } + r.conn.InsertRule(&nftables.Rule{ + Table: chain.Table, + Chain: chain, + Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf}, + &expr.Counter{}, + &expr.Verdict{Kind: expr.VerdictAccept}, + }, + UserData: []byte(userDataAcceptInputRule), + }) +} + +// existingNetbirdRulesInChain returns the set of netbird-owned UserData tags present in a chain; callers must bail on error since InsertRule is additive. +func (r *family) existingNetbirdRulesInChain(chain *nftables.Chain) (map[string]bool, error) { + rules, err := r.conn.GetRules(chain.Table, chain) + if err != nil { + return nil, fmt.Errorf("list rules: %w", err) + } + present := map[string]bool{} + for _, rule := range rules { + if !isNetbirdAcceptRuleTag(rule.UserData) { + continue + } + present[string(rule.UserData)] = true + } + return present, nil +} + +func isNetbirdAcceptRuleTag(userData []byte) bool { + switch string(userData) { + case userDataAcceptForwardRuleIif, + userDataAcceptForwardRuleOif, + userDataAcceptInputRule: + return true + } + return false +} + +func (r *family) removeAcceptFilterRules() error { + var merr *multierror.Error + + if err := r.removeFilterTableRules(); err != nil { + merr = multierror.Append(merr, err) + } + + if err := r.removeExternalChainsRules(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove external chain rules: %w", err)) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) removeFilterTableRules() error { + if r.filterTable == nil { + return nil + } + + ipt, err := iptables.NewWithProtocol(r.iptablesProto()) + if err != nil { + log.Debugf("iptables not available, using nftables to remove filter rules: %v", err) + return r.removeAcceptRulesFromTable(r.filterTable) + } + + if err := r.removeAcceptFilterRulesIptables(ipt); err != nil { + log.Debugf("iptables removal failed (table may be incompatible), falling back to nftables: %v", err) + return r.removeAcceptRulesFromTable(r.filterTable) + } + return nil +} + +func (r *family) removeAcceptRulesFromTable(table *nftables.Table) error { + chains, err := r.conn.ListChainsOfTableFamily(table.Family) + if err != nil { + return fmt.Errorf("list chains: %v", err) + } + + for _, chain := range chains { + if chain.Table.Name != table.Name { + continue + } + + if chain.Name != chainNameForward && chain.Name != chainNameInput { + continue + } + + if err := r.removeAcceptRulesFromChain(table, chain); err != nil { + return err + } + } + + return r.conn.Flush() +} + +func (r *family) removeAcceptRulesFromChain(table *nftables.Table, chain *nftables.Chain) error { + rules, err := r.conn.GetRules(table, chain) + if err != nil { + return fmt.Errorf("get rules from %s/%s: %v", table.Name, chain.Name, err) + } + + for _, rule := range rules { + if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleIif)) || + bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleOif)) || + bytes.Equal(rule.UserData, []byte(userDataAcceptInputRule)) { + if err := r.conn.DelRule(rule); err != nil { + return fmt.Errorf("delete rule from %s/%s: %v", table.Name, chain.Name, err) + } + } + } + return nil +} + +// removeExternalChainsRules removes our accept rules from all external chains. +// This is deterministic - it scans for chains at removal time rather than relying on saved state, +// ensuring cleanup works even after a crash or if chains changed. +func (r *family) removeExternalChainsRules() error { + chains := r.findExternalChains() + if len(chains) == 0 { + return nil + } + + var merr *multierror.Error + for _, chain := range chains { + if err := r.removeAcceptRulesFromChain(chain.Table, chain); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove rules from external chain %s/%s: %w", chain.Table.Name, chain.Name, err)) + continue + } + if err := r.conn.Flush(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("flush external chain %s/%s: %w", chain.Table.Name, chain.Name, err)) + } + } + + return nberrors.FormatErrorOrNil(merr) +} + +// findExternalChains scans for chains from non-netbird tables that have FORWARD or INPUT hooks. +// This is used both at startup (to know where to add rules) and at cleanup (to ensure deterministic removal). +func (r *family) findExternalChains() []*nftables.Chain { + var chains []*nftables.Chain + + families := []nftables.TableFamily{r.af.tableFamily, nftables.TableFamilyINet} + + for _, family := range families { + allChains, err := r.conn.ListChainsOfTableFamily(family) + if err != nil { + log.Debugf("list chains for family %d: %v", family, err) + continue + } + + for _, chain := range allChains { + if r.isExternalChain(chain) { + chains = append(chains, chain) + } + } + } + + return chains +} + +func (r *family) isExternalChain(chain *nftables.Chain) bool { + if r.workTable != nil && chain.Table.Name == r.workTable.Name { + return false + } + + // Skip firewalld-owned chains. Firewalld creates its chains with the + // NFT_CHAIN_OWNER flag, so inserting rules into them returns EPERM. + // We delegate acceptance to firewalld by trusting the interface instead. + if chain.Table.Name == firewalldTableName { + return false + } + + // Skip iptables/ip6tables-managed tables (adding nft-native rules breaks iptables-save compat) + if (chain.Table.Family == nftables.TableFamilyIPv4 || chain.Table.Family == nftables.TableFamilyIPv6) && isIptablesTable(chain.Table.Name) { + return false + } + + if chain.Type != nftables.ChainTypeFilter { + return false + } + + if chain.Hooknum == nil { + return false + } + + return *chain.Hooknum == *nftables.ChainHookForward || *chain.Hooknum == *nftables.ChainHookInput +} + +func isIptablesTable(name string) bool { + switch name { + case tableNameFilter, tableNat, tableMangle, tableRaw, tableSecurity: + return true + } + return false +} + +func (r *family) removeAcceptFilterRulesIptables(ipt *iptables.IPTables) error { + var merr *multierror.Error + + for _, rule := range r.getAcceptForwardRules() { + if err := ipt.DeleteIfExists("filter", chainNameForward, rule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove iptables forward rule: %v", err)) + } + } + + inputRule := r.getAcceptInputRule() + if err := ipt.DeleteIfExists("filter", chainNameInput, inputRule...); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove iptables input rule: %v", err)) + } + + return nberrors.FormatErrorOrNil(merr) +} + +// Flush rule/chain/set operations from the buffer +// +// Method also get all rules after flush and refreshes handle values in the rulesets +func (r *family) Flush() error { + if err := r.flushWithBackoff(); err != nil { + return err + } + + if err := r.refreshRuleHandles(r.chainInputRules, false); err != nil { + log.Errorf("failed to refresh rule handles ipv4 input chain: %v", err) + } + if err := r.refreshRuleHandles(r.chainPrerouting, true); err != nil { + log.Errorf("failed to refresh rule handles prerouting chain: %v", err) + } + + return nil +} + +// queuePreroutingRule builds the prerouting mangle rule that marks +// redirected traffic and queues it on the connection without flushing, +// so the caller can commit it in the same transaction as the rule it +// pairs with. Returns nil when the prerouting chain is absent, in which +// case nothing is queued. +func (r *family) queuePreroutingRule(expressions []expr.Any, userData []byte) *nftables.Rule { + if r.chainPrerouting == nil { + log.Warn("prerouting chain is not created") + return nil + } + + preroutingExprs := slices.Clone(expressions) + + // interface + preroutingExprs = append([]expr.Any{ + &expr.Meta{ + Key: expr.MetaKeyIIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + }, preroutingExprs...) + + // local destination and mark + preroutingExprs = append(preroutingExprs, + &expr.Fib{ + Register: 1, + ResultADDRTYPE: true, + FlagDADDR: true, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(unix.RTN_LOCAL), + }, + + &expr.Immediate{ + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected), + }, + &expr.Meta{ + Key: expr.MetaKeyMARK, + Register: 1, + SourceRegister: true, + }, + ) + + return r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: r.chainPrerouting, + Exprs: preroutingExprs, + UserData: userData, + }) +} + +func (r *family) createDefaultChains() (err error) { + // chainNameInputRules + chain := r.createChain(chainNameInputRules) + err = r.conn.Flush() + if err != nil { + log.Debugf("failed to create chain (%s): %s", chain.Name, err) + return fmt.Errorf(flushError, err) + } + r.chainInputRules = chain + + // netbird-acl-input-filter + // type filter hook input priority filter; policy accept; + chain = r.createFilterChainWithHook(chainNameInputFilter, nftables.ChainHookInput) + r.addJumpRule(chain, r.chainInputRules.Name, expr.MetaKeyIIFNAME) // to netbird-acl-input-rules + r.addDropExpressions(chain, expr.MetaKeyIIFNAME) + err = r.conn.Flush() + if err != nil { + log.Debugf("failed to create chain (%s): %s", chain.Name, err) + return err + } + + // netbird-acl-forward-filter + chainFwFilter := r.createFilterChainWithHook(chainNameForwardFilter, nftables.ChainHookForward) + r.addJumpRulesToRtForward(chainFwFilter) // to netbird-rt-fwd + r.addDropExpressions(chainFwFilter, expr.MetaKeyIIFNAME) + + err = r.conn.Flush() + if err != nil { + log.Debugf("failed to create chain (%s): %s", chainNameForwardFilter, err) + return fmt.Errorf(flushError, err) + } + + if err := r.allowRedirectedTraffic(chainFwFilter); err != nil { + log.Errorf("failed to allow redirected traffic: %s", err) + } + + return nil +} + +// Makes redirected traffic originally destined for the host itself (now subject to the forward filter) +// go through the input filter as well. This will enable e.g. Docker services to keep working by accessing the +// netbird peer IP. +func (r *family) allowRedirectedTraffic(chainFwFilter *nftables.Chain) error { + r.chainPrerouting = r.chains[chainNameManglePrerouting] + + r.addFwmarkToForward(chainFwFilter) + + if err := r.conn.Flush(); err != nil { + return fmt.Errorf(flushError, err) + } + + return nil +} + +func (r *family) addFwmarkToForward(chainFwFilter *nftables.Chain) { + r.conn.InsertRule(&nftables.Rule{ + Table: r.workTable, + Chain: chainFwFilter, + Exprs: []expr.Any{ + &expr.Meta{ + Key: expr.MetaKeyMARK, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkRedirected), + }, + &expr.Verdict{ + Kind: expr.VerdictAccept, + }, + }, + }) +} + +func (r *family) addJumpRulesToRtForward(chainFwFilter *nftables.Chain) { + expressions := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Verdict{ + Kind: expr.VerdictJump, + Chain: r.routingFwChainName, + }, + } + + _ = r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: chainFwFilter, + Exprs: expressions, + }) +} + +func (r *family) createChain(name string) *nftables.Chain { + chain := &nftables.Chain{ + Name: name, + Table: r.workTable, + } + + chain = r.conn.AddChain(chain) + + insertReturnTrafficRule(r.conn, r.workTable, chain) + + return chain +} + +func (r *family) createFilterChainWithHook(name string, hookNum *nftables.ChainHook) *nftables.Chain { + polAccept := nftables.ChainPolicyAccept + chain := &nftables.Chain{ + Name: name, + Table: r.workTable, + Hooknum: hookNum, + Priority: nftables.ChainPriorityFilter, + Type: nftables.ChainTypeFilter, + Policy: &polAccept, + } + + return r.conn.AddChain(chain) +} + +func (r *family) addDropExpressions(chain *nftables.Chain, ifaceKey expr.MetaKey) []expr.Any { + expressions := []expr.Any{ + &expr.Meta{Key: ifaceKey, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Verdict{Kind: expr.VerdictDrop}, + } + _ = r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: chain, + Exprs: expressions, + }) + return nil +} + +func (r *family) addJumpRule(chain *nftables.Chain, to string, ifaceKey expr.MetaKey) { + expressions := []expr.Any{ + &expr.Meta{Key: ifaceKey, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Verdict{ + Kind: expr.VerdictJump, + Chain: to, + }, + } + + _ = r.conn.AddRule(&nftables.Rule{ + Table: chain.Table, + Chain: chain, + Exprs: expressions, + }) +} + +func (r *family) flushWithBackoff() (err error) { + backoff := 4 + backoffTime := 1000 * time.Millisecond + for i := 0; ; i++ { + err = r.conn.Flush() + if err != nil { + log.Debugf("failed to flush nftables: %v", err) + if !strings.Contains(err.Error(), "busy") { + return + } + log.Error("failed to flush nftables, retrying...") + if i == backoff-1 { + return err + } + time.Sleep(backoffTime) + backoffTime *= 2 + continue + } + break + } + return +} + +func (r *family) refreshRuleHandles(chain *nftables.Chain, mangle bool) error { + if r.workTable == nil || chain == nil { + return nil + } + + list, err := r.conn.GetRules(r.workTable, chain) + if err != nil { + return err + } + + for _, rule := range list { + if len(rule.UserData) == 0 { + continue + } + pr, ok := r.filters[firewall.RuleID(rule.UserData)] + if !ok { + continue + } + if mangle { + if pr.mangleRule != nil { + *pr.mangleRule = *rule + } + } else { + *pr.nftRule = *rule + } + } + + return nil +} diff --git a/client/firewall/nftables/dnat_linux.go b/client/firewall/nftables/dnat_linux.go new file mode 100644 index 000000000..c179d60cc --- /dev/null +++ b/client/firewall/nftables/dnat_linux.go @@ -0,0 +1,252 @@ +//go:build !android + +package nftables + +import ( + "fmt" + "net/netip" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + log "github.com/sirupsen/logrus" + + firewall "github.com/netbirdio/netbird/client/firewall/manager" +) + +func (r *family) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + if _, exists := r.rules[ruleID]; exists { + return nil + } + + protoNum, err := r.af.protoNum(protocol) + if err != nil { + return fmt.Errorf("convert protocol to number: %w", err) + } + + exprs := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 2}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 2, + Data: []byte{protoNum}, + }, + &expr.Payload{ + DestRegister: 3, + Base: expr.PayloadBaseTransportHeader, + Offset: 2, + Len: 2, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 3, + Data: binaryutil.BigEndian.PutUint16(originalPort), + }, + } + + bits := 32 + if localAddr.Is6() { + bits = 128 + } + exprs = append(exprs, prefixMatchExprs(r.af, netip.PrefixFrom(localAddr, bits), false)...) + + exprs = append(exprs, + &expr.Immediate{ + Register: 1, + Data: localAddr.AsSlice(), + }, + &expr.Immediate{ + Register: 2, + Data: binaryutil.BigEndian.PutUint16(translatedPort), + }, + &expr.NAT{ + Type: expr.NATTypeDestNAT, + Family: uint32(r.af.tableFamily), + RegAddrMin: 1, + RegProtoMin: 2, + RegProtoMax: 0, + }, + ) + + dnatRule := &nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameRoutingRdr], + Exprs: exprs, + UserData: []byte(ruleID), + } + r.conn.AddRule(dnatRule) + + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("add inbound DNAT rule: %w", err) + } + + r.rules[ruleID] = dnatRule + + return nil +} + +// RemoveInboundDNAT removes an inbound DNAT rule. +func (r *family) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + if err := r.refreshRulesMap(); err != nil { + return fmt.Errorf(refreshRulesMapError, err) + } + + ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + rule, exists := r.rules[ruleID] + if !exists { + return nil + } + + if rule.Handle == 0 { + log.Warnf("inbound DNAT rule %s has no handle, removing stale entry", ruleID) + delete(r.rules, ruleID) + return nil + } + + if err := r.conn.DelRule(rule); err != nil { + return fmt.Errorf("delete inbound DNAT rule %s: %w", ruleID, err) + } + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush delete inbound DNAT rule: %w", err) + } + delete(r.rules, ruleID) + + return nil +} + +// ensureNATOutputChain lazily creates the OUTPUT NAT chain on first use. +func (r *family) ensureNATOutputChain() error { + if _, exists := r.chains[chainNameNATOutput]; exists { + return nil + } + + r.chains[chainNameNATOutput] = r.conn.AddChain(&nftables.Chain{ + Name: chainNameNATOutput, + Table: r.workTable, + Hooknum: nftables.ChainHookOutput, + Priority: nftables.ChainPriorityNATDest, + Type: nftables.ChainTypeNAT, + }) + + if err := r.conn.Flush(); err != nil { + delete(r.chains, chainNameNATOutput) + return fmt.Errorf("create NAT output chain: %w", err) + } + return nil +} + +// AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic. +func (r *family) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + ruleID := firewall.RuleID(fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + if _, exists := r.rules[ruleID]; exists { + return nil + } + + if err := r.ensureNATOutputChain(); err != nil { + return err + } + + protoNum, err := r.af.protoNum(protocol) + if err != nil { + return fmt.Errorf("convert protocol to number: %w", err) + } + + exprs := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{protoNum}, + }, + &expr.Payload{ + DestRegister: 2, + Base: expr.PayloadBaseTransportHeader, + Offset: 2, + Len: 2, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 2, + Data: binaryutil.BigEndian.PutUint16(originalPort), + }, + } + + bits := 32 + if localAddr.Is6() { + bits = 128 + } + exprs = append(exprs, prefixMatchExprs(r.af, netip.PrefixFrom(localAddr, bits), false)...) + + exprs = append(exprs, + &expr.Immediate{ + Register: 1, + Data: localAddr.AsSlice(), + }, + &expr.Immediate{ + Register: 2, + Data: binaryutil.BigEndian.PutUint16(translatedPort), + }, + &expr.NAT{ + Type: expr.NATTypeDestNAT, + Family: uint32(r.af.tableFamily), + RegAddrMin: 1, + RegProtoMin: 2, + }, + ) + + dnatRule := &nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameNATOutput], + Exprs: exprs, + UserData: []byte(ruleID), + } + r.conn.AddRule(dnatRule) + + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("add output DNAT rule: %w", err) + } + + r.rules[ruleID] = dnatRule + + return nil +} + +// RemoveOutputDNAT removes an OUTPUT chain DNAT rule. +func (r *family) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { + if err := r.refreshRulesMap(); err != nil { + return fmt.Errorf(refreshRulesMapError, err) + } + + ruleID := firewall.RuleID(fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) + + rule, exists := r.rules[ruleID] + if !exists { + return nil + } + + if rule.Handle == 0 { + log.Warnf("output DNAT rule %s has no handle, removing stale entry", ruleID) + delete(r.rules, ruleID) + return nil + } + + if err := r.conn.DelRule(rule); err != nil { + return fmt.Errorf("delete output DNAT rule %s: %w", ruleID, err) + } + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush delete output DNAT rule: %w", err) + } + delete(r.rules, ruleID) + + return nil +} diff --git a/client/firewall/nftables/family_linux.go b/client/firewall/nftables/family_linux.go new file mode 100644 index 000000000..4169c9d2d --- /dev/null +++ b/client/firewall/nftables/family_linux.go @@ -0,0 +1,241 @@ +//go:build !android + +package nftables + +import ( + "fmt" + "net/netip" + + "github.com/coreos/go-iptables/iptables" + "github.com/google/nftables" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + "github.com/netbirdio/netbird/client/firewall/firewalld" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/internal/routemanager/ipfwdstate" + "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" +) + +const ( + tableNat = "nat" + tableMangle = "mangle" + tableRaw = "raw" + tableSecurity = "security" + + chainNameRoutingFw = "netbird-rt-fwd" + chainNameRoutingNat = "netbird-rt-postrouting" + chainNameRoutingRdr = "netbird-rt-redirect" + chainNameNATOutput = "netbird-nat-output" + chainNameForward = "FORWARD" + chainNameMangleForward = "netbird-mangle-forward" + + // Peer ACL chain names. + chainNameInputRules = "netbird-acl-input-rules" + chainNameInputFilter = "netbird-acl-input-filter" + chainNameForwardFilter = "netbird-acl-forward-filter" + chainNameManglePrerouting = "netbird-mangle-prerouting" + chainNameManglePostrouting = "netbird-mangle-postrouting" + + flushError = "flush: %w" + + firewalldTableName = "firewalld" + + userDataAcceptForwardRuleIif = "frwacceptiif" + userDataAcceptForwardRuleOif = "frwacceptoif" + userDataAcceptInputRule = "inputaccept" + + // ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation. + ipv4TCPHeaderSize = 40 + // ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation. + ipv6TCPHeaderSize = 60 + + // maxPrefixesSet 1638 prefixes start to fail, taking some margin + maxPrefixesSet = 1500 + refreshRulesMapError = "refresh rules map: %w" +) + +var ( + errFilterTableNotFound = fmt.Errorf("'filter' table not found") +) + +type setInput struct { + set firewall.Set + prefixes []netip.Prefix +} + +// family holds the per-address-family nftables state. One instance +// handles route ACLs, peer ACLs, NAT, DNAT, and MSS clamping for a +// single family; the top-level Manager owns one for v4 and another +// for v6. The name predates the peer-ACL absorption; it's effectively +// the per-family backend now. +type family struct { + conn *nftables.Conn + workTable *nftables.Table + filterTable *nftables.Table + chains map[string]*nftables.Chain + + // filters holds peer + route filter rules keyed by content hash. + // AddFilterRule writes here; DeleteFilterRule looks up by id. + filters map[firewall.RuleID]*Rule + + // rules holds NAT, DNAT, and external accept rules (auxiliary + // plumbing that isn't a filter rule). + rules map[firewall.RuleID]*nftables.Rule + + // Peer ACL chain handles. + chainInputRules *nftables.Chain + chainPrerouting *nftables.Chain + routingFwChainName string + + ipsetCounter *refcounter.Counter[string, setInput, *nftables.Set] + + af addrFamily + wgIface iFaceMapper + ipFwdState *ipfwdstate.IPForwardingState + legacyManagement bool + mtu uint16 +} + +func newFamily(workTable *nftables.Table, wgIface iFaceMapper, mtu uint16) *family { + r := &family{ + conn: &nftables.Conn{}, + workTable: workTable, + chains: make(map[string]*nftables.Chain), + filters: make(map[firewall.RuleID]*Rule), + rules: make(map[firewall.RuleID]*nftables.Rule), + routingFwChainName: chainNameRoutingFw, + af: familyForAddr(workTable.Family == nftables.TableFamilyIPv4), + wgIface: wgIface, + ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()), + mtu: mtu, + } + + r.ipsetCounter = refcounter.New( + r.createIpSet, + r.deleteIpSet, + ) + + var err error + r.filterTable, err = r.loadFilterTable() + if err != nil { + log.Debugf("ip filter table not found: %v", err) + } + + return r +} + +func (r *family) init(workTable *nftables.Table) error { + r.workTable = workTable + + if err := r.removeAcceptFilterRules(); err != nil { + log.Errorf("failed to clean up rules from filter table: %s", err) + } + + if err := r.createContainers(); err != nil { + return fmt.Errorf("create containers: %w", err) + } + + if err := r.setupDataPlaneMark(); err != nil { + log.Errorf("failed to set up data plane mark: %v", err) + } + + if err := r.createDefaultChains(); err != nil { + return fmt.Errorf("create default acl chains: %w", err) + } + + return nil +} + +// Reset cleans existing nftables filter table rules from the system +func (r *family) Reset() error { + // clear without deleting the ipsets, the nf table will be deleted by the caller + r.ipsetCounter.Clear() + + var merr *multierror.Error + + if err := r.removeAcceptFilterRules(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove accept filter rules: %w", err)) + } + + if err := firewalld.UntrustInterface(r.wgIface.Name()); err != nil { + merr = multierror.Append(merr, err) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) loadFilterTable() (*nftables.Table, error) { + tables, err := r.conn.ListTablesOfFamily(r.af.tableFamily) + if err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + + for _, table := range tables { + if table.Name == "filter" { + return table, nil + } + } + + return nil, errFilterTableNotFound +} + +func hookName(hook *nftables.ChainHook) string { + if hook == nil { + return "unknown" + } + switch *hook { + case *nftables.ChainHookForward: + return chainNameForward + case *nftables.ChainHookInput: + return chainNameInput + default: + return fmt.Sprintf("hook(%d)", *hook) + } +} + +func familyName(family nftables.TableFamily) string { + switch family { + case nftables.TableFamilyIPv4: + return "ip" + case nftables.TableFamilyIPv6: + return "ip6" + case nftables.TableFamilyINet: + return "inet" + default: + return fmt.Sprintf("family(%d)", family) + } +} + +func (r *family) iptablesProto() iptables.Protocol { + if r.af.tableFamily == nftables.TableFamilyIPv6 { + return iptables.ProtocolIPv6 + } + return iptables.ProtocolIPv4 +} + +func (r *family) refreshRulesMap() error { + var merr *multierror.Error + newRules := make(map[firewall.RuleID]*nftables.Rule) + for _, chain := range r.chains { + rules, err := r.conn.GetRules(chain.Table, chain) + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("list rules for chain %s: %w", chain.Name, err)) + // preserve existing entries for this chain since we can't verify their state + for k, v := range r.rules { + if v.Chain != nil && v.Chain.Name == chain.Name { + newRules[k] = v + } + } + continue + } + for _, rule := range rules { + if len(rule.UserData) > 0 { + newRules[firewall.RuleID(rule.UserData)] = rule + } + } + } + r.rules = newRules + return nberrors.FormatErrorOrNil(merr) +} diff --git a/client/firewall/nftables/filter_linux.go b/client/firewall/nftables/filter_linux.go new file mode 100644 index 000000000..bb3ac1dfe --- /dev/null +++ b/client/firewall/nftables/filter_linux.go @@ -0,0 +1,535 @@ +//go:build !android + +package nftables + +import ( + "fmt" + "net" + "net/netip" + "slices" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbid "github.com/netbirdio/netbird/client/internal/acl/id" +) + +// AddFilterRule installs one nftables packet-filter rule. With +// destination empty the rule goes to the peer ACL input chain plus a +// paired prerouting mangle rule for the redirect mark. With +// destination set (prefix or named set) it goes to the route ACL +// forward chain. Multi-source rules collapse to one nftables rule +// backed by the shared refcounted hash:net set. +func (r *family) AddFilterRule( + id []byte, + sources []netip.Prefix, + destination firewall.Network, + proto firewall.Protocol, + sPort *firewall.Port, + dPort *firewall.Port, + action firewall.Action, +) (firewall.Rule, error) { + isRoute := !destination.IsZero() + + ruleID := nbid.GenerateRuleID(sources, destination, proto, sPort, dPort, action) + if existing, ok := r.filters[ruleID]; ok { + return existing, nil + } + + srcExprs, err := r.applyNetwork(sourceNetwork(sources), sources, true) + if err != nil { + return nil, fmt.Errorf("apply source: %w", err) + } + + var exprs []expr.Any + if isRoute { + exprs, err = r.buildRouteFilterExprs(srcExprs, destination, proto, sPort, dPort) + } else { + exprs, err = r.buildPeerFilterExprs(srcExprs, proto, sPort, dPort) + } + if err != nil { + r.dropNetworkMatch(srcExprs) + return nil, err + } + + mainExprs := slices.Clone(exprs) + verdict := expr.VerdictAccept + if action == firewall.ActionDrop { + verdict = expr.VerdictDrop + } + mainExprs = append(mainExprs, &expr.Verdict{Kind: verdict}) + + chain := r.chainInputRules + if isRoute { + chain = r.chains[chainNameRoutingFw] + } + + userData := []byte(ruleID) + + // Build the paired prerouting mangle rule before flushing so both + // rules commit in one transaction. An anonymous port set binds to + // exactly one rule, so the mangle rule needs its own expression list + // with fresh sets, not a clone of the main rule's. Guard on the + // prerouting chain first: building the expressions queues the port + // set, so skipping the build when there is no chain to bind it to + // keeps an unbound set out of the connection batch. + var mangleRule *nftables.Rule + if !isRoute && r.chainPrerouting != nil { + mangleExprs, err := r.buildPeerFilterExprs(srcExprs, proto, sPort, dPort) + if err != nil { + r.dropNetworkMatch(exprs) + return nil, fmt.Errorf("build mangle rule: %w", err) + } + mangleRule = r.queuePreroutingRule(mangleExprs, userData) + } + + nftRule := &nftables.Rule{ + Table: r.workTable, + Chain: chain, + Exprs: mainExprs, + UserData: userData, + } + if action == firewall.ActionDrop { + nftRule = r.conn.InsertRule(nftRule) + } else { + nftRule = r.conn.AddRule(nftRule) + } + if err := r.conn.Flush(); err != nil { + r.dropNetworkMatch(exprs) + return nil, fmt.Errorf(flushError, err) + } + + rule := &Rule{ + nftRule: nftRule, + mangleRule: mangleRule, + sources: sources, + id: ruleID, + } + r.filters[ruleID] = rule + + log.Debugf("added filter rule: sources=%v, destination=%v, proto=%v, sPort=%v, dPort=%v, action=%v", + sources, destination, proto, sPort, dPort, action) + return rule, nil +} + +// buildPeerFilterExprs assembles the input-chain (peer ACL) match: the +// IP-header protocol byte read via Payload, then source, then ports +// (no counter), matching the historical peer shape so per-rule kernel +// state is identical to pre-unification. +func (r *family) buildPeerFilterExprs( + srcExprs []expr.Any, + proto firewall.Protocol, + sPort, dPort *firewall.Port, +) ([]expr.Any, error) { + var exprs []expr.Any + + if proto != firewall.ProtocolALL { + protoNum, err := r.af.protoNum(proto) + if err != nil { + return nil, fmt.Errorf("convert protocol to number: %w", err) + } + exprs = append(exprs, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: r.af.protoOffset, + Len: 1, + }, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{protoNum}}, + ) + } + exprs = append(exprs, srcExprs...) + + portExprs, err := r.applyPorts(sPort, dPort) + if err != nil { + return nil, err + } + exprs = append(exprs, portExprs...) + return exprs, nil +} + +// buildRouteFilterExprs assembles the forward-chain (route ACL) match: +// source, then destination, then optional proto/ports, then a counter. +func (r *family) buildRouteFilterExprs( + srcExprs []expr.Any, + destination firewall.Network, + proto firewall.Protocol, + sPort, dPort *firewall.Port, +) ([]expr.Any, error) { + exprs := append([]expr.Any{}, srcExprs...) + + destExprs, err := r.applyNetwork(destination, nil, false) + if err != nil { + return nil, fmt.Errorf("apply destination: %w", err) + } + exprs = append(exprs, destExprs...) + + if proto != firewall.ProtocolALL { + protoNum, err := r.af.protoNum(proto) + if err != nil { + r.dropNetworkMatch(destExprs) + return nil, fmt.Errorf("convert protocol to number: %w", err) + } + exprs = append(exprs, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{protoNum}}, + ) + + portExprs, err := r.applyPorts(sPort, dPort) + if err != nil { + r.dropNetworkMatch(destExprs) + return nil, err + } + exprs = append(exprs, portExprs...) + } + + exprs = append(exprs, &expr.Counter{}) + return exprs, nil +} + +func (r *family) hasRule(id firewall.RuleID) bool { + _, ok := r.filters[id] + return ok +} + +// DeleteFilterRule removes a previously installed filter rule. Source +// set references are recovered from the stored rule's expressions via +// findSets and dropped from the shared refcounter. +func (r *family) DeleteFilterRule(rule firewall.Rule) error { + ruleID := rule.ID() + pr, ok := r.filters[ruleID] + if !ok { + log.Debugf("filter rule %s not found", ruleID) + return nil + } + + // A freshly added rule carries no handle until it is read back from + // the kernel, and Flush only refreshes the peer chains. Pull live + // handles for this rule's chain before deciding it is stale so route + // rules (which Flush never refreshes) can actually be deleted. A + // refresh failure aborts the delete without touching tracking state, + // so the caller can retry while the rule may still exist in the kernel. + if pr.nftRule.Handle == 0 { + if err := r.refreshRuleHandles(pr.nftRule.Chain, false); err != nil { + return fmt.Errorf("refresh handles for chain %s: %w", pr.nftRule.Chain.Name, err) + } + } + // Refresh the mangle handle independently: the main rule's handle can + // be populated while the prerouting refresh during Flush failed, and + // gating the mangle refresh on the main handle would leak the mangle + // rule on delete. + if pr.mangleRule != nil && pr.mangleRule.Handle == 0 { + if err := r.refreshRuleHandles(r.chainPrerouting, true); err != nil { + return fmt.Errorf("refresh mangle handles: %w", err) + } + } + + if pr.nftRule.Handle == 0 { + log.Warnf("filter rule %s has no handle, removing stale entry", ruleID) + // The paired mangle rule can still be in the kernel with a live + // handle. Dropping the tracking entry without removing it would + // leave a prerouting rule that nothing can find again. + if err := r.deleteMangleRule(pr, ruleID); err != nil { + return err + } + r.dropNetworkMatch(pr.nftRule.Exprs) + delete(r.filters, ruleID) + return nil + } + + if err := r.conn.DelRule(pr.nftRule); err != nil { + log.Errorf("queue rule delete: %v", err) + } + r.queueMangleDelete(pr) + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush delete %s: %w", ruleID, err) + } + + r.dropNetworkMatch(pr.nftRule.Exprs) + delete(r.filters, ruleID) + return nil +} + +// deleteMangleRule removes the prerouting rule paired with a filter rule on +// its own, for the paths that drop the filter rule's tracking without queueing +// a delete for it. +func (r *family) deleteMangleRule(pr *Rule, ruleID firewall.RuleID) error { + if pr.mangleRule == nil || pr.mangleRule.Handle == 0 { + return nil + } + + r.queueMangleDelete(pr) + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush mangle delete %s: %w", ruleID, err) + } + return nil +} + +// queueMangleDelete queues the delete of the rule's prerouting counterpart, if +// it has one. The caller commits it. +func (r *family) queueMangleDelete(pr *Rule) { + if pr.mangleRule == nil { + return + } + if err := r.conn.DelRule(pr.mangleRule); err != nil { + log.Errorf("queue mangle rule delete: %v", err) + } +} + +func (r *family) decrementSetCounter(rule *nftables.Rule) error { + if r.ipsetCounter == nil { + return nil + } + sets := findSets(rule) + + var merr *multierror.Error + for _, setName := range sets { + if _, err := r.ipsetCounter.Decrement(setName); err != nil { + merr = multierror.Append(merr, fmt.Errorf("decrement set counter: %w", err)) + } + } + + return nberrors.FormatErrorOrNil(merr) +} + +// dropNetworkMatch undoes whatever the source/destination match +// reserved. Safe to call when the spec is empty or holds only inline +// matchers. +func (r *family) dropNetworkMatch(exprs []expr.Any) { + if r.ipsetCounter == nil { + return + } + for _, e := range exprs { + lookup, ok := e.(*expr.Lookup) + if !ok { + continue + } + if _, err := r.ipsetCounter.Decrement(lookup.SetName); err != nil { + log.Errorf("rollback ipset decrement %s: %v", lookup.SetName, err) + } + } +} + +func (r *family) applyNetwork( + network firewall.Network, + setPrefixes []netip.Prefix, + isSource bool, +) ([]expr.Any, error) { + if network.IsSet() { + exprs, err := r.getIpSet(network.Set, setPrefixes, isSource) + if err != nil { + side := "destination" + if isSource { + side = "source" + } + return nil, fmt.Errorf("%s set: %w", side, err) + } + return exprs, nil + } + + if network.IsPrefix() { + return prefixMatchExprs(r.af, network.Prefix, isSource), nil + } + + return nil, nil +} + +// applyPort builds the transport-header port match. A single value +// compares directly, a range uses a range expression, and multiple +// values go through an anonymous constant set: consecutive cmp +// expressions AND together, so chained equality comparisons could +// never match more than one port. The set is queued on the +// connection and committed by the caller's flush together with the +// rule that binds it. +func (r *family) applyPort(port *firewall.Port, isSource bool) ([]expr.Any, error) { + if port == nil || len(port.Values) == 0 { + return nil, nil + } + + // dst port + offset := uint32(2) + if isSource { + // src port + offset = 0 + } + + exprs := []expr.Any{ + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: offset, + Len: 2, + }, + } + + switch { + case port.IsRange && len(port.Values) == 2: + exprs = append(exprs, &expr.Range{ + Op: expr.CmpOpEq, + Register: 1, + FromData: binaryutil.BigEndian.PutUint16(port.Values[0]), + ToData: binaryutil.BigEndian.PutUint16(port.Values[1]), + }) + case len(port.Values) == 1: + exprs = append(exprs, &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.BigEndian.PutUint16(port.Values[0]), + }) + default: + lookup, err := r.anonymousPortSet(port.Values) + if err != nil { + return nil, err + } + exprs = append(exprs, lookup) + } + + return exprs, nil +} + +// anonymousPortSet queues an anonymous constant set holding the given +// ports on the connection and returns a lookup against it. The set is +// committed by the caller's flush together with the rule that binds it. +func (r *family) anonymousPortSet(values []uint16) (*expr.Lookup, error) { + set := &nftables.Set{ + Anonymous: true, + Constant: true, + Table: r.workTable, + KeyType: nftables.TypeInetService, + } + elements := make([]nftables.SetElement, 0, len(values)) + for _, p := range values { + elements = append(elements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(p)}) + } + if err := r.conn.AddSet(set, elements); err != nil { + return nil, fmt.Errorf("add anonymous port set: %w", err) + } + return &expr.Lookup{ + SourceRegister: 1, + SetID: set.ID, + SetName: set.Name, + }, nil +} + +// applyPorts builds the source then destination port matches. +func (r *family) applyPorts(sPort, dPort *firewall.Port) ([]expr.Any, error) { + sPortExprs, err := r.applyPort(sPort, true) + if err != nil { + return nil, fmt.Errorf("apply source port: %w", err) + } + + dPortExprs, err := r.applyPort(dPort, false) + if err != nil { + return nil, fmt.Errorf("apply destination port: %w", err) + } + + return append(sPortExprs, dPortExprs...), nil +} + +// prefixMatchExprs is the family-aware match sequence for a CIDR +// prefix. /0 returns nil; a host prefix (full bit length for the +// family) skips the bitwise step since the mask is all-ones. Shared +// between family and aclManager so both treat single prefixes +// identically. +func prefixMatchExprs(af addrFamily, prefix netip.Prefix, isSource bool) []expr.Any { + offset := af.dstAddrOffset + if isSource { + offset = af.srcAddrOffset + } + + ones := prefix.Bits() + if ones == 0 { + return nil + } + + payload := &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: offset, + Len: af.addrLen, + } + cmp := &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: prefix.Masked().Addr().AsSlice(), + } + + if ones == af.totalBits { + return []expr.Any{payload, cmp} + } + + mask := net.CIDRMask(ones, af.totalBits) + xor := make([]byte, af.addrLen) + return []expr.Any{ + payload, + &expr.Bitwise{ + DestRegister: 1, + SourceRegister: 1, + Len: af.addrLen, + Mask: mask, + Xor: xor, + }, + cmp, + } +} + +func getCtNewExprs() []expr.Any { + return []expr.Any{ + &expr.Ct{ + Key: expr.CtKeySTATE, + Register: 1, + }, + &expr.Bitwise{ + SourceRegister: 1, + DestRegister: 1, + Len: 4, + Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW), + Xor: binaryutil.NativeEndian.PutUint32(0), + }, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte{0, 0, 0, 0}, + }, + } +} + +// sourceNetwork classifies a source-prefix list into the firewall.Network +// shape the rest of the spec-builder consumes: empty for match-any, a +// single prefix inline, or an ipset for multiple sources. +func sourceNetwork(sources []netip.Prefix) firewall.Network { + switch { + case len(sources) == 0: + return firewall.Network{} + case len(sources) == 1 && sources[0].Bits() == 0: + return firewall.Network{} + case len(sources) == 1: + return firewall.Network{Prefix: sources[0]} + default: + return firewall.Network{Set: firewall.NewPrefixSet(sources)} + } +} + +func ifname(n string) []byte { + b := make([]byte, 16) + copy(b, n+"\x00") + return b +} + +// findSets scans an nftables rule's expressions for expr.Lookup and +// returns the named sets in occurrence order. Used at delete time to +// drop ipsetCounter references; peer and route ACLs go through it. +func findSets(rule *nftables.Rule) []string { + var sets []string + for _, e := range rule.Exprs { + if lookup, ok := e.(*expr.Lookup); ok { + sets = append(sets, lookup.SetName) + } + } + return sets +} diff --git a/client/firewall/nftables/interface_allower_integration_linux_test.go b/client/firewall/nftables/interface_allower_integration_linux_test.go new file mode 100644 index 000000000..4d4bc6187 --- /dev/null +++ b/client/firewall/nftables/interface_allower_integration_linux_test.go @@ -0,0 +1,90 @@ +//go:build privileged + +package nftables + +import ( + "bytes" + "os" + "testing" + + "github.com/google/nftables" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/iface" +) + +// TestInterfaceAllowerInputOnly verifies the userspace-mode allower opens the +// interface on the INPUT hook of foreign chains only (not FORWARD, since the +// userspace router never forwards in the kernel), creates no netbird work +// table, and removes its rules on Close. +func TestInterfaceAllowerInputOnly(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("root required") + } + + require.False(t, ipTableExists(t, getTableName()), "precondition: no stale netbird table") + + conn := &nftables.Conn{} + extTable := conn.AddTable(&nftables.Table{Name: "nbtest_extchains", Family: nftables.TableFamilyINet}) + inputChain := conn.AddChain(&nftables.Chain{ + Name: "ext_input", Table: extTable, + Hooknum: nftables.ChainHookInput, Priority: nftables.ChainPriorityFilter, Type: nftables.ChainTypeFilter, + }) + forwardChain := conn.AddChain(&nftables.Chain{ + Name: "ext_forward", Table: extTable, + Hooknum: nftables.ChainHookForward, Priority: nftables.ChainPriorityFilter, Type: nftables.ChainTypeFilter, + }) + require.NoError(t, conn.Flush(), "create external table and chains") + t.Cleanup(func() { + c := &nftables.Conn{} + c.DelTable(extTable) + _ = c.Flush() + }) + + allower, err := NewInterfaceAllower(ifaceMock, iface.DefaultMTU) + require.NoError(t, err, "create allower") + require.NoError(t, allower.Apply(), "apply") + + require.True(t, chainHasUserData(t, extTable, inputChain, userDataAcceptInputRule), + "external INPUT chain should get the accept rule") + require.Len(t, listRules(t, extTable, forwardChain), 0, + "external FORWARD chain must not be opened in userspace mode") + require.False(t, ipTableExists(t, getTableName()), + "allower must not create a netbird work table") + + require.NoError(t, allower.Close(), "close") + require.False(t, chainHasUserData(t, extTable, inputChain, userDataAcceptInputRule), + "accept rule should be removed on close") +} + +func listRules(t *testing.T, table *nftables.Table, chain *nftables.Chain) []*nftables.Rule { + t.Helper() + c := &nftables.Conn{} + rules, err := c.GetRules(table, chain) + require.NoError(t, err) + return rules +} + +func chainHasUserData(t *testing.T, table *nftables.Table, chain *nftables.Chain, ud string) bool { + for _, r := range listRules(t, table, chain) { + if bytes.Equal(r.UserData, []byte(ud)) { + return true + } + } + return false +} + +func ipTableExists(t *testing.T, name string) bool { + t.Helper() + c := &nftables.Conn{} + for _, fam := range []nftables.TableFamily{nftables.TableFamilyIPv4, nftables.TableFamilyIPv6} { + tbls, err := c.ListTablesOfFamily(fam) + require.NoError(t, err) + for _, tb := range tbls { + if tb.Name == name { + return true + } + } + } + return false +} diff --git a/client/firewall/nftables/interface_allower_linux.go b/client/firewall/nftables/interface_allower_linux.go new file mode 100644 index 000000000..e7232e2bf --- /dev/null +++ b/client/firewall/nftables/interface_allower_linux.go @@ -0,0 +1,107 @@ +package nftables + +import ( + "fmt" + + "github.com/google/nftables" + "github.com/hashicorp/go-multierror" + + nberrors "github.com/netbirdio/netbird/client/errors" +) + +// InterfaceAllower opens the NetBird interface in the kernel's filter table and +// external chains and keeps them reconciled via a netlink monitor, so the host +// firewall doesn't drop traffic the NetBird firewall handles. It is used by the +// userspace firewall, where routing happens in the forwarder, so only INPUT is +// opened (the userspace router never forwards in the kernel). +// +// It owns its own families/connection and never creates a netbird work table. +// firewalld trust is handled by the caller, not here. Its operations are serial +// (Apply before the monitor starts; reconciles run on the single monitor +// goroutine; Close stops the monitor before removing), so it needs no locking. +// +// TODO: this opens nftables and the iptables-nft filter table (detected via +// nft), but not a legacy-iptables ruleset running in parallel with nftables. +// Such a host would keep its legacy filter chains closed for the interface. +type InterfaceAllower struct { + family4 *family + family6 *family + extMonitor *externalChainMonitor +} + +// NewInterfaceAllower builds an allower for the given interface. It returns an +// error when nftables is unavailable (e.g. an iptables-legacy host), so the +// caller can fall back to firewalld trust. +func NewInterfaceAllower(wgIface iFaceMapper, mtu uint16) (*InterfaceAllower, error) { + tableName := getTableName() + + family4 := newFamily(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4}, wgIface, mtu) + + // Probe nftables availability before committing to this backend. + if _, err := family4.conn.ListChainsOfTableFamily(nftables.TableFamilyINet); err != nil { + return nil, fmt.Errorf("nftables not available: %w", err) + } + + a := &InterfaceAllower{family4: family4} + + if wgIface.Address().HasIPv6() { + a.family6 = newFamily(&nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv6}, wgIface, mtu) + } + + a.extMonitor = newExternalChainMonitor(a) + return a, nil +} + +// Apply opens the interface (INPUT only) in the foreign filter chains and starts +// reconciling them on nftables changes. +func (a *InterfaceAllower) Apply() error { + var merr *multierror.Error + for _, f := range a.families() { + // Remove any stale accepts first so a prior unclean exit (e.g. SIGKILL, + // where Close never ran) is recovered deterministically rather than + // accumulating duplicate rules on the iptables filter table. + if err := f.removeAcceptFilterRules(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("clean stale accept rules: %w", err)) + } + if err := f.openInterface(false); err != nil { + merr = multierror.Append(merr, err) + } + } + + a.extMonitor.start() + return nberrors.FormatErrorOrNil(merr) +} + +// families returns the configured address families (v4, and v6 when present). +func (a *InterfaceAllower) families() []*family { + families := []*family{a.family4} + if a.family6 != nil { + families = append(families, a.family6) + } + return families +} + +// reconcileExternalChains re-applies the INPUT accepts to external chains. It +// implements externalChainReconciler for the monitor. +func (a *InterfaceAllower) reconcileExternalChains() error { + var merr *multierror.Error + for _, f := range a.families() { + if err := f.acceptExternalChainsRules(false); err != nil { + merr = multierror.Append(merr, err) + } + } + return nberrors.FormatErrorOrNil(merr) +} + +// Close stops the monitor and removes the accept rules. +func (a *InterfaceAllower) Close() error { + a.extMonitor.stop() + + var merr *multierror.Error + for _, f := range a.families() { + if err := f.removeAcceptFilterRules(); err != nil { + merr = multierror.Append(merr, err) + } + } + return nberrors.FormatErrorOrNil(merr) +} diff --git a/client/firewall/nftables/ipset_linux.go b/client/firewall/nftables/ipset_linux.go new file mode 100644 index 000000000..34783bbeb --- /dev/null +++ b/client/firewall/nftables/ipset_linux.go @@ -0,0 +1,210 @@ +//go:build !android + +package nftables + +import ( + "encoding/binary" + "fmt" + "net/netip" + + "github.com/google/nftables" + "github.com/google/nftables/expr" + log "github.com/sirupsen/logrus" + + firewall "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" +) + +func (r *family) getIpSet(set firewall.Set, prefixes []netip.Prefix, isSource bool) ([]expr.Any, error) { + ref, err := r.ipsetCounter.Increment(set.HashedName(), setInput{ + set: set, + prefixes: prefixes, + }) + if err != nil { + return nil, fmt.Errorf("create or get ipset: %w", err) + } + + return r.getIpSetExprs(ref, isSource) +} + +func (r *family) createIpSet(setName string, input setInput) (*nftables.Set, error) { + // overlapping prefixes will result in an error, so we need to merge them + prefixes := firewall.MergeIPRanges(input.prefixes) + + nfset := &nftables.Set{ + Name: setName, + Comment: input.set.Comment(), + Table: r.workTable, + // required for prefixes + Interval: true, + KeyType: r.af.setKeyType, + } + + elements := r.convertPrefixesToSet(prefixes) + nElements := len(elements) + + maxElements := maxPrefixesSet * 2 + initialElements := elements[:min(maxElements, nElements)] + + if err := r.conn.AddSet(nfset, initialElements); err != nil { + return nil, fmt.Errorf("error adding set %s: %w", setName, err) + } + if err := r.conn.Flush(); err != nil { + return nil, fmt.Errorf("flush error: %w", err) + } + log.Debugf("Created new ipset: %s with %d initial prefixes (total prefixes %d)", setName, len(initialElements)/2, len(prefixes)) + + // The set is committed now. If a later batch fails, destroy it: the + // refcounter records nothing on a create-callback error, so it would + // otherwise leak, and a partial source set fails-open for deny rules. + if err := r.addRemainingElements(nfset, elements, maxElements); err != nil { + if derr := r.deleteIpSet(setName, nfset); derr != nil { + log.Warnf("rollback ipset %s after add failure: %v", setName, derr) + } + return nil, err + } + + log.Infof("Created new ipset: %s with %d prefixes", setName, len(prefixes)) + return nfset, nil +} + +// addRemainingElements adds element batches beyond the initial one in +// maxElements-sized chunks, flushing each. Called after the set has been +// created with its first batch. +func (r *family) addRemainingElements(nfset *nftables.Set, elements []nftables.SetElement, maxElements int) error { + nElements := len(elements) + for subStart := maxElements; subStart < nElements; subStart += maxElements { + subEnd := min(subStart+maxElements, nElements) + subElement := elements[subStart:subEnd] + nSubPrefixes := len(subElement) / 2 + log.Tracef("Adding new prefixes (%d) in ipset: %s", nSubPrefixes, nfset.Name) + if err := r.conn.SetAddElements(nfset, subElement); err != nil { + return fmt.Errorf("error adding prefixes (%d) to set %s: %w", nSubPrefixes, nfset.Name, err) + } + if err := r.conn.Flush(); err != nil { + return fmt.Errorf("flush error: %w", err) + } + log.Debugf("Added new prefixes (%d) in ipset: %s", nSubPrefixes, nfset.Name) + } + return nil +} + +func (r *family) convertPrefixesToSet(prefixes []netip.Prefix) []nftables.SetElement { + var elements []nftables.SetElement + for _, prefix := range prefixes { + // nftables needs half-open intervals [firstIP, lastIP) for prefixes + // e.g. 10.0.0.0/24 becomes [10.0.0.0, 10.0.1.0), 10.1.1.1/32 becomes [10.1.1.1, 10.1.1.2) etc + firstIP := prefix.Addr() + + // For a /0 the last address is the broadcast and its Next() overflows + // to an invalid Addr with an empty key, so wrap to the zero address, + // which nftables reads as the open end of a full-range interval. + var lastKey []byte + if prefix.Bits() == 0 { + lastKey = make([]byte, r.af.addrLen) + } else { + lastKey = calculateLastIP(prefix).Next().AsSlice() + } + + // the nft tool also adds a zero-address IntervalEnd element, see https://github.com/google/nftables/issues/247 + // nftables.SetElement{Key: make([]byte, r.af.addrLen), IntervalEnd: true}, + elements = append(elements, + nftables.SetElement{Key: firstIP.AsSlice()}, + nftables.SetElement{Key: lastKey, IntervalEnd: true}, + ) + } + return elements +} + +// calculateLastIP determines the last IP in a given prefix. +func calculateLastIP(prefix netip.Prefix) netip.Addr { + masked := prefix.Masked() + if masked.Addr().Is4() { + hostMask := ^uint32(0) >> masked.Bits() + lastIP := uint32FromNetipAddr(masked.Addr()) | hostMask + return netip.AddrFrom4(uint32ToBytes(lastIP)) + } + + // IPv6: set host bits to all 1s + b := masked.Addr().As16() + bits := masked.Bits() + for i := bits; i < 128; i++ { + b[i/8] |= 1 << (7 - i%8) + } + return netip.AddrFrom16(b) +} + +// Utility function to convert netip.Addr to uint32. +func uint32FromNetipAddr(addr netip.Addr) uint32 { + b := addr.As4() + return binary.BigEndian.Uint32(b[:]) +} + +// Utility function to convert uint32 to a netip-compatible byte slice. +func uint32ToBytes(ip uint32) [4]byte { + var b [4]byte + binary.BigEndian.PutUint32(b[:], ip) + return b +} + +func (r *family) deleteIpSet(setName string, nfset *nftables.Set) error { + r.conn.DelSet(nfset) + if err := r.conn.Flush(); err != nil { + return fmt.Errorf(flushError, err) + } + + log.Debugf("Deleted unused ipset %s", setName) + return nil +} + +func (r *family) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { + nfset, err := r.conn.GetSetByName(r.workTable, set.HashedName()) + if err != nil { + return fmt.Errorf("get set %s: %w", set.HashedName(), err) + } + + // Overlapping prefixes (e.g. duplicate resolved addresses) make the + // interval set reject the batch, so merge them as createIpSet does. + prefixes = firewall.MergeIPRanges(prefixes) + elements := r.convertPrefixesToSet(prefixes) + + // Add in batches sized like createIpSet so a large update does not + // exceed the netlink message size limit. + maxElements := maxPrefixesSet * 2 + for start := 0; start < len(elements); start += maxElements { + end := min(start+maxElements, len(elements)) + if err := r.conn.SetAddElements(nfset, elements[start:end]); err != nil { + return fmt.Errorf("add elements to set %s: %w", set.HashedName(), err) + } + if err := r.conn.Flush(); err != nil { + return fmt.Errorf(flushError, err) + } + } + + log.Debugf("updated set %s with %d prefixes", set.HashedName(), len(prefixes)) + + return nil +} + +func (r *family) getIpSetExprs(ref refcounter.Ref[*nftables.Set], isSource bool) ([]expr.Any, error) { + // dst offset by default + offset := r.af.dstAddrOffset + if isSource { + // src offset + offset = r.af.srcAddrOffset + } + + return []expr.Any{ + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: offset, + Len: r.af.addrLen, + }, + &expr.Lookup{ + SourceRegister: 1, + SetName: ref.Out.Name, + SetID: ref.Out.ID, + }, + }, nil +} diff --git a/client/firewall/nftables/ipset_linux_test.go b/client/firewall/nftables/ipset_linux_test.go new file mode 100644 index 000000000..7ab2f6c3f --- /dev/null +++ b/client/firewall/nftables/ipset_linux_test.go @@ -0,0 +1,36 @@ +package nftables + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestConvertPrefixesToSetWildcard verifies that a /0 prefix produces a +// usable interval. The last address of a /0 is the broadcast, whose Next() +// overflows to an invalid Addr with an empty key; the IntervalEnd must wrap +// to the zero address instead so nftables sees a full-range interval. +func TestConvertPrefixesToSetWildcard(t *testing.T) { + tests := []struct { + name string + af addrFamily + prefix string + }{ + {"IPv4 /0", afIPv4, "0.0.0.0/0"}, + {"IPv6 /0", afIPv6, "::/0"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &family{af: tt.af} + elements := r.convertPrefixesToSet([]netip.Prefix{netip.MustParsePrefix(tt.prefix)}) + + require.Len(t, elements, 2, "expected start and interval-end element") + assert.False(t, elements[0].IntervalEnd, "first element is the interval start") + assert.True(t, elements[1].IntervalEnd, "second element is the interval end") + assert.Len(t, elements[1].Key, int(tt.af.addrLen), "interval-end key must be a zero address, not empty") + }) + } +} diff --git a/client/firewall/nftables/ipsetstore_linux.go b/client/firewall/nftables/ipsetstore_linux.go deleted file mode 100644 index a6c2e9496..000000000 --- a/client/firewall/nftables/ipsetstore_linux.go +++ /dev/null @@ -1,85 +0,0 @@ -package nftables - -import ( - "net" -) - -type ipsetStore struct { - ipsetReference map[string]int - ipsets map[string]map[string]struct{} // ipsetName -> list of ips -} - -func newIpsetStore() *ipsetStore { - return &ipsetStore{ - ipsetReference: make(map[string]int), - ipsets: make(map[string]map[string]struct{}), - } -} - -func (s *ipsetStore) ips(ipsetName string) (map[string]struct{}, bool) { - r, ok := s.ipsets[ipsetName] - return r, ok -} - -func (s *ipsetStore) newIpset(ipsetName string) map[string]struct{} { - s.ipsetReference[ipsetName] = 0 - ipList := make(map[string]struct{}) - s.ipsets[ipsetName] = ipList - return ipList -} - -func (s *ipsetStore) deleteIpset(ipsetName string) { - delete(s.ipsetReference, ipsetName) - delete(s.ipsets, ipsetName) -} - -func (s *ipsetStore) DeleteIpFromSet(ipsetName string, ip net.IP) { - ipList, ok := s.ipsets[ipsetName] - if !ok { - return - } - delete(ipList, ip.String()) -} - -func (s *ipsetStore) AddIpToSet(ipsetName string, ip net.IP) { - ipList, ok := s.ipsets[ipsetName] - if !ok { - return - } - ipList[ip.String()] = struct{}{} -} - -func (s *ipsetStore) IsIpInSet(ipsetName string, ip net.IP) bool { - ipList, ok := s.ipsets[ipsetName] - if !ok { - return false - } - _, ok = ipList[ip.String()] - return ok -} - -func (s *ipsetStore) AddReferenceToIpset(ipsetName string) { - s.ipsetReference[ipsetName]++ -} - -func (s *ipsetStore) DeleteReferenceFromIpSet(ipsetName string) { - r, ok := s.ipsetReference[ipsetName] - if !ok { - return - } - if r == 0 { - return - } - s.ipsetReference[ipsetName]-- -} - -func (s *ipsetStore) HasReferenceToSet(ipsetName string) bool { - if _, ok := s.ipsetReference[ipsetName]; !ok { - return false - } - if s.ipsetReference[ipsetName] == 0 { - return false - } - - return true -} diff --git a/client/firewall/nftables/legacy_rule_linux_test.go b/client/firewall/nftables/legacy_rule_linux_test.go new file mode 100644 index 000000000..dc2f1c7a0 --- /dev/null +++ b/client/firewall/nftables/legacy_rule_linux_test.go @@ -0,0 +1,60 @@ +package nftables + +import ( + "testing" + + "github.com/google/nftables/expr" + "github.com/stretchr/testify/require" +) + +func TestBuildLegacyRouteRuleExpressions(t *testing.T) { + sourcePayload := &expr.Payload{} + sourceCmp := &expr.Cmp{} + destinationPayload := &expr.Payload{} + destinationCmp := &expr.Cmp{} + nilSourceDestination := &expr.Payload{} + nilDestinationSource := &expr.Cmp{} + + tests := []struct { + name string + source []expr.Any + destination []expr.Any + matches []expr.Any + }{ + { + name: "both non-empty", + source: []expr.Any{sourcePayload, sourceCmp}, + destination: []expr.Any{destinationPayload, destinationCmp}, + matches: []expr.Any{sourcePayload, sourceCmp, destinationPayload, destinationCmp}, + }, + { + name: "nil source", + destination: []expr.Any{nilSourceDestination}, + matches: []expr.Any{nilSourceDestination}, + }, + { + name: "nil destination", + source: []expr.Any{nilDestinationSource}, + matches: []expr.Any{nilDestinationSource}, + }, + { + name: "both nil", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildLegacyRouteRuleExpressions(tt.source, tt.destination) + + require.Len(t, got, len(tt.matches)+2) + for i, match := range tt.matches { + require.Same(t, match, got[i]) + } + + require.IsType(t, &expr.Counter{}, got[len(tt.matches)]) + verdict, ok := got[len(tt.matches)+1].(*expr.Verdict) + require.True(t, ok) + require.Equal(t, expr.VerdictAccept, verdict.Kind) + }) + } +} diff --git a/client/firewall/nftables/manager_linux.go b/client/firewall/nftables/manager_linux.go index fdc7c2f3c..75405e213 100644 --- a/client/firewall/nftables/manager_linux.go +++ b/client/firewall/nftables/manager_linux.go @@ -3,7 +3,6 @@ package nftables import ( "context" "fmt" - "net" "net/netip" "os" "sync" @@ -13,10 +12,8 @@ import ( "github.com/google/nftables/expr" "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" nberrors "github.com/netbirdio/netbird/client/errors" - "github.com/netbirdio/netbird/client/firewall/firewalld" firewall "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/iface/wgaddr" "github.com/netbirdio/netbird/client/internal/statemanager" @@ -45,21 +42,17 @@ type iFaceMapper interface { Address() wgaddr.Address } -// Manager of iptables firewall +// Manager of nftables firewall. Per-family state (peer ACLs, route +// ACLs, NAT, DNAT, MSS clamping) lives on family; Manager dispatches +// by family and provides the public firewall.Manager surface. type Manager struct { mutex sync.Mutex rConn *nftables.Conn wgIface iFaceMapper - router *router - aclManager *AclManager - - // IPv6 counterparts, nil when no v6 overlay - router6 *router - aclManager6 *AclManager - - notrackOutputChain *nftables.Chain - notrackPreroutingChain *nftables.Chain + family4 *family + // IPv6 counterpart, nil when no v6 overlay. + family6 *family extMonitor *externalChainMonitor } @@ -74,21 +67,10 @@ func Create(wgIface iFaceMapper, mtu uint16) (*Manager, error) { tableName := getTableName() workTable := &nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv4} - var err error - m.router, err = newRouter(workTable, wgIface, mtu) - if err != nil { - return nil, fmt.Errorf("create router: %w", err) - } - - m.aclManager, err = newAclManager(workTable, wgIface, chainNameRoutingFw) - if err != nil { - return nil, fmt.Errorf("create acl manager: %w", err) - } + m.family4 = newFamily(workTable, wgIface, mtu) if wgIface.Address().HasIPv6() { - if err := m.createIPv6Components(tableName, wgIface, mtu); err != nil { - return nil, fmt.Errorf("create IPv6 firewall: %w", err) - } + m.createIPv6Components(tableName, wgIface, mtu) } m.extMonitor = newExternalChainMonitor(m) @@ -96,30 +78,19 @@ func Create(wgIface iFaceMapper, mtu uint16) (*Manager, error) { return m, nil } -func (m *Manager) createIPv6Components(tableName string, wgIface iFaceMapper, mtu uint16) error { +func (m *Manager) createIPv6Components(tableName string, wgIface iFaceMapper, mtu uint16) { workTable6 := &nftables.Table{Name: tableName, Family: nftables.TableFamilyIPv6} - var err error - m.router6, err = newRouter(workTable6, wgIface, mtu) - if err != nil { - return fmt.Errorf("create v6 router: %w", err) - } + m.family6 = newFamily(workTable6, wgIface, mtu) - // Share the same IP forwarding state with the v4 router, since - // EnableIPForwarding controls both v4 and v6 sysctls. - m.router6.ipFwdState = m.router.ipFwdState - - m.aclManager6, err = newAclManager(workTable6, wgIface, chainNameRoutingFw) - if err != nil { - return fmt.Errorf("create v6 acl manager: %w", err) - } - - return nil + // Share the per-family forwarding refcounter with the v4 family so a v4 + // rule and a v6 rule against the same state machine cooperate cleanly. + m.family6.ipFwdState = m.family4.ipFwdState } // hasIPv6 reports whether the manager has IPv6 components initialized. func (m *Manager) hasIPv6() bool { - return m.router6 != nil + return m.family6 != nil } func (m *Manager) initIPv6() error { @@ -128,12 +99,8 @@ func (m *Manager) initIPv6() error { return fmt.Errorf("create v6 work table: %w", err) } - if err := m.router6.init(workTable6); err != nil { - return fmt.Errorf("v6 router init: %w", err) - } - - if err := m.aclManager6.init(workTable6); err != nil { - return fmt.Errorf("v6 acl manager init: %w", err) + if err := m.family6.init(workTable6); err != nil { + return fmt.Errorf("v6 family init: %w", err) } return nil @@ -156,19 +123,20 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error { // reconcileExternalChains re-applies passthrough accept rules to external // filter chains for both IPv4 and IPv6 routers. Called by the monitor when -// tables or chains appear (e.g. after firewalld reloads). +// tables or chains appear (e.g. after firewalld reloads). Kernel routing opens +// both INPUT and FORWARD. func (m *Manager) reconcileExternalChains() error { m.mutex.Lock() defer m.mutex.Unlock() var merr *multierror.Error - if m.router != nil { - if err := m.router.acceptExternalChainsRules(); err != nil { + if m.family4 != nil { + if err := m.family4.acceptExternalChainsRules(true); err != nil { merr = multierror.Append(merr, fmt.Errorf("v4: %w", err)) } } if m.hasIPv6() { - if err := m.router6.acceptExternalChainsRules(); err != nil { + if err := m.family6.acceptExternalChainsRules(true); err != nil { merr = multierror.Append(merr, fmt.Errorf("v6: %w", err)) } } @@ -187,12 +155,8 @@ func (m *Manager) initFirewall() (err error) { } }() - if err := m.router.init(workTable); err != nil { - return fmt.Errorf("router init: %w", err) - } - - if err := m.aclManager.init(workTable); err != nil { - return fmt.Errorf("acl manager init: %w", err) + if err := m.family4.init(workTable); err != nil { + return fmt.Errorf("family init: %w", err) } if m.hasIPv6() { @@ -202,10 +166,6 @@ func (m *Manager) initFirewall() (err error) { } } - if err := m.initNoTrackChains(workTable); err != nil { - log.Warnf("raw priority chains not available, notrack rules will be disabled: %v", err) - } - return nil } @@ -220,7 +180,7 @@ func (m *Manager) persistState(stateManager *statemanager.Manager) { InterfaceState: &InterfaceState{ NameStr: m.wgIface.Name(), WGAddress: m.wgIface.Address(), - MTU: m.router.mtu, + MTU: m.family4.mtu, }, }); err != nil { log.Errorf("failed to update state: %v", err) @@ -235,12 +195,12 @@ func (m *Manager) persistState(stateManager *statemanager.Manager) { // rollbackInit performs best-effort cleanup of already-initialized state when Init fails partway through. func (m *Manager) rollbackInit() { - if err := m.router.Reset(); err != nil { - log.Warnf("rollback router: %v", err) + if err := m.family4.Reset(); err != nil { + log.Warnf("rollback family: %v", err) } if m.hasIPv6() { - if err := m.router6.Reset(); err != nil { - log.Warnf("rollback v6 router: %v", err) + if err := m.family6.Reset(); err != nil { + log.Warnf("rollback v6 family: %v", err) } } if err := m.cleanupNetbirdTables(); err != nil { @@ -251,118 +211,67 @@ func (m *Manager) rollbackInit() { } } -// AddPeerFiltering rule to the firewall +// AddFilterRule installs a packet-filtering rule. // -// If comment argument is empty firewall manager should set -// rule ID as comment for the rule -func (m *Manager) AddPeerFiltering( - id []byte, - ip net.IP, - proto firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, - ipsetName string, -) ([]firewall.Rule, error) { - m.mutex.Lock() - defer m.mutex.Unlock() - - if ip.To4() != nil { - return m.aclManager.AddPeerFiltering(id, ip, proto, sPort, dPort, action, ipsetName) - } - - if !m.hasIPv6() { - return nil, fmt.Errorf("add peer filtering for %s: %w", ip, firewall.ErrIPv6NotInitialized) - } - return m.aclManager6.AddPeerFiltering(id, ip, proto, sPort, dPort, action, ipsetName) -} - -func (m *Manager) AddRouteFiltering( +// Destination semantics: zero Network → input chain (peer ACL); +// set Network → forward chain (route ACL). +// +// Sources are a single address family; the rule is dispatched to the +// matching per-family backend. +func (m *Manager) AddFilterRule( id []byte, sources []netip.Prefix, destination firewall.Network, proto firewall.Protocol, - sPort, dPort *firewall.Port, + sPort *firewall.Port, + dPort *firewall.Port, action firewall.Action, ) (firewall.Rule, error) { + if len(sources) == 0 { + return nil, firewall.ErrNoSources + } + m.mutex.Lock() defer m.mutex.Unlock() - if isIPv6RouteRule(sources, destination) { + fam := m.family4 + if isIPv6Rule(sources, destination) { if !m.hasIPv6() { - return nil, fmt.Errorf("add route filtering: %w", firewall.ErrIPv6NotInitialized) + return nil, fmt.Errorf("add filtering: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddRouteFiltering(id, sources, destination, proto, sPort, dPort, action) + fam = m.family6 } - - return m.router.AddRouteFiltering(id, sources, destination, proto, sPort, dPort, action) + return fam.AddFilterRule(id, sources, destination, proto, sPort, dPort, action) } -// DeletePeerRule from the firewall by rule definition -func (m *Manager) DeletePeerRule(rule firewall.Rule) error { +// DeleteFilterRule removes a filtering rule. The owning family is found +// by id in the in-memory filter maps, which are the only tracking for +// filter rules. family.DeleteFilterRule is idempotent when the id is +// absent. +func (m *Manager) DeleteFilterRule(rule firewall.Rule) error { m.mutex.Lock() defer m.mutex.Unlock() - if m.hasIPv6() && isIPv6Rule(rule) { - return m.aclManager6.DeletePeerRule(rule) - } - return m.aclManager.DeletePeerRule(rule) -} - -func isIPv6Rule(rule firewall.Rule) bool { - r, ok := rule.(*Rule) - return ok && r.nftRule != nil && r.nftRule.Table != nil && r.nftRule.Table.Family == nftables.TableFamilyIPv6 -} - -// isIPv6RouteRule determines whether a route rule belongs to the v6 table. -// For static routes, the destination prefix determines the family. For dynamic -// routes (DomainSet), the sources determine the family since management -// duplicates dynamic rules per family. -func isIPv6RouteRule(sources []netip.Prefix, destination firewall.Network) bool { - if destination.IsPrefix() { - return destination.Prefix.Addr().Is6() - } - return len(sources) > 0 && sources[0].Addr().Is6() -} - -// DeleteRouteRule deletes a routing rule. Route rules live in exactly one -// router; the cached maps are normally authoritative, so the kernel is only -// consulted when neither map knows about the rule. -func (m *Manager) DeleteRouteRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - id := rule.ID() - r, err := m.routerForRuleID(id, (*router).hasRule) + fam, err := m.familyForRuleID(rule.ID(), (*family).hasRule) if err != nil { return err } - return r.DeleteRouteRule(rule) + return fam.DeleteFilterRule(rule) } -// routerForRuleID picks the router holding the rule with the given id, using -// the supplied lookup. If the cached maps disagree (or both miss), it refreshes -// from the kernel once and re-checks before falling back to the v4 router. -func (m *Manager) routerForRuleID(id string, has func(*router, string) bool) (*router, error) { - if has(m.router, id) { - return m.router, nil - } - if m.hasIPv6() && has(m.router6, id) { - return m.router6, nil +// familyForRuleID picks the family holding the rule with the given id, using +// the supplied lookup, and falls back to the v4 family on a miss. +func (m *Manager) familyForRuleID(id firewall.RuleID, has func(*family, firewall.RuleID) bool) (*family, error) { + if has(m.family4, id) { + return m.family4, nil } if !m.hasIPv6() { - return m.router, nil + return m.family4, nil } - if err := m.router.refreshRulesMap(); err != nil { - return nil, fmt.Errorf("refresh v4 rules: %w", err) + if has(m.family6, id) { + return m.family6, nil } - if err := m.router6.refreshRulesMap(); err != nil { - return nil, fmt.Errorf("refresh v6 rules: %w", err) - } - if has(m.router6, id) && !has(m.router, id) { - return m.router6, nil - } - return m.router, nil + return m.family4, nil } func (m *Manager) IsServerRouteSupported() bool { @@ -381,10 +290,10 @@ func (m *Manager) AddNatRule(pair firewall.RouterPair) error { if !m.hasIPv6() { return fmt.Errorf("add NAT rule: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddNatRule(pair) + return m.family6.AddNatRule(pair) } - if err := m.router.AddNatRule(pair); err != nil { + if err := m.family4.AddNatRule(pair); err != nil { return err } @@ -396,7 +305,7 @@ func (m *Manager) AddNatRule(pair firewall.RouterPair) error { // so the eventual cleanup still works. if m.hasIPv6() && pair.Dynamic { v6Pair := firewall.ToV6NatPair(pair) - if err := m.router6.AddNatRule(v6Pair); err != nil { + if err := m.family6.AddNatRule(v6Pair); err != nil { return fmt.Errorf("add v6 NAT rule: %w", err) } } @@ -412,18 +321,18 @@ func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error { if !m.hasIPv6() { return nil } - return m.router6.RemoveNatRule(pair) + return m.family6.RemoveNatRule(pair) } var merr *multierror.Error - if err := m.router.RemoveNatRule(pair); err != nil { + if err := m.family4.RemoveNatRule(pair); err != nil { merr = multierror.Append(merr, fmt.Errorf("remove v4 NAT rule: %w", err)) } if m.hasIPv6() && pair.Dynamic { v6Pair := firewall.ToV6NatPair(pair) - if err := m.router6.RemoveNatRule(v6Pair); err != nil { + if err := m.family6.RemoveNatRule(v6Pair); err != nil { merr = multierror.Append(merr, fmt.Errorf("remove v6 NAT rule: %w", err)) } } @@ -431,46 +340,13 @@ func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error { return nberrors.FormatErrorOrNil(merr) } -// AllowNetbird allows netbird interface traffic. -// This is called when USPFilter wraps the native firewall, adding blanket accept -// rules so that packet filtering is handled in userspace instead of by netfilter. -// -// TODO: In USP mode this only adds ACCEPT to the netbird table's own chains, -// which doesn't override DROP rules in external tables (e.g. firewalld). -// Should add passthrough rules to external chains (like the native mode router's -// addExternalChainsRules does) for both the netbird table family and inet tables. -// The netbird table itself is fine (routing chains already exist there), but -// non-netbird tables with INPUT/FORWARD hooks can still DROP our WG traffic. -func (m *Manager) AllowNetbird() error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if err := m.aclManager.createDefaultAllowRules(); err != nil { - return fmt.Errorf("create default allow rules: %w", err) - } - if m.hasIPv6() { - if err := m.aclManager6.createDefaultAllowRules(); err != nil { - return fmt.Errorf("create v6 default allow rules: %w", err) - } - } - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf("flush allow input netbird rules: %w", err) - } - - if err := firewalld.TrustInterface(m.wgIface.Name()); err != nil { - log.Warnf("failed to trust interface in firewalld: %v", err) - } - - return nil -} - // SetLegacyManagement sets the route manager to use legacy management func (m *Manager) SetLegacyManagement(isLegacy bool) error { - if err := firewall.SetLegacyManagement(m.router, isLegacy); err != nil { + if err := firewall.SetLegacyManagement(m.family4, isLegacy); err != nil { return err } if m.hasIPv6() { - return firewall.SetLegacyManagement(m.router6, isLegacy) + return firewall.SetLegacyManagement(m.family6, isLegacy) } return nil } @@ -484,13 +360,13 @@ func (m *Manager) Close(stateManager *statemanager.Manager) error { var merr *multierror.Error - if err := m.router.Reset(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("reset router: %v", err)) + if err := m.family4.Reset(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("reset family: %w", err)) } if m.hasIPv6() { - if err := m.router6.Reset(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("reset v6 router: %v", err)) + if err := m.family6.Reset(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("reset v6 family: %w", err)) } } @@ -530,17 +406,12 @@ func (m *Manager) SetLogLevel(log.Level) { } func (m *Manager) EnableRouting() error { - if err := m.router.ipFwdState.RequestForwarding(); err != nil { - return fmt.Errorf("enable IP forwarding: %w", err) - } - return nil + // v6 only when the overlay actually has v6. + return m.family4.ipFwdState.RequestRouting(m.hasIPv6()) } func (m *Manager) DisableRouting() error { - if err := m.router.ipFwdState.ReleaseForwarding(); err != nil { - return fmt.Errorf("disable IP forwarding: %w", err) - } - return nil + return m.family4.ipFwdState.ReleaseRouting() } // Flush rule/chain/set operations from the buffer @@ -551,49 +422,19 @@ func (m *Manager) Flush() error { m.mutex.Lock() defer m.mutex.Unlock() - if err := m.aclManager.Flush(); err != nil { + if err := m.family4.Flush(); err != nil { return err } if m.hasIPv6() { - if err := m.aclManager6.Flush(); err != nil { - return fmt.Errorf("flush v6 acl: %w", err) + if err := m.family6.Flush(); err != nil { + return fmt.Errorf("flush v6 family: %w", err) } } - if err := m.refreshNoTrackChains(); err != nil { - log.Errorf("failed to refresh notrack chains: %v", err) - } - return nil } -// AddDNATRule adds a DNAT rule -func (m *Manager) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - m.mutex.Lock() - defer m.mutex.Unlock() - - if rule.TranslatedAddress.Is6() { - if !m.hasIPv6() { - return nil, fmt.Errorf("add DNAT rule: %w", firewall.ErrIPv6NotInitialized) - } - return m.router6.AddDNATRule(rule) - } - return m.router.AddDNATRule(rule) -} - -// DeleteDNATRule deletes a DNAT rule -func (m *Manager) DeleteDNATRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - r, err := m.routerForRuleID(rule.ID(), (*router).hasDNATRule) - if err != nil { - return err - } - return r.DeleteDNATRule(rule) -} - // UpdateSet updates the set with the given prefixes func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { m.mutex.Lock() @@ -608,12 +449,12 @@ func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { } } - if err := m.router.UpdateSet(set, v4Prefixes); err != nil { + if err := m.family4.UpdateSet(set, v4Prefixes); err != nil { return err } if m.hasIPv6() && len(v6Prefixes) > 0 { - if err := m.router6.UpdateSet(set, v6Prefixes); err != nil { + if err := m.family6.UpdateSet(set, v6Prefixes); err != nil { return fmt.Errorf("update v6 set: %w", err) } } @@ -630,9 +471,9 @@ func (m *Manager) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protoco if !m.hasIPv6() { return fmt.Errorf("add inbound DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.AddInboundDNAT(localAddr, protocol, originalPort, translatedPort) } // RemoveInboundDNAT removes an inbound DNAT rule. @@ -644,9 +485,9 @@ func (m *Manager) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Prot if !m.hasIPv6() { return fmt.Errorf("remove inbound DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.RemoveInboundDNAT(localAddr, protocol, originalPort, translatedPort) } // AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic. @@ -658,9 +499,9 @@ func (m *Manager) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol if !m.hasIPv6() { return fmt.Errorf("add output DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family4.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) } // RemoveOutputDNAT removes an OUTPUT chain DNAT rule. @@ -672,179 +513,9 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto if !m.hasIPv6() { return fmt.Errorf("remove output DNAT: %w", firewall.ErrIPv6NotInitialized) } - return m.router6.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) + return m.family6.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } - return m.router.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) -} - -const ( - chainNameRawOutput = "netbird-raw-out" - chainNameRawPrerouting = "netbird-raw-pre" -) - -// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic. -// This prevents conntrack from tracking WireGuard proxy traffic on loopback, which -// can interfere with MASQUERADE rules (e.g., from container runtimes like Podman/netavark). -// -// Traffic flows that need NOTRACK: -// -// 1. Egress: WireGuard -> fake endpoint (before eBPF rewrite) -// src=127.0.0.1:wgPort -> dst=127.0.0.1:fakePort -// Matched by: sport=wgPort -// -// 2. Egress: Proxy -> WireGuard (via raw socket) -// src=127.0.0.1:fakePort -> dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 3. Ingress: Packets to WireGuard -// dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 4. Ingress: Packets to proxy (after eBPF rewrite) -// dst=127.0.0.1:proxyPort -// Matched by: dport=proxyPort -// -// Rules are cleaned up when the firewall manager is closed. -func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if m.notrackOutputChain == nil || m.notrackPreroutingChain == nil { - return fmt.Errorf("notrack chains not initialized") - } - - proxyPortBytes := binaryutil.BigEndian.PutUint16(proxyPort) - wgPortBytes := binaryutil.BigEndian.PutUint16(wgPort) - loopback := []byte{127, 0, 0, 1} - - // Egress rules: match outgoing loopback UDP packets - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackOutputChain.Table, - Chain: m.notrackOutputChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // sport=wgPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackOutputChain.Table, - Chain: m.notrackOutputChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // dport=wgPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - - // Ingress rules: match incoming loopback UDP packets - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackPreroutingChain.Table, - Chain: m.notrackPreroutingChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // dport=wgPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackPreroutingChain.Table, - Chain: m.notrackPreroutingChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: proxyPortBytes}, // dport=proxyPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf("flush notrack rules: %w", err) - } - - log.Debugf("set up ebpf proxy notrack rules for ports %d,%d", proxyPort, wgPort) - return nil -} - -func (m *Manager) initNoTrackChains(table *nftables.Table) error { - m.notrackOutputChain = m.rConn.AddChain(&nftables.Chain{ - Name: chainNameRawOutput, - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookOutput, - Priority: nftables.ChainPriorityRaw, - }) - - m.notrackPreroutingChain = m.rConn.AddChain(&nftables.Chain{ - Name: chainNameRawPrerouting, - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityRaw, - }) - - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf("flush chain creation: %w", err) - } - - return nil -} - -func (m *Manager) refreshNoTrackChains() error { - chains, err := m.rConn.ListChainsOfTableFamily(nftables.TableFamilyIPv4) - if err != nil { - return fmt.Errorf("list chains: %w", err) - } - - tableName := getTableName() - for _, c := range chains { - if c.Table.Name != tableName { - continue - } - switch c.Name { - case chainNameRawOutput: - m.notrackOutputChain = c - case chainNameRawPrerouting: - m.notrackPreroutingChain = c - } - } - - return nil + return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } func (m *Manager) createWorkTable() (*nftables.Table, error) { @@ -903,3 +574,14 @@ func getEstablishedExprs(register uint32) []expr.Any { }, } } + +// isIPv6Rule reports whether the rule belongs to the v6 table. For a +// prefix destination the destination family decides; otherwise the +// (single-family) sources do, since management duplicates rules per +// family. +func isIPv6Rule(sources []netip.Prefix, destination firewall.Network) bool { + if destination.IsPrefix() { + return destination.Prefix.Addr().Is6() + } + return len(sources) > 0 && sources[0].Addr().Is6() +} diff --git a/client/firewall/nftables/manager_linux_test.go b/client/firewall/nftables/manager_linux_test.go index 4eb466281..4d6eec3c1 100644 --- a/client/firewall/nftables/manager_linux_test.go +++ b/client/firewall/nftables/manager_linux_test.go @@ -72,13 +72,13 @@ func TestNftablesManager(t *testing.T) { testClient := &nftables.Conn{} - rule, err := manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{53}}, fw.ActionDrop, "") + rule, err := manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{53}}, fw.ActionDrop) require.NoError(t, err, "failed to add rule") err = manager.Flush() require.NoError(t, err, "failed to flush") - rules, err := testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules) + rules, err := testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules) require.NoError(t, err, "failed to get rules") require.Len(t, rules, 2, "expected 2 rules") @@ -149,15 +149,12 @@ func TestNftablesManager(t *testing.T) { // Compare connection tracking rule at position 1 (pushed down by DROP rule insertion) compareExprsIgnoringCounters(t, rules[1].Exprs, expectedExprs1) - for _, r := range rule { - err = manager.DeletePeerRule(r) - require.NoError(t, err, "failed to delete rule") - } + require.NoError(t, manager.DeleteFilterRule(rule), "failed to delete rule") err = manager.Flush() require.NoError(t, err, "failed to flush") - rules, err = testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules) + rules, err = testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules) require.NoError(t, err, "failed to get rules") // established rule remains require.Len(t, rules, 1, "expected 1 rules after deletion") @@ -182,47 +179,39 @@ func TestNftablesManagerRuleOrder(t *testing.T) { testClient := &nftables.Conn{} // Add accept rule first - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "accept-http") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err, "failed to add accept rule") // Add deny rule second for the same traffic - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop, "deny-http") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop) require.NoError(t, err, "failed to add deny rule") err = manager.Flush() require.NoError(t, err, "failed to flush") - rules, err := testClient.GetRules(manager.aclManager.workTable, manager.aclManager.chainInputRules) + rules, err := testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules) require.NoError(t, err, "failed to get rules") t.Logf("Found %d rules in nftables chain", len(rules)) - // Find the accept and deny rules and verify deny comes before accept + // Single-source rules emit a direct payload+cmp on the source IP + // (no set lookup). Match by source-IP + port + verdict instead of + // the legacy per-(action,port) set names ("deny-http"/"accept-http") + // that this test predates. + wantSrc := ip.AsSlice() var acceptRuleIndex, denyRuleIndex = -1, -1 for i, rule := range rules { - hasAcceptHTTPSet := false - hasDenyHTTPSet := false - hasPort80 := false + var hasSrc, hasPort80 bool var action string - for _, e := range rule.Exprs { - // Check for set lookup - if lookup, ok := e.(*expr.Lookup); ok { - switch lookup.SetName { - case "accept-http": - hasAcceptHTTPSet = true - case "deny-http": - hasDenyHTTPSet = true + if cmp, ok := e.(*expr.Cmp); ok && cmp.Op == expr.CmpOpEq { + if bytes.Equal(cmp.Data, wantSrc) { + hasSrc = true } - - } - // Check for port 80 - if cmp, ok := e.(*expr.Cmp); ok { - if cmp.Op == expr.CmpOpEq && len(cmp.Data) == 2 && binary.BigEndian.Uint16(cmp.Data) == 80 { + if len(cmp.Data) == 2 && binary.BigEndian.Uint16(cmp.Data) == 80 { hasPort80 = true } } - // Check for verdict if verdict, ok := e.(*expr.Verdict); ok { switch verdict.Kind { case expr.VerdictAccept: @@ -233,11 +222,15 @@ func TestNftablesManagerRuleOrder(t *testing.T) { } } - if hasAcceptHTTPSet && hasPort80 && action == "ACCEPT" { - t.Logf("Rule [%d]: accept-http set + Port 80 + ACCEPT", i) + if !hasSrc || !hasPort80 { + continue + } + switch action { + case "ACCEPT": + t.Logf("Rule [%d]: src=%s port=80 ACCEPT", i, ip) acceptRuleIndex = i - } else if hasDenyHTTPSet && hasPort80 && action == "DROP" { - t.Logf("Rule [%d]: deny-http set + Port 80 + DROP", i) + case "DROP": + t.Logf("Rule [%d]: src=%s port=80 DROP", i, ip) denyRuleIndex = i } } @@ -281,7 +274,7 @@ func TestNFtablesCreatePerformance(t *testing.T) { start := time.Now() for i := 0; i < testMax; i++ { port := &fw.Port{Values: []uint16{uint16(1000 + i)}} - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, "tcp", nil, port, fw.ActionAccept) require.NoError(t, err, "failed to add rule") if i%100 == 0 { @@ -363,10 +356,10 @@ func TestNftablesManagerCompatibilityWithIptables(t *testing.T) { }) ip := netip.MustParseAddr("100.96.0.1") - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err, "failed to add peer filtering rule") - _, err = manager.AddRouteFiltering( + _, err = manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("192.168.2.0/24")}, fw.Network{Prefix: netip.MustParsePrefix("10.1.0.0/24")}, @@ -385,18 +378,6 @@ func TestNftablesManagerCompatibilityWithIptables(t *testing.T) { err = manager.AddNatRule(pair) require.NoError(t, err, "failed to add NAT rule") - dnatRule, err := manager.AddDNATRule(fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{8080}}, - TranslatedAddress: netip.MustParseAddr("100.96.0.2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - }) - require.NoError(t, err, "failed to add DNAT rule") - - t.Cleanup(func() { - require.NoError(t, manager.DeleteDNATRule(dnatRule), "failed to delete DNAT rule") - }) - stdout, stderr = runIptablesSave(t) verifyIptablesOutput(t, stdout, stderr) } @@ -439,10 +420,10 @@ func TestNftablesManagerIPv6CompatibilityWithIp6tables(t *testing.T) { }) ip := netip.MustParseAddr("fd00::2") - _, err = manager.AddPeerFiltering(nil, ip.AsSlice(), fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "") + _, err = manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err, "add v6 peer filtering rule") - _, err = manager.AddRouteFiltering( + _, err = manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("fd00:1::/64")}, fw.Network{Prefix: netip.MustParsePrefix("2001:db8::/48")}, @@ -460,18 +441,6 @@ func TestNftablesManagerIPv6CompatibilityWithIp6tables(t *testing.T) { }) require.NoError(t, err, "add v6 NAT rule") - dnatRule, err := manager.AddDNATRule(fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{8080}}, - TranslatedAddress: netip.MustParseAddr("fd00::2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - }) - require.NoError(t, err, "add v6 DNAT rule") - - t.Cleanup(func() { - require.NoError(t, manager.DeleteDNATRule(dnatRule), "delete v6 DNAT rule") - }) - stdout, stderr := runIptablesSave(t) verifyIptablesOutput(t, stdout, stderr) @@ -552,7 +521,7 @@ func TestNftablesManagerCompatibilityWithIptablesFor6kPrefixes(t *testing.T) { prefixes = append(prefixes, netip.PrefixFrom(addr, 24)) } } - _, err = manager.AddRouteFiltering( + _, err = manager.AddFilterRule( nil, prefixes, fw.Network{Prefix: netip.MustParsePrefix("10.2.0.0/24")}, @@ -567,7 +536,7 @@ func TestNftablesManagerCompatibilityWithIptablesFor6kPrefixes(t *testing.T) { verifyIptablesOutput(t, stdout, stderr) } -func TestNftablesManagerCompatibilityWithIptablesForEmptyPrefixes(t *testing.T) { +func TestNftablesManagerCompatibilityWithIptablesForWildcardSource(t *testing.T) { if check() != NFTABLES { t.Skip("nftables not supported on this system") } @@ -593,9 +562,9 @@ func TestNftablesManagerCompatibilityWithIptablesForEmptyPrefixes(t *testing.T) verifyIptablesOutput(t, stdout, stderr) }) - _, err = manager.AddRouteFiltering( + _, err = manager.AddFilterRule( nil, - []netip.Prefix{}, + []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}, fw.Network{Prefix: netip.MustParsePrefix("10.2.0.0/24")}, fw.ProtocolTCP, nil, @@ -608,6 +577,73 @@ func TestNftablesManagerCompatibilityWithIptablesForEmptyPrefixes(t *testing.T) verifyIptablesOutput(t, stdout, stderr) } +func TestNftablesManagerMultiPortFilter(t *testing.T) { + if check() != NFTABLES { + t.Skip("nftables not supported on this system") + } + + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + t.Cleanup(func() { + require.NoError(t, manager.Close(nil), "failed to reset manager state") + }) + + ip := netip.MustParseAddr("100.96.0.1") + + rule, err := manager.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80, 443}}, fw.ActionAccept) + require.NoError(t, err, "failed to add multi-port rule") + + testClient := &nftables.Conn{} + rules, err := testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules) + require.NoError(t, err, "failed to get rules") + + var lookup *expr.Lookup + for _, kernelRule := range rules { + if string(kernelRule.UserData) != string(rule.ID()) { + continue + } + for _, e := range kernelRule.Exprs { + if l, ok := e.(*expr.Lookup); ok { + lookup = l + } + } + } + require.NotNil(t, lookup, "multi-port rule must match ports via a set lookup") + + sets, err := testClient.GetSets(manager.family4.workTable) + require.NoError(t, err, "failed to get sets") + + var portSet *nftables.Set + for _, s := range sets { + if s.Name == lookup.SetName { + portSet = s + } + } + require.NotNil(t, portSet, "anonymous port set not found in kernel") + + portSet.Table = manager.family4.workTable + elements, err := testClient.GetSetElements(portSet) + require.NoError(t, err, "failed to get set elements") + + ports := make(map[uint16]bool) + for _, e := range elements { + require.Len(t, e.Key, 2, "port set element key should be 2 bytes") + ports[binary.BigEndian.Uint16(e.Key)] = true + } + require.True(t, ports[80], "port set should contain port 80") + require.True(t, ports[443], "port set should contain port 443") + + require.NoError(t, manager.DeleteFilterRule(rule), "failed to delete rule") + + rules, err = testClient.GetRules(manager.family4.workTable, manager.family4.chainInputRules) + require.NoError(t, err, "failed to get rules after delete") + for _, kernelRule := range rules { + require.NotEqual(t, string(rule.ID()), string(kernelRule.UserData), "rule should be removed from kernel") + } +} + func compareExprsIgnoringCounters(t *testing.T, got, want []expr.Any) { t.Helper() require.Equal(t, len(got), len(want), "expression count mismatch") diff --git a/client/firewall/nftables/router_linux.go b/client/firewall/nftables/router_linux.go deleted file mode 100644 index 4214455a9..000000000 --- a/client/firewall/nftables/router_linux.go +++ /dev/null @@ -1,2244 +0,0 @@ -package nftables - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "net" - "net/netip" - "strings" - - "github.com/coreos/go-iptables/iptables" - "github.com/google/nftables" - "github.com/google/nftables/binaryutil" - "github.com/google/nftables/expr" - "github.com/google/nftables/xt" - "github.com/hashicorp/go-multierror" - log "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" - - nberrors "github.com/netbirdio/netbird/client/errors" - "github.com/netbirdio/netbird/client/firewall/firewalld" - firewall "github.com/netbirdio/netbird/client/firewall/manager" - nbid "github.com/netbirdio/netbird/client/internal/acl/id" - "github.com/netbirdio/netbird/client/internal/routemanager/ipfwdstate" - "github.com/netbirdio/netbird/client/internal/routemanager/refcounter" - nbnet "github.com/netbirdio/netbird/client/net" -) - -const ( - tableNat = "nat" - tableMangle = "mangle" - tableRaw = "raw" - tableSecurity = "security" - - chainNameNatPrerouting = "PREROUTING" - chainNameRoutingFw = "netbird-rt-fwd" - chainNameRoutingNat = "netbird-rt-postrouting" - chainNameRoutingRdr = "netbird-rt-redirect" - chainNameNATOutput = "netbird-nat-output" - chainNameForward = "FORWARD" - chainNameMangleForward = "netbird-mangle-forward" - - firewalldTableName = "firewalld" - - userDataAcceptForwardRuleIif = "frwacceptiif" - userDataAcceptForwardRuleOif = "frwacceptoif" - userDataAcceptInputRule = "inputaccept" - - dnatSuffix = "_dnat" - snatSuffix = "_snat" - - // ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation. - ipv4TCPHeaderSize = 40 - // ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation. - ipv6TCPHeaderSize = 60 - - // maxPrefixesSet 1638 prefixes start to fail, taking some margin - maxPrefixesSet = 1500 - refreshRulesMapError = "refresh rules map: %w" -) - -var ( - errFilterTableNotFound = fmt.Errorf("'filter' table not found") -) - -type setInput struct { - set firewall.Set - prefixes []netip.Prefix -} - -type router struct { - conn *nftables.Conn - workTable *nftables.Table - filterTable *nftables.Table - chains map[string]*nftables.Chain - // rules is useful to avoid duplicates and to get missing attributes that we don't have when adding new rules - rules map[string]*nftables.Rule - ipsetCounter *refcounter.Counter[string, setInput, *nftables.Set] - - af addrFamily - wgIface iFaceMapper - ipFwdState *ipfwdstate.IPForwardingState - legacyManagement bool - mtu uint16 -} - -func newRouter(workTable *nftables.Table, wgIface iFaceMapper, mtu uint16) (*router, error) { - r := &router{ - conn: &nftables.Conn{}, - workTable: workTable, - chains: make(map[string]*nftables.Chain), - rules: make(map[string]*nftables.Rule), - af: familyForAddr(workTable.Family == nftables.TableFamilyIPv4), - wgIface: wgIface, - ipFwdState: ipfwdstate.NewIPForwardingState(), - mtu: mtu, - } - - r.ipsetCounter = refcounter.New( - r.createIpSet, - r.deleteIpSet, - ) - - var err error - r.filterTable, err = r.loadFilterTable() - if err != nil { - log.Debugf("ip filter table not found: %v", err) - } - - return r, nil -} - -func (r *router) init(workTable *nftables.Table) error { - r.workTable = workTable - - if err := r.removeAcceptFilterRules(); err != nil { - log.Errorf("failed to clean up rules from filter table: %s", err) - } - - if err := r.createContainers(); err != nil { - return fmt.Errorf("create containers: %w", err) - } - - if err := r.setupDataPlaneMark(); err != nil { - log.Errorf("failed to set up data plane mark: %v", err) - } - - return nil -} - -// Reset cleans existing nftables filter table rules from the system -func (r *router) Reset() error { - // clear without deleting the ipsets, the nf table will be deleted by the caller - r.ipsetCounter.Clear() - - var merr *multierror.Error - - if err := r.removeAcceptFilterRules(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove accept filter rules: %w", err)) - } - - if err := firewalld.UntrustInterface(r.wgIface.Name()); err != nil { - merr = multierror.Append(merr, err) - } - - if err := r.removeNatPreroutingRules(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove filter prerouting rules: %w", err)) - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) removeNatPreroutingRules() error { - table := &nftables.Table{ - Name: tableNat, - Family: r.af.tableFamily, - } - chain := &nftables.Chain{ - Name: chainNameNatPrerouting, - Table: table, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityNATDest, - Type: nftables.ChainTypeNAT, - } - rules, err := r.conn.GetRules(table, chain) - if err != nil { - return fmt.Errorf("get rules from nat table: %w", err) - } - - var merr *multierror.Error - - // Delete rules that have our UserData suffix - for _, rule := range rules { - if len(rule.UserData) == 0 || !strings.HasSuffix(string(rule.UserData), dnatSuffix) { - continue - } - if err := r.conn.DelRule(rule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete rule %s: %w", rule.UserData, err)) - } - } - - if err := r.conn.Flush(); err != nil { - merr = multierror.Append(merr, fmt.Errorf(flushError, err)) - } - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) loadFilterTable() (*nftables.Table, error) { - tables, err := r.conn.ListTablesOfFamily(r.af.tableFamily) - if err != nil { - return nil, fmt.Errorf("list tables: %w", err) - } - - for _, table := range tables { - if table.Name == "filter" { - return table, nil - } - } - - return nil, errFilterTableNotFound -} - -func hookName(hook *nftables.ChainHook) string { - if hook == nil { - return "unknown" - } - switch *hook { - case *nftables.ChainHookForward: - return chainNameForward - case *nftables.ChainHookInput: - return chainNameInput - default: - return fmt.Sprintf("hook(%d)", *hook) - } -} - -func familyName(family nftables.TableFamily) string { - switch family { - case nftables.TableFamilyIPv4: - return "ip" - case nftables.TableFamilyIPv6: - return "ip6" - case nftables.TableFamilyINet: - return "inet" - default: - return fmt.Sprintf("family(%d)", family) - } -} - -func (r *router) createContainers() error { - r.chains[chainNameRoutingFw] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameRoutingFw, - Table: r.workTable, - }) - - prio := *nftables.ChainPriorityNATSource - 1 - r.chains[chainNameRoutingNat] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameRoutingNat, - Table: r.workTable, - Hooknum: nftables.ChainHookPostrouting, - Priority: &prio, - Type: nftables.ChainTypeNAT, - }) - - r.chains[chainNameRoutingRdr] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameRoutingRdr, - Table: r.workTable, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityNATDest, - Type: nftables.ChainTypeNAT, - }) - - r.chains[chainNameManglePostrouting] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameManglePostrouting, - Table: r.workTable, - Hooknum: nftables.ChainHookPostrouting, - Priority: nftables.ChainPriorityMangle, - Type: nftables.ChainTypeFilter, - }) - - r.chains[chainNameManglePrerouting] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameManglePrerouting, - Table: r.workTable, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityMangle, - Type: nftables.ChainTypeFilter, - }) - - r.chains[chainNameMangleForward] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameMangleForward, - Table: r.workTable, - Hooknum: nftables.ChainHookForward, - Priority: nftables.ChainPriorityMangle, - Type: nftables.ChainTypeFilter, - }) - - insertReturnTrafficRule(r.conn, r.workTable, r.chains[chainNameRoutingFw]) - - r.addPostroutingRules() - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("initialize tables: %v", err) - } - - if err := r.addMSSClampingRules(); err != nil { - log.Errorf("failed to add MSS clamping rules: %s", err) - } - - if err := r.acceptForwardRules(); err != nil { - log.Errorf("failed to add accept rules for the forward chain: %s", err) - } - - if err := firewalld.TrustInterface(r.wgIface.Name()); err != nil { - log.Warnf("failed to trust interface in firewalld: %v", err) - } - - if err := r.refreshRulesMap(); err != nil { - log.Errorf("failed to refresh rules: %s", err) - } - - return nil -} - -// setupDataPlaneMark configures the fwmark for the data plane -func (r *router) setupDataPlaneMark() error { - if r.chains[chainNameManglePrerouting] == nil || r.chains[chainNameManglePostrouting] == nil { - return errors.New("no mangle chains found") - } - - ctNew := getCtNewExprs() - preExprs := []expr.Any{ - &expr.Meta{ - Key: expr.MetaKeyIIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - } - preExprs = append(preExprs, ctNew...) - preExprs = append(preExprs, - &expr.Immediate{ - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(nbnet.DataPlaneMarkIn), - }, - &expr.Ct{ - Key: expr.CtKeyMARK, - Register: 1, - SourceRegister: true, - }, - ) - - preNftRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameManglePrerouting], - Exprs: preExprs, - } - r.conn.AddRule(preNftRule) - - postExprs := []expr.Any{ - &expr.Meta{ - Key: expr.MetaKeyOIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - } - postExprs = append(postExprs, ctNew...) - postExprs = append(postExprs, - &expr.Immediate{ - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(nbnet.DataPlaneMarkOut), - }, - &expr.Ct{ - Key: expr.CtKeyMARK, - Register: 1, - SourceRegister: true, - }, - ) - - postNftRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameManglePostrouting], - Exprs: postExprs, - } - r.conn.AddRule(postNftRule) - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("flush: %w", err) - } - - return nil -} - -// AddRouteFiltering appends a nftables rule to the routing chain -func (r *router) AddRouteFiltering( - id []byte, - sources []netip.Prefix, - destination firewall.Network, - proto firewall.Protocol, - sPort *firewall.Port, - dPort *firewall.Port, - action firewall.Action, -) (firewall.Rule, error) { - - ruleKey := nbid.GenerateRouteRuleKey(sources, destination, proto, sPort, dPort, action) - if _, ok := r.rules[string(ruleKey)]; ok { - return ruleKey, nil - } - - chain := r.chains[chainNameRoutingFw] - var exprs []expr.Any - - var source firewall.Network - switch { - case len(sources) == 1 && sources[0].Bits() == 0: - // If it's 0.0.0.0/0, we don't need to add any source matching - case len(sources) == 1: - // If there's only one source, we can use it directly - source.Prefix = sources[0] - default: - // If there are multiple sources, use a set - source.Set = firewall.NewPrefixSet(sources) - } - - sourceExp, err := r.applyNetwork(source, sources, true) - if err != nil { - return nil, fmt.Errorf("apply source: %w", err) - } - exprs = append(exprs, sourceExp...) - - destExp, err := r.applyNetwork(destination, nil, false) - if err != nil { - return nil, fmt.Errorf("apply destination: %w", err) - } - exprs = append(exprs, destExp...) - - // Handle protocol - if proto != firewall.ProtocolALL { - protoNum, err := r.af.protoNum(proto) - if err != nil { - return nil, fmt.Errorf("convert protocol to number: %w", err) - } - exprs = append(exprs, &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}) - exprs = append(exprs, &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{protoNum}, - }) - - exprs = append(exprs, applyPort(sPort, true)...) - exprs = append(exprs, applyPort(dPort, false)...) - } - - exprs = append(exprs, &expr.Counter{}) - - var verdict expr.VerdictKind - if action == firewall.ActionAccept { - verdict = expr.VerdictAccept - } else { - verdict = expr.VerdictDrop - } - exprs = append(exprs, &expr.Verdict{Kind: verdict}) - - rule := &nftables.Rule{ - Table: r.workTable, - Chain: chain, - Exprs: exprs, - UserData: []byte(ruleKey), - } - - // Insert DROP rules at the beginning, append ACCEPT rules at the end - if action == firewall.ActionDrop { - // TODO: Insert after the established rule - rule = r.conn.InsertRule(rule) - } else { - rule = r.conn.AddRule(rule) - } - - if err := r.conn.Flush(); err != nil { - return nil, fmt.Errorf(flushError, err) - } - - r.rules[string(ruleKey)] = rule - - log.Debugf("added route rule: sources=%v, destination=%v, proto=%v, sPort=%v, dPort=%v, action=%v", sources, destination, proto, sPort, dPort, action) - - return ruleKey, nil -} - -func (r *router) getIpSet(set firewall.Set, prefixes []netip.Prefix, isSource bool) ([]expr.Any, error) { - ref, err := r.ipsetCounter.Increment(set.HashedName(), setInput{ - set: set, - prefixes: prefixes, - }) - if err != nil { - return nil, fmt.Errorf("create or get ipset: %w", err) - } - - return r.getIpSetExprs(ref, isSource) -} - -func (r *router) iptablesProto() iptables.Protocol { - if r.af.tableFamily == nftables.TableFamilyIPv6 { - return iptables.ProtocolIPv6 - } - return iptables.ProtocolIPv4 -} - -func (r *router) hasRule(id string) bool { - _, ok := r.rules[id] - return ok -} - -func (r *router) hasDNATRule(id string) bool { - _, ok := r.rules[id+dnatSuffix] - return ok -} - -func (r *router) DeleteRouteRule(rule firewall.Rule) error { - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - ruleKey := rule.ID() - nftRule, exists := r.rules[ruleKey] - if !exists { - log.Debugf("route rule %s not found", ruleKey) - return nil - } - - if nftRule.Handle == 0 { - log.Warnf("route rule %s has no handle, removing stale entry", ruleKey) - if err := r.decrementSetCounter(nftRule); err != nil { - log.Warnf("decrement set counter for stale rule %s: %v", ruleKey, err) - } - delete(r.rules, ruleKey) - return nil - } - - if err := r.deleteNftRule(nftRule, ruleKey); err != nil { - return fmt.Errorf("delete: %w", err) - } - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf(flushError, err) - } - - if err := r.decrementSetCounter(nftRule); err != nil { - return fmt.Errorf("decrement set counter: %w", err) - } - - return nil -} - -func (r *router) createIpSet(setName string, input setInput) (*nftables.Set, error) { - // overlapping prefixes will result in an error, so we need to merge them - prefixes := firewall.MergeIPRanges(input.prefixes) - - nfset := &nftables.Set{ - Name: setName, - Comment: input.set.Comment(), - Table: r.workTable, - // required for prefixes - Interval: true, - KeyType: r.af.setKeyType, - } - - elements := r.convertPrefixesToSet(prefixes) - nElements := len(elements) - - maxElements := maxPrefixesSet * 2 - initialElements := elements[:min(maxElements, nElements)] - - if err := r.conn.AddSet(nfset, initialElements); err != nil { - return nil, fmt.Errorf("error adding set %s: %w", setName, err) - } - if err := r.conn.Flush(); err != nil { - return nil, fmt.Errorf("flush error: %w", err) - } - log.Debugf("Created new ipset: %s with %d initial prefixes (total prefixes %d)", setName, len(initialElements)/2, len(prefixes)) - - var subEnd int - for subStart := maxElements; subStart < nElements; subStart += maxElements { - subEnd = min(subStart+maxElements, nElements) - subElement := elements[subStart:subEnd] - nSubPrefixes := len(subElement) / 2 - log.Tracef("Adding new prefixes (%d) in ipset: %s", nSubPrefixes, setName) - if err := r.conn.SetAddElements(nfset, subElement); err != nil { - return nil, fmt.Errorf("error adding prefixes (%d) to set %s: %w", nSubPrefixes, setName, err) - } - if err := r.conn.Flush(); err != nil { - return nil, fmt.Errorf("flush error: %w", err) - } - log.Debugf("Added new prefixes (%d) in ipset: %s", nSubPrefixes, setName) - } - - log.Infof("Created new ipset: %s with %d prefixes", setName, len(prefixes)) - return nfset, nil -} - -func (r *router) convertPrefixesToSet(prefixes []netip.Prefix) []nftables.SetElement { - var elements []nftables.SetElement - for _, prefix := range prefixes { - // nftables needs half-open intervals [firstIP, lastIP) for prefixes - // e.g. 10.0.0.0/24 becomes [10.0.0.0, 10.0.1.0), 10.1.1.1/32 becomes [10.1.1.1, 10.1.1.2) etc - firstIP := prefix.Addr() - lastIP := calculateLastIP(prefix).Next() - - elements = append(elements, - // the nft tool also adds a zero-address IntervalEnd element, see https://github.com/google/nftables/issues/247 - // nftables.SetElement{Key: make([]byte, r.af.addrLen), IntervalEnd: true}, - nftables.SetElement{Key: firstIP.AsSlice()}, - nftables.SetElement{Key: lastIP.AsSlice(), IntervalEnd: true}, - ) - } - return elements -} - -// calculateLastIP determines the last IP in a given prefix. -func calculateLastIP(prefix netip.Prefix) netip.Addr { - masked := prefix.Masked() - if masked.Addr().Is4() { - hostMask := ^uint32(0) >> masked.Bits() - lastIP := uint32FromNetipAddr(masked.Addr()) | hostMask - return netip.AddrFrom4(uint32ToBytes(lastIP)) - } - - // IPv6: set host bits to all 1s - b := masked.Addr().As16() - bits := masked.Bits() - for i := bits; i < 128; i++ { - b[i/8] |= 1 << (7 - i%8) - } - return netip.AddrFrom16(b) -} - -// Utility function to convert netip.Addr to uint32. -func uint32FromNetipAddr(addr netip.Addr) uint32 { - b := addr.As4() - return binary.BigEndian.Uint32(b[:]) -} - -// Utility function to convert uint32 to a netip-compatible byte slice. -func uint32ToBytes(ip uint32) [4]byte { - var b [4]byte - binary.BigEndian.PutUint32(b[:], ip) - return b -} - -func (r *router) deleteIpSet(setName string, nfset *nftables.Set) error { - r.conn.DelSet(nfset) - if err := r.conn.Flush(); err != nil { - return fmt.Errorf(flushError, err) - } - - log.Debugf("Deleted unused ipset %s", setName) - return nil -} - -func (r *router) decrementSetCounter(rule *nftables.Rule) error { - sets := r.findSets(rule) - - var merr *multierror.Error - for _, setName := range sets { - if _, err := r.ipsetCounter.Decrement(setName); err != nil { - merr = multierror.Append(merr, fmt.Errorf("decrement set counter: %w", err)) - } - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) findSets(rule *nftables.Rule) []string { - var sets []string - for _, e := range rule.Exprs { - if lookup, ok := e.(*expr.Lookup); ok { - sets = append(sets, lookup.SetName) - } - } - return sets -} - -func (r *router) deleteNftRule(rule *nftables.Rule, ruleKey string) error { - if err := r.conn.DelRule(rule); err != nil { - return fmt.Errorf("delete rule %s: %w", ruleKey, err) - } - delete(r.rules, ruleKey) - - log.Debugf("removed route rule %s", ruleKey) - - return nil -} - -// AddNatRule appends a nftables rule pair to the nat chain -func (r *router) AddNatRule(pair firewall.RouterPair) error { - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - if r.legacyManagement { - log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination) - if err := r.addLegacyRouteRule(pair); err != nil { - return fmt.Errorf("add legacy routing rule: %w", err) - } - } - - if pair.Masquerade { - if err := r.addNatRule(pair); err != nil { - return fmt.Errorf("add nat rule: %w", err) - } - - if err := r.addNatRule(firewall.GetInversePair(pair)); err != nil { - return fmt.Errorf("add inverse nat rule: %w", err) - } - } - - if err := r.conn.Flush(); err != nil { - r.rollbackRules(pair) - return fmt.Errorf("insert rules for %s: %w", pair.Destination, err) - } - - return nil -} - -// rollbackRules cleans up unflushed rules and their set counters after a flush failure. -func (r *router) rollbackRules(pair firewall.RouterPair) { - keys := []string{ - firewall.GenKey(firewall.ForwardingFormat, pair), - firewall.GenKey(firewall.PreroutingFormat, pair), - firewall.GenKey(firewall.PreroutingFormat, firewall.GetInversePair(pair)), - } - for _, key := range keys { - rule, ok := r.rules[key] - if !ok { - continue - } - if err := r.decrementSetCounter(rule); err != nil { - log.Warnf("rollback set counter for %s: %v", key, err) - } - delete(r.rules, key) - } -} - -// addNatRule inserts a nftables rule to the conn client flush queue -func (r *router) addNatRule(pair firewall.RouterPair) error { - sourceExp, err := r.applyNetwork(pair.Source, nil, true) - if err != nil { - return fmt.Errorf("apply source: %w", err) - } - - destExp, err := r.applyNetwork(pair.Destination, nil, false) - if err != nil { - return fmt.Errorf("apply destination: %w", err) - } - - op := expr.CmpOpEq - if pair.Inverse { - op = expr.CmpOpNeq - } - - exprs := []expr.Any{ - &expr.Meta{ - Key: expr.MetaKeyIIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: op, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - } - // We only care about NEW connections to mark them and later identify them in the postrouting chain for masquerading. - // Masquerading will take care of the conntrack state, which means we won't need to mark established connections. - exprs = append(exprs, getCtNewExprs()...) - - exprs = append(exprs, sourceExp...) - exprs = append(exprs, destExp...) - - var markValue uint32 = nbnet.PreroutingFwmarkMasquerade - if pair.Inverse { - markValue = nbnet.PreroutingFwmarkMasqueradeReturn - } - - exprs = append(exprs, - &expr.Immediate{ - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(markValue), - }, - &expr.Meta{ - Key: expr.MetaKeyMARK, - SourceRegister: true, - Register: 1, - }, - ) - - ruleKey := firewall.GenKey(firewall.PreroutingFormat, pair) - - if _, exists := r.rules[ruleKey]; exists { - if err := r.removeNatRule(pair); err != nil { - return fmt.Errorf("remove prerouting rule: %w", err) - } - } - - // Ensure nat rules come first, so the mark can be overwritten. - // Currently overwritten by the dst-type LOCAL rules for redirected traffic. - r.rules[ruleKey] = r.conn.InsertRule(&nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameManglePrerouting], - Exprs: exprs, - UserData: []byte(ruleKey), - }) - - return nil -} - -// addPostroutingRules adds the masquerade rules -func (r *router) addPostroutingRules() { - // First masquerade rule for traffic coming in from WireGuard interface - exprs := []expr.Any{ - // Match on the first fwmark - &expr.Meta{ - Key: expr.MetaKeyMARK, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade), - }, - - // We need to exclude the loopback interface as this changes the ebpf proxy port - &expr.Meta{ - Key: expr.MetaKeyOIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: ifname("lo"), - }, - &expr.Counter{}, - &expr.Masq{}, - } - - r.conn.AddRule(&nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingNat], - Exprs: exprs, - }) - - // Second masquerade rule for traffic going out through WireGuard interface - exprs2 := []expr.Any{ - // Match on the second fwmark - &expr.Meta{ - Key: expr.MetaKeyMARK, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasqueradeReturn), - }, - - // Match WireGuard interface - &expr.Meta{ - Key: expr.MetaKeyOIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Counter{}, - &expr.Masq{}, - } - - r.conn.AddRule(&nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingNat], - Exprs: exprs2, - }) -} - -// addMSSClampingRules adds MSS clamping rules to prevent fragmentation for forwarded traffic. -func (r *router) addMSSClampingRules() error { - overhead := uint16(ipv4TCPHeaderSize) - if r.af.tableFamily == nftables.TableFamilyIPv6 { - overhead = ipv6TCPHeaderSize - } - if r.mtu <= overhead { - log.Debugf("MTU %d too small for MSS clamping (overhead %d), skipping", r.mtu, overhead) - return nil - } - mss := r.mtu - overhead - - exprsOut := []expr.Any{ - &expr.Meta{ - Key: expr.MetaKeyOIFNAME, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Meta{ - Key: expr.MetaKeyL4PROTO, - Register: 1, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{unix.IPPROTO_TCP}, - }, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseTransportHeader, - Offset: 13, - Len: 1, - }, - &expr.Bitwise{ - DestRegister: 1, - SourceRegister: 1, - Len: 1, - Mask: []byte{0x02}, - Xor: []byte{0x00}, - }, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: []byte{0x00}, - }, - &expr.Counter{}, - &expr.Exthdr{ - DestRegister: 1, - Type: 2, - Offset: 2, - Len: 2, - Op: expr.ExthdrOpTcpopt, - }, - &expr.Cmp{ - Op: expr.CmpOpGt, - Register: 1, - Data: binaryutil.BigEndian.PutUint16(uint16(mss)), - }, - &expr.Immediate{ - Register: 1, - Data: binaryutil.BigEndian.PutUint16(uint16(mss)), - }, - &expr.Exthdr{ - SourceRegister: 1, - Type: 2, - Offset: 2, - Len: 2, - Op: expr.ExthdrOpTcpopt, - }, - } - - r.conn.AddRule(&nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameMangleForward], - Exprs: exprsOut, - }) - - return r.conn.Flush() -} - -// addLegacyRouteRule adds a legacy routing rule for mgmt servers pre route acls -func (r *router) addLegacyRouteRule(pair firewall.RouterPair) error { - sourceExp, err := r.applyNetwork(pair.Source, nil, true) - if err != nil { - return fmt.Errorf("apply source: %w", err) - } - - destExp, err := r.applyNetwork(pair.Destination, nil, false) - if err != nil { - return fmt.Errorf("apply destination: %w", err) - } - - exprs := []expr.Any{ - &expr.Counter{}, - &expr.Verdict{ - Kind: expr.VerdictAccept, - }, - } - - exprs = append(exprs, sourceExp...) - exprs = append(exprs, destExp...) - - ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair) - - if _, exists := r.rules[ruleKey]; exists { - if err := r.removeLegacyRouteRule(pair); err != nil { - return fmt.Errorf("remove legacy routing rule: %w", err) - } - } - - r.rules[ruleKey] = r.conn.AddRule(&nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingFw], - Exprs: exprs, - UserData: []byte(ruleKey), - }) - return nil -} - -// removeLegacyRouteRule removes a legacy routing rule for mgmt servers pre route acls -func (r *router) removeLegacyRouteRule(pair firewall.RouterPair) error { - ruleKey := firewall.GenKey(firewall.ForwardingFormat, pair) - - rule, exists := r.rules[ruleKey] - if !exists { - return nil - } - - if rule.Handle == 0 { - log.Warnf("legacy forwarding rule %s has no handle, removing stale entry", ruleKey) - if err := r.decrementSetCounter(rule); err != nil { - log.Warnf("decrement set counter for stale rule %s: %v", ruleKey, err) - } - delete(r.rules, ruleKey) - return nil - } - - if err := r.conn.DelRule(rule); err != nil { - return fmt.Errorf("remove legacy forwarding rule %s -> %s: %w", pair.Source, pair.Destination, err) - } - - log.Debugf("removed legacy forwarding rule %s -> %s", pair.Source, pair.Destination) - - delete(r.rules, ruleKey) - - if err := r.decrementSetCounter(rule); err != nil { - return fmt.Errorf("decrement set counter: %w", err) - } - - return nil -} - -// GetLegacyManagement returns the route manager's legacy management mode -func (r *router) GetLegacyManagement() bool { - return r.legacyManagement -} - -// SetLegacyManagement sets the route manager to use legacy management mode -func (r *router) SetLegacyManagement(isLegacy bool) { - r.legacyManagement = isLegacy -} - -// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls -func (r *router) RemoveAllLegacyRouteRules() error { - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - var merr *multierror.Error - for k, rule := range r.rules { - if !strings.HasPrefix(k, firewall.ForwardingFormatPrefix) { - continue - } - if err := r.conn.DelRule(rule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove legacy forwarding rule: %v", err)) - } else { - delete(r.rules, k) - } - - } - return nberrors.FormatErrorOrNil(merr) -} - -// acceptForwardRules adds iif/oif rules in the filter table/forward chain to make sure -// that our traffic is not dropped by existing rules there. -// The existing FORWARD rules/policies decide outbound traffic towards our interface. -// In case the FORWARD policy is set to "drop", we add an established/related rule to allow return traffic for the inbound rule. -// This method also adds INPUT chain rules to allow traffic to the local interface. -func (r *router) acceptForwardRules() error { - var merr *multierror.Error - - if err := r.acceptFilterTableRules(); err != nil { - merr = multierror.Append(merr, err) - } - - if err := r.acceptExternalChainsRules(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("add accept rules to external chains: %w", err)) - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) acceptFilterTableRules() error { - if r.filterTable == nil { - return nil - } - - fw := "iptables" - - defer func() { - log.Debugf("Used %s to add accept forward and input rules", fw) - }() - - // Try iptables first and fallback to nftables if iptables is not available. - // Use the correct protocol (iptables vs ip6tables) for the address family. - ipt, err := iptables.NewWithProtocol(r.iptablesProto()) - if err != nil { - log.Warnf("Will use nftables to manipulate the filter table because iptables is not available: %v", err) - - fw = "nftables" - return r.acceptFilterRulesNftables(r.filterTable) - } - - if err := r.acceptFilterRulesIptables(ipt); err != nil { - log.Warnf("iptables failed (table may be incompatible), falling back to nftables: %v", err) - fw = "nftables" - return r.acceptFilterRulesNftables(r.filterTable) - } - return nil -} - -func (r *router) acceptFilterRulesIptables(ipt *iptables.IPTables) error { - var merr *multierror.Error - - for _, rule := range r.getAcceptForwardRules() { - if err := ipt.Insert("filter", chainNameForward, 1, rule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("add iptables forward rule: %v", err)) - } else { - log.Debugf("added iptables forward rule: %v", rule) - } - } - - inputRule := r.getAcceptInputRule() - if err := ipt.Insert("filter", chainNameInput, 1, inputRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("add iptables input rule: %v", err)) - } else { - log.Debugf("added iptables input rule: %v", inputRule) - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) getAcceptForwardRules() [][]string { - intf := r.wgIface.Name() - return [][]string{ - {"-i", intf, "-j", "ACCEPT"}, - {"-o", intf, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}, - } -} - -func (r *router) getAcceptInputRule() []string { - return []string{"-i", r.wgIface.Name(), "-j", "ACCEPT"} -} - -// acceptFilterRulesNftables adds accept rules to the ip filter table using nftables. -// This is used when iptables is not available. -func (r *router) acceptFilterRulesNftables(table *nftables.Table) error { - intf := ifname(r.wgIface.Name()) - - forwardChain := &nftables.Chain{ - Name: chainNameForward, - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookForward, - Priority: nftables.ChainPriorityFilter, - } - r.insertForwardAcceptRules(forwardChain, intf) - - inputChain := &nftables.Chain{ - Name: chainNameInput, - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookInput, - Priority: nftables.ChainPriorityFilter, - } - r.insertInputAcceptRule(inputChain, intf) - - return r.conn.Flush() -} - -// acceptExternalChainsRules adds accept rules to external chains (non-netbird, non-iptables tables). -// It dynamically finds chains at call time to handle chains that may have been created after startup. -func (r *router) acceptExternalChainsRules() error { - chains := r.findExternalChains() - if len(chains) == 0 { - return nil - } - - intf := ifname(r.wgIface.Name()) - for _, chain := range chains { - r.applyExternalChainAccept(chain, intf) - } - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("flush external chain rules: %w", err) - } - return nil -} - -func (r *router) applyExternalChainAccept(chain *nftables.Chain, intf []byte) { - if chain.Hooknum == nil { - log.Debugf("skipping external chain %s/%s: hooknum is nil", chain.Table.Name, chain.Name) - return - } - - log.Debugf("adding accept rules to external %s chain: %s %s/%s", - hookName(chain.Hooknum), familyName(chain.Table.Family), chain.Table.Name, chain.Name) - - switch *chain.Hooknum { - case *nftables.ChainHookForward: - r.insertForwardAcceptRules(chain, intf) - case *nftables.ChainHookInput: - r.insertInputAcceptRule(chain, intf) - } -} - -func (r *router) insertForwardAcceptRules(chain *nftables.Chain, intf []byte) { - existing, err := r.existingNetbirdRulesInChain(chain) - if err != nil { - log.Warnf("skip forward accept rules in %s/%s: %v", chain.Table.Name, chain.Name, err) - return - } - r.insertForwardIifRule(chain, intf, existing) - r.insertForwardOifEstablishedRule(chain, intf, existing) -} - -func (r *router) insertForwardIifRule(chain *nftables.Chain, intf []byte, existing map[string]bool) { - if existing[userDataAcceptForwardRuleIif] { - return - } - r.conn.InsertRule(&nftables.Rule{ - Table: chain.Table, - Chain: chain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf}, - &expr.Counter{}, - &expr.Verdict{Kind: expr.VerdictAccept}, - }, - UserData: []byte(userDataAcceptForwardRuleIif), - }) -} - -func (r *router) insertForwardOifEstablishedRule(chain *nftables.Chain, intf []byte, existing map[string]bool) { - if existing[userDataAcceptForwardRuleOif] { - return - } - exprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf}, - } - r.conn.InsertRule(&nftables.Rule{ - Table: chain.Table, - Chain: chain, - Exprs: append(exprs, getEstablishedExprs(2)...), - UserData: []byte(userDataAcceptForwardRuleOif), - }) -} - -func (r *router) insertInputAcceptRule(chain *nftables.Chain, intf []byte) { - existing, err := r.existingNetbirdRulesInChain(chain) - if err != nil { - log.Warnf("skip input accept rule in %s/%s: %v", chain.Table.Name, chain.Name, err) - return - } - if existing[userDataAcceptInputRule] { - return - } - r.conn.InsertRule(&nftables.Rule{ - Table: chain.Table, - Chain: chain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: intf}, - &expr.Counter{}, - &expr.Verdict{Kind: expr.VerdictAccept}, - }, - UserData: []byte(userDataAcceptInputRule), - }) -} - -// existingNetbirdRulesInChain returns the set of netbird-owned UserData tags present in a chain; callers must bail on error since InsertRule is additive. -func (r *router) existingNetbirdRulesInChain(chain *nftables.Chain) (map[string]bool, error) { - rules, err := r.conn.GetRules(chain.Table, chain) - if err != nil { - return nil, fmt.Errorf("list rules: %w", err) - } - present := map[string]bool{} - for _, rule := range rules { - if !isNetbirdAcceptRuleTag(rule.UserData) { - continue - } - present[string(rule.UserData)] = true - } - return present, nil -} - -func isNetbirdAcceptRuleTag(userData []byte) bool { - switch string(userData) { - case userDataAcceptForwardRuleIif, - userDataAcceptForwardRuleOif, - userDataAcceptInputRule: - return true - } - return false -} - -func (r *router) removeAcceptFilterRules() error { - var merr *multierror.Error - - if err := r.removeFilterTableRules(); err != nil { - merr = multierror.Append(merr, err) - } - - if err := r.removeExternalChainsRules(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove external chain rules: %w", err)) - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) removeFilterTableRules() error { - if r.filterTable == nil { - return nil - } - - ipt, err := iptables.NewWithProtocol(r.iptablesProto()) - if err != nil { - log.Debugf("iptables not available, using nftables to remove filter rules: %v", err) - return r.removeAcceptRulesFromTable(r.filterTable) - } - - if err := r.removeAcceptFilterRulesIptables(ipt); err != nil { - log.Debugf("iptables removal failed (table may be incompatible), falling back to nftables: %v", err) - return r.removeAcceptRulesFromTable(r.filterTable) - } - return nil -} - -func (r *router) removeAcceptRulesFromTable(table *nftables.Table) error { - chains, err := r.conn.ListChainsOfTableFamily(table.Family) - if err != nil { - return fmt.Errorf("list chains: %v", err) - } - - for _, chain := range chains { - if chain.Table.Name != table.Name { - continue - } - - if chain.Name != chainNameForward && chain.Name != chainNameInput { - continue - } - - if err := r.removeAcceptRulesFromChain(table, chain); err != nil { - return err - } - } - - return r.conn.Flush() -} - -func (r *router) removeAcceptRulesFromChain(table *nftables.Table, chain *nftables.Chain) error { - rules, err := r.conn.GetRules(table, chain) - if err != nil { - return fmt.Errorf("get rules from %s/%s: %v", table.Name, chain.Name, err) - } - - for _, rule := range rules { - if bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleIif)) || - bytes.Equal(rule.UserData, []byte(userDataAcceptForwardRuleOif)) || - bytes.Equal(rule.UserData, []byte(userDataAcceptInputRule)) { - if err := r.conn.DelRule(rule); err != nil { - return fmt.Errorf("delete rule from %s/%s: %v", table.Name, chain.Name, err) - } - } - } - return nil -} - -// removeExternalChainsRules removes our accept rules from all external chains. -// This is deterministic - it scans for chains at removal time rather than relying on saved state, -// ensuring cleanup works even after a crash or if chains changed. -func (r *router) removeExternalChainsRules() error { - chains := r.findExternalChains() - if len(chains) == 0 { - return nil - } - - for _, chain := range chains { - if err := r.removeAcceptRulesFromChain(chain.Table, chain); err != nil { - log.Warnf("remove rules from external chain %s/%s: %v", chain.Table.Name, chain.Name, err) - } - } - - return r.conn.Flush() -} - -// findExternalChains scans for chains from non-netbird tables that have FORWARD or INPUT hooks. -// This is used both at startup (to know where to add rules) and at cleanup (to ensure deterministic removal). -func (r *router) findExternalChains() []*nftables.Chain { - var chains []*nftables.Chain - - families := []nftables.TableFamily{r.af.tableFamily, nftables.TableFamilyINet} - - for _, family := range families { - allChains, err := r.conn.ListChainsOfTableFamily(family) - if err != nil { - log.Debugf("list chains for family %d: %v", family, err) - continue - } - - for _, chain := range allChains { - if r.isExternalChain(chain) { - chains = append(chains, chain) - } - } - } - - return chains -} - -func (r *router) isExternalChain(chain *nftables.Chain) bool { - if r.workTable != nil && chain.Table.Name == r.workTable.Name { - return false - } - - // Skip firewalld-owned chains. Firewalld creates its chains with the - // NFT_CHAIN_OWNER flag, so inserting rules into them returns EPERM. - // We delegate acceptance to firewalld by trusting the interface instead. - if chain.Table.Name == firewalldTableName { - return false - } - - // Skip iptables/ip6tables-managed tables (adding nft-native rules breaks iptables-save compat) - if (chain.Table.Family == nftables.TableFamilyIPv4 || chain.Table.Family == nftables.TableFamilyIPv6) && isIptablesTable(chain.Table.Name) { - return false - } - - if chain.Type != nftables.ChainTypeFilter { - return false - } - - if chain.Hooknum == nil { - return false - } - - return *chain.Hooknum == *nftables.ChainHookForward || *chain.Hooknum == *nftables.ChainHookInput -} - -func isIptablesTable(name string) bool { - switch name { - case tableNameFilter, tableNat, tableMangle, tableRaw, tableSecurity: - return true - } - return false -} - -func (r *router) removeAcceptFilterRulesIptables(ipt *iptables.IPTables) error { - var merr *multierror.Error - - for _, rule := range r.getAcceptForwardRules() { - if err := ipt.DeleteIfExists("filter", chainNameForward, rule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove iptables forward rule: %v", err)) - } - } - - inputRule := r.getAcceptInputRule() - if err := ipt.DeleteIfExists("filter", chainNameInput, inputRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove iptables input rule: %v", err)) - } - - return nberrors.FormatErrorOrNil(merr) -} - -// RemoveNatRule removes the prerouting mark rule -func (r *router) RemoveNatRule(pair firewall.RouterPair) error { - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - var merr *multierror.Error - - if pair.Masquerade { - if err := r.removeNatRule(pair); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove prerouting rule: %w", err)) - } - - if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove inverse prerouting rule: %w", err)) - } - } - - if err := r.removeLegacyRouteRule(pair); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove legacy routing rule: %w", err)) - } - - // Set counters are decremented in the sub-methods above before flush. If flush fails, - // counters will be off until the next successful removal or refresh cycle. - if err := r.conn.Flush(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("flush remove nat rules %s: %w", pair.Destination, err)) - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) removeNatRule(pair firewall.RouterPair) error { - ruleKey := firewall.GenKey(firewall.PreroutingFormat, pair) - - rule, exists := r.rules[ruleKey] - if !exists { - log.Debugf("prerouting rule %s not found", ruleKey) - return nil - } - - if rule.Handle == 0 { - log.Warnf("prerouting rule %s has no handle, removing stale entry", ruleKey) - if err := r.decrementSetCounter(rule); err != nil { - log.Warnf("decrement set counter for stale rule %s: %v", ruleKey, err) - } - delete(r.rules, ruleKey) - return nil - } - - if err := r.conn.DelRule(rule); err != nil { - return fmt.Errorf("remove prerouting rule %s -> %s: %w", pair.Source, pair.Destination, err) - } - - log.Debugf("removed prerouting rule %s -> %s", pair.Source, pair.Destination) - - delete(r.rules, ruleKey) - - if err := r.decrementSetCounter(rule); err != nil { - return fmt.Errorf("decrement set counter: %w", err) - } - - return nil -} - -// refreshRulesMap rebuilds the rule map from the kernel. This removes stale entries -// (e.g. from failed flushes) and updates handles for all existing rules. -func (r *router) refreshRulesMap() error { - var merr *multierror.Error - newRules := make(map[string]*nftables.Rule) - for _, chain := range r.chains { - rules, err := r.conn.GetRules(chain.Table, chain) - if err != nil { - merr = multierror.Append(merr, fmt.Errorf("list rules for chain %s: %w", chain.Name, err)) - // preserve existing entries for this chain since we can't verify their state - for k, v := range r.rules { - if v.Chain != nil && v.Chain.Name == chain.Name { - newRules[k] = v - } - } - continue - } - for _, rule := range rules { - if len(rule.UserData) > 0 { - newRules[string(rule.UserData)] = rule - } - } - } - r.rules = newRules - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - if err := r.ipFwdState.RequestForwarding(); err != nil { - return nil, err - } - - ruleKey := rule.ID() - if _, exists := r.rules[ruleKey+dnatSuffix]; exists { - return rule, nil - } - - protoNum, err := r.af.protoNum(rule.Protocol) - if err != nil { - return nil, fmt.Errorf("convert protocol to number: %w", err) - } - - if err := r.addDnatRedirect(rule, protoNum, ruleKey); err != nil { - return nil, err - } - - r.addDnatMasq(rule, protoNum, ruleKey) - - // Unlike iptables, there's no point in adding "out" rules in the forward chain here as our policy is ACCEPT. - // To overcome DROP policies in other chains, we'd have to add rules to the chains there. - // We also cannot just add "oif accept" there and filter in our own table as we don't know what is supposed to be allowed. - // TODO: find chains with drop policies and add rules there - - if err := r.conn.Flush(); err != nil { - return nil, fmt.Errorf("flush rules: %w", err) - } - - return &rule, nil -} - -func (r *router) addDnatRedirect(rule firewall.ForwardRule, protoNum uint8, ruleKey string) error { - dnatExprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{protoNum}, - }, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseTransportHeader, - Offset: 2, - Len: 2, - }, - } - dnatExprs = append(dnatExprs, applyPort(&rule.DestinationPort, false)...) - - // shifted translated port is not supported in nftables, so we hand this over to xtables - if rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2 { - if rule.TranslatedPort.Values[0] != rule.DestinationPort.Values[0] || - rule.TranslatedPort.Values[1] != rule.DestinationPort.Values[1] { - return r.addXTablesRedirect(dnatExprs, ruleKey, rule) - } - } - - additionalExprs, regProtoMin, regProtoMax, err := r.handleTranslatedPort(rule) - if err != nil { - return err - } - dnatExprs = append(dnatExprs, additionalExprs...) - - dnatExprs = append(dnatExprs, - &expr.NAT{ - Type: expr.NATTypeDestNAT, - Family: uint32(r.af.tableFamily), - RegAddrMin: 1, - RegProtoMin: regProtoMin, - RegProtoMax: regProtoMax, - }, - ) - - dnatRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingRdr], - Exprs: dnatExprs, - UserData: []byte(ruleKey + dnatSuffix), - } - r.conn.AddRule(dnatRule) - r.rules[ruleKey+dnatSuffix] = dnatRule - - return nil -} - -func (r *router) handleTranslatedPort(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - switch { - case rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2: - return r.handlePortRange(rule) - case len(rule.TranslatedPort.Values) == 0: - return r.handleAddressOnly(rule) - case len(rule.TranslatedPort.Values) == 1: - return r.handleSinglePort(rule) - default: - return nil, 0, 0, fmt.Errorf("invalid translated port: %v", rule.TranslatedPort) - } -} - -func (r *router) handlePortRange(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - exprs := []expr.Any{ - &expr.Immediate{ - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - &expr.Immediate{ - Register: 2, - Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[0]), - }, - &expr.Immediate{ - Register: 3, - Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[1]), - }, - } - return exprs, 2, 3, nil -} - -func (r *router) handleAddressOnly(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - exprs := []expr.Any{ - &expr.Immediate{ - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - } - return exprs, 0, 0, nil -} - -func (r *router) handleSinglePort(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - exprs := []expr.Any{ - &expr.Immediate{ - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - &expr.Immediate{ - Register: 2, - Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[0]), - }, - } - return exprs, 2, 0, nil -} - -func (r *router) addXTablesRedirect(dnatExprs []expr.Any, ruleKey string, rule firewall.ForwardRule) error { - dnatExprs = append(dnatExprs, - &expr.Counter{}, - &expr.Target{ - Name: "DNAT", - Rev: 2, - Info: &xt.NatRange2{ - NatRange: xt.NatRange{ - Flags: uint(xt.NatRangeMapIPs | xt.NatRangeProtoSpecified | xt.NatRangeProtoOffset), - MinIP: rule.TranslatedAddress.AsSlice(), - MaxIP: rule.TranslatedAddress.AsSlice(), - MinPort: rule.TranslatedPort.Values[0], - MaxPort: rule.TranslatedPort.Values[1], - }, - BasePort: rule.DestinationPort.Values[0], - }, - }, - ) - - natTable := &nftables.Table{ - Name: tableNat, - Family: r.af.tableFamily, - } - dnatRule := &nftables.Rule{ - Table: natTable, - Chain: &nftables.Chain{ - Name: chainNameNatPrerouting, - Table: natTable, - Type: nftables.ChainTypeNAT, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityNATDest, - }, - Exprs: dnatExprs, - UserData: []byte(ruleKey + dnatSuffix), - } - r.conn.AddRule(dnatRule) - r.rules[ruleKey+dnatSuffix] = dnatRule - - return nil -} - -func (r *router) addDnatMasq(rule firewall.ForwardRule, protoNum uint8, ruleKey string) { - masqExprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{protoNum}, - }, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: r.af.dstAddrOffset, - Len: r.af.addrLen, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - } - - masqExprs = append(masqExprs, applyPort(&rule.TranslatedPort, false)...) - masqExprs = append(masqExprs, &expr.Masq{}) - - masqRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingNat], - Exprs: masqExprs, - UserData: []byte(ruleKey + snatSuffix), - } - r.conn.AddRule(masqRule) - r.rules[ruleKey+snatSuffix] = masqRule -} - -func (r *router) DeleteDNATRule(rule firewall.Rule) error { - if err := r.ipFwdState.ReleaseForwarding(); err != nil { - log.Errorf("%v", err) - } - - ruleKey := rule.ID() - - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - var merr *multierror.Error - var needsFlush bool - - if dnatRule, exists := r.rules[ruleKey+dnatSuffix]; exists { - if dnatRule.Handle == 0 { - log.Warnf("dnat rule %s has no handle, removing stale entry", ruleKey+dnatSuffix) - delete(r.rules, ruleKey+dnatSuffix) - } else if err := r.conn.DelRule(dnatRule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete dnat rule: %w", err)) - } else { - needsFlush = true - } - } - - if masqRule, exists := r.rules[ruleKey+snatSuffix]; exists { - if masqRule.Handle == 0 { - log.Warnf("snat rule %s has no handle, removing stale entry", ruleKey+snatSuffix) - delete(r.rules, ruleKey+snatSuffix) - } else if err := r.conn.DelRule(masqRule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete snat rule: %w", err)) - } else { - needsFlush = true - } - } - - if needsFlush { - if err := r.conn.Flush(); err != nil { - merr = multierror.Append(merr, fmt.Errorf(flushError, err)) - } - } - - if merr == nil { - delete(r.rules, ruleKey+dnatSuffix) - delete(r.rules, ruleKey+snatSuffix) - } - - return nberrors.FormatErrorOrNil(merr) -} - -func (r *router) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { - nfset, err := r.conn.GetSetByName(r.workTable, set.HashedName()) - if err != nil { - return fmt.Errorf("get set %s: %w", set.HashedName(), err) - } - - elements := r.convertPrefixesToSet(prefixes) - if err := r.conn.SetAddElements(nfset, elements); err != nil { - return fmt.Errorf("add elements to set %s: %w", set.HashedName(), err) - } - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf(flushError, err) - } - - log.Debugf("updated set %s with prefixes %v", set.HashedName(), prefixes) - - return nil -} - -// AddInboundDNAT adds an inbound DNAT rule redirecting traffic from NetBird peers to local services. -func (r *router) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - ruleID := fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - if _, exists := r.rules[ruleID]; exists { - return nil - } - - protoNum, err := r.af.protoNum(protocol) - if err != nil { - return fmt.Errorf("convert protocol to number: %w", err) - } - - exprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 2}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 2, - Data: []byte{protoNum}, - }, - &expr.Payload{ - DestRegister: 3, - Base: expr.PayloadBaseTransportHeader, - Offset: 2, - Len: 2, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 3, - Data: binaryutil.BigEndian.PutUint16(originalPort), - }, - } - - bits := 32 - if localAddr.Is6() { - bits = 128 - } - exprs = append(exprs, r.applyPrefix(netip.PrefixFrom(localAddr, bits), false)...) - - exprs = append(exprs, - &expr.Immediate{ - Register: 1, - Data: localAddr.AsSlice(), - }, - &expr.Immediate{ - Register: 2, - Data: binaryutil.BigEndian.PutUint16(translatedPort), - }, - &expr.NAT{ - Type: expr.NATTypeDestNAT, - Family: uint32(r.af.tableFamily), - RegAddrMin: 1, - RegProtoMin: 2, - RegProtoMax: 0, - }, - ) - - dnatRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingRdr], - Exprs: exprs, - UserData: []byte(ruleID), - } - r.conn.AddRule(dnatRule) - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("add inbound DNAT rule: %w", err) - } - - r.rules[ruleID] = dnatRule - - return nil -} - -// RemoveInboundDNAT removes an inbound DNAT rule. -func (r *router) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - ruleID := fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - rule, exists := r.rules[ruleID] - if !exists { - return nil - } - - if rule.Handle == 0 { - log.Warnf("inbound DNAT rule %s has no handle, removing stale entry", ruleID) - delete(r.rules, ruleID) - return nil - } - - if err := r.conn.DelRule(rule); err != nil { - return fmt.Errorf("delete inbound DNAT rule %s: %w", ruleID, err) - } - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("flush delete inbound DNAT rule: %w", err) - } - delete(r.rules, ruleID) - - return nil -} - -// ensureNATOutputChain lazily creates the OUTPUT NAT chain on first use. -func (r *router) ensureNATOutputChain() error { - if _, exists := r.chains[chainNameNATOutput]; exists { - return nil - } - - r.chains[chainNameNATOutput] = r.conn.AddChain(&nftables.Chain{ - Name: chainNameNATOutput, - Table: r.workTable, - Hooknum: nftables.ChainHookOutput, - Priority: nftables.ChainPriorityNATDest, - Type: nftables.ChainTypeNAT, - }) - - if err := r.conn.Flush(); err != nil { - delete(r.chains, chainNameNATOutput) - return fmt.Errorf("create NAT output chain: %w", err) - } - return nil -} - -// AddOutputDNAT adds an OUTPUT chain DNAT rule for locally-generated traffic. -func (r *router) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - ruleID := fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - if _, exists := r.rules[ruleID]; exists { - return nil - } - - if err := r.ensureNATOutputChain(); err != nil { - return err - } - - protoNum, err := r.af.protoNum(protocol) - if err != nil { - return fmt.Errorf("convert protocol to number: %w", err) - } - - exprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{protoNum}, - }, - &expr.Payload{ - DestRegister: 2, - Base: expr.PayloadBaseTransportHeader, - Offset: 2, - Len: 2, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 2, - Data: binaryutil.BigEndian.PutUint16(originalPort), - }, - } - - bits := 32 - if localAddr.Is6() { - bits = 128 - } - exprs = append(exprs, r.applyPrefix(netip.PrefixFrom(localAddr, bits), false)...) - - exprs = append(exprs, - &expr.Immediate{ - Register: 1, - Data: localAddr.AsSlice(), - }, - &expr.Immediate{ - Register: 2, - Data: binaryutil.BigEndian.PutUint16(translatedPort), - }, - &expr.NAT{ - Type: expr.NATTypeDestNAT, - Family: uint32(r.af.tableFamily), - RegAddrMin: 1, - RegProtoMin: 2, - }, - ) - - dnatRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameNATOutput], - Exprs: exprs, - UserData: []byte(ruleID), - } - r.conn.AddRule(dnatRule) - - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("add output DNAT rule: %w", err) - } - - r.rules[ruleID] = dnatRule - - return nil -} - -// RemoveOutputDNAT removes an OUTPUT chain DNAT rule. -func (r *router) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - ruleID := fmt.Sprintf("output-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort) - - rule, exists := r.rules[ruleID] - if !exists { - return nil - } - - if rule.Handle == 0 { - log.Warnf("output DNAT rule %s has no handle, removing stale entry", ruleID) - delete(r.rules, ruleID) - return nil - } - - if err := r.conn.DelRule(rule); err != nil { - return fmt.Errorf("delete output DNAT rule %s: %w", ruleID, err) - } - if err := r.conn.Flush(); err != nil { - return fmt.Errorf("flush delete output DNAT rule: %w", err) - } - delete(r.rules, ruleID) - - return nil -} - -// applyNetwork generates nftables expressions for networks (CIDR) or sets -func (r *router) applyNetwork( - network firewall.Network, - setPrefixes []netip.Prefix, - isSource bool, -) ([]expr.Any, error) { - if network.IsSet() { - exprs, err := r.getIpSet(network.Set, setPrefixes, isSource) - if err != nil { - return nil, fmt.Errorf("source: %w", err) - } - return exprs, nil - } - - if network.IsPrefix() { - return r.applyPrefix(network.Prefix, isSource), nil - } - - return nil, nil -} - -// applyPrefix generates nftables expressions for a CIDR prefix -func (r *router) applyPrefix(prefix netip.Prefix, isSource bool) []expr.Any { - // dst offset by default - offset := r.af.dstAddrOffset - if isSource { - // src offset - offset = r.af.srcAddrOffset - } - - ones := prefix.Bits() - // unspecified address (/0) doesn't need extra expressions - if ones == 0 { - return nil - } - - mask := net.CIDRMask(ones, r.af.totalBits) - xor := make([]byte, r.af.addrLen) - - return []expr.Any{ - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: offset, - Len: r.af.addrLen, - }, - &expr.Bitwise{ - DestRegister: 1, - SourceRegister: 1, - Len: r.af.addrLen, - Mask: mask, - Xor: xor, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: prefix.Masked().Addr().AsSlice(), - }, - } -} - -func applyPort(port *firewall.Port, isSource bool) []expr.Any { - if port == nil { - return nil - } - - var exprs []expr.Any - - offset := uint32(2) // Default offset for destination port - if isSource { - offset = 0 // Offset for source port - } - - exprs = append(exprs, &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseTransportHeader, - Offset: offset, - Len: 2, - }) - - if port.IsRange && len(port.Values) == 2 { - // Handle port range - exprs = append(exprs, - &expr.Range{ - Op: expr.CmpOpEq, - Register: 1, - FromData: binaryutil.BigEndian.PutUint16(port.Values[0]), - ToData: binaryutil.BigEndian.PutUint16(port.Values[1]), - }, - ) - } else { - // Handle single port or multiple ports - for i, p := range port.Values { - if i > 0 { - // Add a bitwise OR operation between port checks - exprs = append(exprs, &expr.Bitwise{ - SourceRegister: 1, - DestRegister: 1, - Len: 4, - Mask: []byte{0x00, 0x00, 0xff, 0xff}, - Xor: []byte{0x00, 0x00, 0x00, 0x00}, - }) - } - exprs = append(exprs, &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: binaryutil.BigEndian.PutUint16(p), - }) - } - } - - return exprs -} - -func getCtNewExprs() []expr.Any { - return []expr.Any{ - &expr.Ct{ - Key: expr.CtKeySTATE, - Register: 1, - }, - &expr.Bitwise{ - SourceRegister: 1, - DestRegister: 1, - Len: 4, - Mask: binaryutil.NativeEndian.PutUint32(expr.CtStateBitNEW), - Xor: binaryutil.NativeEndian.PutUint32(0), - }, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: []byte{0, 0, 0, 0}, - }, - } -} - -func (r *router) getIpSetExprs(ref refcounter.Ref[*nftables.Set], isSource bool) ([]expr.Any, error) { - // dst offset by default - offset := r.af.dstAddrOffset - if isSource { - // src offset - offset = r.af.srcAddrOffset - } - - return []expr.Any{ - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: offset, - Len: r.af.addrLen, - }, - &expr.Lookup{ - SourceRegister: 1, - SetName: ref.Out.Name, - SetID: ref.Out.ID, - }, - }, nil -} diff --git a/client/firewall/nftables/router_linux_test.go b/client/firewall/nftables/router_linux_test.go index 2fc664d51..49dbfc8f2 100644 --- a/client/firewall/nftables/router_linux_test.go +++ b/client/firewall/nftables/router_linux_test.go @@ -37,7 +37,7 @@ func TestNftablesManager_AddNatRule(t *testing.T) { for _, testCase := range test.InsertRuleTestCases { t.Run(testCase.Name, func(t *testing.T) { - // need fw manager to init both acl mgr and router for all chains to be present + // need fw manager to init both acl mgr and family for all chains to be present manager, err := Create(ifaceMock, iface.DefaultMTU) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) @@ -47,7 +47,7 @@ func TestNftablesManager_AddNatRule(t *testing.T) { nftablesTestingClient := &nftables.Conn{} - rtr := manager.router + rtr := manager.family4 err = rtr.AddNatRule(testCase.InputPair) require.NoError(t, err, "pair should be inserted") @@ -90,9 +90,9 @@ func TestNftablesManager_AddNatRule(t *testing.T) { } // Build CIDR matching expressions - testRouter := &router{af: afIPv4} - sourceExp := testRouter.applyPrefix(testCase.InputPair.Source.Prefix, true) - destExp := testRouter.applyPrefix(testCase.InputPair.Destination.Prefix, false) + testRouter := &family{af: afIPv4} + sourceExp := prefixMatchExprs(testRouter.af, testCase.InputPair.Source.Prefix, true) + destExp := prefixMatchExprs(testRouter.af, testCase.InputPair.Destination.Prefix, false) // Combine all expressions in the correct order // nolint:gocritic @@ -100,14 +100,14 @@ func TestNftablesManager_AddNatRule(t *testing.T) { testingExpression = append(testingExpression, sourceExp...) testingExpression = append(testingExpression, destExp...) - natRuleKey := firewall.GenKey(firewall.PreroutingFormat, testCase.InputPair) + natRuleKey := testCase.InputPair.GenKey(firewall.PreroutingFormat) found := 0 for _, chain := range rtr.chains { if chain.Name == chainNameManglePrerouting { rules, err := nftablesTestingClient.GetRules(chain.Table, chain) require.NoError(t, err, "should list rules for %s table and %s chain", chain.Table.Name, chain.Name) for _, rule := range rules { - if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey { + if len(rule.UserData) > 0 && firewall.RuleID(rule.UserData) == natRuleKey { // Compare expressions up to the mark setting expressions require.ElementsMatchf(t, rule.Exprs[:len(testingExpression)], testingExpression, "prerouting nat rule elements should match") found = 1 @@ -135,19 +135,19 @@ func TestNftablesManager_RemoveNatRule(t *testing.T) { require.NoError(t, err) require.NoError(t, manager.Init(nil)) - rtr := manager.router + rtr := manager.family4 - // First add the NAT rule using the router's method + // First add the NAT rule using the family's method err = rtr.AddNatRule(testCase.InputPair) require.NoError(t, err, "should add NAT rule") // Verify the rule was added - natRuleKey := firewall.GenKey(firewall.PreroutingFormat, testCase.InputPair) + natRuleKey := testCase.InputPair.GenKey(firewall.PreroutingFormat) found := false rules, err := rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNameManglePrerouting]) require.NoError(t, err, "should list rules") for _, rule := range rules { - if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey { + if len(rule.UserData) > 0 && firewall.RuleID(rule.UserData) == natRuleKey { found = true break } @@ -163,7 +163,7 @@ func TestNftablesManager_RemoveNatRule(t *testing.T) { rules, err = rtr.conn.GetRules(rtr.workTable, rtr.chains[chainNameManglePrerouting]) require.NoError(t, err, "should list rules after removal") for _, rule := range rules { - if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey { + if len(rule.UserData) > 0 && firewall.RuleID(rule.UserData) == natRuleKey { found = true break } @@ -200,11 +200,10 @@ func TestRouter_AddRouteFiltering(t *testing.T) { defer deleteWorkTable() - r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU) - require.NoError(t, err, "Failed to create router") + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) require.NoError(t, r.init(workTable)) - defer func(r *router) { + defer func(r *family) { require.NoError(t, r.Reset(), "Failed to reset rules") }(r) @@ -314,16 +313,16 @@ func TestRouter_AddRouteFiltering(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ruleKey, err := r.AddRouteFiltering(nil, tt.sources, firewall.Network{Prefix: tt.destination}, tt.proto, tt.sPort, tt.dPort, tt.action) - require.NoError(t, err, "AddRouteFiltering failed") + ruleKey, err := r.AddFilterRule(nil, tt.sources, firewall.Network{Prefix: tt.destination}, tt.proto, tt.sPort, tt.dPort, tt.action) + require.NoError(t, err, "AddFilterRule failed") t.Cleanup(func() { - require.NoError(t, r.DeleteRouteRule(ruleKey), "Failed to delete rule") + require.NoError(t, r.DeleteFilterRule(ruleKey), "Failed to delete rule") }) - // Check if the rule is in the internal map - rule, ok := r.rules[ruleKey.ID()] - assert.True(t, ok, "Rule not found in internal map") + stored, ok := r.filters[id.RuleID(ruleKey.ID())] + require.True(t, ok, "Rule not found in filters map") + rule := stored.nftRule t.Log("Internal rule expressions:") for i, expr := range rule.Exprs { @@ -339,7 +338,7 @@ func TestRouter_AddRouteFiltering(t *testing.T) { var nftRule *nftables.Rule for _, rule := range rules { - if string(rule.UserData) == ruleKey.ID() { + if firewall.RuleID(rule.UserData) == ruleKey.ID() { nftRule = rule break } @@ -367,12 +366,11 @@ func TestNftablesCreateIpSet(t *testing.T) { defer deleteWorkTable() - r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU) - require.NoError(t, err, "Failed to create router") + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) require.NoError(t, r.init(workTable)) defer func() { - require.NoError(t, r.Reset(), "Failed to reset router") + require.NoError(t, r.Reset(), "Failed to reset family") }() tests := []struct { @@ -509,6 +507,58 @@ func TestNftablesCreateIpSet(t *testing.T) { } } +// TestNftablesUpdateSetMergesOverlapping verifies that UpdateSet merges +// overlapping prefixes before adding them. An interval set rejects +// overlapping elements, so without the merge a batch holding a /32 already +// covered by a /24, or a duplicate address as DNS resolution can produce, +// would fail. +func TestNftablesUpdateSetMergesOverlapping(t *testing.T) { + if check() != NFTABLES { + t.Skip("nftables not supported on this system") + } + + workTable, err := createWorkTable() + require.NoError(t, err, "create work table") + defer deleteWorkTable() + + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) + require.NoError(t, r.init(workTable)) + defer func() { + require.NoError(t, r.Reset(), "reset family") + }() + + initial := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")} + set := firewall.NewPrefixSet(initial) + + created, err := r.createIpSet(set.HashedName(), setInput{prefixes: initial}) + require.NoError(t, err, "create ip set") + require.NotNil(t, created) + + overlapping := []netip.Prefix{ + netip.MustParsePrefix("192.168.1.0/24"), + netip.MustParsePrefix("192.168.1.1/32"), + netip.MustParsePrefix("192.168.1.1/32"), + } + require.NoError(t, r.UpdateSet(set, overlapping), "UpdateSet must merge overlapping prefixes") + + fetchedSet, err := r.conn.GetSetByName(r.workTable, set.HashedName()) + require.NoError(t, err, "fetch updated set") + elements, err := r.conn.GetSetElements(fetchedSet) + require.NoError(t, err, "get set elements") + + starts := make(map[string]bool) + for _, elem := range elements { + if elem.IntervalEnd { + continue + } + starts[netip.AddrFrom4(*(*[4]byte)(elem.Key)).String()] = true + } + // The /32s are covered by the /24, so the update adds one interval and + // leaves the one created earlier in place. + assert.Equal(t, map[string]bool{"10.0.0.0": true, "192.168.1.0": true}, starts, + "merged set must hold the original and the merged interval") +} + func TestNftablesCreateIpSet_IPv6(t *testing.T) { if check() != NFTABLES { t.Skip("nftables not supported on this system") @@ -518,11 +568,10 @@ func TestNftablesCreateIpSet_IPv6(t *testing.T) { require.NoError(t, err, "Failed to create v6 work table") defer deleteWorkTableIPv6() - r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU) - require.NoError(t, err, "Failed to create router") + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) require.NoError(t, r.init(workTable)) defer func() { - require.NoError(t, r.Reset(), "Failed to reset router") + require.NoError(t, r.Reset(), "Failed to reset family") }() tests := []struct { @@ -748,6 +797,14 @@ func containsPort(exprs []expr.Any, port *firewall.Port, isSource bool) bool { } } } + case *expr.Lookup: + // Multiple discrete ports compile to an anonymous set lookup + // rather than a chain of comparisons. The set's id and name are + // assigned dynamically, so matching the lookup is enough here; + // the set elements are verified separately. + if !port.IsRange && len(port.Values) > 1 { + portMatchFound = true + } } if payloadFound && portMatchFound { return true @@ -861,13 +918,12 @@ func TestRouter_RefreshRulesMap_RemovesStaleEntries(t *testing.T) { require.NoError(t, err) defer deleteWorkTable() - r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU) - require.NoError(t, err) + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) require.NoError(t, r.init(workTable)) defer func() { require.NoError(t, r.Reset()) }() // Add a real rule to the kernel - ruleKey, err := r.AddRouteFiltering( + ruleKey, err := r.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("192.168.1.0/24")}, firewall.Network{Prefix: netip.MustParsePrefix("10.0.0.0/24")}, @@ -878,11 +934,11 @@ func TestRouter_RefreshRulesMap_RemovesStaleEntries(t *testing.T) { ) require.NoError(t, err) t.Cleanup(func() { - require.NoError(t, r.DeleteRouteRule(ruleKey)) + require.NoError(t, r.DeleteFilterRule(ruleKey)) }) // Inject a stale entry with Handle=0 (simulates store-before-flush failure) - staleKey := "stale-rule-that-does-not-exist" + staleKey := firewall.RuleID("stale-rule-that-does-not-exist") r.rules[staleKey] = &nftables.Rule{ Table: r.workTable, Chain: r.chains[chainNameRoutingFw], @@ -902,6 +958,54 @@ func TestRouter_RefreshRulesMap_RemovesStaleEntries(t *testing.T) { assert.NotZero(t, realRule.Handle, "real rule should have a valid handle") } +// TestRouter_DeleteRouteRule_RemovesKernelRule verifies a route filter +// rule is actually removed from the kernel on delete. The route chain is +// not refreshed by Flush, so the stored rule carries a zero handle; +// DeleteFilterRule must pull live handles itself before issuing the +// delete or the kernel rule leaks. Regression test for that path. +func TestRouter_DeleteRouteRule_RemovesKernelRule(t *testing.T) { + if check() != NFTABLES { + t.Skip("nftables not supported on this system") + } + + workTable, err := createWorkTable() + require.NoError(t, err) + defer deleteWorkTable() + + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) + require.NoError(t, r.init(workTable)) + defer func() { require.NoError(t, r.Reset()) }() + + ruleKey, err := r.AddFilterRule( + nil, + []netip.Prefix{netip.MustParsePrefix("192.168.1.0/24")}, + firewall.Network{Prefix: netip.MustParsePrefix("10.0.0.0/24")}, + firewall.ProtocolTCP, + nil, + &firewall.Port{Values: []uint16{80}}, + firewall.ActionAccept, + ) + require.NoError(t, err) + + countKernelRules := func() int { + list, err := r.conn.GetRules(r.workTable, r.chains[chainNameRoutingFw]) + require.NoError(t, err) + n := 0 + for _, rule := range list { + if string(rule.UserData) == string(ruleKey.ID()) { + n++ + } + } + return n + } + + require.Equal(t, 1, countKernelRules(), "rule should be present in the kernel after add") + + require.NoError(t, r.DeleteFilterRule(ruleKey)) + assert.Equal(t, 0, countKernelRules(), "rule must be removed from the kernel after delete") + assert.NotContains(t, r.filters, ruleKey.ID(), "filters map entry should be cleared") +} + func TestRouter_DeleteRouteRule_StaleHandle(t *testing.T) { if check() != NFTABLES { t.Skip("nftables not supported on this system") @@ -911,24 +1015,27 @@ func TestRouter_DeleteRouteRule_StaleHandle(t *testing.T) { require.NoError(t, err) defer deleteWorkTable() - r, err := newRouter(workTable, ifaceMock, iface.DefaultMTU) - require.NoError(t, err) + r := newFamily(workTable, ifaceMock, iface.DefaultMTU) require.NoError(t, r.init(workTable)) defer func() { require.NoError(t, r.Reset()) }() // Inject a stale entry with Handle=0 - staleKey := "stale-route-rule" - r.rules[staleKey] = &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingFw], - Handle: 0, - UserData: []byte(staleKey), + staleKey := id.RuleID("stale-route-rule") + staleRule := &Rule{ + nftRule: &nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameRoutingFw], + Handle: 0, + UserData: []byte(staleKey), + }, + id: staleKey, } + r.filters[staleKey] = staleRule - // DeleteRouteRule should not return an error for stale handles - err = r.DeleteRouteRule(id.RuleID(staleKey)) + // DeleteFilterRule should not return an error for stale handles + err = r.DeleteFilterRule(staleRule) assert.NoError(t, err, "deleting a stale rule should not error") - assert.NotContains(t, r.rules, staleKey, "stale entry should be cleaned up") + assert.NotContains(t, r.filters, staleKey, "stale entry should be cleaned up") } func TestRouter_AddNatRule_WithStaleEntry(t *testing.T) { @@ -950,7 +1057,7 @@ func TestRouter_AddNatRule_WithStaleEntry(t *testing.T) { Masquerade: true, } - rtr := manager.router + rtr := manager.family4 // First add succeeds err = rtr.AddNatRule(pair) @@ -960,11 +1067,11 @@ func TestRouter_AddNatRule_WithStaleEntry(t *testing.T) { }) // Corrupt the handle to simulate stale state - natRuleKey := firewall.GenKey(firewall.PreroutingFormat, pair) + natRuleKey := pair.GenKey(firewall.PreroutingFormat) if rule, exists := rtr.rules[natRuleKey]; exists { rule.Handle = 0 } - inverseKey := firewall.GenKey(firewall.PreroutingFormat, firewall.GetInversePair(pair)) + inverseKey := firewall.GetInversePair(pair).GenKey(firewall.PreroutingFormat) if rule, exists := rtr.rules[inverseKey]; exists { rule.Handle = 0 } @@ -979,7 +1086,7 @@ func TestRouter_AddNatRule_WithStaleEntry(t *testing.T) { found := 0 for _, rule := range rules { - if len(rule.UserData) > 0 && string(rule.UserData) == natRuleKey { + if len(rule.UserData) > 0 && firewall.RuleID(rule.UserData) == natRuleKey { found++ } } @@ -1010,7 +1117,7 @@ func TestCalculateLastIP(t *testing.T) { } func TestConvertPrefixesToSet_IPv6(t *testing.T) { - r := &router{af: afIPv6} + r := &family{af: afIPv6} prefixes := []netip.Prefix{ netip.MustParsePrefix("fd00::/64"), netip.MustParsePrefix("2001:db8::1/128"), diff --git a/client/firewall/nftables/routing_linux.go b/client/firewall/nftables/routing_linux.go new file mode 100644 index 000000000..e98471e8f --- /dev/null +++ b/client/firewall/nftables/routing_linux.go @@ -0,0 +1,523 @@ +//go:build !android + +package nftables + +import ( + "fmt" + "strings" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" + + nberrors "github.com/netbirdio/netbird/client/errors" + firewall "github.com/netbirdio/netbird/client/firewall/manager" + nbnet "github.com/netbirdio/netbird/client/net" +) + +func (r *family) AddNatRule(pair firewall.RouterPair) error { + if err := r.refreshRulesMap(); err != nil { + return fmt.Errorf(refreshRulesMapError, err) + } + + // Resolve every rule's match expressions before queueing any of them: a + // message buffered on the shared connection cannot be un-queued, so + // returning an error after queueing would leave the next caller's Flush + // to commit a rule nothing tracks. + var legacyExprs []expr.Any + if r.legacyManagement { + log.Warnf("This peer is connected to a NetBird Management service with an older version. Allowing all traffic for %s", pair.Destination) + + var err error + legacyExprs, err = r.legacyRouteRuleExprs(pair) + if err != nil { + return fmt.Errorf("build legacy routing rule: %w", err) + } + } + + inverse := firewall.GetInversePair(pair) + var natExprs, inverseExprs []expr.Any + if pair.Masquerade { + var err error + natExprs, err = r.natRuleExprs(pair) + if err != nil { + r.dropNetworkMatch(legacyExprs) + return fmt.Errorf("build nat rule: %w", err) + } + + inverseExprs, err = r.natRuleExprs(inverse) + if err != nil { + r.dropNetworkMatch(legacyExprs) + r.dropNetworkMatch(natExprs) + return fmt.Errorf("build inverse nat rule: %w", err) + } + } + + if legacyExprs != nil { + r.queueLegacyRouteRule(pair, legacyExprs) + } + if pair.Masquerade { + r.queueNatRule(pair, natExprs) + r.queueNatRule(inverse, inverseExprs) + } + + if err := r.conn.Flush(); err != nil { + r.rollbackRules(pair) + return fmt.Errorf("insert rules for %s: %w", pair.Destination, err) + } + + return nil +} + +// rollbackRules cleans up unflushed rules and their set counters after a flush failure. +func (r *family) rollbackRules(pair firewall.RouterPair) { + keys := []firewall.RuleID{ + pair.GenKey(firewall.ForwardingFormat), + pair.GenKey(firewall.PreroutingFormat), + firewall.GetInversePair(pair).GenKey(firewall.PreroutingFormat), + } + for _, key := range keys { + rule, ok := r.rules[key] + if !ok { + continue + } + if err := r.decrementSetCounter(rule); err != nil { + log.Warnf("rollback set counter for %s: %v", key, err) + } + delete(r.rules, key) + } +} + +// natRuleExprs resolves the match expressions of the pair's prerouting +// marking rule. It reserves the ipset references the matches need but queues +// nothing on the connection, so its error paths leave the connection clean. +func (r *family) natRuleExprs(pair firewall.RouterPair) ([]expr.Any, error) { + sourceExp, err := r.applyNetwork(pair.Source, nil, true) + if err != nil { + return nil, fmt.Errorf("apply source: %w", err) + } + + destExp, err := r.applyNetwork(pair.Destination, nil, false) + if err != nil { + r.dropNetworkMatch(sourceExp) + return nil, fmt.Errorf("apply destination: %w", err) + } + + op := expr.CmpOpEq + if pair.Inverse { + op = expr.CmpOpNeq + } + + exprs := []expr.Any{ + &expr.Meta{ + Key: expr.MetaKeyIIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: op, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + } + // We only care about NEW connections to mark them and later identify them in the postrouting chain for masquerading. + // Masquerading will take care of the conntrack state, which means we won't need to mark established connections. + exprs = append(exprs, getCtNewExprs()...) + + exprs = append(exprs, sourceExp...) + exprs = append(exprs, destExp...) + + markValue := nbnet.PreroutingFwmarkMasquerade + if pair.Inverse { + markValue = nbnet.PreroutingFwmarkMasqueradeReturn + } + + exprs = append(exprs, + &expr.Immediate{ + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(markValue), + }, + &expr.Meta{ + Key: expr.MetaKeyMARK, + SourceRegister: true, + Register: 1, + }, + ) + + return exprs, nil +} + +// queueNatRule replaces any tracked rule for the pair and queues the new +// prerouting marking rule on the connection. Failures are logged rather than +// returned: the caller has already queued messages that only a Flush can +// commit, so it must not return early. +func (r *family) queueNatRule(pair firewall.RouterPair, exprs []expr.Any) { + ruleID := pair.GenKey(firewall.PreroutingFormat) + + if _, exists := r.rules[ruleID]; exists { + if err := r.removeNatRule(pair); err != nil { + // The rule this replaces may still be in the kernel. Keep tracking + // it and skip the new one: overwriting the entry would leave the old + // rule installed with nothing that can find it again, while keeping + // it lets the next update retry the whole replacement. + log.Errorf("replace prerouting rule %s: %v", ruleID, err) + r.dropNetworkMatch(exprs) + return + } + } + + // Ensure nat rules come first, so the mark can be overwritten. + // Currently overwritten by the dst-type LOCAL rules for redirected traffic. + r.rules[ruleID] = r.conn.InsertRule(&nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameManglePrerouting], + Exprs: exprs, + UserData: []byte(ruleID), + }) +} + +func (r *family) addPostroutingRules() { + // First masquerade rule for traffic coming in from WireGuard interface + exprs := []expr.Any{ + // Match on the first fwmark + &expr.Meta{ + Key: expr.MetaKeyMARK, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade), + }, + + // We need to exclude the loopback interface as this changes the wg proxy port + &expr.Meta{ + Key: expr.MetaKeyOIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: ifname("lo"), + }, + &expr.Counter{}, + &expr.Masq{}, + } + + r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameRoutingNat], + Exprs: exprs, + }) + + // Second masquerade rule for traffic going out through WireGuard interface + exprs2 := []expr.Any{ + // Match on the second fwmark + &expr.Meta{ + Key: expr.MetaKeyMARK, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasqueradeReturn), + }, + + // Match WireGuard interface + &expr.Meta{ + Key: expr.MetaKeyOIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Counter{}, + &expr.Masq{}, + } + + r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameRoutingNat], + Exprs: exprs2, + }) +} + +// addMSSClampingRules adds MSS clamping rules to prevent fragmentation for forwarded traffic. +func (r *family) addMSSClampingRules() error { + overhead := uint16(ipv4TCPHeaderSize) + if r.af.tableFamily == nftables.TableFamilyIPv6 { + overhead = ipv6TCPHeaderSize + } + if r.mtu <= overhead { + log.Debugf("MTU %d too small for MSS clamping (overhead %d), skipping", r.mtu, overhead) + return nil + } + mss := r.mtu - overhead + + exprsOut := []expr.Any{ + &expr.Meta{ + Key: expr.MetaKeyOIFNAME, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.wgIface.Name()), + }, + &expr.Meta{ + Key: expr.MetaKeyL4PROTO, + Register: 1, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.IPPROTO_TCP}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 13, + Len: 1, + }, + &expr.Bitwise{ + DestRegister: 1, + SourceRegister: 1, + Len: 1, + Mask: []byte{0x02}, + Xor: []byte{0x00}, + }, + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte{0x00}, + }, + &expr.Counter{}, + &expr.Exthdr{ + DestRegister: 1, + Type: 2, + Offset: 2, + Len: 2, + Op: expr.ExthdrOpTcpopt, + }, + &expr.Cmp{ + Op: expr.CmpOpGt, + Register: 1, + Data: binaryutil.BigEndian.PutUint16(uint16(mss)), + }, + &expr.Immediate{ + Register: 1, + Data: binaryutil.BigEndian.PutUint16(uint16(mss)), + }, + &expr.Exthdr{ + SourceRegister: 1, + Type: 2, + Offset: 2, + Len: 2, + Op: expr.ExthdrOpTcpopt, + }, + } + + r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameMangleForward], + Exprs: exprsOut, + }) + + return r.conn.Flush() +} + +func buildLegacyRouteRuleExpressions(sourceExp, destExp []expr.Any) []expr.Any { + exprs := make([]expr.Any, 0, len(sourceExp)+len(destExp)+2) + exprs = append(exprs, sourceExp...) + exprs = append(exprs, destExp...) + exprs = append(exprs, + &expr.Counter{}, + &expr.Verdict{Kind: expr.VerdictAccept}, + ) + return exprs +} + +// legacyRouteRuleExprs resolves the match expressions of the pair's legacy +// forwarding rule, queueing nothing on the connection. +func (r *family) legacyRouteRuleExprs(pair firewall.RouterPair) ([]expr.Any, error) { + sourceExp, err := r.applyNetwork(pair.Source, nil, true) + if err != nil { + return nil, fmt.Errorf("apply source: %w", err) + } + + destExp, err := r.applyNetwork(pair.Destination, nil, false) + if err != nil { + r.dropNetworkMatch(sourceExp) + return nil, fmt.Errorf("apply destination: %w", err) + } + + return buildLegacyRouteRuleExpressions(sourceExp, destExp), nil +} + +// queueLegacyRouteRule replaces any tracked rule for the pair and queues the +// new legacy forwarding rule. Failures are logged for the same reason as in +// queueNatRule. +func (r *family) queueLegacyRouteRule(pair firewall.RouterPair, exprs []expr.Any) { + ruleID := pair.GenKey(firewall.ForwardingFormat) + + if _, exists := r.rules[ruleID]; exists { + if err := r.removeLegacyRouteRule(pair); err != nil { + // Keep the old rule tracked instead of losing it, as in queueNatRule. + log.Errorf("replace legacy forwarding rule %s: %v", ruleID, err) + r.dropNetworkMatch(exprs) + return + } + } + + r.rules[ruleID] = r.conn.AddRule(&nftables.Rule{ + Table: r.workTable, + Chain: r.chains[chainNameRoutingFw], + Exprs: exprs, + UserData: []byte(ruleID), + }) +} + +// removeLegacyRouteRule removes a legacy routing rule for mgmt servers pre route acls +func (r *family) removeLegacyRouteRule(pair firewall.RouterPair) error { + ruleID := pair.GenKey(firewall.ForwardingFormat) + + rule, exists := r.rules[ruleID] + if !exists { + return nil + } + + return r.deleteLegacyRuleEntry(ruleID, rule) +} + +// deleteLegacyRuleEntry removes one legacy forwarding rule and drops its +// ipset references. It also clears stale entries that never got a handle. +func (r *family) deleteLegacyRuleEntry(ruleID firewall.RuleID, rule *nftables.Rule) error { + if rule.Handle == 0 { + log.Warnf("legacy forwarding rule %s has no handle, removing stale entry", ruleID) + if err := r.decrementSetCounter(rule); err != nil { + log.Warnf("decrement set counter for stale rule %s: %v", ruleID, err) + } + delete(r.rules, ruleID) + return nil + } + + if err := r.conn.DelRule(rule); err != nil { + return fmt.Errorf("remove legacy forwarding rule %s: %w", ruleID, err) + } + + delete(r.rules, ruleID) + + if err := r.decrementSetCounter(rule); err != nil { + return fmt.Errorf("decrement set counter: %w", err) + } + + return nil +} + +// GetLegacyManagement returns the route manager's legacy management mode +func (r *family) GetLegacyManagement() bool { + return r.legacyManagement +} + +// SetLegacyManagement sets the route manager to use legacy management mode +func (r *family) SetLegacyManagement(isLegacy bool) { + r.legacyManagement = isLegacy +} + +// RemoveAllLegacyRouteRules removes all legacy routing rules for mgmt servers pre route acls +func (r *family) RemoveAllLegacyRouteRules() error { + if err := r.refreshRulesMap(); err != nil { + return fmt.Errorf(refreshRulesMapError, err) + } + + var merr *multierror.Error + var found bool + for k, rule := range r.rules { + if !strings.HasPrefix(string(k), firewall.ForwardingFormatPrefix) { + continue + } + found = true + if err := r.deleteLegacyRuleEntry(k, rule); err != nil { + merr = multierror.Append(merr, err) + } + } + + // Commit the queued deletes here instead of leaving them for whichever + // caller flushes next: the tracking entries are already gone, so an + // uncommitted delete would leave a rule in the kernel that nothing can + // find again. + if found { + if err := r.conn.Flush(); err != nil { + merr = multierror.Append(merr, fmt.Errorf(flushError, err)) + } + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) RemoveNatRule(pair firewall.RouterPair) error { + if err := r.refreshRulesMap(); err != nil { + return fmt.Errorf(refreshRulesMapError, err) + } + + var merr *multierror.Error + + if pair.Masquerade { + if err := r.removeNatRule(pair); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove prerouting rule: %w", err)) + } + + if err := r.removeNatRule(firewall.GetInversePair(pair)); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove inverse prerouting rule: %w", err)) + } + } + + if err := r.removeLegacyRouteRule(pair); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove legacy routing rule: %w", err)) + } + + // Set counters are decremented in the sub-methods above before flush. If flush fails, + // counters will be off until the next successful removal or refresh cycle. + if err := r.conn.Flush(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("flush remove nat rules %s: %w", pair.Destination, err)) + } + + return nberrors.FormatErrorOrNil(merr) +} + +func (r *family) removeNatRule(pair firewall.RouterPair) error { + ruleID := pair.GenKey(firewall.PreroutingFormat) + + rule, exists := r.rules[ruleID] + if !exists { + log.Debugf("prerouting rule %s not found", ruleID) + return nil + } + + if rule.Handle == 0 { + log.Warnf("prerouting rule %s has no handle, removing stale entry", ruleID) + if err := r.decrementSetCounter(rule); err != nil { + log.Warnf("decrement set counter for stale rule %s: %v", ruleID, err) + } + delete(r.rules, ruleID) + return nil + } + + if err := r.conn.DelRule(rule); err != nil { + return fmt.Errorf("remove prerouting rule %s -> %s: %w", pair.Source, pair.Destination, err) + } + + log.Debugf("removed prerouting rule %s -> %s", pair.Source, pair.Destination) + + delete(r.rules, ruleID) + + if err := r.decrementSetCounter(rule); err != nil { + return fmt.Errorf("decrement set counter: %w", err) + } + + return nil +} diff --git a/client/firewall/nftables/rule_linux.go b/client/firewall/nftables/rule_linux.go index a90b74e36..8f3c0aebc 100644 --- a/client/firewall/nftables/rule_linux.go +++ b/client/firewall/nftables/rule_linux.go @@ -1,21 +1,26 @@ package nftables import ( - "net" + "net/netip" "github.com/google/nftables" + + "github.com/netbirdio/netbird/client/firewall/manager" ) -// Rule to handle management of rules +// Rule wraps an installed filter rule (peer or route). Source set +// membership is encoded in the rule's expressions; DeleteFilterRule +// recovers the set name via findSets so the refcounter can drop the +// right reference. mangleRule is set only for peer rules. type Rule struct { nftRule *nftables.Rule mangleRule *nftables.Rule - nftSet *nftables.Set - ruleID string - ip net.IP + // sources is the canonical source list this rule was created for. + sources []netip.Prefix + id manager.RuleID } -// GetRuleID returns the rule id -func (r *Rule) ID() string { - return r.ruleID +// ID returns the rule id +func (r *Rule) ID() manager.RuleID { + return r.id } diff --git a/client/firewall/nftables/testhelpers_linux_test.go b/client/firewall/nftables/testhelpers_linux_test.go new file mode 100644 index 000000000..72db3f7d2 --- /dev/null +++ b/client/firewall/nftables/testhelpers_linux_test.go @@ -0,0 +1,27 @@ +//go:build privileged + +package nftables + +import ( + "fmt" + "net" + "net/netip" +) + +func pfx(ip net.IP) []netip.Prefix { + if ip == nil { + return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + } + if ip.IsUnspecified() { + if ip.To4() != nil { + return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + } + return []netip.Prefix{netip.PrefixFrom(netip.IPv6Unspecified(), 0)} + } + a, ok := netip.AddrFromSlice(ip) + if !ok { + panic(fmt.Sprintf("invalid IP length: %d", len(ip))) + } + a = a.Unmap() + return []netip.Prefix{netip.PrefixFrom(a, a.BitLen())} +} diff --git a/client/firewall/uspfilter/allow_netbird.go b/client/firewall/uspfilter/allow_netbird.go deleted file mode 100644 index b120cdf12..000000000 --- a/client/firewall/uspfilter/allow_netbird.go +++ /dev/null @@ -1,37 +0,0 @@ -//go:build !windows - -package uspfilter - -import ( - log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/client/firewall/firewalld" - "github.com/netbirdio/netbird/client/internal/statemanager" -) - -// Close cleans up the firewall manager by removing all rules and closing trackers -func (m *Manager) Close(stateManager *statemanager.Manager) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - m.resetState() - - if m.nativeFirewall != nil { - return m.nativeFirewall.Close(stateManager) - } - if err := firewalld.UntrustInterface(m.wgIface.Name()); err != nil { - log.Warnf("failed to untrust interface in firewalld: %v", err) - } - return nil -} - -// AllowNetbird allows netbird interface traffic -func (m *Manager) AllowNetbird() error { - if m.nativeFirewall != nil { - return m.nativeFirewall.AllowNetbird() - } - if err := firewalld.TrustInterface(m.wgIface.Name()); err != nil { - log.Warnf("failed to trust interface in firewalld: %v", err) - } - return nil -} diff --git a/client/firewall/uspfilter/common/iface.go b/client/firewall/uspfilter/common/iface.go deleted file mode 100644 index 9c06eb3f7..000000000 --- a/client/firewall/uspfilter/common/iface.go +++ /dev/null @@ -1,17 +0,0 @@ -package common - -import ( - wgdevice "golang.zx2c4.com/wireguard/device" - - "github.com/netbirdio/netbird/client/iface/device" - "github.com/netbirdio/netbird/client/iface/wgaddr" -) - -// IFaceMapper defines subset methods of interface required for manager -type IFaceMapper interface { - Name() string - SetFilter(device.PacketFilter) error - Address() wgaddr.Address - GetWGDevice() *wgdevice.Device - GetDevice() *device.FilteredDevice -} diff --git a/client/firewall/uspfilter/filter.go b/client/firewall/uspfilter/filter.go index 91866dcab..0c73400f1 100644 --- a/client/firewall/uspfilter/filter.go +++ b/client/firewall/uspfilter/filter.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "errors" "fmt" - "net" "net/netip" "os" "slices" @@ -20,14 +19,18 @@ import ( "github.com/google/uuid" "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" + wgdevice "golang.zx2c4.com/wireguard/device" nberrors "github.com/netbirdio/netbird/client/errors" + "github.com/netbirdio/netbird/client/firewall/firewalld" firewall "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/firewall/uspfilter/common" "github.com/netbirdio/netbird/client/firewall/uspfilter/conntrack" "github.com/netbirdio/netbird/client/firewall/uspfilter/forwarder" nblog "github.com/netbirdio/netbird/client/firewall/uspfilter/log" + "github.com/netbirdio/netbird/client/iface/device" "github.com/netbirdio/netbird/client/iface/netstack" + "github.com/netbirdio/netbird/client/iface/wgaddr" nbid "github.com/netbirdio/netbird/client/internal/acl/id" nftypes "github.com/netbirdio/netbird/client/internal/netflow/types" "github.com/netbirdio/netbird/client/internal/statemanager" @@ -58,7 +61,10 @@ const ( // EnvDisableMSSClamping disables TCP MSS clamping for forwarded traffic. EnvDisableMSSClamping = "NB_DISABLE_MSS_CLAMPING" - // EnvForceUserspaceRouter forces userspace routing even if native routing is available. + // EnvForceUserspaceRouter is a deprecated alias for + // NB_FORCE_USERSPACE_FIREWALL: the userspace firewall always routes in + // userspace, so forcing one forces the other. Kept for backward + // compatibility. EnvForceUserspaceRouter = "NB_FORCE_USERSPACE_ROUTER" // EnvEnableLocalForwarding enables forwarding of local traffic to the native stack for internal (non-NetBird) interfaces. @@ -70,14 +76,20 @@ const ( EnvEnableNetstackLocalForwarding = "NB_ENABLE_NETSTACK_LOCAL_FORWARDING" ) -var errNatNotSupported = errors.New("nat not supported with userspace firewall") +// errNotSupported is returned by firewall operations that only make sense with +// a kernel firewall (kernel NAT/DNAT, eBPF) and are not implemented in +// userspace mode, where they should not be called. +var errNotSupported = errors.New("not supported with userspace firewall") -// RuleSet is a set of rules grouped by a string key -type RuleSet map[string]PeerRule +// peerRules is the canonical list-based storage for peer ACL rules. +// Drop and accept rules live in separate slices; drop-before-accept +// ordering comes from consulting the deny slice (and its index) before +// the accept one. +type peerRules []*PeerRule -type RouteRules []*RouteRule +type routeRules []*RouteRule -func (r RouteRules) Sort() { +func (r routeRules) Sort() { slices.SortStableFunc(r, func(a, b *RouteRule) int { // Deny rules come first if a.action == firewall.ActionDrop && b.action != firewall.ActionDrop { @@ -86,22 +98,74 @@ func (r RouteRules) Sort() { if a.action != firewall.ActionDrop && b.action == firewall.ActionDrop { return 1 } - return strings.Compare(a.id, b.id) + return strings.Compare(string(a.id), string(b.id)) }) } +// peerRuleSpec carries the parameters that define a peer filter rule, +// threaded together through the build path so the builders take a single +// argument instead of a long parameter list. +type peerRuleSpec struct { + mgmtID []byte + sources []netip.Prefix + ipLayer gopacket.LayerType + proto firewall.Protocol + sPort *firewall.Port + dPort *firewall.Port + action firewall.Action +} + +// Iface is the network interface the userspace firewall attaches to: the +// methods of the WireGuard device it actually uses. +type Iface interface { + Name() string + Address() wgaddr.Address + SetFilter(device.PacketFilter) error + GetWGDevice() *wgdevice.Device +} + +// InterfaceAllower opens the NetBird interface in the host firewall so it +// doesn't drop traffic the userspace firewall handles, without taking over +// packet filtering. Implementations (nftables, iptables, firewalld, the windows +// netsh rule) are selected per platform and injected into Create; Apply runs at +// creation and Close on teardown. +type InterfaceAllower interface { + Apply() error + Close() error +} + +// Config holds the dependencies and options for the userspace firewall. +type Config struct { + // IFace is the overlay interface the filter attaches to. + IFace Iface + // InterfaceAllower opens the NetBird interface in foreign kernel filter + // chains so the kernel doesn't drop traffic the userspace firewall handles. + // Nil in netstack mode, on non-Linux platforms without a backend, or when + // neither nftables nor iptables is available. firewalld trust is applied by + // the manager regardless, since firewalld owns its own chains and we cannot + // insert into them. + InterfaceAllower InterfaceAllower + // DisableServerRoutes indicates whether server routes are disabled. + DisableServerRoutes bool + FlowLogger nftypes.FlowLogger + MTU uint16 +} + // Manager userspace firewall manager type Manager struct { - outgoingRules map[netip.Addr]RuleSet - incomingDenyRules map[netip.Addr]RuleSet - incomingRules map[netip.Addr]RuleSet - routeRules RouteRules - routeRulesMap map[nbid.RuleID]*RouteRule - decoders sync.Pool - wgIface common.IFaceMapper - nativeFirewall firewall.Manager + decoders sync.Pool + wgIface Iface + ifaceAllower InterfaceAllower + mutex sync.RWMutex - mutex sync.RWMutex + incomingDenyRules peerRules + incomingAcceptRules peerRules + incomingDenyIndex peerRuleIndex + incomingAcceptIndex peerRuleIndex + peerRulesMap map[nbid.RuleID]*PeerRule + + routeRules routeRules + routeRulesMap map[nbid.RuleID]*RouteRule // indicates whether server routes are disabled disableServerRoutes bool @@ -121,6 +185,7 @@ type Manager struct { udpTracker *conntrack.UDPTracker icmpTracker *conntrack.ICMPTracker tcpTracker *conntrack.TCPTracker + fragments *fragmentTracker forwarder atomic.Pointer[forwarder.Forwarder] pendingCapture atomic.Pointer[forwarder.PacketCapture] logger *nblog.Logger @@ -183,22 +248,39 @@ func (d *decoder) decodePacket(data []byte) error { } } -// Create userspace firewall manager constructor -func Create(iface common.IFaceMapper, disableServerRoutes bool, flowLogger nftypes.FlowLogger, mtu uint16) (*Manager, error) { - return create(iface, nil, disableServerRoutes, flowLogger, mtu) -} - -func CreateWithNativeFirewall(iface common.IFaceMapper, nativeFirewall firewall.Manager, disableServerRoutes bool, flowLogger nftypes.FlowLogger, mtu uint16) (*Manager, error) { - if nativeFirewall == nil { - return nil, errors.New("native firewall is nil") +// decodeTransport decodes the transport header of a first fragment (which +// gopacket leaves undecoded) into the decoder and appends its layer type to +// decoded, so the ACL pipeline can evaluate it like a normal packet. It returns +// false if the protocol is unsupported or the header is truncated. +func (d *decoder) decodeTransport(proto layers.IPProtocol, payload []byte) bool { + var l4 gopacket.DecodingLayer + var layerType gopacket.LayerType + var minLen int + switch proto { + case layers.IPProtocolTCP: + l4, layerType, minLen = &d.tcp, layers.LayerTypeTCP, 20 + case layers.IPProtocolUDP: + l4, layerType, minLen = &d.udp, layers.LayerTypeUDP, 8 + case layers.IPProtocolICMPv4: + l4, layerType, minLen = &d.icmp4, layers.LayerTypeICMPv4, 8 + case layers.IPProtocolICMPv6: + l4, layerType, minLen = &d.icmp6, layers.LayerTypeICMPv6, 8 + default: + return false } - mgr, err := create(iface, nativeFirewall, disableServerRoutes, flowLogger, mtu) - if err != nil { - return nil, err + // Reject a fragment too small to hold the full transport header before + // decoding: it can't be ACL-evaluated (tiny-fragment attack), and skipping + // the decode avoids gopacket allocating an error on the drop path. + if len(payload) < minLen { + return false } - return mgr, nil + if err := l4.DecodeFromBytes(payload, gopacket.NilDecodeFeedback); err != nil { + return false + } + d.decoded = append(d.decoded, layerType) + return true } func parseCreateEnv() (bool, bool, bool) { @@ -231,7 +313,7 @@ func parseCreateEnv() (bool, bool, bool) { return disableConntrack, enableLocalForwarding, disableMSSClamping } -func create(iface common.IFaceMapper, nativeFirewall firewall.Manager, disableServerRoutes bool, flowLogger nftypes.FlowLogger, mtu uint16) (*Manager, error) { +func Create(cfg Config) (_ *Manager, err error) { disableConntrack, enableLocalForwarding, disableMSSClamping := parseCreateEnv() m := &Manager{ @@ -254,62 +336,133 @@ func create(iface common.IFaceMapper, nativeFirewall firewall.Manager, disableSe return d }, }, - nativeFirewall: nativeFirewall, - outgoingRules: make(map[netip.Addr]RuleSet), - incomingDenyRules: make(map[netip.Addr]RuleSet), - incomingRules: make(map[netip.Addr]RuleSet), - wgIface: iface, + wgIface: cfg.IFace, + ifaceAllower: cfg.InterfaceAllower, localipmanager: newLocalIPManager(), - disableServerRoutes: disableServerRoutes, + disableServerRoutes: cfg.DisableServerRoutes, stateful: !disableConntrack, logger: nblog.NewFromLogrus(log.StandardLogger()), - flowLogger: flowLogger, + flowLogger: cfg.FlowLogger, netstack: netstack.IsEnabled(), localForwarding: enableLocalForwarding, + peerRulesMap: make(map[nbid.RuleID]*PeerRule), routeRulesMap: make(map[nbid.RuleID]*RouteRule), dnatMappings: make(map[netip.Addr]netip.Addr), portDNATRules: []portDNATRule{}, netstackServices: make(map[serviceKey]struct{}), - mtu: mtu, + mtu: cfg.MTU, } m.routingEnabled.Store(false) + // Release the allower (and its monitor) if setup fails after it was wired in. + defer func() { + if err != nil { + m.closeAllowerOnError() + } + }() + if !disableMSSClamping { - m.mssClampEnabled = true - if mtu > ipv4TCPHeaderMinSize { - m.mssClampValueIPv4 = mtu - ipv4TCPHeaderMinSize - } - if mtu > ipv6TCPHeaderMinSize { - m.mssClampValueIPv6 = mtu - ipv6TCPHeaderMinSize - } + m.enableMSSClamping(cfg.MTU) } - if err := m.localipmanager.UpdateLocalIPs(iface); err != nil { + if err := m.localipmanager.UpdateLocalIPs(cfg.IFace); err != nil { return nil, fmt.Errorf("update local IPs: %w", err) } - if disableConntrack { - log.Info("conntrack is disabled") - } else { - m.udpTracker = conntrack.NewUDPTracker(conntrack.DefaultUDPTimeout, m.logger, flowLogger) - m.icmpTracker = conntrack.NewICMPTracker(conntrack.DefaultICMPTimeout, m.logger, flowLogger) - m.tcpTracker = conntrack.NewTCPTracker(conntrack.DefaultTCPTimeout, m.logger, flowLogger) - } + m.fragments = newFragmentTracker(m.logger) + m.setupConntrack(disableConntrack) if m.netstack && m.localForwarding { if err := m.initForwarder(); err != nil { log.Errorf("failed to initialize forwarder: %v", err) } } - if err := iface.SetFilter(m); err != nil { + if err := cfg.IFace.SetFilter(m); err != nil { + m.fragments.Close() return nil, fmt.Errorf("set filter: %w", err) } + + m.openHostFirewall(cfg.IFace.Name()) + return m, nil } +// closeAllowerOnError releases the allower (and its monitor) when Create fails +// after the allower was wired in. +func (m *Manager) closeAllowerOnError() { + if m.ifaceAllower == nil { + return + } + if err := m.ifaceAllower.Close(); err != nil { + log.Warnf("close interface allower after failed firewall setup: %v", err) + } +} + +// enableMSSClamping enables MSS clamping and computes the per-family clamp values. +func (m *Manager) enableMSSClamping(mtu uint16) { + m.mssClampEnabled = true + if mtu > ipv4TCPHeaderMinSize { + m.mssClampValueIPv4 = mtu - ipv4TCPHeaderMinSize + } + if mtu > ipv6TCPHeaderMinSize { + m.mssClampValueIPv6 = mtu - ipv6TCPHeaderMinSize + } +} + +// setupConntrack initializes the stateful trackers unless conntrack is disabled. +func (m *Manager) setupConntrack(disabled bool) { + if disabled { + log.Info("conntrack is disabled") + return + } + m.udpTracker = conntrack.NewUDPTracker(conntrack.DefaultUDPTimeout, m.logger, m.flowLogger) + m.icmpTracker = conntrack.NewICMPTracker(conntrack.DefaultICMPTimeout, m.logger, m.flowLogger) + m.tcpTracker = conntrack.NewTCPTracker(conntrack.DefaultTCPTimeout, m.logger, m.flowLogger) +} + +// openHostFirewall opens the NetBird interface in the kernel firewall so it +// doesn't drop traffic the userspace firewall handles. Best-effort: failures +// here shouldn't prevent the firewall from coming up. +func (m *Manager) openHostFirewall(ifaceName string) { + if m.ifaceAllower != nil { + if err := m.ifaceAllower.Apply(); err != nil { + log.Errorf("failed to allow netbird interface traffic: %v", err) + } + } + // firewalld owns its own chains we can't insert into, so trust the interface + // there in addition to the allower. Netstack has no kernel interface. + if !m.netstack { + if err := firewalld.TrustInterface(ifaceName); err != nil { + log.Warnf("failed to trust interface in firewalld: %v", err) + } + } +} + +// Close cleans up the firewall manager: removes rules, closes trackers, and +// closes the interface allower. +func (m *Manager) Close(*statemanager.Manager) error { + m.mutex.Lock() + defer m.mutex.Unlock() + + m.resetState() + + var merr *multierror.Error + if m.ifaceAllower != nil { + if err := m.ifaceAllower.Close(); err != nil { + merr = multierror.Append(merr, fmt.Errorf("close interface allower: %w", err)) + } + } + if !m.netstack { + if err := firewalld.UntrustInterface(m.wgIface.Name()); err != nil { + merr = multierror.Append(merr, fmt.Errorf("untrust interface in firewalld: %w", err)) + } + } + return nberrors.FormatErrorOrNil(merr) +} + // blockInvalidRouted installs drop rules for traffic to the wg overlay that // arrives via the routing path. v4 and v6 are independent: a v6 install // failure leaves v4 protection in place (and vice versa) so the returned // slice always contains whatever was successfully installed, even on error. // Callers must persist the slice so DisableRouting can clean partial state. -func (m *Manager) blockInvalidRouted(iface common.IFaceMapper) ([]firewall.Rule, error) { +func (m *Manager) blockInvalidRouted(iface Iface) ([]firewall.Rule, error) { wgPrefix := iface.Address().Network log.Debugf("blocking invalid routed traffic for %s", wgPrefix) @@ -320,7 +473,7 @@ func (m *Manager) blockInvalidRouted(iface common.IFaceMapper) ([]firewall.Rule, } var rules []firewall.Rule - v4Rule, err := m.addRouteFiltering( + v4Rule, err := m.addRouteRule( nil, sources, firewall.Network{Prefix: wgPrefix}, @@ -336,7 +489,7 @@ func (m *Manager) blockInvalidRouted(iface common.IFaceMapper) ([]firewall.Rule, if v6Net.IsValid() { log.Debugf("blocking invalid routed traffic for %s", v6Net) - v6Rule, err := m.addRouteFiltering( + v6Rule, err := m.addRouteRule( nil, sources, firewall.Network{Prefix: v6Net}, @@ -357,20 +510,14 @@ func (m *Manager) blockInvalidRouted(iface common.IFaceMapper) ([]firewall.Rule, } func (m *Manager) determineRouting() error { - var disableUspRouting, forceUserspaceRouter bool - var err error + var disableUspRouting bool if val := os.Getenv(EnvDisableUserspaceRouting); val != "" { + var err error disableUspRouting, err = strconv.ParseBool(val) if err != nil { log.Warnf("failed to parse %s: %v", EnvDisableUserspaceRouting, err) } } - if val := os.Getenv(EnvForceUserspaceRouter); val != "" { - forceUserspaceRouter, err = strconv.ParseBool(val) - if err != nil { - log.Warnf("failed to parse %s: %v", EnvForceUserspaceRouter, err) - } - } switch { case disableUspRouting: @@ -385,26 +532,11 @@ func (m *Manager) determineRouting() error { log.Info("server routes are disabled") - case forceUserspaceRouter: - m.routingEnabled.Store(true) - m.nativeRouter.Store(false) - - log.Info("userspace routing is forced") - - case !m.netstack && m.nativeFirewall != nil: - // if the OS supports routing natively, then we don't need to filter/route ourselves - // netstack mode won't support native routing as there is no interface - - m.routingEnabled.Store(true) - m.nativeRouter.Store(true) - - log.Info("native routing is enabled") - default: m.routingEnabled.Store(true) m.nativeRouter.Store(false) - log.Info("userspace routing enabled by default") + log.Info("userspace routing enabled") } if m.routingEnabled.Load() && !m.nativeRouter.Load() { @@ -470,96 +602,118 @@ func (m *Manager) IsStateful() bool { return m.stateful } -func (m *Manager) AddNatRule(pair firewall.RouterPair) error { - if m.nativeRouter.Load() && m.nativeFirewall != nil { - return m.nativeFirewall.AddNatRule(pair) - } - +func (m *Manager) AddNatRule(firewall.RouterPair) error { // userspace routed packets are always SNATed to the inbound direction // TODO: implement outbound SNAT return nil } // RemoveNatRule removes a routing firewall rule -func (m *Manager) RemoveNatRule(pair firewall.RouterPair) error { - if m.nativeRouter.Load() && m.nativeFirewall != nil { - return m.nativeFirewall.RemoveNatRule(pair) - } +func (m *Manager) RemoveNatRule(firewall.RouterPair) error { return nil } -// AddPeerFiltering rule to the firewall -// -// If comment argument is empty firewall manager should set -// rule ID as comment for the rule -func (m *Manager) AddPeerFiltering( +// addPeerRule installs an input-chain rule that matches packets +// by source only. Called from AddFilterRule when the caller doesn't +// specify a destination. Sources are expected to share one address +// family; the family selects the ipLayer so the ICMP variant matches +// what the decoder produces. +func (m *Manager) addPeerRule( id []byte, - ip net.IP, + sources []netip.Prefix, proto firewall.Protocol, sPort *firewall.Port, dPort *firewall.Port, action firewall.Action, - _ string, -) ([]firewall.Rule, error) { - // TODO: fix in upper layers - i, ok := netip.AddrFromSlice(ip) - if !ok { - return nil, fmt.Errorf("invalid IP: %s", ip) - } - - i = i.Unmap() - r := PeerRule{ - id: uuid.New().String(), - mgmtId: id, - ip: i, - ipLayer: layers.LayerTypeIPv6, - matchByIP: true, - drop: action == firewall.ActionDrop, - } - if i.Is4() { - r.ipLayer = layers.LayerTypeIPv4 - } - - if s := r.ip.String(); s == "0.0.0.0" || s == "::" { - r.matchByIP = false - } - - r.sPort = sPort - r.dPort = dPort - - r.protoLayer = protoToLayer(proto, r.ipLayer) - - m.mutex.Lock() - var targetMap map[netip.Addr]RuleSet - if r.drop { - targetMap = m.incomingDenyRules - } else { - targetMap = m.incomingRules - } - - if _, ok := targetMap[r.ip]; !ok { - targetMap[r.ip] = make(RuleSet) - } - targetMap[r.ip][r.id] = r - m.mutex.Unlock() - return []firewall.Rule{&r}, nil -} - -func (m *Manager) AddRouteFiltering( - id []byte, - sources []netip.Prefix, - destination firewall.Network, - proto firewall.Protocol, - sPort, dPort *firewall.Port, - action firewall.Action, ) (firewall.Rule, error) { m.mutex.Lock() defer m.mutex.Unlock() - return m.addRouteFiltering(id, sources, destination, proto, sPort, dPort, action) + // Sources are a single family; normalize v4-mapped prefixes to plain + // v4 and pick the matching IP layer. A /0 source matches any address + // of its own family only, mirroring the kernel backends. + normalized := make([]netip.Prefix, len(sources)) + ipLayer := layers.LayerTypeIPv4 + for i, p := range sources { + normalized[i] = firewall.UnmapPrefix(p) + if normalized[i].Addr().Is6() { + ipLayer = layers.LayerTypeIPv6 + } + } + spec := peerRuleSpec{ + mgmtID: id, + sources: normalized, + ipLayer: ipLayer, + proto: proto, + sPort: sPort, + dPort: dPort, + action: action, + } + return m.addOnePeerRule(spec), nil } -func (m *Manager) addRouteFiltering( +// addOnePeerRule builds and registers a single-family peer rule, or +// returns the existing rule when one with the same content key is +// already installed. The caller must hold m.mutex. The content key is +// the shared GenerateRuleID with an empty destination, so peer rules +// dedup the same way route rules and the kernel backends do; it is +// order-independent, so callers passing the same sources in any order +// dedup to one rule. +// +// There is no refcount: a content key is installed once and deleted on +// the first DeleteFilterRule for that key. The caller must therefore +// key its own tracking by the returned rule id so add and delete stay +// balanced per content key; the acl manager does this via +// peerRulesPairs. +func (m *Manager) addOnePeerRule(spec peerRuleSpec) *PeerRule { + ruleID := nbid.GenerateRuleID(spec.sources, firewall.Network{}, spec.proto, spec.sPort, spec.dPort, spec.action) + if existing, ok := m.peerRulesMap[ruleID]; ok { + return existing + } + + rule := m.buildPeerRule(ruleID, spec) + m.registerPeerRule(rule) + return rule +} + +func (m *Manager) buildPeerRule(ruleID nbid.RuleID, spec peerRuleSpec) *PeerRule { + r := &PeerRule{ + id: ruleID, + mgmtId: spec.mgmtID, + sources: spec.sources, + action: spec.action, + srcPort: spec.sPort, + dstPort: spec.dPort, + } + r.sourceAddrs = make(map[netip.Addr]struct{}, len(spec.sources)) + for _, p := range spec.sources { + if p.Bits() == p.Addr().BitLen() { + r.sourceAddrs[p.Addr()] = struct{}{} + } + } + r.protoLayer = protoToLayer(spec.proto, spec.ipLayer) + return r +} + +// registerPeerRule records a freshly built peer rule in the matching +// slice, index, and dedup map. The caller must hold m.mutex. +func (m *Manager) registerPeerRule(r *PeerRule) { + if r.action == firewall.ActionDrop { + m.incomingDenyRules = append(m.incomingDenyRules, r) + m.incomingDenyIndex.add(r) + } else { + m.incomingAcceptRules = append(m.incomingAcceptRules, r) + m.incomingAcceptIndex.add(r) + } + m.peerRulesMap[r.id] = r +} + +// AddFilterRule is the unified entry point for both peer (input chain) +// and route (forward chain) filtering rules. The destination +// distinguishes the two semantics: a zero Network installs an +// input-side rule that matches by source only; a set Network installs +// a forward-side rule that also matches the destination. +func (m *Manager) AddFilterRule( id []byte, sources []netip.Prefix, destination firewall.Network, @@ -567,19 +721,49 @@ func (m *Manager) addRouteFiltering( sPort, dPort *firewall.Port, action firewall.Action, ) (firewall.Rule, error) { - if m.nativeRouter.Load() && m.nativeFirewall != nil { - return m.nativeFirewall.AddRouteFiltering(id, sources, destination, proto, sPort, dPort, action) + if len(sources) == 0 { + return nil, firewall.ErrNoSources } - ruleKey := nbid.GenerateRouteRuleKey(sources, destination, proto, sPort, dPort, action) + if destination.IsZero() { + return m.addPeerRule(id, sources, proto, sPort, dPort, action) + } - if existingRule, ok := m.routeRulesMap[ruleKey]; ok { + m.mutex.Lock() + defer m.mutex.Unlock() + return m.addRouteRule(id, sources, destination, proto, sPort, dPort, action) +} + +// DeleteFilterRule deletes a filtering rule. The rule's underlying type +// is used to route to the correct internal path. +func (m *Manager) DeleteFilterRule(rule firewall.Rule) error { + m.mutex.Lock() + defer m.mutex.Unlock() + + if r, ok := rule.(*PeerRule); ok { + return m.deletePeerRuleLocked(r) + } + + // Anything else is a route rule (matched on the forward path). + return m.deleteRouteRule(rule) +} + +func (m *Manager) addRouteRule( + id []byte, + sources []netip.Prefix, + destination firewall.Network, + proto firewall.Protocol, + sPort, dPort *firewall.Port, + action firewall.Action, +) (firewall.Rule, error) { + ruleID := nbid.GenerateRuleID(sources, destination, proto, sPort, dPort, action) + + if existingRule, ok := m.routeRulesMap[ruleID]; ok { return existingRule, nil } rule := RouteRule{ - // TODO: consolidate these IDs - id: string(ruleKey), + id: ruleID, mgmtId: id, sources: sources, dstSet: destination.Set, @@ -594,78 +778,58 @@ func (m *Manager) addRouteFiltering( m.routeRules = append(m.routeRules, &rule) m.routeRules.Sort() - m.routeRulesMap[ruleKey] = &rule + m.routeRulesMap[ruleID] = &rule return &rule, nil } -func (m *Manager) DeleteRouteRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - return m.deleteRouteRule(rule) -} - func (m *Manager) deleteRouteRule(rule firewall.Rule) error { - if m.nativeRouter.Load() && m.nativeFirewall != nil { - return m.nativeFirewall.DeleteRouteRule(rule) + ruleID := rule.ID() + trimmed, _, ok := removeRuleByID(m.routeRules, ruleID) + if !ok { + return fmt.Errorf("route rule not found: %s", ruleID) } - - ruleKey := nbid.RuleID(rule.ID()) - if _, ok := m.routeRulesMap[ruleKey]; !ok { - return fmt.Errorf("route rule not found: %s", ruleKey) - } - - idx := slices.IndexFunc(m.routeRules, func(r *RouteRule) bool { - return r.id == string(ruleKey) - }) - if idx < 0 { - return fmt.Errorf("route rule not found in slice: %s", ruleKey) - } - - m.routeRules = slices.Delete(m.routeRules, idx, idx+1) - delete(m.routeRulesMap, ruleKey) + m.routeRules = trimmed + delete(m.routeRulesMap, ruleID) return nil } -// DeletePeerRule from the firewall by rule definition -func (m *Manager) DeletePeerRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() +// deletePeerRuleLocked removes a peer rule from the matching slice, +// index, and dedup map. The caller must hold m.mutex. +func (m *Manager) deletePeerRuleLocked(r *PeerRule) error { + target, index := &m.incomingAcceptRules, &m.incomingAcceptIndex + if r.action == firewall.ActionDrop { + target, index = &m.incomingDenyRules, &m.incomingDenyIndex + } - r, ok := rule.(*PeerRule) + trimmed, stored, ok := removeRuleByID(*target, r.id) if !ok { - return fmt.Errorf("delete rule: invalid rule type: %T", rule) - } - - var sourceMap map[netip.Addr]RuleSet - if r.drop { - sourceMap = m.incomingDenyRules - } else { - sourceMap = m.incomingRules - } - - if ruleset, ok := sourceMap[r.ip]; ok { - if _, exists := ruleset[r.id]; !exists { - return fmt.Errorf("delete rule: no rule with such id: %v", r.id) - } - delete(ruleset, r.id) - if len(ruleset) == 0 { - delete(sourceMap, r.ip) - } - } else { return fmt.Errorf("delete rule: no rule with such id: %v", r.id) } - + *target = trimmed + index.remove(stored) + delete(m.peerRulesMap, r.id) return nil } -// SetLegacyManagement doesn't need to be implemented for this manager -func (m *Manager) SetLegacyManagement(isLegacy bool) error { - if m.nativeFirewall == nil { - return nil +// removeRuleByID removes the first rule whose id matches ruleID from +// rules, preserving order. It returns the trimmed slice, the removed +// rule, and whether a match was found. +func removeRuleByID[S ~[]T, T firewall.Rule](rules S, ruleID firewall.RuleID) (S, T, bool) { + idx := slices.IndexFunc(rules, func(r T) bool { return r.ID() == ruleID }) + var removed T + if idx < 0 { + return rules, removed, false } - return m.nativeFirewall.SetLegacyManagement(isLegacy) + removed = rules[idx] + return slices.Delete(rules, idx, idx+1), removed, true +} + +// SetLegacyManagement is a no-op for the userspace firewall: it only matters +// when an old management server can't send route firewall rules, which the +// userspace router doesn't rely on. +func (m *Manager) SetLegacyManagement(bool) error { + return nil } // Flush doesn't need to be implemented for this manager @@ -674,11 +838,14 @@ func (m *Manager) Flush() error { return nil } // resetState clears all firewall rules and closes connection trackers. // Must be called with m.mutex held. func (m *Manager) resetState() { - clear(m.outgoingRules) - clear(m.incomingDenyRules) - clear(m.incomingRules) + m.incomingDenyRules = m.incomingDenyRules[:0] + m.incomingAcceptRules = m.incomingAcceptRules[:0] + m.incomingDenyIndex.reset() + m.incomingAcceptIndex.reset() + clear(m.peerRulesMap) clear(m.routeRulesMap) m.routeRules = m.routeRules[:0] + m.blockRules = nil m.udpHookOut.Store(nil) m.tcpHookOut.Store(nil) @@ -694,6 +861,10 @@ func (m *Manager) resetState() { m.tcpTracker.Close() } + if m.fragments != nil { + m.fragments.Close() + } + if fwder := m.forwarder.Load(); fwder != nil { fwder.SetCapture(nil) fwder.Stop() @@ -708,21 +879,9 @@ func (m *Manager) resetState() { } } -// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic. -func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error { - if m.nativeFirewall == nil { - return nil - } - return m.nativeFirewall.SetupEBPFProxyNoTrack(proxyPort, wgPort) -} - // UpdateSet updates the rule destinations associated with the given set // by merging the existing prefixes with the new ones, then deduplicating. func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { - if m.nativeRouter.Load() && m.nativeFirewall != nil { - return m.nativeFirewall.UpdateSet(set, prefixes) - } - m.mutex.Lock() defer m.mutex.Unlock() @@ -820,11 +979,11 @@ func (m *Manager) extractIPs(d *decoder) (srcIP, dstIP netip.Addr) { case layers.LayerTypeIPv4: src, _ := netip.AddrFromSlice(d.ip4.SrcIP) dst, _ := netip.AddrFromSlice(d.ip4.DstIP) - return src, dst + return src.Unmap(), dst.Unmap() case layers.LayerTypeIPv6: src, _ := netip.AddrFromSlice(d.ip6.SrcIP) dst, _ := netip.AddrFromSlice(d.ip6.DstIP) - return src, dst + return src.Unmap(), dst.Unmap() default: return netip.Addr{}, netip.Addr{} } @@ -1046,19 +1205,20 @@ func (m *Manager) filterInbound(packetData []byte, size int) bool { return true } - // TODO: pass fragments of routed packets to forwarder + // gopacket does not decode the transport header of any IP fragment, so + // fragments take a dedicated path: the first fragment's header is decoded + // and ACL-evaluated here, and the remaining fragments inherit its verdict. if fragment { - if m.logger.Enabled(nblog.LevelTrace) { - if d.decoded[0] == layers.LayerTypeIPv4 { - m.logger.Trace4("packet is a fragment: src=%v dst=%v id=%v flags=%v", - srcIP, dstIP, d.ip4.Id, d.ip4.Flags) - } else { - m.logger.Trace2("packet is an IPv6 fragment: src=%v dst=%v", srcIP, dstIP) - } - } - return false + return m.filterInboundFragment(d, srcIP, dstIP, size) } + return m.filterInboundDecoded(d, srcIP, dstIP, packetData, size) +} + +// filterInboundDecoded runs the ACL, DNAT and conntrack pipeline on a fully +// decoded (non-fragment) inbound packet. It returns true if the packet should +// be dropped. +func (m *Manager) filterInboundDecoded(d *decoder, srcIP, dstIP netip.Addr, packetData []byte, size int) bool { // TODO: optimize port DNAT by caching matched rules in conntrack if translated := m.translateInboundPortDNAT(packetData, d, srcIP, dstIP); translated { // Re-decode after port DNAT translation to update port information @@ -1089,33 +1249,226 @@ func (m *Manager) filterInbound(packetData []byte, size int) bool { return m.handleRoutedTraffic(d, srcIP, dstIP, packetData, size) } +// fragmentMeta holds the reassembly identity and layout of an IP fragment, +// extracted uniformly for IPv4 and IPv6. +type fragmentMeta struct { + key fragmentKey + // offset is the fragment offset in 8-byte units (zero for the first + // fragment). + offset uint16 + // moreFragments is the More Fragments bit. A first fragment with it unset is + // an IPv6 atomic fragment (a complete datagram, RFC 6946): it has no trailing + // fragments to inherit a verdict, so it must not be recorded. + moreFragments bool + proto layers.IPProtocol + // l4payload is the fragmentable payload of this fragment. For the first + // fragment it starts with the transport header. + l4payload []byte + // headerEndOctets is the first fragment's payload length in 8-byte units: + // the smallest offset a trailing fragment may start at without overlapping + // the inspected transport header. + headerEndOctets uint16 +} + +// fragmentMetadata extracts the fragment identity and layout from a decoded IP +// fragment. It returns false for fragments it can't interpret (e.g. an IPv6 +// fragment header shorter than 8 bytes), which are then dropped. +func fragmentMetadata(d *decoder, srcIP, dstIP netip.Addr) (fragmentMeta, bool) { + switch d.decoded[0] { + case layers.LayerTypeIPv4: + payload := d.ip4.Payload + return fragmentMeta{ + key: fragmentKey{srcIP: srcIP, dstIP: dstIP, id: uint32(d.ip4.Id), proto: uint8(d.ip4.Protocol)}, + offset: d.ip4.FragOffset, + moreFragments: d.ip4.Flags&layers.IPv4MoreFragments != 0, + proto: d.ip4.Protocol, + l4payload: payload, + headerEndOctets: octets(len(payload)), + }, true + + case layers.LayerTypeIPv6: + // IPv6 fragment extension header: 8 bytes, followed by the fragmentable + // payload. Layout: next header (1), reserved (1), offset+flags (2), id (4). + payload := d.ip6.Payload + if len(payload) < 8 { + return fragmentMeta{}, false + } + nextHeader := layers.IPProtocol(payload[0]) + offsetFlags := binary.BigEndian.Uint16(payload[2:4]) + id := binary.BigEndian.Uint32(payload[4:8]) + l4 := payload[8:] + return fragmentMeta{ + key: fragmentKey{srcIP: srcIP, dstIP: dstIP, id: id, proto: uint8(nextHeader)}, + offset: offsetFlags >> 3, + moreFragments: offsetFlags&1 != 0, + proto: nextHeader, + l4payload: l4, + headerEndOctets: octets(len(l4)), + }, true + + default: + return fragmentMeta{}, false + } +} + +// octets rounds a byte length up to whole 8-byte units, the granularity of the +// IP fragment offset field. +func octets(nbytes int) uint16 { + return uint16((nbytes + 7) / 8) +} + +// filterInboundFragment decides the fate of an IP fragment. gopacket stops +// decoding at the network layer for every fragment, so the first fragment's +// transport header is decoded and ACL-evaluated here and its verdict recorded; +// the remaining (headerless) fragments inherit that verdict. Anything that +// cannot be tied to an allowed, non-overlapping first fragment is dropped. +func (m *Manager) filterInboundFragment(d *decoder, srcIP, dstIP netip.Addr, size int) bool { + meta, ok := fragmentMetadata(d, srcIP, dstIP) + if !ok { + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace2("dropping unsupported fragment: src=%v dst=%v", srcIP, dstIP) + } + return true + } + + if meta.offset != 0 { + return m.filterTrailingFragment(meta, srcIP, dstIP) + } + + // A new first fragment supersedes any recorded verdict for this datagram, so + // a re-sent or overlapping offset-zero fragment can't inherit the old one. + m.fragments.poison(meta.key) + + // First fragment: decode its transport header so the ACL can evaluate it. A + // decode failure means the fragment is too small to hold the full transport + // header (RFC 1858 §3 tiny-fragment attack); it can't be evaluated, so drop it. + if !d.decodeTransport(meta.proto, meta.l4payload) { + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace3("dropping first fragment without full L4 header: src=%v dst=%v id=%v", + srcIP, dstIP, meta.key.id) + } + return true + } + + return m.filterFirstFragment(d, meta, srcIP, dstIP, size) +} + +// filterTrailingFragment applies a recorded first-fragment verdict to a +// non-first fragment. +func (m *Manager) filterTrailingFragment(meta fragmentMeta, srcIP, dstIP netip.Addr) bool { + switch m.fragments.verdict(meta.key, meta.offset) { + case fragmentAllow: + return false + case fragmentOverlap: + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace3("dropping overlapping fragment rewriting inspected header: src=%v dst=%v id=%v", + srcIP, dstIP, meta.key.id) + } + return true + default: + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace3("dropping fragment with no allowed first fragment: src=%v dst=%v id=%v", + srcIP, dstIP, meta.key.id) + } + return true + } +} + +// filterFirstFragment runs the verdict part of the inbound pipeline on a first +// fragment with its transport header decoded. It mirrors filterInboundDecoded +// but skips DNAT (port rewriting on fragments is unsupported) and forwarder +// injection (fragments are left to the stack to reassemble, not forwarded). +// Allowed fragments have their verdict recorded so the datagram's trailing +// fragments inherit it. +func (m *Manager) filterFirstFragment(d *decoder, meta fragmentMeta, srcIP, dstIP netip.Addr, size int) bool { + if m.stateful && m.isValidTrackedConnection(d, srcIP, dstIP, size) { + m.recordFirstFragment(meta) + return false + } + + if m.localipmanager.IsLocalIP(dstIP) { + ruleID, blocked := m.peerACLsBlock(srcIP, d, nil) + if blocked { + m.storeDropFlow("Dropping local first fragment (ACL denied): rule_id=%s proto=%v src=%s:%d dst=%s:%d", + d, srcIP, dstIP, ruleID, size) + return true + } + m.trackInbound(d, srcIP, dstIP, ruleID, size) + m.recordFirstFragment(meta) + return false + } + + if !m.routingEnabled.Load() { + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace2("Dropping routed fragment (routing disabled): src=%s dst=%s", srcIP, dstIP) + } + return true + } + if m.nativeRouter.Load() { + m.trackInbound(d, srcIP, dstIP, nil, size) + m.recordFirstFragment(meta) + return false + } + + // TODO: pass fragments of routed packets to the forwarder; until then + // allowed routed fragments go to the native stack. + srcPort, dstPort := getPortsFromPacket(d) + ruleID, pass := m.routeACLsPass(srcIP, dstIP, d.decoded[1], srcPort, dstPort) + if !pass { + m.storeDropFlow("Dropping routed first fragment (ACL denied): rule_id=%s proto=%v src=%s:%d dst=%s:%d", + d, srcIP, dstIP, ruleID, size) + return true + } + + m.recordFirstFragment(meta) + return false +} + +// recordFirstFragment caches an allowed first fragment's verdict for its +// trailing fragments to inherit. Atomic fragments (no More Fragments bit) are +// complete datagrams with no trailing fragments, so they are not cached and +// cannot exhaust the verdict table. +func (m *Manager) recordFirstFragment(meta fragmentMeta) { + if !meta.moreFragments { + return + } + m.fragments.recordAllowed(meta.key, meta.headerEndOctets) +} + +// storeDropFlow logs and records a netflow drop event for an inbound packet +// denied by the ACLs. msg is the trace format taking rule id, protocol, source +// and destination. +func (m *Manager) storeDropFlow(msg string, d *decoder, srcIP, dstIP netip.Addr, ruleID []byte, size int) { + pnum := getProtocolFromPacket(d) + srcPort, dstPort := getPortsFromPacket(d) + + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace6(msg, ruleID, pnum, srcIP, srcPort, dstIP, dstPort) + } + + m.flowLogger.StoreEvent(nftypes.EventFields{ + FlowID: uuid.New(), + Type: nftypes.TypeDrop, + RuleID: ruleID, + Direction: nftypes.Ingress, + Protocol: pnum, + SourceIP: srcIP, + DestIP: dstIP, + SourcePort: srcPort, + DestPort: dstPort, + // TODO: icmp type/code + RxPackets: 1, + RxBytes: uint64(size), + }) +} + // handleLocalTraffic handles local traffic. // If it returns true, the packet should be dropped. func (m *Manager) handleLocalTraffic(d *decoder, srcIP, dstIP netip.Addr, packetData []byte, size int) bool { ruleID, blocked := m.peerACLsBlock(srcIP, d, packetData) if blocked { - pnum := getProtocolFromPacket(d) - srcPort, dstPort := getPortsFromPacket(d) - - if m.logger.Enabled(nblog.LevelTrace) { - m.logger.Trace6("Dropping local packet (ACL denied): rule_id=%s proto=%v src=%s:%d dst=%s:%d", - ruleID, pnum, srcIP, srcPort, dstIP, dstPort) - } - - m.flowLogger.StoreEvent(nftypes.EventFields{ - FlowID: uuid.New(), - Type: nftypes.TypeDrop, - RuleID: ruleID, - Direction: nftypes.Ingress, - Protocol: pnum, - SourceIP: srcIP, - DestIP: dstIP, - SourcePort: srcPort, - DestPort: dstPort, - // TODO: icmp type/code - RxPackets: 1, - RxBytes: uint64(size), - }) + m.storeDropFlow("Dropping local packet (ACL denied): rule_id=%s proto=%v src=%s:%d dst=%s:%d", + d, srcIP, dstIP, ruleID, size) return true } @@ -1168,27 +1521,8 @@ func (m *Manager) handleRoutedTraffic(d *decoder, srcIP, dstIP netip.Addr, packe ruleID, pass := m.routeACLsPass(srcIP, dstIP, protoLayer, srcPort, dstPort) if !pass { - proto := getProtocolFromPacket(d) - - if m.logger.Enabled(nblog.LevelTrace) { - m.logger.Trace6("Dropping routed packet (ACL denied): rule_id=%s proto=%v src=%s:%d dst=%s:%d", - ruleID, proto, srcIP, srcPort, dstIP, dstPort) - } - - m.flowLogger.StoreEvent(nftypes.EventFields{ - FlowID: uuid.New(), - Type: nftypes.TypeDrop, - RuleID: ruleID, - Direction: nftypes.Ingress, - Protocol: proto, - SourceIP: srcIP, - DestIP: dstIP, - SourcePort: srcPort, - DestPort: dstPort, - // TODO: icmp type/code - RxPackets: 1, - RxBytes: uint64(size), - }) + m.storeDropFlow("Dropping routed packet (ACL denied): rule_id=%s proto=%v src=%s:%d dst=%s:%d", + d, srcIP, dstIP, ruleID, size) return true } @@ -1404,20 +1738,12 @@ func (m *Manager) peerACLsBlock(srcIP netip.Addr, d *decoder, packetData []byte) return nil, false } - if mgmtId, filter, ok := validateRule(srcIP, packetData, m.incomingDenyRules[srcIP], d); ok { + if mgmtId, filter, ok := m.incomingDenyIndex.match(srcIP, d); ok { return mgmtId, filter } - - if mgmtId, filter, ok := validateRule(srcIP, packetData, m.incomingRules[srcIP], d); ok { + if mgmtId, filter, ok := m.incomingAcceptIndex.match(srcIP, d); ok { return mgmtId, filter } - if mgmtId, filter, ok := validateRule(srcIP, packetData, m.incomingRules[netip.IPv4Unspecified()], d); ok { - return mgmtId, filter - } - if mgmtId, filter, ok := validateRule(srcIP, packetData, m.incomingRules[netip.IPv6Unspecified()], d); ok { - return mgmtId, filter - } - return nil, true } @@ -1438,39 +1764,6 @@ func portsMatch(rulePort *firewall.Port, packetPort uint16) bool { return false } -func validateRule(ip netip.Addr, packetData []byte, rules map[string]PeerRule, d *decoder) ([]byte, bool, bool) { - payloadLayer := d.decoded[1] - - for _, rule := range rules { - if rule.matchByIP && ip.Compare(rule.ip) != 0 { - continue - } - - if rule.protoLayer == layerTypeAll { - return rule.mgmtId, rule.drop, true - } - - if !protoLayerMatches(rule.protoLayer, payloadLayer) { - continue - } - - switch payloadLayer { - case layers.LayerTypeTCP: - if portsMatch(rule.sPort, uint16(d.tcp.SrcPort)) && portsMatch(rule.dPort, uint16(d.tcp.DstPort)) { - return rule.mgmtId, rule.drop, true - } - case layers.LayerTypeUDP: - if portsMatch(rule.sPort, uint16(d.udp.SrcPort)) && portsMatch(rule.dPort, uint16(d.udp.DstPort)) { - return rule.mgmtId, rule.drop, true - } - case layers.LayerTypeICMPv4, layers.LayerTypeICMPv6: - return rule.mgmtId, rule.drop, true - } - } - - return nil, false, false -} - // routeACLsPass returns true if the packet is allowed by the route ACLs func (m *Manager) routeACLsPass(srcIP, dstIP netip.Addr, protoLayer gopacket.LayerType, srcPort, dstPort uint16) ([]byte, bool) { m.mutex.RLock() @@ -1547,10 +1840,13 @@ func (m *Manager) EnableRouting() error { } rules, err := m.blockInvalidRouted(m.wgIface) - // Persist whatever was installed even on partial failure, so DisableRouting - // can clean it up later. m.blockRules = rules if err != nil { + // Roll back so forwarding can't stay active without the full set of + // block rules. + if derr := m.disableRouting(); derr != nil { + log.Warnf("roll back routing after block rule failure: %v", derr) + } return fmt.Errorf("block invalid routed: %w", err) } @@ -1561,6 +1857,10 @@ func (m *Manager) DisableRouting() error { m.mutex.Lock() defer m.mutex.Unlock() + return m.disableRouting() +} + +func (m *Manager) disableRouting() error { fwder := m.forwarder.Load() if fwder == nil { return nil diff --git a/client/firewall/uspfilter/filter_bench_test.go b/client/firewall/uspfilter/filter_bench_test.go index 4dccb0f65..72f3417f2 100644 --- a/client/firewall/uspfilter/filter_bench_test.go +++ b/client/firewall/uspfilter/filter_bench_test.go @@ -94,7 +94,7 @@ func BenchmarkCoreFiltering(b *testing.B) { stateful: false, setupFunc: func(m *Manager) { // Single rule allowing all traffic - _, err := m.AddPeerFiltering(nil, net.ParseIP("0.0.0.0"), fw.ProtocolALL, nil, nil, fw.ActionAccept, "") + _, err := m.AddFilterRule(nil, pfx(net.ParseIP("0.0.0.0")), fw.Network{}, fw.ProtocolALL, nil, nil, fw.ActionAccept) require.NoError(b, err) }, desc: "Baseline: Single 'allow all' rule without connection tracking", @@ -114,15 +114,13 @@ func BenchmarkCoreFiltering(b *testing.B) { // Add explicit rules matching return traffic pattern for i := 0; i < 1000; i++ { // Simulate realistic ruleset size ip := generateRandomIPs(1)[0] - _, err := m.AddPeerFiltering( + _, err := m.AddFilterRule( nil, - ip, + pfx(ip), fw.Network{}, fw.ProtocolTCP, &fw.Port{Values: []uint16{uint16(1024 + i)}}, &fw.Port{Values: []uint16{80}}, - fw.ActionAccept, - "", - ) + fw.ActionAccept) require.NoError(b, err) } }, @@ -133,15 +131,13 @@ func BenchmarkCoreFiltering(b *testing.B) { stateful: true, setupFunc: func(m *Manager) { // Add some basic rules but rely on state for established connections - _, err := m.AddPeerFiltering( + _, err := m.AddFilterRule( nil, - net.ParseIP("0.0.0.0"), + pfx(net.ParseIP("0.0.0.0")), fw.Network{}, fw.ProtocolTCP, nil, nil, - fw.ActionDrop, - "", - ) + fw.ActionDrop) require.NoError(b, err) }, desc: "Connection tracking with established connections", @@ -168,9 +164,12 @@ func BenchmarkCoreFiltering(b *testing.B) { } // Create manager and basic setup - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) defer b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) @@ -208,9 +207,12 @@ func BenchmarkStateScaling(b *testing.B) { for _, count := range connCounts { b.Run(fmt.Sprintf("conns_%d", count), func(b *testing.B) { - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) @@ -251,9 +253,12 @@ func BenchmarkEstablishmentOverhead(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) @@ -409,9 +414,12 @@ func BenchmarkRoutedNetworkReturn(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) @@ -536,9 +544,12 @@ func BenchmarkLongLivedConnections(b *testing.B) { require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) } - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) defer b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) @@ -546,7 +557,7 @@ func BenchmarkLongLivedConnections(b *testing.B) { // Setup initial state based on scenario if sc.rules { // Single rule to allow all return traffic from port 80 - _, err := manager.AddPeerFiltering(nil, net.ParseIP("0.0.0.0"), fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept, "") + _, err := manager.AddFilterRule(nil, pfx(net.ParseIP("0.0.0.0")), fw.Network{}, fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept) require.NoError(b, err) } @@ -619,9 +630,12 @@ func BenchmarkShortLivedConnections(b *testing.B) { require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) } - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) defer b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) @@ -629,7 +643,7 @@ func BenchmarkShortLivedConnections(b *testing.B) { // Setup initial state based on scenario if sc.rules { // Single rule to allow all return traffic from port 80 - _, err := manager.AddPeerFiltering(nil, net.ParseIP("0.0.0.0"), fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept, "") + _, err := manager.AddFilterRule(nil, pfx(net.ParseIP("0.0.0.0")), fw.Network{}, fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept) require.NoError(b, err) } @@ -730,16 +744,19 @@ func BenchmarkParallelLongLivedConnections(b *testing.B) { require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) } - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) defer b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) // Setup initial state based on scenario if sc.rules { - _, err := manager.AddPeerFiltering(nil, net.ParseIP("0.0.0.0"), fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept, "") + _, err := manager.AddFilterRule(nil, pfx(net.ParseIP("0.0.0.0")), fw.Network{}, fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept) require.NoError(b, err) } @@ -810,15 +827,18 @@ func BenchmarkParallelShortLivedConnections(b *testing.B) { require.NoError(b, os.Unsetenv("NB_DISABLE_CONNTRACK")) } - manager, _ := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) defer b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) if sc.rules { - _, err := manager.AddPeerFiltering(nil, net.ParseIP("0.0.0.0"), fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept, "") + _, err := manager.AddFilterRule(nil, pfx(net.ParseIP("0.0.0.0")), fw.Network{}, fw.ProtocolTCP, &fw.Port{Values: []uint16{80}}, nil, fw.ActionAccept) require.NoError(b, err) } @@ -931,7 +951,7 @@ func BenchmarkRouteACLs(b *testing.B) { for _, r := range rules { dst := fw.Network{Prefix: r.dest} - _, err := manager.AddRouteFiltering(nil, r.sources, dst, r.proto, nil, r.port, fw.ActionAccept) + _, err := manager.AddFilterRule(nil, r.sources, dst, r.proto, nil, r.port, fw.ActionAccept) if err != nil { b.Fatal(err) } @@ -1014,9 +1034,11 @@ func BenchmarkMSSClamping(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) @@ -1079,9 +1101,11 @@ func BenchmarkMSSClampingOverhead(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) @@ -1134,9 +1158,11 @@ func BenchmarkMSSClampingMemory(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) diff --git a/client/firewall/uspfilter/filter_filter_test.go b/client/firewall/uspfilter/filter_filter_test.go index a64c83138..57b59c9bc 100644 --- a/client/firewall/uspfilter/filter_filter_test.go +++ b/client/firewall/uspfilter/filter_filter_test.go @@ -5,7 +5,7 @@ import ( "net/netip" "testing" - "github.com/golang/mock/gomock" + "go.uber.org/mock/gomock" "github.com/google/gopacket" "github.com/google/gopacket/layers" "github.com/stretchr/testify/require" @@ -32,7 +32,7 @@ func TestPeerACLFiltering(t *testing.T) { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) require.NotNil(t, manager) @@ -496,40 +496,32 @@ func TestPeerACLFiltering(t *testing.T) { t.Run(tc.name, func(t *testing.T) { if tc.ruleAction == fw.ActionDrop { // add general accept rule for the same IP to test drop rule precedence - rules, err := manager.AddPeerFiltering( + rules, err := manager.AddFilterRule( nil, - net.ParseIP(tc.ruleIP), + pfx(net.ParseIP(tc.ruleIP)), fw.Network{}, fw.ProtocolALL, nil, nil, - fw.ActionAccept, - "", - ) + fw.ActionAccept) require.NoError(t, err) - require.NotEmpty(t, rules) + require.NotNil(t, rules) t.Cleanup(func() { - for _, rule := range rules { - require.NoError(t, manager.DeletePeerRule(rule)) - } + require.NoError(t, manager.DeleteFilterRule(rules)) }) } - rules, err := manager.AddPeerFiltering( + rules, err := manager.AddFilterRule( nil, - net.ParseIP(tc.ruleIP), + pfx(net.ParseIP(tc.ruleIP)), fw.Network{}, tc.ruleProto, tc.ruleSrcPort, tc.ruleDstPort, - tc.ruleAction, - "", - ) + tc.ruleAction) require.NoError(t, err) - require.NotEmpty(t, rules) + require.NotNil(t, rules) t.Cleanup(func() { - for _, rule := range rules { - require.NoError(t, manager.DeletePeerRule(rule)) - } + require.NoError(t, manager.DeleteFilterRule(rules)) }) packet := createTestPacket(t, tc.srcIP, tc.dstIP, tc.proto, tc.srcPort, tc.dstPort) @@ -557,7 +549,7 @@ func TestPeerACLFilteringIPv6(t *testing.T) { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) }) @@ -652,14 +644,24 @@ func TestPeerACLFilteringIPv6(t *testing.T) { shouldBeBlocked: false, }, { - name: "IPv6: v4 wildcard ICMP rule matches ICMPv6 via protoLayerMatches", + name: "IPv6: v4 wildcard ICMP rule does not match ICMPv6", srcIP: "fd00::1", dstIP: "fd00::100", proto: fw.ProtocolICMP, ruleIP: "0.0.0.0", ruleProto: fw.ProtocolICMP, ruleAction: fw.ActionAccept, - shouldBeBlocked: false, + shouldBeBlocked: true, + }, + { + name: "IPv4: v6 wildcard ICMP rule does not match ICMPv4", + srcIP: "100.10.0.1", + dstIP: "100.10.0.100", + proto: fw.ProtocolICMP, + ruleIP: "::", + ruleProto: fw.ProtocolICMP, + ruleAction: fw.ActionAccept, + shouldBeBlocked: true, }, } @@ -672,22 +674,18 @@ func TestPeerACLFilteringIPv6(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.ruleAction == fw.ActionDrop { - rules, err := manager.AddPeerFiltering(nil, net.ParseIP(tc.ruleIP), fw.ProtocolALL, nil, nil, fw.ActionAccept, "") + rules, err := manager.AddFilterRule(nil, pfx(net.ParseIP(tc.ruleIP)), fw.Network{}, fw.ProtocolALL, nil, nil, fw.ActionAccept) require.NoError(t, err) t.Cleanup(func() { - for _, rule := range rules { - require.NoError(t, manager.DeletePeerRule(rule)) - } + require.NoError(t, manager.DeleteFilterRule(rules)) }) } - rules, err := manager.AddPeerFiltering(nil, net.ParseIP(tc.ruleIP), tc.ruleProto, nil, tc.ruleDstPort, tc.ruleAction, "") + rules, err := manager.AddFilterRule(nil, pfx(net.ParseIP(tc.ruleIP)), fw.Network{}, tc.ruleProto, nil, tc.ruleDstPort, tc.ruleAction) require.NoError(t, err) - require.NotEmpty(t, rules) + require.NotNil(t, rules) t.Cleanup(func() { - for _, rule := range rules { - require.NoError(t, manager.DeletePeerRule(rule)) - } + require.NoError(t, manager.DeleteFilterRule(rules)) }) packet := createTestPacket(t, tc.srcIP, tc.dstIP, tc.proto, tc.srcPort, tc.dstPort) @@ -800,7 +798,7 @@ func setupRoutedManager(tb testing.TB, network string) *Manager { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(tb, err) require.NoError(tb, manager.EnableRouting()) require.NotNil(tb, manager) @@ -1405,7 +1403,7 @@ func TestRouteACLFiltering(t *testing.T) { t.Run(tc.name, func(t *testing.T) { if tc.rule.action == fw.ActionDrop { // add general accept rule to test drop rule - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}, fw.Network{Prefix: netip.MustParsePrefix("0.0.0.0/0")}, @@ -1415,13 +1413,13 @@ func TestRouteACLFiltering(t *testing.T) { fw.ActionAccept, ) require.NoError(t, err) - require.NotNil(t, rule) + require.NotEmpty(t, rule) t.Cleanup(func() { - require.NoError(t, manager.DeleteRouteRule(rule)) + require.NoError(t, manager.DeleteFilterRule(rule)) }) } - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( nil, tc.rule.sources, tc.rule.dest, @@ -1431,10 +1429,10 @@ func TestRouteACLFiltering(t *testing.T) { tc.rule.action, ) require.NoError(t, err) - require.NotNil(t, rule) + require.NotEmpty(t, rule) t.Cleanup(func() { - require.NoError(t, manager.DeleteRouteRule(rule)) + require.NoError(t, manager.DeleteFilterRule(rule)) }) srcIP := netip.MustParseAddr(tc.srcIP) @@ -1602,9 +1600,9 @@ func TestRouteACLOrder(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - var rules []fw.Rule + var addedRules []fw.Rule for _, r := range tc.rules { - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( nil, r.sources, r.dest, @@ -1615,12 +1613,12 @@ func TestRouteACLOrder(t *testing.T) { ) require.NoError(t, err) require.NotNil(t, rule) - rules = append(rules, rule) + addedRules = append(addedRules, rule) } t.Cleanup(func() { - for _, rule := range rules { - require.NoError(t, manager.DeleteRouteRule(rule)) + for _, rule := range addedRules { + require.NoError(t, manager.DeleteFilterRule(rule)) } }) @@ -1646,7 +1644,7 @@ func TestRouteACLSet(t *testing.T) { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) @@ -1655,7 +1653,7 @@ func TestRouteACLSet(t *testing.T) { set := fw.NewDomainSet(domain.List{"example.org"}) // Add rule that uses the set (initially empty) - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}, fw.Network{Set: set}, @@ -1689,7 +1687,7 @@ func TestRouteACLFilteringIPv6(t *testing.T) { manager := setupRoutedManager(t, "10.10.0.100/16") v6Dst := netip.MustParsePrefix("fd00:dead:beef::/48") - _, err := manager.AddRouteFiltering( + _, err := manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("fd00::/16")}, fw.Network{Prefix: v6Dst}, @@ -1700,7 +1698,7 @@ func TestRouteACLFilteringIPv6(t *testing.T) { ) require.NoError(t, err) - _, err = manager.AddRouteFiltering( + _, err = manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("fd00::/16")}, fw.Network{Prefix: netip.MustParsePrefix("fd00:dead:beef:1::/64")}, diff --git a/client/firewall/uspfilter/filter_routeacl_test.go b/client/firewall/uspfilter/filter_routeacl_test.go index 449554d8b..2a0980e83 100644 --- a/client/firewall/uspfilter/filter_routeacl_test.go +++ b/client/firewall/uspfilter/filter_routeacl_test.go @@ -4,7 +4,7 @@ import ( "net/netip" "testing" - "github.com/golang/mock/gomock" + "go.uber.org/mock/gomock" "github.com/google/gopacket/layers" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -29,7 +29,7 @@ func TestAddRouteFilteringReturnsExistingRule(t *testing.T) { destination := fw.Network{Prefix: netip.MustParsePrefix("192.168.1.0/24")} // Add rule first time - rule1, err := manager.AddRouteFiltering( + rule1, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -42,7 +42,7 @@ func TestAddRouteFilteringReturnsExistingRule(t *testing.T) { require.NotNil(t, rule1) // Add the same rule again - rule2, err := manager.AddRouteFiltering( + rule2, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -74,7 +74,7 @@ func TestAddRouteFilteringDifferentRulesGetDifferentIDs(t *testing.T) { sources := []netip.Prefix{netip.MustParsePrefix("100.64.1.0/24")} // Add first rule - rule1, err := manager.AddRouteFiltering( + rule1, err := manager.AddFilterRule( []byte("policy-1"), sources, fw.Network{Prefix: netip.MustParsePrefix("192.168.1.0/24")}, @@ -86,7 +86,7 @@ func TestAddRouteFilteringDifferentRulesGetDifferentIDs(t *testing.T) { require.NoError(t, err) // Add different rule (different destination) - rule2, err := manager.AddRouteFiltering( + rule2, err := manager.AddFilterRule( []byte("policy-2"), sources, fw.Network{Prefix: netip.MustParsePrefix("192.168.2.0/24")}, // Different! @@ -115,7 +115,7 @@ func TestRouteRuleUpdateDoesNotCauseGap(t *testing.T) { sources := []netip.Prefix{netip.MustParsePrefix("100.64.1.0/24")} destination := fw.Network{Prefix: netip.MustParsePrefix("192.168.1.0/24")} - rule1, err := manager.AddRouteFiltering( + rule1, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -132,7 +132,7 @@ func TestRouteRuleUpdateDoesNotCauseGap(t *testing.T) { require.True(t, pass, "Traffic should pass with rule in place") // Re-add same rule (simulates network map update) - rule2, err := manager.AddRouteFiltering( + rule2, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -147,7 +147,7 @@ func TestRouteRuleUpdateDoesNotCauseGap(t *testing.T) { // won't delete rule1 during cleanup. If IDs differed, deleting rule1 // would remove the only matching rule and cause a traffic gap. if rule1.ID() != rule2.ID() { - err = manager.DeleteRouteRule(rule1) + err = manager.DeleteFilterRule(rule1) require.NoError(t, err) } @@ -156,6 +156,59 @@ func TestRouteRuleUpdateDoesNotCauseGap(t *testing.T) { "Traffic should still pass after rule update - no gap should occur") } +// TestBlockInvalidRoutedDualStack verifies that when the interface has an +// IPv6 overlay address, blockInvalidRouted installs a block rule for both +// the v4 and v6 WG prefixes and that routed traffic to the v6 prefix is +// denied. The v4-only soft-skip path is covered by +// TestBlockInvalidRoutedIdempotent, whose mock leaves IPv6Net invalid. +func TestBlockInvalidRoutedDualStack(t *testing.T) { + ctrl := gomock.NewController(t) + dev := mocks.NewMockDevice(ctrl) + dev.EXPECT().MTU().Return(1500, nil).AnyTimes() + + wgNet := netip.MustParsePrefix("100.64.0.1/16") + wgNet6 := netip.MustParsePrefix("fd00:1234::1/64") + + ifaceMock := &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + AddressFunc: func() wgaddr.Address { + return wgaddr.Address{ + IP: wgNet.Addr(), + Network: wgNet, + IPv6: wgNet6.Addr(), + IPv6Net: wgNet6, + } + }, + GetDeviceFunc: func() *device.FilteredDevice { + return &device.FilteredDevice{Device: dev} + }, + GetWGDeviceFunc: func() *wgdevice.Device { + return &wgdevice.Device{} + }, + } + + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, manager.Close(nil)) + }) + + rules, err := manager.blockInvalidRouted(ifaceMock) + require.NoError(t, err) + require.Len(t, rules, 2, "dual-stack interface must produce a v4 and a v6 block rule") + + manager.mutex.RLock() + ruleCount := len(manager.routeRules) + manager.mutex.RUnlock() + assert.Equal(t, 2, ruleCount, "should have one block rule per family") + + // v6 routed traffic to the WG prefix must be denied. + srcIP := netip.MustParseAddr("2001:db8::1") + dstIP := netip.MustParseAddr("fd00:1234::50") + _, pass := manager.routeACLsPass(srcIP, dstIP, layers.LayerTypeTCP, 12345, 80) + assert.False(t, pass, "block rule should deny routed traffic to the v6 WG prefix") +} + // TestBlockInvalidRoutedIdempotent verifies that blockInvalidRouted creates // exactly one drop rule for the WireGuard network prefix, and calling it again // returns the same rule without duplicating. @@ -182,7 +235,7 @@ func TestBlockInvalidRoutedIdempotent(t *testing.T) { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) @@ -245,7 +298,7 @@ func TestBlockRuleNotAccumulatedOnRepeatedEnableRouting(t *testing.T) { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) @@ -274,7 +327,7 @@ func TestRouteRuleCountStableAcrossUpdates(t *testing.T) { // Simulate 5 network map updates with the same route rule for i := 0; i < 5; i++ { - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -304,7 +357,7 @@ func TestDeleteRouteRuleAfterIdempotentAdd(t *testing.T) { destination := fw.Network{Prefix: netip.MustParsePrefix("192.168.1.0/24")} // Add same rule twice - rule1, err := manager.AddRouteFiltering( + rule1, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -315,7 +368,7 @@ func TestDeleteRouteRuleAfterIdempotentAdd(t *testing.T) { ) require.NoError(t, err) - rule2, err := manager.AddRouteFiltering( + rule2, err := manager.AddFilterRule( []byte("policy-1"), sources, destination, @@ -329,7 +382,7 @@ func TestDeleteRouteRuleAfterIdempotentAdd(t *testing.T) { require.Equal(t, rule1.ID(), rule2.ID(), "Should return same rule ID") // Delete using first reference - err = manager.DeleteRouteRule(rule1) + err = manager.DeleteFilterRule(rule1) require.NoError(t, err) // Verify traffic no longer passes @@ -364,7 +417,7 @@ func setupTestManager(t *testing.T) *Manager { }, } - manager, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) require.NoError(t, manager.EnableRouting()) diff --git a/client/firewall/uspfilter/filter_test.go b/client/firewall/uspfilter/filter_test.go index f19c4bb56..280ce9311 100644 --- a/client/firewall/uspfilter/filter_test.go +++ b/client/firewall/uspfilter/filter_test.go @@ -78,18 +78,19 @@ func TestManagerCreate(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) if err != nil { t.Errorf("failed to create Manager: %v", err) return } + t.Cleanup(func() { require.NoError(t, m.Close(nil)) }) if m == nil { t.Error("Manager is nil") } } -func TestManagerAddPeerFiltering(t *testing.T) { +func TestManagerAddFilterRule(t *testing.T) { isSetFilterCalled := false ifaceMock := &IFaceMock{ SetFilterFunc: func(device.PacketFilter) error { @@ -98,18 +99,19 @@ func TestManagerAddPeerFiltering(t *testing.T) { }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) if err != nil { t.Errorf("failed to create Manager: %v", err) return } + t.Cleanup(func() { require.NoError(t, m.Close(nil)) }) ip := net.ParseIP("192.168.1.1") proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionDrop - rule, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + rule, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) if err != nil { t.Errorf("failed to add filtering: %v", err) return @@ -131,74 +133,47 @@ func TestManagerDeleteRule(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) if err != nil { t.Errorf("failed to create Manager: %v", err) return } + t.Cleanup(func() { require.NoError(t, m.Close(nil)) }) ip := netip.MustParseAddr("192.168.1.1") proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionDrop - rule2, err := m.AddPeerFiltering(nil, ip.AsSlice(), proto, nil, port, action, "") + rule2, err := m.AddFilterRule(nil, pfx(ip.AsSlice()), fw.Network{}, proto, nil, port, action) if err != nil { t.Errorf("failed to add filtering: %v", err) return } - // Check rules exist in appropriate maps - for _, r := range rule2 { - peerRule, ok := r.(*PeerRule) - if !ok { - t.Errorf("rule should be a PeerRule") - continue - } - // Check if rule exists in deny or allow maps based on action - var found bool - if peerRule.drop { - _, found = m.incomingDenyRules[ip][r.ID()] - } else { - _, found = m.incomingRules[ip][r.ID()] - } - if !found { - t.Errorf("rule2 is not in the expected rules map") + peerRule, ok := rule2.(*PeerRule) + require.True(t, ok, "rule should be a peer rule") + + inMap := func() bool { + if peerRule.action == fw.ActionDrop { + return findRuleByID(m.incomingDenyRules, ip, rule2.ID()) } + return findRuleByID(m.incomingAcceptRules, ip, rule2.ID()) } - for _, r := range rule2 { - err = m.DeletePeerRule(r) - if err != nil { - t.Errorf("failed to delete rule: %v", err) - return - } - } + require.True(t, inMap(), "rule2 should be in the expected rules list") - // Check rules are removed from appropriate maps - for _, r := range rule2 { - peerRule, ok := r.(*PeerRule) - if !ok { - t.Errorf("rule should be a PeerRule") - continue - } - // Check if rule is removed from deny or allow maps based on action - var found bool - if peerRule.drop { - _, found = m.incomingDenyRules[ip][r.ID()] - } else { - _, found = m.incomingRules[ip][r.ID()] - } - if found { - t.Errorf("rule2 should be removed from the rules map") - } - } + require.NoError(t, m.DeleteFilterRule(rule2), "failed to delete rule") + + require.False(t, inMap(), "rule2 should be removed from the rules list") } func TestSetUDPPacketHook(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) }) @@ -220,9 +195,11 @@ func TestSetUDPPacketHook(t *testing.T) { } func TestSetTCPPacketHook(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) }) @@ -250,7 +227,7 @@ func TestPeerRuleLifecycleDenyRules(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, m.Close(nil)) @@ -260,36 +237,34 @@ func TestPeerRuleLifecycleDenyRules(t *testing.T) { addr := netip.MustParseAddr("192.168.1.1") // Add multiple deny rules for different ports - rule1, err := m.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{22}}, fw.ActionDrop, "") + rule1, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{22}}, fw.ActionDrop) require.NoError(t, err) - rule2, err := m.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{80}}, fw.ActionDrop, "") + rule2, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop) require.NoError(t, err) m.mutex.RLock() - denyCount := len(m.incomingDenyRules[addr]) + denyCount := countRulesForAddr(m.incomingDenyRules, addr) m.mutex.RUnlock() require.Equal(t, 2, denyCount, "Should have exactly 2 deny rules") // Delete the first deny rule - err = m.DeletePeerRule(rule1[0]) + err = m.DeleteFilterRule(rule1) require.NoError(t, err) m.mutex.RLock() - denyCount = len(m.incomingDenyRules[addr]) + denyCount = countRulesForAddr(m.incomingDenyRules, addr) m.mutex.RUnlock() require.Equal(t, 1, denyCount, "Should have 1 deny rule after deleting first") // Delete the second deny rule - err = m.DeletePeerRule(rule2[0]) + err = m.DeleteFilterRule(rule2) require.NoError(t, err) m.mutex.RLock() - _, exists := m.incomingDenyRules[addr] + exists := countRulesForAddr(m.incomingDenyRules, addr) > 0 m.mutex.RUnlock() - require.False(t, exists, "Deny rules IP entry should be cleaned up when empty") + require.False(t, exists, "Deny rules should be cleaned up when empty") } // TestPeerRuleAddAndDeleteDontLeak verifies that repeatedly adding and deleting @@ -299,7 +274,7 @@ func TestPeerRuleAddAndDeleteDontLeak(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, m.Close(nil)) @@ -311,27 +286,21 @@ func TestPeerRuleAddAndDeleteDontLeak(t *testing.T) { // Simulate 10 network map updates: add rule, delete old, add new for i := 0; i < 10; i++ { // Add a deny rule - rules, err := m.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{22}}, fw.ActionDrop, "") + rules, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{22}}, fw.ActionDrop) require.NoError(t, err) // Add an allow rule - allowRules, err := m.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "") + allowRules, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err) // Delete them (simulating ACL manager cleanup) - for _, r := range rules { - require.NoError(t, m.DeletePeerRule(r)) - } - for _, r := range allowRules { - require.NoError(t, m.DeletePeerRule(r)) - } + require.NoError(t, m.DeleteFilterRule(rules)) + require.NoError(t, m.DeleteFilterRule(allowRules)) } m.mutex.RLock() - denyCount := len(m.incomingDenyRules[addr]) - allowCount := len(m.incomingRules[addr]) + denyCount := countRulesForAddr(m.incomingDenyRules, addr) + allowCount := countRulesForAddr(m.incomingAcceptRules, addr) m.mutex.RUnlock() require.Equal(t, 0, denyCount, "No deny rules should remain after cleanup") @@ -345,7 +314,7 @@ func TestMixedAllowDenyRulesSameIP(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, m.Close(nil)) @@ -354,41 +323,39 @@ func TestMixedAllowDenyRulesSameIP(t *testing.T) { ip := net.ParseIP("192.168.1.1") // Add allow rule for port 80 - allowRule, err := m.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{80}}, fw.ActionAccept, "") + allowRule, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err) // Add deny rule for port 22 - denyRule, err := m.AddPeerFiltering(nil, ip, fw.ProtocolTCP, nil, - &fw.Port{Values: []uint16{22}}, fw.ActionDrop, "") + denyRule, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{22}}, fw.ActionDrop) require.NoError(t, err) addr := netip.MustParseAddr("192.168.1.1") m.mutex.RLock() - allowCount := len(m.incomingRules[addr]) - denyCount := len(m.incomingDenyRules[addr]) + allowCount := countRulesForAddr(m.incomingAcceptRules, addr) + denyCount := countRulesForAddr(m.incomingDenyRules, addr) m.mutex.RUnlock() require.Equal(t, 1, allowCount, "Should have 1 allow rule") require.Equal(t, 1, denyCount, "Should have 1 deny rule") // Delete allow rule should not affect deny rule - err = m.DeletePeerRule(allowRule[0]) + err = m.DeleteFilterRule(allowRule) require.NoError(t, err) m.mutex.RLock() - denyCountAfter := len(m.incomingDenyRules[addr]) + denyCountAfter := countRulesForAddr(m.incomingDenyRules, addr) m.mutex.RUnlock() require.Equal(t, 1, denyCountAfter, "Deny rule should still exist after deleting allow rule") // Delete deny rule - err = m.DeletePeerRule(denyRule[0]) + err = m.DeleteFilterRule(denyRule) require.NoError(t, err) m.mutex.RLock() - _, denyExists := m.incomingDenyRules[addr] - _, allowExists := m.incomingRules[addr] + denyExists := countRulesForAddr(m.incomingDenyRules, addr) > 0 + allowExists := countRulesForAddr(m.incomingAcceptRules, addr) > 0 m.mutex.RUnlock() require.False(t, denyExists, "Deny rules should be empty") @@ -400,7 +367,7 @@ func TestManagerReset(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) if err != nil { t.Errorf("failed to create Manager: %v", err) return @@ -411,7 +378,7 @@ func TestManagerReset(t *testing.T) { port := &fw.Port{Values: []uint16{80}} action := fw.ActionDrop - _, err = m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err = m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) if err != nil { t.Errorf("failed to add filtering: %v", err) return @@ -423,7 +390,7 @@ func TestManagerReset(t *testing.T) { return } - if len(m.outgoingRules) != 0 || len(m.incomingRules) != 0 || len(m.incomingDenyRules) != 0 { + if len(m.incomingAcceptRules) != 0 || len(m.incomingDenyRules) != 0 { t.Errorf("rules are not empty") } } @@ -439,7 +406,7 @@ func TestNotMatchByIP(t *testing.T) { }, } - m, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) if err != nil { t.Errorf("failed to create Manager: %v", err) return @@ -449,7 +416,7 @@ func TestNotMatchByIP(t *testing.T) { proto := fw.ProtocolUDP action := fw.ActionAccept - _, err = m.AddPeerFiltering(nil, ip, proto, nil, nil, action, "") + _, err = m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, nil, action) if err != nil { t.Errorf("failed to add filtering: %v", err) return @@ -502,7 +469,7 @@ func TestRemovePacketHook(t *testing.T) { } // creating manager instance - manager, err := Create(iface, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{IFace: iface, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) if err != nil { t.Fatalf("Failed to create Manager: %s", err) } @@ -519,9 +486,11 @@ func TestRemovePacketHook(t *testing.T) { } func TestProcessOutgoingHooks(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) manager.udpTracker.Close() @@ -606,7 +575,7 @@ func TestUSPFilterCreatePerformance(t *testing.T) { ifaceMock := &IFaceMock{ SetFilterFunc: func(device.PacketFilter) error { return nil }, } - manager, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) time.Sleep(time.Second) @@ -621,7 +590,7 @@ func TestUSPFilterCreatePerformance(t *testing.T) { start := time.Now() for i := 0; i < testMax; i++ { port := &fw.Port{Values: []uint16{uint16(1000 + i)}} - _, err = manager.AddPeerFiltering(nil, ip, "tcp", nil, port, fw.ActionAccept, "") + _, err = manager.AddFilterRule(nil, pfx(ip), fw.Network{}, "tcp", nil, port, fw.ActionAccept) require.NoError(t, err, "failed to add rule") } @@ -631,9 +600,11 @@ func TestUSPFilterCreatePerformance(t *testing.T) { } func TestStatefulFirewall_UDPTracking(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) manager.udpTracker.Close() // Close the existing tracker @@ -845,7 +816,7 @@ func TestUpdateSetMerge(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - manager, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) @@ -858,7 +829,7 @@ func TestUpdateSetMerge(t *testing.T) { netip.MustParsePrefix("192.168.1.0/24"), } - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}, fw.Network{Set: set}, @@ -931,7 +902,7 @@ func TestUpdateSetDeduplication(t *testing.T) { SetFilterFunc: func(device.PacketFilter) error { return nil }, } - manager, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, manager.Close(nil)) @@ -939,7 +910,7 @@ func TestUpdateSetDeduplication(t *testing.T) { set := fw.NewDomainSet(domain.List{"example.org"}) - rule, err := manager.AddRouteFiltering( + rule, err := manager.AddFilterRule( nil, []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0")}, fw.Network{Set: set}, @@ -1051,7 +1022,7 @@ func TestMSSClamping(t *testing.T) { }, } - manager, err := Create(ifaceMock, false, flowLogger, 1280) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: 1280}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) @@ -1243,7 +1214,7 @@ func TestShouldForward(t *testing.T) { return wgaddr.Address{IP: wgIP, Network: netip.PrefixFrom(wgIP, 24)} } - manager, err := Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + manager, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) @@ -1358,7 +1329,7 @@ func TestShouldForward(t *testing.T) { // Re-create manager to pick up the new address with IPv6 require.NoError(t, manager.Close(nil)) - manager, err = Create(ifaceMock, false, flowLogger, nbiface.DefaultMTU) + manager, err = Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) require.NoError(t, err) v6Cases := []struct { diff --git a/client/firewall/uspfilter/forwarder/forwarder.go b/client/firewall/uspfilter/forwarder/forwarder.go index 6291eb285..7308b38bd 100644 --- a/client/firewall/uspfilter/forwarder/forwarder.go +++ b/client/firewall/uspfilter/forwarder/forwarder.go @@ -5,11 +5,14 @@ import ( "fmt" "net" "net/netip" + "os" "runtime" + "strconv" "sync" "time" log "github.com/sirupsen/logrus" + wgdevice "golang.zx2c4.com/wireguard/device" "gvisor.dev/gvisor/pkg/buffer" "gvisor.dev/gvisor/pkg/tcpip" "gvisor.dev/gvisor/pkg/tcpip/header" @@ -20,9 +23,9 @@ import ( "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" "gvisor.dev/gvisor/pkg/tcpip/transport/udp" - "github.com/netbirdio/netbird/client/firewall/uspfilter/common" "github.com/netbirdio/netbird/client/firewall/uspfilter/conntrack" nblog "github.com/netbirdio/netbird/client/firewall/uspfilter/log" + "github.com/netbirdio/netbird/client/iface/wgaddr" nftypes "github.com/netbirdio/netbird/client/internal/netflow/types" ) @@ -31,8 +34,19 @@ const ( defaultMaxInFlight = 1024 iosReceiveWindow = 16384 iosMaxInFlight = 256 + + // envForceTCPRACK overrides the platform default for gVisor's RACK loss + // detection. Set to a truthy value to force RACK on, or a falsy value to + // force it off, on any platform. + envForceTCPRACK = "NB_FORCE_TCP_RACK" ) +// IFace provides the WireGuard device and overlay addresses the forwarder needs. +type IFace interface { + GetWGDevice() *wgdevice.Device + Address() wgaddr.Address +} + type Forwarder struct { logger *nblog.Logger flowLogger nftypes.FlowLogger @@ -51,7 +65,7 @@ type Forwarder struct { pingSemaphore chan struct{} } -func New(iface common.IFaceMapper, logger *nblog.Logger, flowLogger nftypes.FlowLogger, netstack bool, mtu uint16) (*Forwarder, error) { +func New(iface IFace, logger *nblog.Logger, flowLogger nftypes.FlowLogger, netstack bool, mtu uint16) (*Forwarder, error) { s := stack.New(stack.Options{ NetworkProtocols: []stack.NetworkProtocolFactory{ ipv4.NewProtocol, @@ -152,6 +166,8 @@ func New(iface common.IFaceMapper, logger *nblog.Logger, flowLogger nftypes.Flow maxInFlight = iosMaxInFlight } + configureTCPRecovery(s) + tcpForwarder := tcp.NewForwarder(s, receiveWindow, maxInFlight, f.handleTCP) s.SetTransportProtocolHandler(tcp.ProtocolNumber, tcpForwarder.HandlePacket) @@ -466,3 +482,31 @@ func probeRawICMP(network, addr string, logger *nblog.Logger) bool { logger.Debug1("forwarder: raw %s socket access available", network) return true } + +// configureTCPRecovery disables gVisor's RACK loss detection on Windows, where +// it interacts poorly with the host and collapses throughput on routed TCP +// connections (gVisor issue #9778). Other platforms keep the default. The +// EnvForceTCPRACK environment variable overrides the platform default. +func configureTCPRecovery(s *stack.Stack) { + disableRACK := runtime.GOOS == "windows" + + if val := os.Getenv(envForceTCPRACK); val != "" { + force, err := strconv.ParseBool(val) + if err != nil { + log.Warnf("parse %s: %v", envForceTCPRACK, err) + } else { + disableRACK = !force + } + } + + if !disableRACK { + return + } + + opt := tcpip.TCPRecovery(0) + if err := s.SetTransportProtocolOption(tcp.ProtocolNumber, &opt); err != nil { + log.Warnf("disable TCP RACK loss detection: %v", err) + return + } + log.Info("forwarder: TCP RACK loss detection disabled") +} diff --git a/client/firewall/uspfilter/fragment.go b/client/firewall/uspfilter/fragment.go new file mode 100644 index 000000000..accc54365 --- /dev/null +++ b/client/firewall/uspfilter/fragment.go @@ -0,0 +1,204 @@ +package uspfilter + +import ( + "context" + "net/netip" + "os" + "strconv" + "sync" + "time" + + nblog "github.com/netbirdio/netbird/client/firewall/uspfilter/log" +) + +const ( + // defaultFragmentTimeout bounds how long a first-fragment verdict is kept + // while the remaining fragments arrive. It mirrors the Linux IP reassembly + // timeout (net.ipv4.ipfrag_time). + defaultFragmentTimeout = 30 * time.Second + // fragmentCleanupInterval is how often expired verdicts are purged. + fragmentCleanupInterval = 10 * time.Second + // defaultMaxFragmentEntries caps the number of concurrently tracked + // fragmented datagrams. The table stays bounded because each datagram is a + // single small entry regardless of how many fragments it is split into, and + // the 13-bit IPv4 fragment-offset field limits any datagram to 64 KiB. + defaultMaxFragmentEntries = 16384 + + // EnvFragmentMaxEntries overrides defaultMaxFragmentEntries. + EnvFragmentMaxEntries = "NB_FRAGMENT_MAX_ENTRIES" +) + +// fragmentVerdict is the decision for a trailing (headerless) fragment. +type fragmentVerdict int + +const ( + // fragmentDeny drops the fragment: no allowed first fragment is on record. + fragmentDeny fragmentVerdict = iota + // fragmentAllow passes the fragment: it belongs to an allowed datagram and + // does not overlap the already-inspected transport header. + fragmentAllow + // fragmentOverlap drops the fragment and poisons its datagram: it overlaps + // the transport header the ACL inspected (RFC 1858 §4, RFC 3128; RFC 5722 + // requires discarding the whole datagram on overlap for IPv6). + fragmentOverlap +) + +// fragmentKey identifies a fragmented datagram. It matches the RFC 791 / RFC +// 8200 reassembly key: source, destination, protocol and identification. The id +// is 32-bit to hold both the IPv4 (16-bit) and IPv6 (32-bit) identification. +type fragmentKey struct { + srcIP netip.Addr + dstIP netip.Addr + id uint32 + proto uint8 +} + +// fragmentEntry records the verdict of an allowed first fragment. +type fragmentEntry struct { + // headerEndOctets is the offset, in 8-byte units, at which the first + // fragment's payload ended. A trailing fragment starting before this + // overlaps bytes the ACL already inspected and is rejected. + headerEndOctets uint16 + // recordedAt is when the first fragment was accepted. The verdict expires a + // fixed timeout later and is not refreshed, mirroring the kernel reassembly + // timer so a trailing-fragment flood can't keep a datagram alive. + recordedAt time.Time +} + +// fragmentTracker records the ACL verdict of a datagram's first fragment so the +// remaining fragments, which carry no L4 header, can inherit the decision +// without reassembling the datagram. Only allowed first fragments are stored; +// anything that cannot be tied to an allowed, non-overlapping first fragment is +// dropped (fail closed). +type fragmentTracker struct { + logger *nblog.Logger + mutex sync.Mutex + entries map[fragmentKey]fragmentEntry + timeout time.Duration + // maxEntries caps the table; atCapacity dedups the capacity warning until + // the table drains below the cap again. + maxEntries int + atCapacity bool + cleanupTicker *time.Ticker + cancel context.CancelFunc +} + +func newFragmentTracker(logger *nblog.Logger) *fragmentTracker { + ctx, cancel := context.WithCancel(context.Background()) + t := &fragmentTracker{ + logger: logger, + entries: make(map[fragmentKey]fragmentEntry), + timeout: defaultFragmentTimeout, + maxEntries: fragmentMaxEntries(logger), + cleanupTicker: time.NewTicker(fragmentCleanupInterval), + cancel: cancel, + } + go t.cleanupRoutine(ctx) + return t +} + +func fragmentMaxEntries(logger *nblog.Logger) int { + v := os.Getenv(EnvFragmentMaxEntries) + if v == "" { + return defaultMaxFragmentEntries + } + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + logger.Warn2("invalid %s=%q, using default", EnvFragmentMaxEntries, v) + return defaultMaxFragmentEntries + } + return n +} + +// recordAllowed stores the verdict of an allowed first fragment. headerEndOctets +// is the first fragment's payload length in 8-byte units. When the table is full +// the record is dropped, which fails closed: the datagram's trailing fragments +// will be denied. +func (t *fragmentTracker) recordAllowed(key fragmentKey, headerEndOctets uint16) { + t.mutex.Lock() + defer t.mutex.Unlock() + + if t.entries == nil { + return + } + if _, ok := t.entries[key]; !ok && len(t.entries) >= t.maxEntries { + if !t.atCapacity { + t.atCapacity = true + t.logger.Warn2("fragment verdict table at capacity (%d/%d): trailing fragments of new datagrams will be dropped", + len(t.entries), t.maxEntries) + } + return + } + t.entries[key] = fragmentEntry{ + headerEndOctets: headerEndOctets, + recordedAt: time.Now(), + } +} + +// poison drops any recorded verdict for a datagram, so its later fragments are +// denied until a new allowed first fragment is recorded. Called on every +// offset-zero fragment to defeat offset-zero overlap rewrites (RFC 3128). +func (t *fragmentTracker) poison(key fragmentKey) { + t.mutex.Lock() + defer t.mutex.Unlock() + delete(t.entries, key) +} + +// verdict decides the fate of a trailing fragment at fragOffsetOctets (the IPv4 +// fragment offset, in 8-byte units). A fragment overlapping the inspected +// header poisons the datagram: the entry is removed so all further fragments of +// that datagram are denied too. +func (t *fragmentTracker) verdict(key fragmentKey, fragOffsetOctets uint16) fragmentVerdict { + t.mutex.Lock() + defer t.mutex.Unlock() + + entry, ok := t.entries[key] + if !ok { + return fragmentDeny + } + if time.Since(entry.recordedAt) > t.timeout { + delete(t.entries, key) + return fragmentDeny + } + if fragOffsetOctets < entry.headerEndOctets { + delete(t.entries, key) + return fragmentOverlap + } + return fragmentAllow +} + +func (t *fragmentTracker) cleanupRoutine(ctx context.Context) { + defer t.cleanupTicker.Stop() + for { + select { + case <-t.cleanupTicker.C: + t.cleanup() + case <-ctx.Done(): + return + } + } +} + +func (t *fragmentTracker) cleanup() { + t.mutex.Lock() + defer t.mutex.Unlock() + + for key, entry := range t.entries { + if time.Since(entry.recordedAt) > t.timeout { + delete(t.entries, key) + } + } + + if len(t.entries) < t.maxEntries { + t.atCapacity = false + } +} + +// Close stops the cleanup routine and releases resources. +func (t *fragmentTracker) Close() { + t.cancel() + + t.mutex.Lock() + t.entries = nil + t.mutex.Unlock() +} diff --git a/client/firewall/uspfilter/fragment_bench_test.go b/client/firewall/uspfilter/fragment_bench_test.go new file mode 100644 index 000000000..a9e6d2d13 --- /dev/null +++ b/client/firewall/uspfilter/fragment_bench_test.go @@ -0,0 +1,115 @@ +package uspfilter + +import ( + "encoding/binary" + "testing" +) + +// benchFilterInbound drives filterInbound over a fixed packet in a tight loop. +// Packets are built once, outside the timed region, so the benchmark measures +// only pipeline cost, which is what an attacker can amplify. +func benchFilterInbound(b *testing.B, pkt []byte) { + b.Helper() + b.ReportAllocs() + b.SetBytes(int64(len(pkt))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + m := benchManager + m.filterInbound(pkt, len(pkt)) + } +} + +// benchManager is a package-level manager reused across fragment benchmarks so +// setup cost stays out of the timed region. +var benchManager *Manager + +func setupBenchManager(b *testing.B) *Manager { + b.Helper() + m := newFragmentTestManager(b) + allowUDP(b, m, 8080) + // Disable conntrack so the allowed-first-fragment path measures transport + // decode + ACL every iteration instead of matching the connection tracked + // on the first iteration. + m.stateful = false + benchManager = m + return m +} + +// BenchmarkInbound_NormalPacket is the baseline: a full, non-fragmented UDP +// packet that passes the ACL. Fragment paths should stay comparable to this. +func BenchmarkInbound_NormalPacket(b *testing.B) { + setupBenchManager(b) + pkt := normalUDPPacket(b, 8080, 32) + benchFilterInbound(b, pkt) +} + +// BenchmarkInbound_FirstFragmentAllowed measures the first-fragment path: +// transport decode + ACL evaluation + verdict record. +func BenchmarkInbound_FirstFragmentAllowed(b *testing.B) { + setupBenchManager(b) + pkt := firstFragmentUDP(b, 0x2000, 8080, 32) + benchFilterInbound(b, pkt) +} + +// BenchmarkInbound_TrailingFragmentAllowed measures the common trailing-fragment +// path: a single map lookup after the first fragment is on record. +func BenchmarkInbound_TrailingFragmentAllowed(b *testing.B) { + m := setupBenchManager(b) + first := firstFragmentUDP(b, 0x3000, 8080, 32) + m.filterInbound(first, len(first)) + pkt := trailingFragment(b, 0x3000, 5, false, 24) + benchFilterInbound(b, pkt) +} + +// BenchmarkInbound_TrailingFragmentNoFirst is the primary DoS vector: an +// attacker floods trailing fragments with no first fragment on record. Each is +// a map miss and must be cheap. +func BenchmarkInbound_TrailingFragmentNoFirst(b *testing.B) { + setupBenchManager(b) + pkt := trailingFragment(b, 0x4000, 185, false, 40) + benchFilterInbound(b, pkt) +} + +// BenchmarkInbound_TinyFirstFragment measures the tiny-fragment drop path: a +// first fragment too small to decode a transport header. +func BenchmarkInbound_TinyFirstFragment(b *testing.B) { + setupBenchManager(b) + pkt := trailingFragment(b, 0x5000, 0, true, 4) + benchFilterInbound(b, pkt) +} + +// BenchmarkInbound_TrailingFragmentDistinctIDs is the worst case for the +// verdict table: an attacker varies the datagram id on every packet so no first +// fragment ever matches. Verdict lookups always miss and nothing is recorded, +// so the table cannot grow. Each iteration rewrites the id field in place. +func BenchmarkInbound_TrailingFragmentDistinctIDs(b *testing.B) { + setupBenchManager(b) + pkt := trailingFragment(b, 0x6000, 185, false, 40) + m := benchManager + + b.ReportAllocs() + b.SetBytes(int64(len(pkt))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + // IPv4 identification field is at bytes 4:6. + binary.BigEndian.PutUint16(pkt[4:6], uint16(i)) + m.filterInbound(pkt, len(pkt)) + } +} + +// BenchmarkInbound_FirstFragmentDistinctIDs measures sustained first-fragment +// pressure with distinct ids: transport decode + ACL + verdict insert until the +// table caps, exercising the map growth and capacity guard. +func BenchmarkInbound_FirstFragmentDistinctIDs(b *testing.B) { + setupBenchManager(b) + pkt := firstFragmentUDP(b, 0x7000, 8080, 32) + m := benchManager + + b.ReportAllocs() + b.SetBytes(int64(len(pkt))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + binary.BigEndian.PutUint16(pkt[4:6], uint16(i)) + m.filterInbound(pkt, len(pkt)) + } +} diff --git a/client/firewall/uspfilter/fragment_test.go b/client/firewall/uspfilter/fragment_test.go new file mode 100644 index 000000000..8f99864be --- /dev/null +++ b/client/firewall/uspfilter/fragment_test.go @@ -0,0 +1,554 @@ +package uspfilter + +import ( + "encoding/binary" + "net" + "net/netip" + "testing" + "time" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/stretchr/testify/require" + + fw "github.com/netbirdio/netbird/client/firewall/manager" + nbiface "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/device" + "github.com/netbirdio/netbird/client/iface/wgaddr" +) + +const ( + fragTestSrc = "100.10.0.1" + fragTestDst = "100.10.0.100" + fragTestSrcV6 = "fd00::1" + fragTestDstV6 = "fd00::100" +) + +func newFragmentTestManager(tb testing.TB) *Manager { + tb.Helper() + + ifaceMock := &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + AddressFunc: func() wgaddr.Address { + return wgaddr.Address{ + IP: netip.MustParseAddr(fragTestDst), + Network: netip.MustParsePrefix("100.10.0.0/16"), + IPv6: netip.MustParseAddr(fragTestDstV6), + IPv6Net: netip.MustParsePrefix("fd00::/64"), + } + }, + } + + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) + require.NoError(tb, err) + require.NoError(tb, m.UpdateLocalIPs()) + tb.Cleanup(func() { require.NoError(tb, m.Close(nil)) }) + return m +} + +// firstFragmentUDPTo builds the first fragment of a fragmented UDP datagram to +// the given destination: it carries the full UDP header plus payloadLen bytes +// of data, with the More Fragments flag set and offset zero. +func firstFragmentUDPTo(tb testing.TB, dst string, id uint16, dstPort uint16, payloadLen int) []byte { + tb.Helper() + + ip := &layers.IPv4{ + Version: 4, + TTL: 64, + Id: id, + Protocol: layers.IPProtocolUDP, + SrcIP: net.ParseIP(fragTestSrc), + DstIP: net.ParseIP(dst), + Flags: layers.IPv4MoreFragments, + } + udp := &layers.UDP{SrcPort: 40000, DstPort: layers.UDPPort(dstPort)} + require.NoError(tb, udp.SetNetworkLayerForChecksum(ip)) + + buf := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{ComputeChecksums: true, FixLengths: true} + require.NoError(tb, gopacket.SerializeLayers(buf, opts, ip, udp, gopacket.Payload(make([]byte, payloadLen)))) + return buf.Bytes() +} + +func firstFragmentUDP(tb testing.TB, id uint16, dstPort uint16, payloadLen int) []byte { + tb.Helper() + return firstFragmentUDPTo(tb, fragTestDst, id, dstPort, payloadLen) +} + +// firstFragmentTCP builds the first fragment of a fragmented TCP datagram: the +// full 20-byte TCP header plus 12 bytes of data, with the More Fragments flag +// set and offset zero. +func firstFragmentTCP(tb testing.TB, id uint16, dstPort uint16) []byte { + tb.Helper() + + ip := &layers.IPv4{ + Version: 4, + TTL: 64, + Id: id, + Protocol: layers.IPProtocolTCP, + SrcIP: net.ParseIP(fragTestSrc), + DstIP: net.ParseIP(fragTestDst), + Flags: layers.IPv4MoreFragments, + } + tcp := &layers.TCP{SrcPort: 40000, DstPort: layers.TCPPort(dstPort), SYN: true, Window: 64240} + require.NoError(tb, tcp.SetNetworkLayerForChecksum(ip)) + + buf := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{ComputeChecksums: true, FixLengths: true} + require.NoError(tb, gopacket.SerializeLayers(buf, opts, ip, tcp, gopacket.Payload(make([]byte, 12)))) + return buf.Bytes() +} + +// trailingFragmentTo builds a non-first fragment to the given destination: an +// IPv4 header at the given fragment offset (in 8-byte units) carrying raw +// payload and no L4 header. +func trailingFragmentTo(tb testing.TB, dst string, proto layers.IPProtocol, id uint16, fragOffsetOctets uint16, moreFragments bool, payloadLen int) []byte { + tb.Helper() + + ip := &layers.IPv4{ + Version: 4, + TTL: 64, + Id: id, + Protocol: proto, + SrcIP: net.ParseIP(fragTestSrc), + DstIP: net.ParseIP(dst), + FragOffset: fragOffsetOctets, + } + if moreFragments { + ip.Flags = layers.IPv4MoreFragments + } + + buf := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{FixLengths: true} + require.NoError(tb, gopacket.SerializeLayers(buf, opts, ip, gopacket.Payload(make([]byte, payloadLen)))) + return buf.Bytes() +} + +func trailingFragment(tb testing.TB, id uint16, fragOffsetOctets uint16, moreFragments bool, payloadLen int) []byte { + tb.Helper() + return trailingFragmentTo(tb, fragTestDst, layers.IPProtocolUDP, id, fragOffsetOctets, moreFragments, payloadLen) +} + +// outboundUDPPacket builds a complete outbound UDP packet from the local +// address, used to establish conntrack state for reply-direction tests. +func outboundUDPPacket(tb testing.TB, srcPort, dstPort uint16) []byte { + tb.Helper() + + ip := &layers.IPv4{ + Version: 4, + TTL: 64, + Id: 1, + Protocol: layers.IPProtocolUDP, + SrcIP: net.ParseIP(fragTestDst), + DstIP: net.ParseIP(fragTestSrc), + } + udp := &layers.UDP{SrcPort: layers.UDPPort(srcPort), DstPort: layers.UDPPort(dstPort)} + require.NoError(tb, udp.SetNetworkLayerForChecksum(ip)) + + buf := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{ComputeChecksums: true, FixLengths: true} + require.NoError(tb, gopacket.SerializeLayers(buf, opts, ip, udp, gopacket.Payload(make([]byte, 16)))) + return buf.Bytes() +} + +// normalUDPPacket builds a complete, non-fragmented UDP packet for baseline +// comparisons against the fragment paths. +func normalUDPPacket(tb testing.TB, dstPort uint16, payloadLen int) []byte { + tb.Helper() + + ip := &layers.IPv4{ + Version: 4, + TTL: 64, + Id: 1, + Protocol: layers.IPProtocolUDP, + SrcIP: net.ParseIP(fragTestSrc), + DstIP: net.ParseIP(fragTestDst), + } + udp := &layers.UDP{SrcPort: 40000, DstPort: layers.UDPPort(dstPort)} + require.NoError(tb, udp.SetNetworkLayerForChecksum(ip)) + + buf := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{ComputeChecksums: true, FixLengths: true} + require.NoError(tb, gopacket.SerializeLayers(buf, opts, ip, udp, gopacket.Payload(make([]byte, payloadLen)))) + return buf.Bytes() +} + +func allowUDP(tb testing.TB, m *Manager, dstPort uint16) { + tb.Helper() + _, err := m.AddFilterRule(nil, pfx(net.ParseIP(fragTestSrc)), fw.Network{}, fw.ProtocolUDP, nil, + &fw.Port{Values: []uint16{dstPort}}, fw.ActionAccept) + require.NoError(tb, err) +} + +// TestFragment_TrailingWithoutFirstDropped is the core bypass repro: a trailing +// fragment with no allowed first fragment on record must be dropped. Before the +// fix, filterInbound returned false (allow) for any fragment. +func TestFragment_TrailingWithoutFirstDropped(t *testing.T) { + m := newFragmentTestManager(t) + + frag := trailingFragment(t, 0x1234, 185, false, 40) + require.True(t, m.filterInbound(frag, len(frag)), + "trailing fragment without an allowed first fragment must be dropped") +} + +// TestFragment_AllowedFirstPassesTrailing verifies that once a first fragment +// passes the ACL, its trailing fragments inherit the allow verdict. +func TestFragment_AllowedFirstPassesTrailing(t *testing.T) { + m := newFragmentTestManager(t) + allowUDP(t, m, 8080) + + // First fragment: UDP header (8) + 32 payload = 40 octets -> headerEnd = 5. + first := firstFragmentUDP(t, 0x2222, 8080, 32) + require.False(t, m.filterInbound(first, len(first)), + "allowed first fragment should pass and be recorded") + + trailing := trailingFragment(t, 0x2222, 5, false, 24) + require.False(t, m.filterInbound(trailing, len(trailing)), + "trailing fragment of an allowed datagram should pass") +} + +// TestFragment_DeniedFirstDropsTrailing verifies that a first fragment blocked +// by the ACL leaves no verdict, so its trailing fragments are dropped. +func TestFragment_DeniedFirstDropsTrailing(t *testing.T) { + m := newFragmentTestManager(t) + // No accept rule: local traffic defaults to deny. + + first := firstFragmentUDP(t, 0x3333, 9999, 32) + require.True(t, m.filterInbound(first, len(first)), + "first fragment to a blocked port should be dropped by the ACL") + + trailing := trailingFragment(t, 0x3333, 5, false, 24) + require.True(t, m.filterInbound(trailing, len(trailing)), + "trailing fragment of a denied datagram must be dropped") +} + +// TestFragment_OverlappingHeaderDropped covers the RFC 1858 §4 / RFC 3128 +// overlapping-fragment rewrite: a trailing fragment starting inside the range +// the ACL already inspected is dropped and poisons the datagram. TCP is used so +// the overlap lands on real header bytes (the flags at byte 13). +func TestFragment_OverlappingHeaderDropped(t *testing.T) { + m := newFragmentTestManager(t) + _, err := m.AddFilterRule(nil, pfx(net.ParseIP(fragTestSrc)), fw.Network{}, fw.ProtocolTCP, nil, + &fw.Port{Values: []uint16{8080}}, fw.ActionAccept) + require.NoError(t, err) + + // First fragment: TCP header (20) + 12 data = 32 bytes -> headerEnd = 4 octets. + first := firstFragmentTCP(t, 0x4444, 8080) + require.False(t, m.filterInbound(first, len(first))) + + // Overlapping fragment at offset 1 (byte 8) falls inside the inspected TCP + // header, so it could rewrite the flags or port on reassembly. + overlap := trailingFragmentTo(t, fragTestDst, layers.IPProtocolTCP, 0x4444, 1, true, 32) + require.True(t, m.filterInbound(overlap, len(overlap)), + "fragment overlapping the inspected header must be dropped") + + // The datagram is now poisoned: a later, non-overlapping fragment is also + // dropped because the verdict was removed. + later := trailingFragmentTo(t, fragTestDst, layers.IPProtocolTCP, 0x4444, 4, false, 24) + require.True(t, m.filterInbound(later, len(later)), + "fragments after an overlap must be dropped (datagram poisoned)") +} + +// TestFragment_OffsetZeroOverlapPoisons covers the RFC 3128 offset-zero rewrite: +// an allowed first fragment followed by a denied offset-zero fragment for the +// same datagram must not leave the earlier allow verdict in place. +func TestFragment_OffsetZeroOverlapPoisons(t *testing.T) { + m := newFragmentTestManager(t) + allowUDP(t, m, 8080) + + allowed := firstFragmentUDP(t, 0x5A5A, 8080, 32) + require.False(t, m.filterInbound(allowed, len(allowed)), + "allowed first fragment should pass and be recorded") + + // A second offset-zero fragment to a denied port supersedes the datagram's + // verdict; it is dropped and must not leave the allow in place. + denied := firstFragmentUDP(t, 0x5A5A, 9999, 32) + require.True(t, m.filterInbound(denied, len(denied)), + "denied offset-zero fragment must be dropped") + + trailing := trailingFragment(t, 0x5A5A, 5, false, 24) + require.True(t, m.filterInbound(trailing, len(trailing)), + "trailing fragment must be denied after the datagram was poisoned") +} + +// TestFragment_TinyFirstDropped covers the tiny-fragment attack: a first +// fragment too small to contain the full transport header can't be +// ACL-evaluated and must be dropped. +func TestFragment_TinyFirstDropped(t *testing.T) { + m := newFragmentTestManager(t) + allowUDP(t, m, 8080) + + // IPv4 header + 4 raw bytes, MF set, offset 0: too small for the 8-byte UDP + // header, so it decodes to L3 only. + tiny := trailingFragment(t, 0x5555, 0, true, 4) + require.True(t, m.filterInbound(tiny, len(tiny)), + "tiny first fragment without a full L4 header must be dropped") +} + +// TestFragment_TCPFirstFragment verifies the TCP arm of the transport decode: a +// first fragment carrying the full 20-byte TCP header is ACL-evaluated and its +// trailing fragments inherit the verdict. +func TestFragment_TCPFirstFragment(t *testing.T) { + m := newFragmentTestManager(t) + _, err := m.AddFilterRule(nil, pfx(net.ParseIP(fragTestSrc)), fw.Network{}, fw.ProtocolTCP, nil, + &fw.Port{Values: []uint16{8080}}, fw.ActionAccept) + require.NoError(t, err) + + // TCP header (20) + 12 data = 32 bytes -> headerEnd = 4 octets. + first := firstFragmentTCP(t, 0x6666, 8080) + require.False(t, m.filterInbound(first, len(first)), + "allowed TCP first fragment should pass and be recorded") + + trailing := trailingFragmentTo(t, fragTestDst, layers.IPProtocolTCP, 0x6666, 4, false, 24) + require.False(t, m.filterInbound(trailing, len(trailing)), + "trailing fragment of an allowed TCP datagram should pass") +} + +// TestFragment_TCPTinyFirstDropped verifies the TCP minimum header length: 12 +// bytes would satisfy a UDP header but falls short of the 20-byte TCP header. +func TestFragment_TCPTinyFirstDropped(t *testing.T) { + m := newFragmentTestManager(t) + _, err := m.AddFilterRule(nil, pfx(net.ParseIP(fragTestSrc)), fw.Network{}, fw.ProtocolTCP, nil, + &fw.Port{Values: []uint16{8080}}, fw.ActionAccept) + require.NoError(t, err) + + tiny := trailingFragmentTo(t, fragTestDst, layers.IPProtocolTCP, 0x7777, 0, true, 12) + require.True(t, m.filterInbound(tiny, len(tiny)), + "first fragment shorter than the TCP header must be dropped") +} + +// TestFragment_ConntrackAllowsFirstFragment verifies the conntrack branch: reply +// fragments of an outbound-established UDP flow pass without any inbound rule. +func TestFragment_ConntrackAllowsFirstFragment(t *testing.T) { + m := newFragmentTestManager(t) + + out := outboundUDPPacket(t, 12345, 40000) + require.False(t, m.filterOutbound(out, len(out))) + + first := firstFragmentUDP(t, 0x8888, 12345, 32) + require.False(t, m.filterInbound(first, len(first)), + "reply first fragment should pass via conntrack") + + trailing := trailingFragment(t, 0x8888, 5, false, 24) + require.False(t, m.filterInbound(trailing, len(trailing)), + "trailing fragment of a tracked flow should pass") +} + +// TestFragment_RoutingDisabledDropsFragment verifies routed first fragments are +// dropped when routing is disabled. +func TestFragment_RoutingDisabledDropsFragment(t *testing.T) { + m := newFragmentTestManager(t) + m.routingEnabled.Store(false) + + first := firstFragmentUDPTo(t, "198.51.100.10", 0x9999, 8080, 32) + require.True(t, m.filterInbound(first, len(first)), + "routed first fragment must be dropped when routing is disabled") +} + +// TestFragment_RouteACL verifies the route-ACL branch: fragments to a non-local +// destination follow the route rules, allowed datagrams pass their trailing +// fragments and denied ones don't. +func TestFragment_RouteACL(t *testing.T) { + m := newFragmentTestManager(t) + m.routingEnabled.Store(true) + m.nativeRouter.Store(false) + + _, err := m.AddFilterRule( + []byte("rt-1"), + []netip.Prefix{netip.MustParsePrefix("100.10.0.0/16")}, + fw.Network{Prefix: netip.MustParsePrefix("198.51.100.0/24")}, + fw.ProtocolUDP, + nil, + &fw.Port{Values: []uint16{8080}}, + fw.ActionAccept, + ) + require.NoError(t, err) + + first := firstFragmentUDPTo(t, "198.51.100.10", 0xAAAA, 8080, 32) + require.False(t, m.filterInbound(first, len(first)), + "route-ACL-allowed first fragment should pass") + trailing := trailingFragmentTo(t, "198.51.100.10", layers.IPProtocolUDP, 0xAAAA, 5, false, 24) + require.False(t, m.filterInbound(trailing, len(trailing)), + "trailing fragment of an allowed routed datagram should pass") + + denied := firstFragmentUDPTo(t, "198.51.100.10", 0xBBBB, 9999, 32) + require.True(t, m.filterInbound(denied, len(denied)), + "route-ACL-denied first fragment must be dropped") + deniedTrailing := trailingFragmentTo(t, "198.51.100.10", layers.IPProtocolUDP, 0xBBBB, 5, false, 24) + require.True(t, m.filterInbound(deniedTrailing, len(deniedTrailing)), + "trailing fragment of a denied routed datagram must be dropped") +} + +// TestFragment_ExpiredVerdictDropsTrailing verifies a verdict older than the +// tracker timeout no longer admits trailing fragments. +func TestFragment_ExpiredVerdictDropsTrailing(t *testing.T) { + m := newFragmentTestManager(t) + allowUDP(t, m, 8080) + + first := firstFragmentUDP(t, 0xCCCC, 8080, 32) + require.False(t, m.filterInbound(first, len(first))) + + m.fragments.mutex.Lock() + for key, entry := range m.fragments.entries { + entry.recordedAt = time.Now().Add(-defaultFragmentTimeout - time.Second) + m.fragments.entries[key] = entry + } + m.fragments.mutex.Unlock() + + trailing := trailingFragment(t, 0xCCCC, 5, false, 24) + require.True(t, m.filterInbound(trailing, len(trailing)), + "trailing fragment after verdict expiry must be dropped") +} + +// TestFragment_CapacityFailsClosed verifies the table cap: at capacity, new +// datagram verdicts are not recorded (their trailing fragments are dropped) +// while already-recorded datagrams keep working. +func TestFragment_CapacityFailsClosed(t *testing.T) { + m := newFragmentTestManager(t) + allowUDP(t, m, 8080) + + m.fragments.mutex.Lock() + m.fragments.maxEntries = 1 + m.fragments.mutex.Unlock() + + first1 := firstFragmentUDP(t, 0x0101, 8080, 32) + require.False(t, m.filterInbound(first1, len(first1))) + + first2 := firstFragmentUDP(t, 0x0202, 8080, 32) + require.False(t, m.filterInbound(first2, len(first2)), + "first fragment itself still passes at capacity") + + trailing2 := trailingFragment(t, 0x0202, 5, false, 24) + require.True(t, m.filterInbound(trailing2, len(trailing2)), + "trailing fragment of an unrecorded datagram must be dropped at capacity") + + trailing1 := trailingFragment(t, 0x0101, 5, false, 24) + require.False(t, m.filterInbound(trailing1, len(trailing1)), + "already-recorded datagram should keep passing at capacity") +} + +// v6FragmentHeader builds the 8-byte IPv6 fragment extension header for the +// given inner protocol, offset (8-byte units), More Fragments bit and id. +func v6FragmentHeader(proto layers.IPProtocol, offsetOctets uint16, moreFragments bool, id uint32) []byte { + offsetFlags := offsetOctets << 3 + if moreFragments { + offsetFlags |= 1 + } + hdr := make([]byte, 8) + hdr[0] = uint8(proto) + binary.BigEndian.PutUint16(hdr[2:4], offsetFlags) + binary.BigEndian.PutUint32(hdr[4:8], id) + return hdr +} + +func v6UDPHeader(dstPort uint16, dataLen int) []byte { + hdr := make([]byte, 8) + binary.BigEndian.PutUint16(hdr[0:2], 40000) + binary.BigEndian.PutUint16(hdr[2:4], dstPort) + binary.BigEndian.PutUint16(hdr[4:6], uint16(8+dataLen)) + return hdr +} + +// firstFragmentUDPv6 builds the first fragment of a fragmented IPv6 UDP +// datagram: fragment header (offset 0, More Fragments set) + full UDP header + +// data. +func firstFragmentUDPv6(tb testing.TB, id uint32, dstPort uint16, dataLen int) []byte { + tb.Helper() + return fragmentUDPv6(tb, id, dstPort, dataLen, true) +} + +// fragmentUDPv6 builds an offset-zero IPv6 UDP fragment. With moreFragments +// false it is an atomic fragment (a complete datagram, RFC 6946). +func fragmentUDPv6(tb testing.TB, id uint32, dstPort uint16, dataLen int, moreFragments bool) []byte { + tb.Helper() + + ip := &layers.IPv6{ + Version: 6, + NextHeader: layers.IPProtocolIPv6Fragment, + HopLimit: 64, + SrcIP: net.ParseIP(fragTestSrcV6), + DstIP: net.ParseIP(fragTestDstV6), + } + payload := append(v6FragmentHeader(layers.IPProtocolUDP, 0, moreFragments, id), v6UDPHeader(dstPort, dataLen)...) + payload = append(payload, make([]byte, dataLen)...) + + buf := gopacket.NewSerializeBuffer() + require.NoError(tb, gopacket.SerializeLayers(buf, gopacket.SerializeOptions{FixLengths: true}, ip, gopacket.Payload(payload))) + return buf.Bytes() +} + +// trailingFragmentV6 builds a non-first IPv6 fragment: fragment header at the +// given offset carrying raw data and no transport header. +func trailingFragmentV6(tb testing.TB, id uint32, offsetOctets uint16, moreFragments bool, dataLen int) []byte { + tb.Helper() + + ip := &layers.IPv6{ + Version: 6, + NextHeader: layers.IPProtocolIPv6Fragment, + HopLimit: 64, + SrcIP: net.ParseIP(fragTestSrcV6), + DstIP: net.ParseIP(fragTestDstV6), + } + payload := append(v6FragmentHeader(layers.IPProtocolUDP, offsetOctets, moreFragments, id), make([]byte, dataLen)...) + + buf := gopacket.NewSerializeBuffer() + require.NoError(tb, gopacket.SerializeLayers(buf, gopacket.SerializeOptions{FixLengths: true}, ip, gopacket.Payload(payload))) + return buf.Bytes() +} + +// TestFragmentV6_TrailingWithoutFirstDropped verifies the IPv6 bypass is closed: +// a trailing fragment with no allowed first fragment is dropped. +func TestFragmentV6_TrailingWithoutFirstDropped(t *testing.T) { + m := newFragmentTestManager(t) + + frag := trailingFragmentV6(t, 0xAABBCCDD, 100, false, 40) + require.True(t, m.filterInbound(frag, len(frag)), + "IPv6 trailing fragment without an allowed first fragment must be dropped") +} + +// TestFragmentV6_AllowedFirstPassesTrailing verifies IPv6 fragments are +// evaluated like IPv4: an allowed first fragment lets its trailing fragments +// through. +func TestFragmentV6_AllowedFirstPassesTrailing(t *testing.T) { + m := newFragmentTestManager(t) + _, err := m.AddFilterRule(nil, pfx(net.ParseIP(fragTestSrcV6)), fw.Network{}, fw.ProtocolUDP, nil, + &fw.Port{Values: []uint16{8080}}, fw.ActionAccept) + require.NoError(t, err) + + // First fragment: UDP header (8) + 32 data = 40 octets -> headerEnd = 5. + first := firstFragmentUDPv6(t, 0xAABBCCDD, 8080, 32) + require.False(t, m.filterInbound(first, len(first)), + "allowed IPv6 first fragment should pass and be recorded") + + trailing := trailingFragmentV6(t, 0xAABBCCDD, 5, false, 24) + require.False(t, m.filterInbound(trailing, len(trailing)), + "trailing fragment of an allowed IPv6 datagram should pass") +} + +// TestFragmentV6_AtomicNotCached verifies an IPv6 atomic fragment (fragment +// header with offset 0 and no More Fragments, a complete datagram per RFC 6946) +// is evaluated but not recorded, so a flood of allowed atomic fragments can't +// exhaust the verdict table. +func TestFragmentV6_AtomicNotCached(t *testing.T) { + m := newFragmentTestManager(t) + _, err := m.AddFilterRule(nil, pfx(net.ParseIP(fragTestSrcV6)), fw.Network{}, fw.ProtocolUDP, nil, + &fw.Port{Values: []uint16{8080}}, fw.ActionAccept) + require.NoError(t, err) + + atomic := fragmentUDPv6(t, 0xA70301C, 8080, 16, false) + require.False(t, m.filterInbound(atomic, len(atomic)), + "allowed IPv6 atomic fragment should pass") + + m.fragments.mutex.Lock() + n := len(m.fragments.entries) + m.fragments.mutex.Unlock() + require.Zero(t, n, "atomic fragment must not create a verdict entry") + + // A genuine fragmented datagram (More Fragments set) is still recorded. + first := fragmentUDPv6(t, 0xBEEF, 8080, 32, true) + require.False(t, m.filterInbound(first, len(first))) + m.fragments.mutex.Lock() + n = len(m.fragments.entries) + m.fragments.mutex.Unlock() + require.Equal(t, 1, n, "genuine first fragment must record a verdict") +} diff --git a/client/firewall/uspfilter/allow_netbird_windows.go b/client/firewall/uspfilter/interface_allower_windows.go similarity index 68% rename from client/firewall/uspfilter/allow_netbird_windows.go rename to client/firewall/uspfilter/interface_allower_windows.go index 10a2b9116..4cd0fe969 100644 --- a/client/firewall/uspfilter/allow_netbird_windows.go +++ b/client/firewall/uspfilter/interface_allower_windows.go @@ -9,7 +9,7 @@ import ( log "github.com/sirupsen/logrus" nberrors "github.com/netbirdio/netbird/client/errors" - "github.com/netbirdio/netbird/client/internal/statemanager" + "github.com/netbirdio/netbird/client/internal/wincmd" ) type action string @@ -20,35 +20,20 @@ const ( firewallRuleName = "Netbird" ) -// Close cleans up the firewall manager by removing all rules and closing trackers -func (m *Manager) Close(*statemanager.Manager) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - m.resetState() - - if !isWindowsFirewallReachable() { - return nil - } - - var merr *multierror.Error - if isFirewallRuleActive(firewallRuleName) { - if err := manageFirewallRule(firewallRuleName, deleteRule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove windows firewall rule: %w", err)) - } - } - - if isFirewallRuleActive(firewallRuleName + "-v6") { - if err := manageFirewallRule(firewallRuleName+"-v6", deleteRule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove windows v6 firewall rule: %w", err)) - } - } - - return nberrors.FormatErrorOrNil(merr) +// WindowsInterfaceAllower opens the NetBird interface in the Windows firewall +// via netsh advfirewall rules. It implements InterfaceAllower for the userspace +// firewall on Windows. +type WindowsInterfaceAllower struct { + iface Iface } -// AllowNetbird allows netbird interface traffic -func (m *Manager) AllowNetbird() error { +// NewWindowsInterfaceAllower builds the Windows netsh-based interface allower. +func NewWindowsInterfaceAllower(iface Iface) *WindowsInterfaceAllower { + return &WindowsInterfaceAllower{iface: iface} +} + +// Apply adds inbound-allow netsh rules for the interface's addresses. +func (a *WindowsInterfaceAllower) Apply() error { if !isWindowsFirewallReachable() { return nil } @@ -60,13 +45,13 @@ func (m *Manager) AllowNetbird() error { "enable=yes", "action=allow", "profile=any", - "localip="+m.wgIface.Address().IP.String(), + "localip="+a.iface.Address().IP.String(), ); err != nil { return err } } - if v6 := m.wgIface.Address().IPv6; v6.IsValid() && !isFirewallRuleActive(firewallRuleName+"-v6") { + if v6 := a.iface.Address().IPv6; v6.IsValid() && !isFirewallRuleActive(firewallRuleName+"-v6") { if err := manageFirewallRule(firewallRuleName+"-v6", addRule, "dir=in", @@ -82,13 +67,32 @@ func (m *Manager) AllowNetbird() error { return nil } -func manageFirewallRule(ruleName string, action action, extraArgs ...string) error { +// Close removes the netsh rules added by Apply. +func (a *WindowsInterfaceAllower) Close() error { + if !isWindowsFirewallReachable() { + return nil + } + var merr *multierror.Error + if isFirewallRuleActive(firewallRuleName) { + if err := manageFirewallRule(firewallRuleName, deleteRule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove windows firewall rule: %w", err)) + } + } + if isFirewallRuleActive(firewallRuleName + "-v6") { + if err := manageFirewallRule(firewallRuleName+"-v6", deleteRule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove windows v6 firewall rule: %w", err)) + } + } + return nberrors.FormatErrorOrNil(merr) +} + +func manageFirewallRule(ruleName string, action action, extraArgs ...string) error { args := []string{"advfirewall", "firewall", string(action), "rule", "name=" + ruleName} if action == addRule { args = append(args, extraArgs...) } - netshCmd := GetSystem32Command("netsh") + netshCmd := wincmd.System32("netsh") cmd := exec.Command(netshCmd, args...) cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true} return cmd.Run() @@ -97,7 +101,7 @@ func manageFirewallRule(ruleName string, action action, extraArgs ...string) err func isWindowsFirewallReachable() bool { args := []string{"advfirewall", "show", "allprofiles", "state"} - netshCmd := GetSystem32Command("netsh") + netshCmd := wincmd.System32("netsh") cmd := exec.Command(netshCmd, args...) cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true} @@ -114,23 +118,10 @@ func isWindowsFirewallReachable() bool { func isFirewallRuleActive(ruleName string) bool { args := []string{"advfirewall", "firewall", "show", "rule", "name=" + ruleName} - netshCmd := GetSystem32Command("netsh") + netshCmd := wincmd.System32("netsh") cmd := exec.Command(netshCmd, args...) cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true} _, err := cmd.Output() return err == nil } - -// GetSystem32Command checks if a command can be found in the system path and returns it. In case it can't find it -// in the path it will return the full path of a command assuming C:\windows\system32 as the base path. -func GetSystem32Command(command string) string { - _, err := exec.LookPath(command) - if err == nil { - return command - } - - log.Tracef("Command %s not found in PATH, using C:\\windows\\system32\\%s.exe path", command, command) - - return "C:\\windows\\system32\\" + command + ".exe" -} diff --git a/client/firewall/uspfilter/localip.go b/client/firewall/uspfilter/localip.go index b35be56c6..869832732 100644 --- a/client/firewall/uspfilter/localip.go +++ b/client/firewall/uspfilter/localip.go @@ -7,8 +7,6 @@ import ( "sync/atomic" log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/client/firewall/uspfilter/common" ) // localIPSnapshot is an immutable snapshot of local IP addresses, swapped @@ -60,7 +58,7 @@ func processInterface(iface net.Interface, ips map[netip.Addr]struct{}, addresse } // UpdateLocalIPs rebuilds the local IP snapshot and swaps it in atomically. -func (m *localIPManager) UpdateLocalIPs(iface common.IFaceMapper) (err error) { +func (m *localIPManager) UpdateLocalIPs(iface Iface) (err error) { defer func() { if r := recover(); r != nil { err = fmt.Errorf("panic: %v", r) diff --git a/client/firewall/uspfilter/nat.go b/client/firewall/uspfilter/nat.go index 5d51c1538..49c26766a 100644 --- a/client/firewall/uspfilter/nat.go +++ b/client/firewall/uspfilter/nat.go @@ -486,22 +486,6 @@ func incrementalUpdate(oldChecksum uint16, oldBytes, newBytes []byte) uint16 { return ^uint16(sum) } -// AddDNATRule adds outbound DNAT rule for forwarding external traffic to NetBird network. -func (m *Manager) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - if m.nativeFirewall == nil { - return nil, errNatNotSupported - } - return m.nativeFirewall.AddDNATRule(rule) -} - -// DeleteDNATRule deletes outbound DNAT rule. -func (m *Manager) DeleteDNATRule(rule firewall.Rule) error { - if m.nativeFirewall == nil { - return errNatNotSupported - } - return m.nativeFirewall.DeleteDNATRule(rule) -} - // addPortRedirection adds a port redirection rule. func (m *Manager) addPortRedirection(targetIP netip.Addr, protocol gopacket.LayerType, originalPort, translatedPort uint16) error { m.portDNATMutex.Lock() @@ -521,7 +505,6 @@ func (m *Manager) addPortRedirection(targetIP netip.Addr, protocol gopacket.Laye } // AddInboundDNAT adds an inbound DNAT rule redirecting traffic from NetBird peers to local services. -// TODO: also delegate to nativeFirewall when available for kernel WG mode func (m *Manager) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { var layerType gopacket.LayerType switch protocol { @@ -567,20 +550,16 @@ func (m *Manager) RemoveInboundDNAT(localAddr netip.Addr, protocol firewall.Prot return m.removePortRedirection(localAddr, layerType, originalPort, translatedPort) } -// AddOutputDNAT delegates to the native firewall if available. -func (m *Manager) AddOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - if m.nativeFirewall == nil { - return fmt.Errorf("output DNAT not supported without native firewall") - } - return m.nativeFirewall.AddOutputDNAT(localAddr, protocol, originalPort, translatedPort) +// AddOutputDNAT is not supported by the userspace firewall: it backs kernel DNS +// redirection, but userspace DNS is served in-process on the gVisor netstack, so +// this should never be called. +func (m *Manager) AddOutputDNAT(netip.Addr, firewall.Protocol, uint16, uint16) error { + return errNotSupported } -// RemoveOutputDNAT delegates to the native firewall if available. -func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { - if m.nativeFirewall == nil { - return nil - } - return m.nativeFirewall.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) +// RemoveOutputDNAT is a no-op for the userspace firewall (see AddOutputDNAT). +func (m *Manager) RemoveOutputDNAT(netip.Addr, firewall.Protocol, uint16, uint16) error { + return nil } // translateInboundPortDNAT applies port-specific DNAT translation to inbound packets. diff --git a/client/firewall/uspfilter/nat_bench_test.go b/client/firewall/uspfilter/nat_bench_test.go index 1e15c8c0c..422c6b849 100644 --- a/client/firewall/uspfilter/nat_bench_test.go +++ b/client/firewall/uspfilter/nat_bench_test.go @@ -64,9 +64,11 @@ func BenchmarkDNATTranslation(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) @@ -124,9 +126,11 @@ func BenchmarkDNATTranslation(b *testing.B) { // BenchmarkDNATConcurrency tests DNAT performance under concurrent load func BenchmarkDNATConcurrency(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) @@ -196,9 +200,11 @@ func BenchmarkDNATScaling(b *testing.B) { for _, count := range mappingCounts { b.Run(fmt.Sprintf("mappings_%d", count), func(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) @@ -308,9 +314,11 @@ func BenchmarkChecksumUpdate(b *testing.B) { // BenchmarkDNATMemoryAllocations checks for memory allocations in DNAT operations func BenchmarkDNATMemoryAllocations(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) @@ -481,9 +489,11 @@ func BenchmarkPortDNAT(b *testing.B) { for _, sc := range scenarios { b.Run(sc.name, func(b *testing.B) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(b, err) defer func() { require.NoError(b, manager.Close(nil)) diff --git a/client/firewall/uspfilter/nat_stateful_test.go b/client/firewall/uspfilter/nat_stateful_test.go index 21c6da06e..5fa5da027 100644 --- a/client/firewall/uspfilter/nat_stateful_test.go +++ b/client/firewall/uspfilter/nat_stateful_test.go @@ -13,9 +13,11 @@ import ( // TestPortDNATBasic tests basic port DNAT functionality func TestPortDNATBasic(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) @@ -49,9 +51,11 @@ func TestPortDNATBasic(t *testing.T) { // TestPortDNATMultipleRules tests multiple port DNAT rules func TestPortDNATMultipleRules(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) diff --git a/client/firewall/uspfilter/nat_test.go b/client/firewall/uspfilter/nat_test.go index 4598c3901..5b5840383 100644 --- a/client/firewall/uspfilter/nat_test.go +++ b/client/firewall/uspfilter/nat_test.go @@ -15,9 +15,11 @@ import ( // TestDNATTranslationCorrectness verifies DNAT translation works correctly func TestDNATTranslationCorrectness(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) @@ -104,9 +106,11 @@ func parsePacket(t testing.TB, packetData []byte) *decoder { // TestDNATMappingManagement tests adding/removing DNAT mappings func TestDNATMappingManagement(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) @@ -152,9 +156,11 @@ func TestDNATMappingManagement(t *testing.T) { } func TestInboundPortDNAT(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) @@ -202,9 +208,11 @@ func TestInboundPortDNAT(t *testing.T) { } func TestInboundPortDNATNegative(t *testing.T) { - manager, err := Create(&IFaceMock{ - SetFilterFunc: func(device.PacketFilter) error { return nil }, - }, false, flowLogger, iface.DefaultMTU) + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) defer func() { require.NoError(t, manager.Close(nil)) diff --git a/client/firewall/uspfilter/peer_acl_bench_test.go b/client/firewall/uspfilter/peer_acl_bench_test.go new file mode 100644 index 000000000..bcb0ca5c2 --- /dev/null +++ b/client/firewall/uspfilter/peer_acl_bench_test.go @@ -0,0 +1,333 @@ +//go:build uspbench + +package uspfilter + +import ( + "fmt" + "io" + "math/rand" + "net" + "net/netip" + "runtime" + "testing" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + + fw "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/device" + "github.com/netbirdio/netbird/client/iface/wgaddr" +) + +// BenchmarkPeerACLMatch measures the per-packet cost of the peer ACL +// matcher (peerACLsBlock) across realistic shapes: M distinct policy +// rules, each with K source peers in its set. +// +// With the reverse-source index, miss cost is independent of M and +// hit cost grows only with the number of rules touching a single +// srcIP, not with total rule count. +func BenchmarkPeerACLMatch(b *testing.B) { + shapes := []struct{ M, K int }{ + {1, 100}, {10, 100}, {50, 100}, {100, 100}, {100, 1000}, + } + families := []struct { + name string + v6 bool + }{{"v4", false}, {"v6", true}} + + for _, fam := range families { + for _, s := range shapes { + b.Run(fmt.Sprintf("%s/M=%d/K=%d/hit", fam.name, s.M, s.K), func(b *testing.B) { + runPeerACLBench(b, s.M, s.K, true, fam.v6) + }) + b.Run(fmt.Sprintf("%s/M=%d/K=%d/miss", fam.name, s.M, s.K), func(b *testing.B) { + runPeerACLBench(b, s.M, s.K, false, fam.v6) + }) + } + } +} + +func runPeerACLBench(b *testing.B, m, k int, hit, v6 bool) { + log.SetOutput(io.Discard) // keep manager logs out of the benchmark output + + // Miss packets are dropped, so they always traverse the full peer + // ACL matcher (every bucket) without short-circuiting and without + // touching conntrack. Disable conntrack for the miss case so it + // measures the matcher, not established-state lookups. The hit case + // keeps conntrack on: an accepted packet reaches trackInbound, which + // needs the trackers conntrack creates. + if !hit { + b.Setenv("NB_DISABLE_CONNTRACK", "1") + } + + bits := 32 + genPkt := generatePacket + addrs := uniqueAddrs + if v6 { + bits = 128 + genPkt = generatePacket6 + addrs = uniqueAddrs6 + } + + // dstIP must be a local IP so filterInbound takes the local-traffic + // path (handleLocalTraffic → peerACLsBlock) we are measuring; an + // address the manager doesn't own would be treated as routed and + // short-circuit before the peer matcher. + dstIP := addrs(1, 2)[0] + mockAddr := wgaddr.Address{IP: dstIP, Network: netip.PrefixFrom(dstIP, bits)} + if v6 { + // The local-IP manager needs a valid v4 address too; expose the v6 + // dst as the interface's IPv6 so IsLocalIP recognizes it. + mockAddr = wgaddr.Address{ + IP: netip.MustParseAddr("100.64.0.1"), + Network: netip.MustParsePrefix("100.64.0.0/16"), + IPv6: dstIP, + IPv6Net: netip.PrefixFrom(dstIP, bits), + } + } + manager, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + AddressFunc: func() wgaddr.Address { return mockAddr }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) + b.Cleanup(func() { require.NoError(b, manager.Close(nil)) }) + + // Generate M policies × K source peers, all distinct. + all := addrs(m*k, 1) + for i := 0; i < m; i++ { + sources := make([]netip.Prefix, k) + for j, a := range all[i*k : (i+1)*k] { + sources[j] = netip.PrefixFrom(a, bits) + } + _, err := manager.AddFilterRule( + nil, sources, fw.Network{}, fw.ProtocolTCP, nil, + &fw.Port{Values: []uint16{uint16(80 + i)}}, + fw.ActionAccept) + require.NoError(b, err) + } + + // Hit: cycle through real sources, picking the matching policy's port. + // Miss: a source from a disjoint range, port 80 (matches no policy). + var pktFn func(i int) []byte + if hit { + pktFn = func(i int) []byte { + policy := i % m + src := all[policy*k+(i%k)] + return genPkt(b, src.AsSlice(), dstIP.AsSlice(), + uint16(1024+i%60000), uint16(80+policy), layers.IPProtocolTCP) + } + } else { + miss := addrs(4096, 99) + pktFn = func(i int) []byte { + return genPkt(b, miss[i%len(miss)].AsSlice(), dstIP.AsSlice(), + uint16(1024+i%60000), 80, layers.IPProtocolTCP) + } + } + + // Pre-build a pool to avoid allocations dominating the measurement. + pool := make([][]byte, 1024) + for i := range pool { + pool[i] = pktFn(i) + } + + // Confirm the matcher is actually exercised: a hit packet must be + // allowed and a miss packet dropped. Without this the benchmark + // could silently time the routed early-return instead. + require.Equal(b, !hit, manager.filterInbound(pool[0], 0), + "benchmark must reach the peer ACL matcher") + + b.ResetTimer() + for i := 0; i < b.N; i++ { + manager.filterInbound(pool[i%len(pool)], 0) + } +} + +// BenchmarkPeerACLIndexMemory reports the resident memory cost of +// the source-keyed index across realistic deployment shapes. Two +// dimensions matter: (M, K), the number of policies × peers-per-policy, +// and overlap, the fraction of peers shared between policies. +// +// The output uses ReportMetric("bytes/rule") so the cost can be +// compared across shapes directly. Total bytes = bytes/rule * M. +func BenchmarkPeerACLIndexMemory(b *testing.B) { + cases := []struct { + name string + M, K int + overlapFrac float64 // 0 = disjoint per-policy sources, 1 = all share the same pool + }{ + {"M=10/K=100/disjoint", 10, 100, 0}, + {"M=100/K=100/disjoint", 100, 100, 0}, + {"M=100/K=1000/disjoint", 100, 1000, 0}, + {"M=100/K=1000/overlap=0.5", 100, 1000, 0.5}, + {"M=100/K=1000/overlap=1.0", 100, 1000, 1.0}, + {"M=1000/K=100/overlap=1.0", 1000, 100, 1.0}, + } + + for _, c := range cases { + b.Run(c.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + mgr, err := Create(Config{ + IFace: &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + }, + FlowLogger: flowLogger, MTU: iface.DefaultMTU}) + require.NoError(b, err) + + populateIndexedRules(b, mgr, c.M, c.K, c.overlapFrac) + + runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + before := ms.HeapAlloc + + // Drop the manager's external roots so we can isolate + // the index cost. We hold the manager itself live; the + // index is what we measure on the second pass. + mgr.incomingAcceptIndex.reset() + mgr.incomingDenyIndex.reset() + mgr.incomingAcceptRules = mgr.incomingAcceptRules[:0] + mgr.incomingDenyRules = mgr.incomingDenyRules[:0] + runtime.GC() + runtime.ReadMemStats(&ms) + after := ms.HeapAlloc + + delta := int64(before) - int64(after) + if delta < 0 { + delta = 0 + } + b.ReportMetric(float64(delta)/float64(c.M), "bytes/rule") + b.ReportMetric(float64(delta), "bytes/total") + + require.NoError(b, mgr.Close(nil)) + } + }) + } +} + +func populateIndexedRules(b *testing.B, mgr *Manager, m, k int, overlapFrac float64) { + b.Helper() + pool := uniqueAddrs(k+m*k, 1) // big enough universe + sharedLen := int(float64(k) * overlapFrac) + if sharedLen > k { + sharedLen = k + } + shared := pool[:sharedLen] + uniquePool := pool[sharedLen:] + + for i := 0; i < m; i++ { + sources := make([]netip.Prefix, 0, k) + for _, a := range shared { + sources = append(sources, netip.PrefixFrom(a, 32)) + } + // each policy gets (k-sharedLen) addresses unique to it from the unique pool + unique := uniquePool[i*(k-sharedLen) : (i+1)*(k-sharedLen)] + for _, a := range unique { + sources = append(sources, netip.PrefixFrom(a, 32)) + } + _, err := mgr.AddFilterRule( + nil, sources, fw.Network{}, fw.ProtocolTCP, nil, + &fw.Port{Values: []uint16{uint16(80 + i)}}, + fw.ActionAccept) + require.NoError(b, err) + } +} + +// uniqueAddrs returns n distinct addrs. Seeds 1, 2 are used for +// policy sources / dst; seed 99 puts misses in 10/8. +func uniqueAddrs(n int, seed int64) []netip.Addr { + out := make([]netip.Addr, 0, n) + seen := make(map[netip.Addr]struct{}, n) + r := rand.New(rand.NewSource(seed)) + miss := seed == 99 + for len(out) < n { + var b [4]byte + if miss { + b[0] = 10 + b[1] = byte(r.Intn(256)) + } else { + b[0] = 100 + b[1] = byte(64 + r.Intn(63)) + } + b[2] = byte(r.Intn(256)) + b[3] = byte(1 + r.Intn(254)) + a := netip.AddrFrom4(b) + if _, ok := seen[a]; ok { + continue + } + seen[a] = struct{}{} + out = append(out, a) + } + return out +} + +// uniqueAddrs6 mirrors uniqueAddrs for IPv6: sources come from the ULA +// range fd00::/8, the miss set (seed 99) from 2001:db8::/32 so it is +// disjoint from any source. +func uniqueAddrs6(n int, seed int64) []netip.Addr { + out := make([]netip.Addr, 0, n) + seen := make(map[netip.Addr]struct{}, n) + r := rand.New(rand.NewSource(seed)) + miss := seed == 99 + for len(out) < n { + var b [16]byte + if miss { + b[0], b[1], b[2], b[3] = 0x20, 0x01, 0x0d, 0xb8 + } else { + b[0] = 0xfd + } + for x := 8; x < 16; x++ { + b[x] = byte(r.Intn(256)) + } + a := netip.AddrFrom16(b) + if _, ok := seen[a]; ok { + continue + } + seen[a] = struct{}{} + out = append(out, a) + } + return out +} + +// generatePacket6 builds an IPv6 TCP/UDP packet, mirroring +// generatePacket for the v4 case. +func generatePacket6(b *testing.B, srcIP, dstIP net.IP, srcPort, dstPort uint16, protocol layers.IPProtocol) []byte { + b.Helper() + + ipv6 := &layers.IPv6{ + Version: 6, + HopLimit: 64, + NextHeader: protocol, + SrcIP: srcIP, + DstIP: dstIP, + } + + var transportLayer gopacket.SerializableLayer + switch protocol { + case layers.IPProtocolTCP: + tcp := &layers.TCP{ + SrcPort: layers.TCPPort(srcPort), + DstPort: layers.TCPPort(dstPort), + SYN: true, + } + require.NoError(b, tcp.SetNetworkLayerForChecksum(ipv6)) + transportLayer = tcp + case layers.IPProtocolUDP: + udp := &layers.UDP{ + SrcPort: layers.UDPPort(srcPort), + DstPort: layers.UDPPort(dstPort), + } + require.NoError(b, udp.SetNetworkLayerForChecksum(ipv6)) + transportLayer = udp + } + + buf := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{ComputeChecksums: true, FixLengths: true} + require.NoError(b, gopacket.SerializeLayers(buf, opts, ipv6, transportLayer, gopacket.Payload("test"))) + return buf.Bytes() +} diff --git a/client/firewall/uspfilter/peer_acl_dedup_test.go b/client/firewall/uspfilter/peer_acl_dedup_test.go new file mode 100644 index 000000000..696766d80 --- /dev/null +++ b/client/firewall/uspfilter/peer_acl_dedup_test.go @@ -0,0 +1,150 @@ +package uspfilter + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + fw "github.com/netbirdio/netbird/client/firewall/manager" + nbiface "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/device" +) + +func newTestManager(t *testing.T) *Manager { + t.Helper() + ifaceMock := &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + } + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) + require.NoError(t, err, "create manager") + t.Cleanup(func() { require.NoError(t, m.Close(nil)) }) + return m +} + +// TestAddPeerFiltering_DeduplicatesIdenticalRules verifies that adding +// the same peer rule twice does not create two backing rules. The acl +// manager keys its own cache, but the firewall backend must be +// idempotent on its own so a double-apply cannot leak rules, matching +// the route path and the kernel backends. +func TestAddPeerFiltering_DeduplicatesIdenticalRules(t *testing.T) { + m := newTestManager(t) + + ip := net.ParseIP("192.168.1.1") + proto := fw.ProtocolTCP + port := &fw.Port{Values: []uint16{80}} + action := fw.ActionDrop + + first, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err, "first add") + + second, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err, "second add") + + assert.Equal(t, first.ID(), second.ID(), "duplicate add should return the same rule id") + assert.Len(t, m.incomingDenyRules, 1, "duplicate add must not create a second backing rule") +} + +// TestDeletePeerFiltering_NoRefcountSingleDeleteRemoves locks the +// backend's owner accounting for the same-owner case: a content key +// installed twice by the same owner registers one owner claim, so the +// first DeleteFilterRule removes the rule. Owner counting only kicks +// in for distinct management rule IDs (see the peer owner tests); the +// acl manager keys its tracking per (policy, content) and deletes once +// per key, so adds and deletes stay balanced. +func TestDeletePeerFiltering_NoRefcountSingleDeleteRemoves(t *testing.T) { + m := newTestManager(t) + + ip := net.ParseIP("192.168.1.1") + proto := fw.ProtocolTCP + port := &fw.Port{Values: []uint16{80}} + action := fw.ActionDrop + + first, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err, "first add") + + second, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err, "second add") + require.Equal(t, first.ID(), second.ID(), "dedup to one rule") + require.Len(t, m.incomingDenyRules, 1, "still one backing rule after duplicate add") + + require.NoError(t, m.DeleteFilterRule(first), "delete once") + assert.Empty(t, m.incomingDenyRules, "single delete removes the backing rule (no refcount)") + assert.NotContains(t, m.peerRulesMap, first.ID(), "dedup map entry cleared") +} + +// TestAddPeerFiltering_DeterministicID verifies the peer rule id is a +// content hash, not a random UUID: identical inputs produce the same id +// across independent managers. A random id breaks caller-side dedup. +func TestAddPeerFiltering_DeterministicID(t *testing.T) { + ip := net.ParseIP("10.0.0.5") + proto := fw.ProtocolUDP + port := &fw.Port{Values: []uint16{53}} + action := fw.ActionAccept + + m1 := newTestManager(t) + r1, err := m1.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err) + + m2 := newTestManager(t) + r2, err := m2.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err) + + assert.Equal(t, r1.ID(), r2.ID(), "same inputs must produce the same rule id") +} + +// TestAddPeerFiltering_DistinctRulesNotDeduped verifies that rules +// differing only by port are kept separate. +func TestAddPeerFiltering_DistinctRulesNotDeduped(t *testing.T) { + m := newTestManager(t) + + ip := net.ParseIP("192.168.1.1") + proto := fw.ProtocolTCP + action := fw.ActionAccept + + r80, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, &fw.Port{Values: []uint16{80}}, action) + require.NoError(t, err) + + r443, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, &fw.Port{Values: []uint16{443}}, action) + require.NoError(t, err) + + assert.NotEqual(t, r80.ID(), r443.ID(), "different ports must produce different rule ids") + assert.Len(t, m.incomingAcceptRules, 2, "distinct rules must both be stored") +} + +// TestAddPeerFiltering_SourceVsDestPortNotDeduped verifies that a rule +// matching on source port and one matching on destination port for the +// same selector do not collide: the port lands in a different slot, so +// the content key must differ. +func TestAddPeerFiltering_SourceVsDestPortNotDeduped(t *testing.T) { + m := newTestManager(t) + + ip := net.ParseIP("192.168.1.1") + proto := fw.ProtocolTCP + port := &fw.Port{Values: []uint16{80}} + action := fw.ActionAccept + + dPortRule, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) + require.NoError(t, err) + + sPortRule, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, port, nil, action) + require.NoError(t, err) + + assert.NotEqual(t, dPortRule.ID(), sPortRule.ID(), "source-port and dest-port matches must produce different rule ids") +} + +// TestAddFilterRule_EmptySourcesRejected verifies that an empty source +// list is rejected rather than treated as "match any". "Match any" must +// be an explicit /0, so a zeroed list can never silently widen a rule to +// every source. +func TestAddFilterRule_EmptySourcesRejected(t *testing.T) { + m := newTestManager(t) + + proto := fw.ProtocolTCP + port := &fw.Port{Values: []uint16{80}} + + _, err := m.AddFilterRule(nil, nil, fw.Network{}, proto, nil, port, fw.ActionAccept) + require.ErrorIs(t, err, fw.ErrNoSources, "empty sources must be rejected") + assert.Empty(t, m.incomingAcceptRules, "no rule should be stored for empty sources") +} diff --git a/client/firewall/uspfilter/peer_acl_ipv6_test.go b/client/firewall/uspfilter/peer_acl_ipv6_test.go new file mode 100644 index 000000000..282b44ebb --- /dev/null +++ b/client/firewall/uspfilter/peer_acl_ipv6_test.go @@ -0,0 +1,105 @@ +package uspfilter + +import ( + "net" + "net/netip" + "testing" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + fw "github.com/netbirdio/netbird/client/firewall/manager" + nbiface "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/device" + "github.com/netbirdio/netbird/client/iface/wgaddr" +) + +func newV6TestManager(t *testing.T, localV6 string) *Manager { + t.Helper() + ifaceMock := &IFaceMock{ + SetFilterFunc: func(device.PacketFilter) error { return nil }, + AddressFunc: func() wgaddr.Address { + return wgaddr.Address{ + IP: netip.MustParseAddr("100.10.0.100"), + Network: netip.MustParsePrefix("100.10.0.0/16"), + IPv6: netip.MustParseAddr(localV6), + IPv6Net: netip.MustParsePrefix("fd00::/64"), + } + }, + } + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: nbiface.DefaultMTU}) + require.NoError(t, err, "create manager") + t.Cleanup(func() { require.NoError(t, m.Close(nil)) }) + return m +} + +func v6UDPPacket(t *testing.T, src, dst string, dstPort uint16) []byte { + t.Helper() + ip6 := &layers.IPv6{ + Version: 6, + HopLimit: 64, + NextHeader: layers.IPProtocolUDP, + SrcIP: net.ParseIP(src), + DstIP: net.ParseIP(dst), + } + udp := &layers.UDP{SrcPort: 51334, DstPort: layers.UDPPort(dstPort)} + require.NoError(t, udp.SetNetworkLayerForChecksum(ip6)) + + buf := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{ComputeChecksums: true, FixLengths: true} + require.NoError(t, gopacket.SerializeLayers(buf, opts, ip6, udp, gopacket.Payload("test"))) + return buf.Bytes() +} + +// TestPeerACL_IPv6HostRule verifies the source index resolves /128 v6 +// rules: a matching v6 source is accepted, a non-matching one is +// denied by the default. This is the end-to-end proof that the index +// is not v4-only. +func TestPeerACL_IPv6HostRule(t *testing.T) { + m := newV6TestManager(t, "fd00::100") + + src := net.ParseIP("fd00::1") + _, err := m.AddFilterRule(nil, pfx(src), fw.Network{}, fw.ProtocolUDP, nil, &fw.Port{Values: []uint16{53}}, fw.ActionAccept) + require.NoError(t, err, "add v6 accept rule") + + require.False(t, m.filterInbound(v6UDPPacket(t, "fd00::1", "fd00::100", 53), 0), + "v6 packet from the allowed /128 source must be accepted") + require.True(t, m.filterInbound(v6UDPPacket(t, "fd00::2", "fd00::100", 53), 0), + "v6 packet from an unlisted source must be denied by default") +} + +// TestPeerACL_IPv6IndexBuckets verifies that v6 sources land in the +// right index bucket: a /128 in bySource keyed by its address, and +// coarser prefixes (including ::/0) in the nonHost slice. +func TestPeerACL_IPv6IndexBuckets(t *testing.T) { + m := newV6TestManager(t, "fd00::100") + port := &fw.Port{Values: []uint16{53}} + + host := netip.MustParseAddr("fd00::1") + _, err := m.AddFilterRule(nil, []netip.Prefix{netip.PrefixFrom(host, 128)}, fw.Network{}, fw.ProtocolUDP, nil, port, fw.ActionAccept) + require.NoError(t, err) + assert.Contains(t, m.incomingAcceptIndex.bySource, host, "/128 v6 source must be indexed by address") + + _, err = m.AddFilterRule(nil, []netip.Prefix{netip.MustParsePrefix("fd00:dead::/64")}, fw.Network{}, fw.ProtocolUDP, nil, port, fw.ActionAccept) + require.NoError(t, err) + require.Len(t, m.incomingAcceptIndex.nonHost, 1, "coarser v6 prefix must land in nonHost") + + _, err = m.AddFilterRule(nil, []netip.Prefix{netip.MustParsePrefix("::/0")}, fw.Network{}, fw.ProtocolUDP, nil, port, fw.ActionAccept) + require.NoError(t, err) + require.Len(t, m.incomingAcceptIndex.nonHost, 2, "::/0 source must also land in nonHost") +} + +// TestPeerACL_IPv4MappedSourceNormalized verifies a v4-mapped v6 +// source prefix is normalized to v4 so a plain v4 packet matches it. +func TestPeerACL_IPv4MappedSourceNormalized(t *testing.T) { + m := newTestManager(t) + + mapped := netip.MustParseAddr("::ffff:192.168.1.1") + _, err := m.AddFilterRule(nil, []netip.Prefix{netip.PrefixFrom(mapped, mapped.BitLen())}, fw.Network{}, fw.ProtocolUDP, nil, &fw.Port{Values: []uint16{53}}, fw.ActionAccept) + require.NoError(t, err) + + v4 := netip.MustParseAddr("192.168.1.1") + assert.Contains(t, m.incomingAcceptIndex.bySource, v4, "v4-mapped v6 source must be indexed as plain v4") +} diff --git a/client/firewall/uspfilter/peer_family_scope_test.go b/client/firewall/uspfilter/peer_family_scope_test.go new file mode 100644 index 000000000..1cf3498fb --- /dev/null +++ b/client/firewall/uspfilter/peer_family_scope_test.go @@ -0,0 +1,104 @@ +package uspfilter + +import ( + "net" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + fw "github.com/netbirdio/netbird/client/firewall/manager" +) + +// peerACLCheck decodes the packet and runs it through the peer ACLs, +// returning the attributed management rule id and the drop verdict. +func peerACLCheck(t *testing.T, m *Manager, packet []byte) ([]byte, bool) { + t.Helper() + d := m.decoders.Get().(*decoder) + defer m.decoders.Put(d) + require.NoError(t, d.decodePacket(packet)) + src, _ := m.extractIPs(d) + return m.peerACLsBlock(src, d, packet) +} + +// TestPeerACL_MultiValuePortMatchesEachListedPort guards the multi-value +// port path: a rule listing several discrete destination ports must +// match a packet to each listed port and drop one that is not listed. +// Management currently splits a multi-port policy into one rule per port +// (and the wire format carries a single port), so this list shape is not +// emitted today; the test locks correct matching in case that changes. +func TestPeerACL_MultiValuePortMatchesEachListedPort(t *testing.T) { + m := newTestManager(t) + + src := net.ParseIP("192.168.1.1") + ports := &fw.Port{Values: []uint16{80, 443}} + _, err := m.AddFilterRule(nil, pfx(src), fw.Network{}, fw.ProtocolTCP, nil, ports, fw.ActionAccept) + require.NoError(t, err, "add multi-value port rule") + + for _, p := range []uint16{80, 443} { + _, blocked := peerACLCheck(t, m, createTestPacket(t, "192.168.1.1", "10.0.0.2", fw.ProtocolTCP, 12345, p)) + assert.False(t, blocked, "packet to listed port %d must match the rule", p) + } + + _, blocked := peerACLCheck(t, m, createTestPacket(t, "192.168.1.1", "10.0.0.2", fw.ProtocolTCP, 12345, 8080)) + assert.True(t, blocked, "packet to a port not in the list must not match the rule") +} + +// TestPeerACL_MatchAnyIsFamilyScoped verifies that a /0 source matches +// only packets of its own family: 0.0.0.0/0 must not match IPv6 packets +// and ::/0 must not match IPv4 packets, matching kernel backend +// semantics. +func TestPeerACL_MatchAnyIsFamilyScoped(t *testing.T) { + m := newTestManager(t) + + v4Packet := createTestPacket(t, "10.0.0.1", "10.0.0.2", fw.ProtocolUDP, 12345, 53) + v6Packet := v6UDPPacket(t, "fd00::1", "fd00::100", 53) + + v4Any := []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + rule, err := m.AddFilterRule(nil, v4Any, fw.Network{}, fw.ProtocolALL, nil, nil, fw.ActionAccept) + require.NoError(t, err, "add v4 /0 rule") + + _, blocked := peerACLCheck(t, m, v4Packet) + assert.False(t, blocked, "0.0.0.0/0 must match IPv4 packets") + _, blocked = peerACLCheck(t, m, v6Packet) + assert.True(t, blocked, "0.0.0.0/0 must not match IPv6 packets") + + require.NoError(t, m.DeleteFilterRule(rule)) + + v6Any := []netip.Prefix{netip.PrefixFrom(netip.IPv6Unspecified(), 0)} + _, err = m.AddFilterRule(nil, v6Any, fw.Network{}, fw.ProtocolALL, nil, nil, fw.ActionAccept) + require.NoError(t, err, "add v6 /0 rule") + + _, blocked = peerACLCheck(t, m, v6Packet) + assert.False(t, blocked, "::/0 must match IPv6 packets") + _, blocked = peerACLCheck(t, m, v4Packet) + assert.True(t, blocked, "::/0 must not match IPv4 packets") +} + +// TestRouteACL_MixedFamilyZeroSourcesStayFamilySafe verifies the route +// path keeps per-prefix family matching when a single rule carries both +// 0.0.0.0/0 and ::/0 sources, as blockInvalidRouted does. +func TestRouteACL_MixedFamilyZeroSourcesStayFamilySafe(t *testing.T) { + m := newTestManager(t) + + sources := []netip.Prefix{ + netip.PrefixFrom(netip.IPv4Unspecified(), 0), + netip.PrefixFrom(netip.IPv6Unspecified(), 0), + } + + _, err := m.AddFilterRule(nil, sources, fw.Network{Prefix: netip.MustParsePrefix("10.0.0.0/24")}, + fw.ProtocolALL, nil, nil, fw.ActionAccept) + require.NoError(t, err) + _, err = m.AddFilterRule(nil, sources, fw.Network{Prefix: netip.MustParsePrefix("fd00:1::/64")}, + fw.ProtocolALL, nil, nil, fw.ActionAccept) + require.NoError(t, err) + + v4Src := netip.MustParseAddr("192.168.1.1") + v6Src := netip.MustParseAddr("fd00::1") + + _, pass := m.routeACLsPass(v4Src, netip.MustParseAddr("10.0.0.5"), 255, 0, 0) + assert.True(t, pass, "v4 source must match the v4 destination rule via 0.0.0.0/0") + _, pass = m.routeACLsPass(v6Src, netip.MustParseAddr("fd00:1::5"), 255, 0, 0) + assert.True(t, pass, "v6 source must match the v6 destination rule via ::/0") +} diff --git a/client/firewall/uspfilter/peer_index.go b/client/firewall/uspfilter/peer_index.go new file mode 100644 index 000000000..552ffd2d7 --- /dev/null +++ b/client/firewall/uspfilter/peer_index.go @@ -0,0 +1,140 @@ +package uspfilter + +import ( + "net/netip" + "slices" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + + firewall "github.com/netbirdio/netbird/client/firewall/manager" +) + +// peerRuleIndex is the source-side dispatcher consulted on the packet +// hot path. It splits rules into two buckets by the shape of their +// source list: +// +// - bySource: every source is a host prefix (/32 for v4, /128 for +// v6). Keyed by the concrete source address, so a hit guarantees +// the source filter passes and the matcher goes straight to +// proto/port checks. This is the common case for peer ACLs. +// - nonHost: any source list with a prefix coarser than a host, +// including a /0 "match any". Walked linearly with a per-rule +// Contains() check. Expected small or empty for typical peer ACLs. +// +// Maintained incrementally by add/remove, never rebuilt. +type peerRuleIndex struct { + bySource map[netip.Addr][]*PeerRule + nonHost []*PeerRule +} + +func (i *peerRuleIndex) add(r *PeerRule) { + if hasNonHostSource(r) { + i.nonHost = append(i.nonHost, r) + return + } + if i.bySource == nil { + i.bySource = make(map[netip.Addr][]*PeerRule) + } + for a := range r.sourceAddrs { + i.bySource[a] = append(i.bySource[a], r) + } +} + +func (i *peerRuleIndex) remove(r *PeerRule) { + if hasNonHostSource(r) { + i.nonHost = slices.DeleteFunc(i.nonHost, eqRule(r)) + return + } + if i.bySource == nil { + return + } + for a := range r.sourceAddrs { + entries := slices.DeleteFunc(i.bySource[a], eqRule(r)) + if len(entries) == 0 { + delete(i.bySource, a) + } else { + i.bySource[a] = entries + } + } +} + +func (i *peerRuleIndex) reset() { + i.bySource = nil + i.nonHost = i.nonHost[:0] +} + +// match returns the first rule matching src and the decoded packet. +// Host rules are found by direct map lookup; nonHost rules run a +// per-rule source Contains() check. Containment is family-scoped, so +// a /0 source matches every address of its own family only (0.0.0.0/0 +// never matches v6 sources and ::/0 never matches v4). Within either +// bucket the matcher runs the proto/port filter. +func (i *peerRuleIndex) match(src netip.Addr, d *decoder) ([]byte, bool, bool) { + payloadLayer := d.decoded[1] + + for _, rule := range i.bySource[src] { + if id, drop, ok := matchProto(rule, d, payloadLayer); ok { + return id, drop, true + } + } + for _, rule := range i.nonHost { + if !prefixesContain(rule.sources, src) { + continue + } + if id, drop, ok := matchProto(rule, d, payloadLayer); ok { + return id, drop, true + } + } + return nil, false, false +} + +func eqRule(target *PeerRule) func(*PeerRule) bool { + return func(p *PeerRule) bool { return p == target } +} + +// hasNonHostSource reports whether the rule has any source prefix +// that is not a single host address. Called only at add/remove time, +// not on the packet path. +func hasNonHostSource(r *PeerRule) bool { + for _, p := range r.sources { + if p.Bits() != p.Addr().BitLen() { + return true + } + } + return false +} + +// matchProto applies the proto/port half of a rule against the +// decoded packet. Source matching is the caller's responsibility. +func matchProto(rule *PeerRule, d *decoder, payloadLayer gopacket.LayerType) ([]byte, bool, bool) { + drop := rule.action == firewall.ActionDrop + if rule.protoLayer == layerTypeAll { + return rule.mgmtId, drop, true + } + if !protoLayerMatches(rule.protoLayer, payloadLayer) { + return nil, false, false + } + switch payloadLayer { + case layers.LayerTypeTCP: + if portsMatch(rule.srcPort, uint16(d.tcp.SrcPort)) && portsMatch(rule.dstPort, uint16(d.tcp.DstPort)) { + return rule.mgmtId, drop, true + } + case layers.LayerTypeUDP: + if portsMatch(rule.srcPort, uint16(d.udp.SrcPort)) && portsMatch(rule.dstPort, uint16(d.udp.DstPort)) { + return rule.mgmtId, drop, true + } + case layers.LayerTypeICMPv4, layers.LayerTypeICMPv6: + return rule.mgmtId, drop, true + } + return nil, false, false +} + +func prefixesContain(sources []netip.Prefix, src netip.Addr) bool { + for _, p := range sources { + if p.Contains(src) { + return true + } + } + return false +} diff --git a/client/firewall/uspfilter/rule.go b/client/firewall/uspfilter/rule.go index 08d68a78e..6d73b19ac 100644 --- a/client/firewall/uspfilter/rule.go +++ b/client/firewall/uspfilter/rule.go @@ -10,24 +10,43 @@ import ( // PeerRule to handle management of rules type PeerRule struct { - id string - mgmtId []byte - ip netip.Addr - ipLayer gopacket.LayerType - matchByIP bool - protoLayer gopacket.LayerType - sPort *firewall.Port - dPort *firewall.Port - drop bool + id firewall.RuleID + mgmtId []byte + // sources is the canonical list of source prefixes this rule + // matches against. + sources []netip.Prefix + // sourceAddrs is a fast-path membership set for host-prefix + // sources (/32 v4, /128 v6). Populated alongside sources; + // consulted before falling back to prefix scan. + sourceAddrs map[netip.Addr]struct{} + protoLayer gopacket.LayerType + srcPort *firewall.Port + dstPort *firewall.Port + action firewall.Action +} + +// matchesSource reports whether the given source address is covered +// by this rule's source list. Prefix containment is family-scoped, so +// a /0 source matches every address of its own family only. +func (r *PeerRule) matchesSource(src netip.Addr) bool { + if _, ok := r.sourceAddrs[src]; ok { + return true + } + for _, p := range r.sources { + if p.Contains(src) { + return true + } + } + return false } // ID returns the rule id -func (r *PeerRule) ID() string { +func (r *PeerRule) ID() firewall.RuleID { return r.id } type RouteRule struct { - id string + id firewall.RuleID mgmtId []byte sources []netip.Prefix dstSet firewall.Set @@ -39,6 +58,6 @@ type RouteRule struct { } // ID returns the rule id -func (r *RouteRule) ID() string { +func (r *RouteRule) ID() firewall.RuleID { return r.id } diff --git a/client/firewall/uspfilter/testhelpers_test.go b/client/firewall/uspfilter/testhelpers_test.go new file mode 100644 index 000000000..a0760f5ba --- /dev/null +++ b/client/firewall/uspfilter/testhelpers_test.go @@ -0,0 +1,50 @@ +package uspfilter + +import ( + "net" + "net/netip" + + firewall "github.com/netbirdio/netbird/client/firewall/manager" +) + +// countRulesForAddr reports how many rules in the given slice match +// the supplied source address. +func countRulesForAddr(rules peerRules, src netip.Addr) int { + n := 0 + for _, r := range rules { + if r.matchesSource(src) { + n++ + } + } + return n +} + +// findRuleByID returns true if the rules slice contains a rule with +// the given id whose source set covers src. +func findRuleByID(rules peerRules, src netip.Addr, id firewall.RuleID) bool { + for _, r := range rules { + if r.id == id && r.matchesSource(src) { + return true + } + } + return false +} + +// pfx converts a single net.IP into the []netip.Prefix form +// AddFilterRule expects. A nil or unspecified address becomes a /0 +// ("match any") prefix in the matching family; any other address +// becomes its /32 (or /128) host prefix. +func pfx(ip net.IP) []netip.Prefix { + if ip == nil { + return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + } + if ip.IsUnspecified() { + if ip.To4() != nil { + return []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)} + } + return []netip.Prefix{netip.PrefixFrom(netip.IPv6Unspecified(), 0)} + } + a, _ := netip.AddrFromSlice(ip) + a = a.Unmap() + return []netip.Prefix{netip.PrefixFrom(a, a.BitLen())} +} diff --git a/client/firewall/uspfilter/tracer.go b/client/firewall/uspfilter/tracer.go index 696489e95..3c081314c 100644 --- a/client/firewall/uspfilter/tracer.go +++ b/client/firewall/uspfilter/tracer.go @@ -285,6 +285,14 @@ func (m *Manager) TracePacket(packetData []byte, direction fw.RuleDirection) *Pa trace.SourceIP = srcIP trace.DestinationIP = dstIP + // A fragment or otherwise truncated packet has no transport layer. + // The inbound datapath drops these via isValidPacket; the tracer must + // guard explicitly since every downstream stage reads d.decoded[1]. + if len(d.decoded) < 2 { + trace.AddResult(StageReceived, "Packet has no transport layer (fragment or unsupported protocol)", false) + return trace + } + // Determine protocol and ports switch d.decoded[1] { case layers.LayerTypeTCP: diff --git a/client/firewall/uspfilter/tracer_test.go b/client/firewall/uspfilter/tracer_test.go index 657f96fc0..27b5e3f9e 100644 --- a/client/firewall/uspfilter/tracer_test.go +++ b/client/firewall/uspfilter/tracer_test.go @@ -45,7 +45,7 @@ func TestTracePacket(t *testing.T) { }, } - m, err := Create(ifaceMock, false, flowLogger, iface.DefaultMTU) + m, err := Create(Config{IFace: ifaceMock, FlowLogger: flowLogger, MTU: iface.DefaultMTU}) require.NoError(t, err) if !statefulMode { @@ -97,7 +97,7 @@ func TestTracePacket(t *testing.T) { proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionAccept - _, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -121,7 +121,7 @@ func TestTracePacket(t *testing.T) { proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionDrop - _, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -150,7 +150,7 @@ func TestTracePacket(t *testing.T) { proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionAccept - _, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -178,7 +178,7 @@ func TestTracePacket(t *testing.T) { proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionAccept - _, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -205,7 +205,7 @@ func TestTracePacket(t *testing.T) { src := netip.PrefixFrom(netip.AddrFrom4([4]byte{1, 1, 1, 1}), 32) dst := netip.PrefixFrom(netip.AddrFrom4([4]byte{192, 168, 17, 2}), 32) - _, err := m.AddRouteFiltering(nil, []netip.Prefix{src}, fw.Network{Prefix: dst}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) + _, err := m.AddFilterRule(nil, []netip.Prefix{src}, fw.Network{Prefix: dst}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionAccept) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -231,7 +231,7 @@ func TestTracePacket(t *testing.T) { src := netip.PrefixFrom(netip.AddrFrom4([4]byte{1, 1, 1, 1}), 32) dst := netip.PrefixFrom(netip.AddrFrom4([4]byte{192, 168, 17, 2}), 32) - _, err := m.AddRouteFiltering(nil, []netip.Prefix{src}, fw.Network{Prefix: dst}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop) + _, err := m.AddFilterRule(nil, []netip.Prefix{src}, fw.Network{Prefix: dst}, fw.ProtocolTCP, nil, &fw.Port{Values: []uint16{80}}, fw.ActionDrop) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -332,7 +332,7 @@ func TestTracePacket(t *testing.T) { ip := net.ParseIP("1.1.1.1") proto := fw.ProtocolICMP action := fw.ActionAccept - _, err := m.AddPeerFiltering(nil, ip, proto, nil, nil, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, nil, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -355,7 +355,7 @@ func TestTracePacket(t *testing.T) { ip := net.ParseIP("1.1.1.1") proto := fw.ProtocolICMP action := fw.ActionDrop - _, err := m.AddPeerFiltering(nil, ip, proto, nil, nil, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, nil, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -379,7 +379,7 @@ func TestTracePacket(t *testing.T) { proto := fw.ProtocolUDP port := &fw.Port{Values: []uint16{53}} action := fw.ActionAccept - _, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { @@ -423,7 +423,7 @@ func TestTracePacket(t *testing.T) { proto := fw.ProtocolTCP port := &fw.Port{Values: []uint16{80}} action := fw.ActionDrop - _, err := m.AddPeerFiltering(nil, ip, proto, nil, port, action, "") + _, err := m.AddFilterRule(nil, pfx(ip), fw.Network{}, proto, nil, port, action) require.NoError(t, err) }, packetBuilder: func() *PacketBuilder { diff --git a/client/grpc/dialer_generic.go b/client/grpc/dialer_generic.go index 479575996..72b3538ba 100644 --- a/client/grpc/dialer_generic.go +++ b/client/grpc/dialer_generic.go @@ -6,38 +6,54 @@ import ( "context" "fmt" "net" - "os/user" + "os" "runtime" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - log "github.com/sirupsen/logrus" "google.golang.org/grpc" nbnet "github.com/netbirdio/netbird/client/net" + "github.com/netbirdio/netbird/client/netevents/sweep" ) +// Sweeper registers in-flight dials for the network change sweep. +type Sweeper interface { + StartDial(ctx context.Context) *sweep.Dial +} + func WithCustomDialer(_ bool, _ string) grpc.DialOption { + return grpc.WithContextDialer(dialContext) +} + +// WithSweeper dials like WithCustomDialer but registers connections and +// dials with the sweeper. Append it after WithCustomDialer: gRPC applies +// dial options in order, so the later context dialer wins. +func WithSweeper(sweeper Sweeper) grpc.DialOption { return grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { - if runtime.GOOS == "linux" { - currentUser, err := user.Current() - if err != nil { - return nil, status.Errorf(codes.FailedPrecondition, "failed to get current user: %v", err) - } + dial := sweeper.StartDial(ctx) + defer dial.Release() - // the custom dialer requires root permissions which are not required for use cases run as non-root - if currentUser.Uid != "0" { - log.Debug("Not running as root, using standard dialer") - dialer := &net.Dialer{} - return dialer.DialContext(ctx, "tcp", addr) - } - } - - conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr) + conn, err := dialContext(dial.Ctx(), addr) if err != nil { - return nil, fmt.Errorf("nbnet.NewDialer().DialContext: %w", err) + return nil, err } - return conn, nil + return dial.WrapConn(conn) }) } + +func dialContext(ctx context.Context, addr string) (net.Conn, error) { + // The custom dialer requires root permissions which are not required for + // use cases run as non-root. The effective UID is read directly: a passwd + // lookup fails without cgo for a UID that has no entry, as on OpenShift. + if runtime.GOOS == "linux" && os.Geteuid() != 0 { + log.Debug("Not running as root, using standard dialer") + dialer := &net.Dialer{} + return dialer.DialContext(ctx, "tcp", addr) + } + + conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr) + if err != nil { + return nil, fmt.Errorf("nbnet.NewDialer().DialContext: %w", err) + } + return conn, nil +} diff --git a/client/grpc/dialer_js.go b/client/grpc/dialer_js.go index b89ec3c21..4ff4ceb20 100644 --- a/client/grpc/dialer_js.go +++ b/client/grpc/dialer_js.go @@ -1,13 +1,26 @@ package grpc import ( + "context" + "google.golang.org/grpc" + "github.com/netbirdio/netbird/client/netevents/sweep" "github.com/netbirdio/netbird/util/wsproxy/client" ) +// Sweeper registers in-flight dials for the network change sweep. +type Sweeper interface { + StartDial(ctx context.Context) *sweep.Dial +} + // WithCustomDialer returns a gRPC dial option that uses WebSocket transport for WASM/JS environments. // The component parameter specifies the WebSocket proxy component path (e.g., "/management", "/signal"). func WithCustomDialer(tlsEnabled bool, component string) grpc.DialOption { return client.WithWebSocketDialer(tlsEnabled, component) } + +// WithSweeper is a no-op on WASM/JS: there is no network change signal. +func WithSweeper(_ Sweeper) grpc.DialOption { + return grpc.EmptyDialOption{} +} diff --git a/client/grpc/retry.go b/client/grpc/retry.go new file mode 100644 index 000000000..0bb6037bf --- /dev/null +++ b/client/grpc/retry.go @@ -0,0 +1,56 @@ +package grpc + +import ( + "context" + "errors" + "time" + + "github.com/cenkalti/backoff/v4" +) + +// ChangeWatcher exposes OS network availability transitions. +type ChangeWatcher interface { + Changed() <-chan struct{} +} + +// Retry mirrors backoff.Retry, but the sleep between attempts also wakes on +// OS network availability transitions: an operation cut down by a network +// change retries the moment the network settles instead of sleeping through +// the recovery. A nil watcher never fires, leaving plain backoff.Retry +// behavior. +func Retry(ctx context.Context, operation backoff.Operation, bo backoff.BackOff, watcher ChangeWatcher) error { + bo.Reset() + for { + err := operation() + if err == nil { + return nil + } + + var permanent *backoff.PermanentError + if errors.As(err, &permanent) { + return permanent.Err + } + + next := bo.NextBackOff() + if next == backoff.Stop { + if cerr := ctx.Err(); cerr != nil { + return cerr + } + return err + } + + var changed <-chan struct{} + if watcher != nil { + changed = watcher.Changed() + } + timer := time.NewTimer(next) + select { + case <-timer.C: + case <-changed: + timer.Stop() + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + } + } +} diff --git a/client/grpc/retry_test.go b/client/grpc/retry_test.go new file mode 100644 index 000000000..266bb93e5 --- /dev/null +++ b/client/grpc/retry_test.go @@ -0,0 +1,91 @@ +package grpc + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/netevents/netstate" +) + +func TestRetryWakesOnNetworkChange(t *testing.T) { + ns := netstate.New() + attempts := 0 + operation := func() error { + attempts++ + if attempts == 1 { + return errors.New("cut by network change") + } + return nil + } + + go func() { + time.Sleep(20 * time.Millisecond) + ns.Set(false) + }() + + start := time.Now() + err := Retry(context.Background(), operation, backoff.NewConstantBackOff(time.Minute), ns) + + require.NoError(t, err) + assert.Equal(t, 2, attempts, "network change must cause one immediate retry") + assert.Less(t, time.Since(start), time.Second, "the transition must cut the minute-long sleep short") +} + +func TestRetryPermanentError(t *testing.T) { + sentinel := errors.New("permission denied") + operation := func() error { + return backoff.Permanent(sentinel) + } + + err := Retry(context.Background(), operation, backoff.NewConstantBackOff(time.Millisecond), nil) + assert.ErrorIs(t, err, sentinel, "permanent errors must stop retries") +} + +func TestRetryNilNetState(t *testing.T) { + attempts := 0 + operation := func() error { + attempts++ + if attempts < 3 { + return errors.New("transient") + } + return nil + } + + err := Retry(context.Background(), operation, backoff.NewConstantBackOff(time.Millisecond), nil) + require.NoError(t, err) + assert.Equal(t, 3, attempts, "nil network state must preserve timed retries") +} + +func TestRetryStops(t *testing.T) { + failure := errors.New("still failing") + operation := func() error { + return failure + } + + err := Retry(context.Background(), operation, &backoff.StopBackOff{}, nil) + assert.ErrorIs(t, err, failure, "stop backoff must return the operation error") +} + +func TestRetryCtxCancelDuringSleep(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + operation := func() error { + return errors.New("failing") + } + + go func() { + time.Sleep(20 * time.Millisecond) + cancel() + }() + + start := time.Now() + err := Retry(ctx, operation, backoff.NewConstantBackOff(time.Minute), netstate.New()) + + assert.ErrorIs(t, err, context.Canceled, "context cancellation must stop the retry loop") + assert.Less(t, time.Since(start), time.Second, "context cancellation must interrupt backoff sleep") +} diff --git a/client/iface/bind/ice_bind.go b/client/iface/bind/ice_bind.go index 156450c61..cd1ed12f1 100644 --- a/client/iface/bind/ice_bind.go +++ b/client/iface/bind/ice_bind.go @@ -22,6 +22,16 @@ import ( nbnet "github.com/netbirdio/netbird/client/net" ) +const ( + // wgMsgTypeHandshakeInitiation is the lowest WireGuard message type. + wgMsgTypeHandshakeInitiation uint32 = 1 + // wgMsgTypeTransport is the highest WireGuard message type. + wgMsgTypeTransport uint32 = 4 + // wgMinMsgSize is the smallest WireGuard message: transport data with an empty + // payload, which is what a keepalive is. + wgMinMsgSize = 32 +) + type receiverCreator struct { iceBind *ICEBind } @@ -47,10 +57,13 @@ type ICEBind struct { endpoints map[netip.Addr]net.Conn endpointsMu sync.Mutex recvChan chan recvMessage - // every time when Close() is called (i.e. BindUpdate()) we need to close exit from the receiveRelayed and create a - // new closed channel. With the closedChanMu we can safely close the channel and create a new one + // Close() (i.e. BindUpdate()) closes closedChan to release receiveRelayed, + // and the following Open() installs a fresh one. closedChanMu guards both + // closedChan and closed: readers only ever hold it long enough to copy the + // channel, never across a blocking receive, so Open cannot be starved by a + // parked receiver. closedChan chan struct{} - closedChanMu sync.RWMutex // protect the closeChan recreation from reading from it. + closedChanMu sync.RWMutex closed bool activityRecorder *ActivityRecorder @@ -82,24 +95,41 @@ func NewICEBind(transportNet transport.Net, address wgaddr.Address, mtu uint16) } func (s *ICEBind) Open(uport uint16) ([]wgConn.ReceiveFunc, uint16, error) { - s.closed = false s.closedChanMu.Lock() - s.closedChan = make(chan struct{}) - s.closedChanMu.Unlock() + defer s.closedChanMu.Unlock() + + // Open the underlying bind before touching any state, so a failure leaves + // the current generation exactly as it was. Publishing the new generation + // first would strand it: StdNetBind rejects an Open while it is already + // open, and a Close arriving in that window would mark the bind closed + // while this call went on to install live sockets, after which every later + // Close returns early and never shuts them down. fns, port, err := s.StdNetBind.Open(uport) if err != nil { return nil, 0, err } + + // Release whoever is parked on the outgoing generation before replacing it. + // An Open that follows an Open rather than a Close would otherwise leave + // them waiting on a channel no later Close can reach. + if !s.closed { + close(s.closedChan) + } + s.closed = false + s.closedChan = make(chan struct{}) + fns = append(fns, s.receiveRelayed) return fns, port, nil } func (s *ICEBind) Close() error { + s.closedChanMu.Lock() + defer s.closedChanMu.Unlock() + if s.closed { return nil } s.closed = true - close(s.closedChan) s.muUDPMux.Lock() @@ -111,6 +141,15 @@ func (s *ICEBind) Close() error { return s.StdNetBind.Close() } +// currentClosedChan copies the channel that signals the current Open +// generation is closing. Callers select on the copy so the lock is never held +// across a blocking receive, which would otherwise stall the next Open. +func (s *ICEBind) currentClosedChan() chan struct{} { + s.closedChanMu.RLock() + defer s.closedChanMu.RUnlock() + return s.closedChan +} + func (s *ICEBind) ActivityRecorder() *ActivityRecorder { return s.activityRecorder } @@ -140,8 +179,10 @@ func (b *ICEBind) RemoveEndpoint(fakeIP netip.Addr) { } func (b *ICEBind) ReceiveFromEndpoint(ctx context.Context, ep *Endpoint, buf []byte) { + closedChan := b.currentClosedChan() + select { - case <-b.closedChan: + case <-closedChan: return case <-ctx.Done(): return @@ -216,8 +257,15 @@ func (s *ICEBind) createReceiverFn(pc wgConn.BatchReader, conn *net.UDPConn, rxO for i := 0; i < numMsgs; i++ { msg := &(*msgs)[i] - // todo: handle err - if ok, _ := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr); ok { + if ok, err := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr); ok { + if err != nil { + log.Debugf("failed to handle STUN packet from %s: %v", msg.Addr, err) + } + // WireGuard reuses sizes and eps across reads and only skips a slot + // whose size is below the minimum message size. Leaving a consumed + // slot untouched makes it process this buffer again under the + // previous packet's length and endpoint. + sizes[i] = 0 continue } sizes[i] = msg.N @@ -271,11 +319,16 @@ func (s *ICEBind) createOrUpdateMux() { func (s *ICEBind) filterOutStunMessages(buffers [][]byte, n int, addr net.Addr) (bool, error) { for i := range buffers { - if !stun.IsMessage(buffers[i]) { + if n > len(buffers[i]) { + continue + } + pkt := buffers[i][:n] + + if isWireGuardMsg(pkt) || !stun.IsMessage(pkt) { continue } - msg, err := s.parseSTUNMessage(buffers[i][:n]) + msg, err := s.parseSTUNMessage(pkt) if err != nil { buffers[i] = []byte{} return true, err @@ -311,11 +364,10 @@ func (s *ICEBind) parseSTUNMessage(raw []byte) (*stun.Message, error) { // receiveRelayed is a receive function that is used to receive packets from the relayed connection and forward to the // WireGuard. Critical part is do not block if the Closed() has been called. func (c *ICEBind) receiveRelayed(buffs [][]byte, sizes []int, eps []wgConn.Endpoint) (int, error) { - c.closedChanMu.RLock() - defer c.closedChanMu.RUnlock() + closedChan := c.currentClosedChan() select { - case <-c.closedChan: + case <-closedChan: return 0, net.ErrClosed case msg, ok := <-c.recvChan: if !ok { @@ -347,18 +399,34 @@ func putMessages(msgs *[]ipv6.Message, msgsPool *sync.Pool) { msgsPool.Put(msgs) } -func isTransportPkg(buffers [][]byte, n int) bool { - // The first buffer should contain at least 4 bytes for type - if len(buffers[0]) < 4 { - return true +// isWireGuardMsg reports whether the packet carries a WireGuard message header: a +// little-endian uint32 message type in the range 1..4, which leaves the three bytes +// after the type byte zero, in a packet long enough to hold any WireGuard message. +// +// A well formed STUN message cannot take that shape. Its length field sits in the two +// bytes the type must leave zero, and for a message of at least wgMinMsgSize bytes that +// field holds at least 12, so the two framings do not overlap. The test has to be this +// tight because stun.IsMessage only looks at the magic cookie, which in a WireGuard +// message overlaps the receiver index: a session whose index happens to equal the cookie +// would otherwise have all of its inbound data misrouted to the STUN handler until the +// next rekey. +func isWireGuardMsg(pkt []byte) bool { + if len(pkt) < wgMinMsgSize { + return false } - // WireGuard packet type is a little-endian uint32 at start - packetType := binary.LittleEndian.Uint32(buffers[0][:4]) + msgType := binary.LittleEndian.Uint32(pkt[:4]) + return msgType >= wgMsgTypeHandshakeInitiation && msgType <= wgMsgTypeTransport +} - // Check if packetType matches known WireGuard message types - if packetType == 4 && n > 32 { - return true +// isTransportPkg reports whether the packet is WireGuard transport data carrying a +// payload, which is what counts as peer activity. A keepalive holds no payload and is +// exactly wgMinMsgSize bytes. +func isTransportPkg(buffers [][]byte, n int) bool { + if n < 4 || n > len(buffers[0]) { + return false } - return false + + msgType := binary.LittleEndian.Uint32(buffers[0][:4]) + return msgType == wgMsgTypeTransport && n > wgMinMsgSize } diff --git a/client/iface/bind/ice_bind_close_test.go b/client/iface/bind/ice_bind_close_test.go new file mode 100644 index 000000000..829d27663 --- /dev/null +++ b/client/iface/bind/ice_bind_close_test.go @@ -0,0 +1,261 @@ +package bind + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + wgConn "golang.zx2c4.com/wireguard/conn" +) + +// startReceivers runs every receive function the way wireguard-go's device +// does: one goroutine per function, all tracked by a single WaitGroup. After +// calling Bind.Close, closeBindLocked waits on exactly that WaitGroup while +// holding device.net, so a receive function that never returns wedges the +// device and every goroutine that needs the same lock. +func startReceivers(fns []wgConn.ReceiveFunc) *sync.WaitGroup { + wg, _ := startReceiversEntered(fns) + return wg +} + +// startReceiversEntered also returns a channel closed once every receive +// function has been called at least once. A receive function that has been +// entered is either inside its blocking receive or about to be, which is a +// stronger signal to synchronise on than a bare sleep. +func startReceiversEntered(fns []wgConn.ReceiveFunc) (*sync.WaitGroup, <-chan struct{}) { + var wg sync.WaitGroup + var entered sync.WaitGroup + wg.Add(len(fns)) + entered.Add(len(fns)) + + for i := range fns { + go func(fn wgConn.ReceiveFunc) { + defer wg.Done() + buffs := [][]byte{make([]byte, 1500)} + sizes := make([]int, 1) + eps := make([]wgConn.Endpoint, 1) + first := true + for { + if first { + entered.Done() + first = false + } + if _, err := fn(buffs, sizes, eps); err != nil { + return + } + } + }(fns[i]) + } + + allEntered := make(chan struct{}) + go func() { + entered.Wait() + close(allEntered) + }() + return &wg, allEntered +} + +// closeBounded runs Close off the caller's goroutine so a regression that +// wedges it fails the test instead of hanging teardown, and reports whether it +// returned in time. +func closeBounded(iceBind *ICEBind, timeout time.Duration) bool { + done := make(chan struct{}) + go func() { + _ = iceBind.Close() + close(done) + }() + select { + case <-done: + return true + case <-time.After(timeout): + return false + } +} + +// isClosed reports whether the bind's current generation channel is closed. +func isClosed(iceBind *ICEBind) bool { + iceBind.closedChanMu.RLock() + ch := iceBind.closedChan + iceBind.closedChanMu.RUnlock() + select { + case <-ch: + return true + default: + return false + } +} + +// receiversStopped reports whether every receive function returned before the +// timeout, mirroring device.net.stopping.Wait() inside closeBindLocked. +func receiversStopped(wg *sync.WaitGroup, timeout time.Duration) bool { + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + return true + case <-time.After(timeout): + return false + } +} + +// TestICEBindCloseReleasesReceivers covers the contract closeBindLocked relies +// on: once Close returns, every receive function handed out by Open must stop. +func TestICEBindCloseReleasesReceivers(t *testing.T) { + iceBind := setupICEBind(t) + + fns, _, err := iceBind.Open(0) + require.NoError(t, err, "opening the bind must succeed") + + wg := startReceivers(fns) + require.NoError(t, iceBind.Close()) + + require.True(t, receiversStopped(wg, 5*time.Second), + "every receive function must return once Close returns") +} + +// TestICEBindOpenDoesNotBlockOnParkedReceiver reproduces the deadlock that +// wedges interface creation. +// +// receiveRelayed used to hold closedChanMu for the whole of its blocking +// select, so a parked receiver kept the read lock indefinitely. Open takes the +// same mutex for writing to install a fresh closedChan, so it could never +// acquire it while a receiver was parked. wireguard-go reaches Open from +// Device.IpcSet and Device.Up with device.net held, so the stall takes the +// device's lock with it and every other device goroutine queues behind it. +// +// Failing here means Open never returned. +func TestICEBindOpenDoesNotBlockOnParkedReceiver(t *testing.T) { + iceBind := setupICEBind(t) + + fns, _, err := iceBind.Open(0) + require.NoError(t, err, "the first Open must succeed") + + wg, entered := startReceiversEntered(fns) + t.Cleanup(func() { + // Both bounded: a regression that wedges Close must surface as the + // assertion below, not as a hung teardown. + if !closeBounded(iceBind, 5*time.Second) { + t.Error("Close did not return during teardown; the bind lifecycle is wedged even though the assertion above passed") + } + if !receiversStopped(wg, 5*time.Second) { + t.Error("receive functions were still running after teardown Close, which is what closeBindLocked blocks on") + } + }) + + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("receive functions never started") + } + // Entered is not yet parked, so still allow the blocking receive to be + // reached. Parking takes microseconds; this margin is six orders larger. + time.Sleep(500 * time.Millisecond) + + reopened := make(chan struct{}) + go func() { + // The error is irrelevant; StdNetBind rejects a second Open. What + // matters is that the call returns at all. + _, _, _ = iceBind.Open(0) + close(reopened) + }() + + select { + case <-reopened: + case <-time.After(10 * time.Second): + t.Fatal("Open blocked while a receive function was parked; wireguard-go makes this call with device.net held, which is what stalls interface creation") + } +} + +// TestICEBindConcurrentOpenClose exercises Open and Close from separate +// goroutines, the way Device.IpcSet and Device.Up reach the bind, and is meant +// to be run under -race. +// +// closed and closedChan must be updated together. When they were not, Close +// could observe a stale closed and either skip close(closedChan) and +// StdNetBind.Close entirely, leaving the receive functions running, or race a +// second Close and close the same channel twice. +// +// Receive functions are deliberately not started here: this test is about the +// shared state, and parking them would turn a race report into a hang. +func TestICEBindConcurrentOpenClose(t *testing.T) { + iceBind := setupICEBind(t) + + var wg sync.WaitGroup + wg.Add(2) + + // Release both loops together so the calls genuinely interleave rather + // than depending on goroutine start order. + start := make(chan struct{}) + + go func() { + defer wg.Done() + <-start + for i := 0; i < 200; i++ { + _, _, _ = iceBind.Open(0) + } + }() + go func() { + defer wg.Done() + <-start + for i := 0; i < 200; i++ { + _ = iceBind.Close() + } + }() + + close(start) + wg.Wait() + require.NoError(t, iceBind.Close()) + require.True(t, isClosed(iceBind), "the final Close must leave the current generation channel closed") +} + +// TestICEBindCloseReleasesReceiversUnderConcurrentClose runs full Open, receive, +// Close cycles with a second Close and an Open racing the first Close. Any +// iteration where the receive functions outlive Close, or where the surviving +// generation channel is left open, is the state closeBindLocked deadlocks on. +func TestICEBindCloseReleasesReceiversUnderConcurrentClose(t *testing.T) { + if testing.Short() { + t.Skip("stress test") + } + + for i := 0; i < 200; i++ { + iceBind := setupICEBind(t) + + fns, _, err := iceBind.Open(0) + require.NoError(t, err, "iteration %d: opening the bind must succeed", i) + wg := startReceivers(fns) + + start := make(chan struct{}) + var racers sync.WaitGroup + racers.Add(3) + for c := 0; c < 2; c++ { + go func() { + defer racers.Done() + <-start + _ = iceBind.Close() + }() + } + // An Open overlapping the Closes is what produces a generation whose + // channel outlives the flag saying the bind is closed. + go func() { + defer racers.Done() + <-start + _, _, _ = iceBind.Open(0) + }() + close(start) + racers.Wait() + + // Settle on a closed bind whatever order the racers landed in. + _ = iceBind.Close() + + if !receiversStopped(wg, 5*time.Second) { + t.Fatalf("iteration %d: receive functions still running after Close; closeBindLocked would block here on device.net.stopping.Wait", i) + } + if !isClosed(iceBind) { + t.Fatalf("iteration %d: Close returned with the current generation channel still open, so nothing will ever release its receivers", i) + } + } +} diff --git a/client/iface/bind/ice_bind_test.go b/client/iface/bind/ice_bind_test.go index 0b8db7640..8dd82d7df 100644 --- a/client/iface/bind/ice_bind_test.go +++ b/client/iface/bind/ice_bind_test.go @@ -2,17 +2,22 @@ package bind import ( "fmt" + "io" "net" "net/netip" "sync" + "sync/atomic" "testing" "time" + "github.com/pion/transport/v3" "github.com/pion/transport/v3/stdnet" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" + "golang.zx2c4.com/wireguard/device" + "golang.zx2c4.com/wireguard/tun/netstack" "github.com/netbirdio/netbird/client/iface/wgaddr" ) @@ -326,3 +331,86 @@ func listenUDP(t *testing.T, network, addr string) *net.UDPConn { require.NoError(t, err) return conn } + +type countingNet struct { + transport.Net + calls atomic.Int64 + ifaces []*transport.Interface + before func() +} + +func (n *countingNet) Interfaces() ([]*transport.Interface, error) { + n.calls.Add(1) + if n.before != nil { + n.before() + } + return n.ifaces, nil +} + +func TestICEBind_OpenDoesNotDiscoverInterfaces(t *testing.T) { + n := newCountingNet() + iceBind := NewICEBind(n, testAddress(), 1280) + + for i := 0; i < 2; i++ { + _, port, err := iceBind.Open(0) + require.NoError(t, err) + require.NotZero(t, port, "the bind listens on a port") + assert.Equal(t, int64(0), n.calls.Load(), "opening the bind must not enumerate interfaces, attempt %d", i+1) + require.NoError(t, iceBind.Close()) + } + + _, _, err := iceBind.Open(0) + require.NoError(t, err) + t.Cleanup(func() { _ = iceBind.Close() }) + + mux, err := iceBind.GetICEMux() + require.NoError(t, err) + assert.NotEmpty(t, mux.SingleSocketUDPMux.GetListenAddresses(), "listen addresses come from the discovered interfaces") + assert.Equal(t, int64(1), n.calls.Load(), "interfaces are discovered when the listen addresses are asked for") +} + +func TestICEBind_DeviceUpDoesNotWaitOnInterfaceDiscovery(t *testing.T) { + tunDev, _, err := netstack.CreateNetTUN([]netip.Addr{netip.MustParseAddr("100.64.0.1")}, nil, 1280) + require.NoError(t, err) + + var dev *device.Device + n := newCountingNet() + n.before = func() { + _ = dev.IpcGetOperation(io.Discard) + } + iceBind := NewICEBind(n, testAddress(), 1280) + dev = device.NewDevice(tunDev, iceBind, device.NewLogger(device.LogLevelSilent, "")) + + upDone := make(chan error, 1) + go func() { + upDone <- dev.Up() + }() + + select { + case err := <-upDone: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("device Up did not return: interface discovery ran under the device lock and waited on the device itself") + } + t.Cleanup(dev.Close) + + assert.Equal(t, int64(0), n.calls.Load(), "bringing the device up must not enumerate interfaces") + + mux, err := iceBind.GetICEMux() + require.NoError(t, err) + assert.NotEmpty(t, mux.SingleSocketUDPMux.GetListenAddresses(), "discovery that talks to the device succeeds once the device is up") + assert.Equal(t, int64(1), n.calls.Load(), "interfaces are discovered outside the device lock") +} + +func newCountingNet() *countingNet { + ifc := transport.NewInterface(net.Interface{Index: 1, MTU: 1500, Name: "eth0", Flags: net.FlagUp | net.FlagMulticast}) + ifc.AddAddress(&net.IPNet{IP: net.IPv4(192, 168, 1, 10), Mask: net.CIDRMask(24, 32)}) + return &countingNet{ifaces: []*transport.Interface{ifc}} +} + +func testAddress() wgaddr.Address { + return wgaddr.Address{ + IP: netip.MustParseAddr("100.64.0.1"), + Network: netip.MustParsePrefix("100.64.0.0/10"), + } +} diff --git a/client/iface/bind/stun_filter_test.go b/client/iface/bind/stun_filter_test.go new file mode 100644 index 000000000..0e118e0fd --- /dev/null +++ b/client/iface/bind/stun_filter_test.go @@ -0,0 +1,215 @@ +//go:build !js + +package bind + +import ( + "encoding/binary" + "net" + "testing" + "time" + + "github.com/pion/stun/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/ipv4" + wgConn "golang.zx2c4.com/wireguard/conn" +) + +// magicCookieBytes is the STUN magic cookie as it appears on the wire. In a +// WireGuard message the same offset holds the receiver (or sender) index, which is +// a random uint32, so a session can draw exactly this value. +var magicCookieBytes = []byte{0x21, 0x12, 0xA4, 0x42} + +const testBufSize = 1500 + +// wgMsg builds a WireGuard message of the given type and size, with the index field +// at bytes 4:8 set to index. +func wgMsg(msgType uint32, size int, index []byte) []byte { + pkt := make([]byte, size) + binary.LittleEndian.PutUint32(pkt[:4], msgType) + copy(pkt[4:8], index) + return pkt +} + +// intoBuffer copies pkt into a full-size receive buffer, the way the kernel read +// does, so tests see the same buffer/length split as the hot path. +func intoBuffer(pkt []byte) [][]byte { + buf := make([]byte, testBufSize) + copy(buf, pkt) + return [][]byte{buf} +} + +func TestFilterOutStunMessages_PassesWireGuardWithCookieShapedIndex(t *testing.T) { + tests := []struct { + name string + msgType uint32 + size int + }{ + {"transport data", wgMsgTypeTransport, 128}, + {"keepalive", wgMsgTypeTransport, wgMinMsgSize}, + {"handshake initiation", wgMsgTypeHandshakeInitiation, 148}, + {"handshake response", 2, 92}, + {"cookie reply", 3, 64}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pkt := wgMsg(tc.msgType, tc.size, magicCookieBytes) + require.True(t, stun.IsMessage(pkt), "precondition: pion sees this as STUN") + + buffers := intoBuffer(pkt) + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, tc.size, &net.UDPAddr{}) + assert.NoError(t, err) + assert.False(t, filtered, "WireGuard message must be handed to WireGuard, not the STUN handler") + assert.Len(t, buffers[0], testBufSize, "buffer must be left intact for WireGuard") + }) + } +} + +func TestFilterOutStunMessages_FiltersRealSTUNMessage(t *testing.T) { + msg, err := stun.Build(stun.BindingRequest, stun.TransactionID, stun.Fingerprint) + require.NoError(t, err) + + buffers := intoBuffer(msg.Raw) + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, len(msg.Raw), &net.UDPAddr{}) + assert.NoError(t, err) + assert.True(t, filtered, "STUN message must be consumed by the STUN handler") + assert.Empty(t, buffers[0], "consumed buffer must be emptied so WireGuard does not see it") +} + +// TestIsWireGuardMsg_DisjointFromSTUN locks the invariant the filter relies on: a +// well formed STUN message long enough to be a WireGuard message always has a +// non-zero length field, so it cannot be mistaken for a WireGuard header. +func TestIsWireGuardMsg_DisjointFromSTUN(t *testing.T) { + types := []stun.MessageType{ + stun.BindingRequest, + stun.BindingSuccess, + stun.BindingError, + {Method: stun.MethodBinding, Class: stun.ClassIndication}, + } + + for _, msgType := range types { + // Long enough that the length guard is not what makes this pass. + msg, err := stun.Build(msgType, stun.TransactionID, + stun.NewUsername("remoteUfrag:localUfrag"), stun.Fingerprint) + require.NoError(t, err) + require.GreaterOrEqual(t, len(msg.Raw), wgMinMsgSize, "precondition: %s", msgType) + assert.False(t, isWireGuardMsg(msg.Raw), + "%s must not look like a WireGuard message", msgType) + } +} + +func TestIsWireGuardMsg(t *testing.T) { + tests := []struct { + name string + pkt []byte + want bool + }{ + {"transport data", wgMsg(wgMsgTypeTransport, 128, nil), true}, + {"handshake initiation", wgMsg(wgMsgTypeHandshakeInitiation, 148, nil), true}, + {"unknown type 5", wgMsg(5, 128, nil), false}, + {"type 0", wgMsg(0, 128, nil), false}, + {"non-zero reserved byte", []byte{0x04, 0x00, 0x01, 0x00}, false}, + {"too short", []byte{0x04, 0x00, 0x00}, false}, + {"empty", nil, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isWireGuardMsg(tc.pkt), "wrong classification for %s", tc.name) + }) + } +} + +// TestFilterOutStunMessages_IgnoresBytesBeyondPacket guards against classifying on +// buffer contents left over from an earlier, longer packet. +func TestFilterOutStunMessages_IgnoresBytesBeyondPacket(t *testing.T) { + buf := make([]byte, testBufSize) + copy(buf[4:8], magicCookieBytes) + buffers := [][]byte{buf} + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, 2, &net.UDPAddr{}) + assert.NoError(t, err) + assert.False(t, filtered, "a 2 byte packet must not be classified from stale buffer bytes") +} + +// TestReceiveFn_ClearsSizeOfConsumedPacket covers the accounting WireGuard relies +// on: sizes is reused across reads, so a slot whose packet was consumed as STUN must +// be reported as empty. Otherwise WireGuard reprocesses the same buffer under the +// previous packet's length, which for a WireGuard-shaped packet means it is handled +// twice. +func TestReceiveFn_ClearsSizeOfConsumedPacket(t *testing.T) { + conn := listenUDP(t, "udp4", "127.0.0.1:0") + defer conn.Close() + + recvFn := receiverCreator{setupICEBind(t)}.CreateReceiverFn( + ipv4.NewPacketConn(conn), conn, false, createMsgPool(), + ) + + msg, err := stun.Build(stun.BindingRequest, stun.TransactionID, stun.Fingerprint) + require.NoError(t, err) + + sender := listenUDP(t, "udp4", "127.0.0.1:0") + defer sender.Close() + _, err = sender.WriteTo(msg.Raw, conn.LocalAddr()) + require.NoError(t, err) + + require.NoError(t, conn.SetReadDeadline(time.Now().Add(3*time.Second))) + + bufs := [][]byte{make([]byte, 1500)} + // A leftover size from an earlier read, which is what makes the missing reset + // observable. + sizes := []int{148} + eps := make([]wgConn.Endpoint, 1) + + n, err := recvFn(bufs, sizes, eps) + require.NoError(t, err) + require.Equal(t, 1, n) + assert.Zero(t, sizes[0], "consumed STUN packet must not leave a size behind for WireGuard") +} + +func TestIsTransportPkg(t *testing.T) { + tests := []struct { + name string + pkt []byte + n int + want bool + }{ + {"transport data with payload", wgMsg(wgMsgTypeTransport, 128, nil), 128, true}, + {"keepalive", wgMsg(wgMsgTypeTransport, wgMinMsgSize, nil), wgMinMsgSize, false}, + {"handshake initiation", wgMsg(wgMsgTypeHandshakeInitiation, 148, nil), 148, false}, + {"stale type bytes beyond packet", wgMsg(wgMsgTypeTransport, 128, nil), 2, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isTransportPkg(intoBuffer(tc.pkt), tc.n), + "wrong activity classification for %s", tc.name) + }) + } +} + +// TestFilterOutStunMessages_ConsumesSTUNWithWireGuardShapedType covers the one STUN +// encoding whose leading bytes collide with a WireGuard message type: method 0x080 as a +// request encodes to 0x0200, so the type byte reads as a handshake response and the byte +// after it is zero. Only the length check keeps such a message out of WireGuard's hands. +// pion implements no method in that range, so this is a synthetic worst case rather than +// traffic ICE produces. +func TestFilterOutStunMessages_ConsumesSTUNWithWireGuardShapedType(t *testing.T) { + msg, err := stun.Build(stun.NewType(stun.Method(0x080), stun.ClassRequest), stun.TransactionID) + require.NoError(t, err) + require.Equal(t, []byte{0x02, 0x00, 0x00, 0x00}, msg.Raw[:4], + "precondition: the leading bytes read as a WireGuard message type") + + buffers := intoBuffer(msg.Raw) + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, len(msg.Raw), &net.UDPAddr{}) + assert.NoError(t, err) + assert.True(t, filtered, "STUN message must be consumed despite its WireGuard-shaped type") +} diff --git a/client/iface/configurer/allowedips.go b/client/iface/configurer/allowedips.go new file mode 100644 index 000000000..08a766ee0 --- /dev/null +++ b/client/iface/configurer/allowedips.go @@ -0,0 +1,201 @@ +//go:build (linux && !android) || freebsd + +package configurer + +import ( + "net" + "net/netip" + "slices" + "sync" + + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +// allowedIPStore mirrors the allowed IPs configured on each peer of a device. +// +// A configurer is the only writer of its device's peer set, so the mirror is authoritative +// by construction. It spares the paths that have to rewrite one peer's allowed IPs a full +// device dump just to recover prefixes the process already configured itself. +// +// An allowed IP belongs to exactly one peer: configuring a prefix on a peer takes it away +// from whichever peer held it before, and the configurer leaves that handover to the device +// rather than removing the prefix from the previous holder itself. The store tracks the +// owner of each prefix and performs the same handover, so rewriting one peer's list never +// takes a prefix back from the peer that owns it now. +// +// Its own lock guards the map alone, not the device write it accompanies. Consistency +// between the two rests on the caller serializing every configurer call, which WGIface +// does with its mutex; two unserialized writers would interleave a device write with the +// record of a different one. +// +// An operator reconfiguring the device out of band, through `wg set` or the UAPI socket, +// is the one way the mirror can still go stale. A peer missing from it falls back to the +// device, which reseats that peer's prefixes and their ownership; a peer that is present +// does not, so one recorded from empty while the device already held prefixes keeps only +// what was recorded, and the next endpoint removal drops the rest. +type allowedIPStore struct { + mu sync.RWMutex + peers map[wgtypes.Key][]netip.Prefix + owners map[netip.Prefix]wgtypes.Key +} + +func newAllowedIPStore() *allowedIPStore { + return &allowedIPStore{ + peers: make(map[wgtypes.Key][]netip.Prefix), + owners: make(map[netip.Prefix]wgtypes.Key), + } +} + +// get returns the prefixes recorded for a peer, and whether the peer is known at all. +// The caller receives a copy and may retain or modify it freely. +func (s *allowedIPStore) get(key wgtypes.Key) ([]netip.Prefix, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + prefixes, ok := s.peers[key] + if !ok { + return nil, false + } + return slices.Clone(prefixes), true +} + +// set replaces the prefixes recorded for a peer. +func (s *allowedIPStore) set(key wgtypes.Key, prefixes []netip.Prefix) { + s.mu.Lock() + defer s.mu.Unlock() + + k := key + s.releaseLocked(k) + + normalized := normalizePrefixes(prefixes) + for _, prefix := range normalized { + s.claimLocked(k, prefix) + } + s.peers[k] = normalized +} + +// add records prefixes on a peer without dropping the ones already there, matching the +// union semantics of a peer update that does not replace its allowed IPs. It records the +// peer if it is not known yet, so it belongs to the operations that create a peer on the +// device rather than to the update-only ones. +func (s *allowedIPStore) add(key wgtypes.Key, prefixes []netip.Prefix) { + s.mu.Lock() + defer s.mu.Unlock() + + s.mergeLocked(key, prefixes) +} + +// addExisting is add for an update-only device operation. Such an operation is a silent +// no-op when the peer is absent, so recording a peer here would leave the store claiming +// prefixes the device never took, and the peer would then be recreated by the next endpoint +// removal, stealing those allowed IPs from the peer that legitimately holds them. +func (s *allowedIPStore) addExisting(key wgtypes.Key, prefixes []netip.Prefix) { + s.mu.Lock() + defer s.mu.Unlock() + + k := key + if _, ok := s.peers[k]; !ok { + return + } + s.mergeLocked(k, prefixes) +} + +// ensure records a peer with no prefixes unless it is already known. A device operation +// that is not update-only creates the peer when it is absent, so it has to be recorded even +// when it configures nothing else; otherwise the peer exists on the device while the store +// treats it as unknown, and a prefix later handed over to it is not accounted for. +func (s *allowedIPStore) ensure(key wgtypes.Key) { + s.mu.Lock() + defer s.mu.Unlock() + + k := key + if _, ok := s.peers[k]; !ok { + s.peers[k] = nil + } +} + +// forget drops every prefix recorded for a peer. +func (s *allowedIPStore) forget(key wgtypes.Key) { + s.mu.Lock() + defer s.mu.Unlock() + + k := key + s.releaseLocked(k) + delete(s.peers, k) +} + +// reset drops every peer, mirroring a device reconfiguration that replaces the peer set. +func (s *allowedIPStore) reset() { + s.mu.Lock() + defer s.mu.Unlock() + + s.peers = make(map[wgtypes.Key][]netip.Prefix) + s.owners = make(map[netip.Prefix]wgtypes.Key) +} + +// mergeLocked unions normalized prefixes into a peer and transfers their ownership. +// The caller must hold s.mu for writing. +func (s *allowedIPStore) mergeLocked(k wgtypes.Key, prefixes []netip.Prefix) { + merged := s.peers[k] + for _, prefix := range prefixes { + prefix = normalizePrefix(prefix) + s.claimLocked(k, prefix) + if !slices.Contains(merged, prefix) { + merged = append(merged, prefix) + } + } + s.peers[k] = merged +} + +// claimLocked hands a prefix over to a peer, taking it from its previous owner the way the +// device does when the same prefix is configured on a second peer. +func (s *allowedIPStore) claimLocked(k wgtypes.Key, prefix netip.Prefix) { + if owner, ok := s.owners[prefix]; ok && owner != k { + s.peers[owner] = slices.DeleteFunc(s.peers[owner], func(p netip.Prefix) bool { + return p == prefix + }) + } + s.owners[prefix] = k +} + +// releaseLocked drops a peer's claim on every prefix it currently holds. +func (s *allowedIPStore) releaseLocked(k wgtypes.Key) { + for _, prefix := range s.peers[k] { + if s.owners[prefix] == k { + delete(s.owners, prefix) + } + } +} + +// ipNetsToPrefixes converts addresses read back from a device. Unmap keeps a v4-mapped v6 +// address comparable to the plain v4 prefix the configurer was given. +func ipNetsToPrefixes(ipNets []net.IPNet) []netip.Prefix { + prefixes := make([]netip.Prefix, 0, len(ipNets)) + for _, ipNet := range ipNets { + addr, ok := netip.AddrFromSlice(ipNet.IP) + if !ok { + continue + } + + ones, maskBits := ipNet.Mask.Size() + // A device may report a v4 prefix as a v4-mapped address. Align the address form with + // the mask rather than unmapping on sight: a 32 bit mask always describes v4, while a + // 128 bit mask describes v4 only when it covers the mapped prefix, so a genuine v6 + // prefix inside the mapped range stays v6 instead of being dropped as invalid. + if addr.Is4In6() { + switch { + case maskBits == 32: + addr = addr.Unmap() + case maskBits == 128 && ones >= 96: + addr, ones = addr.Unmap(), ones-96 + } + } + + prefix := netip.PrefixFrom(addr, ones) + if !prefix.IsValid() { + continue + } + prefixes = append(prefixes, prefix.Masked()) + } + return prefixes +} diff --git a/client/iface/configurer/allowedips_test.go b/client/iface/configurer/allowedips_test.go new file mode 100644 index 000000000..405609e29 --- /dev/null +++ b/client/iface/configurer/allowedips_test.go @@ -0,0 +1,192 @@ +//go:build (linux && !android) || freebsd + +package configurer + +import ( + "net" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +// The store keys on the parsed key, so the tests use two distinct ones rather than names. +var ( + testPeer = wgtypes.Key{1} + otherPeer = wgtypes.Key{2} +) + +func TestAllowedIPStoreUnknownPeer(t *testing.T) { + s := newAllowedIPStore() + + prefixes, ok := s.get(testPeer) + assert.False(t, ok, "an unconfigured peer must be reported as unknown, not as one without prefixes") + assert.Nil(t, prefixes, "an unknown peer has no prefixes") +} + +func TestAllowedIPStoreAddUnions(t *testing.T) { + s := newAllowedIPStore() + overlay := netip.MustParsePrefix("100.64.0.1/32") + routed := netip.MustParsePrefix("10.20.0.0/16") + + s.set(testPeer, []netip.Prefix{overlay}) + // A peer update does not replace allowed IPs, and a repeated prefix must not be doubled. + s.add(testPeer, []netip.Prefix{overlay, routed}) + + prefixes, ok := s.get(testPeer) + require.True(t, ok, "peer must be known after set") + assert.Equal(t, []netip.Prefix{overlay, routed}, prefixes, "add must union rather than replace") +} + +func TestAllowedIPStoreGetReturnsCopy(t *testing.T) { + s := newAllowedIPStore() + overlay := netip.MustParsePrefix("100.64.0.1/32") + s.set(testPeer, []netip.Prefix{overlay}) + + prefixes, ok := s.get(testPeer) + require.True(t, ok, "peer must be known after set") + prefixes[0] = netip.MustParsePrefix("0.0.0.0/0") + + stored, _ := s.get(testPeer) + assert.Equal(t, []netip.Prefix{overlay}, stored, "a caller mutating the returned slice must not corrupt the store") +} + +func TestAllowedIPStoreForgetAndReset(t *testing.T) { + s := newAllowedIPStore() + s.set(testPeer, []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}) + s.set(otherPeer, []netip.Prefix{netip.MustParsePrefix("100.64.0.2/32")}) + + s.forget(testPeer) + _, ok := s.get(testPeer) + assert.False(t, ok, "a forgotten peer must be unknown") + _, ok = s.get(otherPeer) + assert.True(t, ok, "forgetting one peer must not touch the others") + + s.reset() + _, ok = s.get(otherPeer) + assert.False(t, ok, "reset must drop every peer") +} + +func TestIPNetsToPrefixes(t *testing.T) { + tests := []struct { + name string + ipNet net.IPNet + want string + }{ + { + name: "v4", + ipNet: net.IPNet{IP: net.IP{10, 20, 0, 0}, Mask: net.CIDRMask(16, 32)}, + want: "10.20.0.0/16", + }, + { + name: "v4 mapped under a 128 bit mask", + ipNet: net.IPNet{IP: net.ParseIP("10.20.0.0"), Mask: net.CIDRMask(112, 128)}, + want: "10.20.0.0/16", + }, + { + name: "v6", + ipNet: net.IPNet{IP: net.ParseIP("fd00::"), Mask: net.CIDRMask(64, 128)}, + want: "fd00::/64", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := ipNetsToPrefixes([]net.IPNet{tc.ipNet}) + require.Len(t, got, 1, "the address must be converted, not dropped") + assert.Equal(t, tc.want, got[0].String(), "converted prefix") + }) + } +} + +func TestIPNetsToPrefixesRoundTrip(t *testing.T) { + prefixes := []netip.Prefix{ + netip.MustParsePrefix("100.64.0.1/32"), + netip.MustParsePrefix("10.20.0.0/16"), + netip.MustParsePrefix("fd00::/64"), + } + + assert.Equal(t, prefixes, ipNetsToPrefixes(prefixesToIPNets(prefixes)), + "prefixes handed to a device must come back unchanged") +} + +func TestAllowedIPStoreNormalizesMappedPrefixes(t *testing.T) { + s := newAllowedIPStore() + v4 := netip.MustParsePrefix("10.20.0.0/16") + mapped := netip.PrefixFrom(netip.AddrFrom16(v4.Addr().As16()), 112) + + s.set(testPeer, []netip.Prefix{mapped}) + // A v4 rule only matches a v4-mapped address once it has been unmapped, so the store must + // hold the plain form and recognise the two spellings as the same prefix. + s.add(testPeer, []netip.Prefix{v4}) + + prefixes, ok := s.get(testPeer) + require.True(t, ok, "peer must be known after set") + assert.Equal(t, []netip.Prefix{v4}, prefixes, "a mapped prefix must be stored unmapped and not duplicated") +} + +func TestAllowedIPStoreAddExistingDoesNotCreate(t *testing.T) { + s := newAllowedIPStore() + routed := netip.MustParsePrefix("10.20.0.0/16") + + // An update-only device operation on an absent peer is a silent no-op, so nothing may be + // recorded for a peer the store does not already know. + s.addExisting(testPeer, []netip.Prefix{routed}) + _, ok := s.get(testPeer) + assert.False(t, ok, "addExisting must not record an unknown peer") + + overlay := netip.MustParsePrefix("100.64.0.1/32") + s.set(testPeer, []netip.Prefix{overlay}) + s.addExisting(testPeer, []netip.Prefix{routed}) + + prefixes, _ := s.get(testPeer) + assert.Equal(t, []netip.Prefix{overlay, routed}, prefixes, "addExisting must union onto a known peer") +} + +func TestAllowedIPStoreHandsPrefixOverToTheNewOwner(t *testing.T) { + s := newAllowedIPStore() + routed := netip.MustParsePrefix("10.20.0.0/16") + other := otherPeer + + s.set(testPeer, []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32"), routed}) + s.set(other, []netip.Prefix{netip.MustParsePrefix("100.64.0.2/32")}) + + // The device takes an allowed IP away from its previous holder when it is configured on + // another peer, so the store must do the same rather than list it under both. + s.addExisting(other, []netip.Prefix{routed}) + + previous, _ := s.get(testPeer) + assert.NotContains(t, previous, routed, "the previous owner must lose the prefix") + current, _ := s.get(other) + assert.Contains(t, current, routed, "the new owner must hold the prefix") +} + +func TestAllowedIPStoreForgetReleasesOwnership(t *testing.T) { + s := newAllowedIPStore() + routed := netip.MustParsePrefix("10.20.0.0/16") + + s.set(testPeer, []netip.Prefix{routed}) + s.forget(testPeer) + s.set(otherPeer, []netip.Prefix{routed}) + + // A forgotten peer must not be resurrected as a key in the peer map by a later claim. + _, ok := s.get(testPeer) + assert.False(t, ok, "the forgotten peer must stay unknown") + current, _ := s.get(otherPeer) + assert.Equal(t, []netip.Prefix{routed}, current, "the new owner must hold the prefix") +} + +func TestIPNetsToPrefixesKeepsV6InTheMappedRange(t *testing.T) { + // ::ffff:0:0/64 reads as v4-mapped but is a genuine v6 prefix: unmapping it would leave a + // v4 address under a 64 bit mask, which is invalid, and the allowed IP would be dropped. + got := ipNetsToPrefixes([]net.IPNet{{ + IP: net.ParseIP("::ffff:0:0"), + Mask: net.CIDRMask(64, 128), + }}) + + require.Len(t, got, 1, "the prefix must be converted, not dropped") + assert.False(t, got[0].Addr().Is4(), "a v6 prefix in the mapped range must not become v4") + assert.Equal(t, 64, got[0].Bits(), "the prefix length must survive the conversion") +} diff --git a/client/iface/configurer/common.go b/client/iface/configurer/common.go index 10162d703..02d2a7e6a 100644 --- a/client/iface/configurer/common.go +++ b/client/iface/configurer/common.go @@ -3,29 +3,48 @@ package configurer import ( "net" "net/netip" - - "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) -// buildPresharedKeyConfig creates a wgtypes.Config for setting a preshared key on a peer. -// This is a shared helper used by both kernel and userspace configurers. -func buildPresharedKeyConfig(peerKey wgtypes.Key, psk wgtypes.Key, updateOnly bool) wgtypes.Config { - return wgtypes.Config{ - Peers: []wgtypes.PeerConfig{{ - PublicKey: peerKey, - PresharedKey: &psk, - UpdateOnly: updateOnly, - }}, - } -} - +// prefixesToIPNets converts prefixes on their way to a device. It is the only place that +// conversion happens, so it also normalizes: the device is then given the same form the +// store records, and a v4-mapped prefix cannot reach net.IPNet, which prints such an +// address as v4 while taking the length from its 16 byte mask and so turns +// ::ffff:10.1.2.3/64 into 10.1.2.3/0 — an allowed IP matching every v4 address. func prefixesToIPNets(prefixes []netip.Prefix) []net.IPNet { ipNets := make([]net.IPNet, len(prefixes)) for i, prefix := range prefixes { + normalized := normalizePrefix(prefix) ipNets[i] = net.IPNet{ - IP: prefix.Addr().AsSlice(), // Convert netip.Addr to net.IP - Mask: net.CIDRMask(prefix.Bits(), prefix.Addr().BitLen()), // Create subnet mask + IP: normalized.Addr().AsSlice(), + Mask: net.CIDRMask(normalized.Bits(), normalized.Addr().BitLen()), } } return ipNets } + +// normalizePrefix puts a prefix into the form the store recognises it by. It clears the +// host bits, which a device does on its own, so a caller passing 10.20.0.1/16 still matches +// the 10.20.0.0/16 read back from the device; and it unmaps a v4-mapped prefix so that it +// compares equal to, and marshals like, the plain v4 prefix for the same network. +// +// Masking comes first because it also decides the address family: only a prefix at least 96 +// bits long keeps the mapped marker through the mask, so a shorter prefix inside the mapped +// range is a genuine v6 prefix and unmapping it would yield an invalid v4 prefix. +func normalizePrefix(prefix netip.Prefix) netip.Prefix { + masked := prefix.Masked() + + addr := masked.Addr() + if !addr.Is4In6() { + return masked + } + return netip.PrefixFrom(addr.Unmap(), masked.Bits()-96) +} + +// normalizePrefixes returns a normalized copy without changing the caller's slice. +func normalizePrefixes(prefixes []netip.Prefix) []netip.Prefix { + normalized := make([]netip.Prefix, len(prefixes)) + for i, prefix := range prefixes { + normalized[i] = normalizePrefix(prefix) + } + return normalized +} diff --git a/client/iface/configurer/common_test.go b/client/iface/configurer/common_test.go new file mode 100644 index 000000000..487c1a734 --- /dev/null +++ b/client/iface/configurer/common_test.go @@ -0,0 +1,81 @@ +package configurer + +import ( + "net" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizePrefix(t *testing.T) { + v4 := netip.MustParsePrefix("10.20.0.0/16") + v6 := netip.MustParsePrefix("fd00::/64") + + assert.Equal(t, v4, normalizePrefix(v4), "a plain v4 prefix is unchanged") + assert.Equal(t, v6, normalizePrefix(v6), "a real v6 prefix is unchanged") + assert.Equal(t, v4, normalizePrefix(netip.PrefixFrom(netip.AddrFrom16(v4.Addr().As16()), 112)), + "a mapped prefix under a 128 bit mask becomes plain v4") + // A prefix shorter than /96 inside the mapped range is a genuine v6 prefix. Unmapping it + // would pair a v4 address with a v6 sized mask, which is invalid, and the store would then + // record a zero prefix that can never recreate the allowed IP. + for _, tc := range []string{"::ffff:0:0/64", "::ffff:1.2.3.4/80", "::ffff:1.2.3.4/95"} { + got := normalizePrefix(netip.MustParsePrefix(tc)) + assert.True(t, got.IsValid(), "%s must normalize to a valid prefix", tc) + assert.False(t, got.Addr().Is4(), "%s must stay v6", tc) + } +} + +func TestNormalizePrefixClearsHostBits(t *testing.T) { + // A device stores a prefix masked, so a caller passing host bits must still match what a + // device fallback seeded, otherwise that prefix could never be removed by value. + assert.Equal(t, netip.MustParsePrefix("10.20.0.0/16"), + normalizePrefix(netip.MustParsePrefix("10.20.0.1/16")), "host bits must be cleared") + assert.Equal(t, netip.MustParsePrefix("fd00::/64"), + normalizePrefix(netip.MustParsePrefix("fd00::1/64")), "host bits must be cleared for v6") +} + +func TestPrefixesToIPNetsNormalizes(t *testing.T) { + // net.IPNet prints a v4-mapped address as v4 but takes the length from its 16 byte + // mask, so an unnormalized ::ffff:10.1.2.3/64 reaches a userspace device as 10.1.2.3/0, + // an allowed IP that matches every v4 address. + tests := []struct { + name string + given string + want string + }{ + {name: "mapped below /96", given: "::ffff:10.1.2.3/64", want: "::/64"}, + {name: "mapped at /112", given: "::ffff:10.1.2.3/112", want: "10.1.0.0/16"}, + {name: "host bits are cleared", given: "10.20.0.1/16", want: "10.20.0.0/16"}, + {name: "v6 is untouched", given: "fd00::1/64", want: "fd00::/64"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := prefixesToIPNets([]netip.Prefix{netip.MustParsePrefix(tc.given)}) + require.Len(t, got, 1, "the prefix must be converted, not dropped") + assert.Equal(t, tc.want, got[0].String(), "what the device is given") + assert.NotEqual(t, 0, mustOnes(t, got[0]), "a device must never be given a zero length allowed IP") + }) + } +} + +func TestPrefixesToIPNetsAgreesWithTheStore(t *testing.T) { + for _, given := range []string{"::ffff:10.1.2.3/64", "::ffff:10.1.2.3/112", "10.20.0.1/16", "fd00::1/64"} { + prefix := netip.MustParsePrefix(given) + + toDevice := prefixesToIPNets([]netip.Prefix{prefix}) + recorded := normalizePrefix(prefix) + + assert.Equal(t, recorded.String(), toDevice[0].String(), + "%s must reach the device in the form the store records", given) + } +} + +func mustOnes(t *testing.T, ipNet net.IPNet) int { + t.Helper() + + ones, _ := ipNet.Mask.Size() + return ones +} diff --git a/client/iface/configurer/kernel_unix.go b/client/iface/configurer/kernel_unix.go index da69c2a35..afe1300d5 100644 --- a/client/iface/configurer/kernel_unix.go +++ b/client/iface/configurer/kernel_unix.go @@ -3,9 +3,12 @@ package configurer import ( + "errors" "fmt" "net" "net/netip" + "slices" + "sync" "time" log "github.com/sirupsen/logrus" @@ -15,19 +18,45 @@ import ( "github.com/netbirdio/netbird/monotime" ) -type KernelConfigurer struct { - deviceName string - statsCache *statsCache +var errConfigurerClosed = errors.New("configurer closed") + +type wgClient interface { + Device(name string) (*wgtypes.Device, error) + ConfigureDevice(name string, cfg wgtypes.Config) error + Close() error } +type KernelConfigurer struct { + deviceName string + statsCache *statsCache + allowedIPs *allowedIPStore + newWgctrlFn func() (wgClient, error) + + clientMu sync.Mutex + client wgClient + closed bool +} + +// NewKernelConfigurer creates a configurer with an empty allowed IP mirror +// and a statistics cache for the named kernel device. func NewKernelConfigurer(deviceName string) *KernelConfigurer { + return newKernelConfigurer(deviceName, func() (wgClient, error) { + return wgctrl.New() + }) +} + +func newKernelConfigurer(deviceName string, newWgctrlFn func() (wgClient, error)) *KernelConfigurer { c := &KernelConfigurer{ - deviceName: deviceName, + deviceName: deviceName, + allowedIPs: newAllowedIPStore(), + newWgctrlFn: newWgctrlFn, } c.statsCache = newStatsCache(statsCacheTTL, c.fetchStats) return c } +// ConfigureInterface sets the device key, port and firewall mark, replacing all peers. +// The allowed IP mirror is reset only after the device accepts the configuration. func (c *KernelConfigurer) ConfigureInterface(privateKey string, port int) error { log.Debugf("adding Wireguard private key") key, err := wgtypes.ParseKey(privateKey) @@ -46,6 +75,8 @@ func (c *KernelConfigurer) ConfigureInterface(privateKey string, port int) error if err != nil { return fmt.Errorf(`received error "%w" while configuring interface %s with port %d`, err, c.deviceName, port) } + + c.allowedIPs.reset() return nil } @@ -58,9 +89,20 @@ func (c *KernelConfigurer) SetPresharedKey(peerKey string, psk wgtypes.Key, upda } cfg := buildPresharedKeyConfig(parsedPeerKey, psk, updateOnly) - return c.configure(cfg) + if err := c.configure(cfg); err != nil { + return err + } + + // Without updateOnly this creates the peer when it is absent, so the store has to + // know about it even though no allowed IP was configured. + if !updateOnly { + c.allowedIPs.ensure(parsedPeerKey) + } + return nil } +// UpdatePeer creates or updates a peer, merging allowed IPs with its existing set. +// Prefixes assigned to this peer are transferred from their previous owners. func (c *KernelConfigurer) UpdatePeer(peerKey string, allowedIps []netip.Prefix, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { @@ -83,19 +125,23 @@ func (c *KernelConfigurer) UpdatePeer(peerKey string, allowedIps []netip.Prefix, if err != nil { return fmt.Errorf(`received error "%w" while updating peer on interface %s with settings: allowed ips %s, endpoint %s`, err, c.deviceName, allowedIps, endpoint.String()) } + + c.allowedIPs.add(peerKeyParsed, allowedIps) return nil } +// RemoveEndpointAddress clears the endpoint of a peer while keeping it configured. +// Neither the netlink API nor the userspace one can clear an endpoint in place, so the peer +// is removed and re-added with the allowed IPs it already had. func (c *KernelConfigurer) RemoveEndpointAddress(peerKey string) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return err } - // Get the existing peer to preserve its allowed IPs - existingPeer, err := c.getPeer(c.deviceName, peerKey) + allowedIPs, err := c.peerAllowedIPs(peerKeyParsed) if err != nil { - return fmt.Errorf("get peer: %w", err) + return err } removePeerCfg := wgtypes.PeerConfig{ @@ -104,26 +150,27 @@ func (c *KernelConfigurer) RemoveEndpointAddress(peerKey string) error { } if err := c.configure(wgtypes.Config{Peers: []wgtypes.PeerConfig{removePeerCfg}}); err != nil { - return fmt.Errorf(`error removing peer %s from interface %s: %w`, peerKey, c.deviceName, err) + return fmt.Errorf("remove peer %s from interface %s: %w", peerKey, c.deviceName, err) } - //Re-add the peer without the endpoint but same AllowedIPs reAddPeerCfg := wgtypes.PeerConfig{ PublicKey: peerKeyParsed, - AllowedIPs: existingPeer.AllowedIPs, + AllowedIPs: prefixesToIPNets(allowedIPs), ReplaceAllowedIPs: true, } if err := c.configure(wgtypes.Config{Peers: []wgtypes.PeerConfig{reAddPeerCfg}}); err != nil { + c.allowedIPs.forget(peerKeyParsed) return fmt.Errorf( - `error re-adding peer %s to interface %s with allowed IPs %v: %w`, - peerKey, c.deviceName, existingPeer.AllowedIPs, err, + "re-add peer %s to interface %s with allowed IPs %v: %w", + peerKey, c.deviceName, allowedIPs, err, ) } return nil } +// RemovePeer removes a peer and forgets its allowed IPs after a successful device write. func (c *KernelConfigurer) RemovePeer(peerKey string) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { @@ -142,15 +189,13 @@ func (c *KernelConfigurer) RemovePeer(peerKey string) error { if err != nil { return fmt.Errorf(`received error "%w" while removing peer %s from interface %s`, err, peerKey, c.deviceName) } + + c.allowedIPs.forget(peerKeyParsed) return nil } +// AddAllowedIP adds a prefix to an existing peer; an absent peer is a silent no-op. func (c *KernelConfigurer) AddAllowedIP(peerKey string, allowedIP netip.Prefix) error { - ipNet := net.IPNet{ - IP: allowedIP.Addr().AsSlice(), - Mask: net.CIDRMask(allowedIP.Bits(), allowedIP.Addr().BitLen()), - } - peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return err @@ -159,7 +204,7 @@ func (c *KernelConfigurer) AddAllowedIP(peerKey string, allowedIP netip.Prefix) PublicKey: peerKeyParsed, UpdateOnly: true, ReplaceAllowedIPs: false, - AllowedIPs: []net.IPNet{ipNet}, + AllowedIPs: prefixesToIPNets([]netip.Prefix{allowedIP}), } config := wgtypes.Config{ @@ -169,69 +214,75 @@ func (c *KernelConfigurer) AddAllowedIP(peerKey string, allowedIP netip.Prefix) if err != nil { return fmt.Errorf(`received error "%w" while adding allowed Ip to peer on interface %s with settings: allowed ips %s`, err, c.deviceName, allowedIP) } + + c.allowedIPs.addExisting(peerKeyParsed, []netip.Prefix{allowedIP}) return nil } +// RemoveAllowedIP removes a prefix while preserving the peer's other allowed IPs. +// A prefix not assigned to the peer is a no-op. func (c *KernelConfigurer) RemoveAllowedIP(peerKey string, allowedIP netip.Prefix) error { - ipNet := net.IPNet{ - IP: allowedIP.Addr().AsSlice(), - Mask: net.CIDRMask(allowedIP.Bits(), allowedIP.Addr().BitLen()), - } - peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return fmt.Errorf("parse peer key: %w", err) } - existingPeer, err := c.getPeer(c.deviceName, peerKey) + currentAllowedIPs, err := c.peerAllowedIPs(peerKeyParsed) if err != nil { - return fmt.Errorf("get peer: %w", err) + return err } - newAllowedIPs := existingPeer.AllowedIPs - - for i, existingAllowedIP := range existingPeer.AllowedIPs { - if existingAllowedIP.String() == ipNet.String() { - newAllowedIPs = append(existingPeer.AllowedIPs[:i], existingPeer.AllowedIPs[i+1:]...) //nolint:gocritic - break - } + idx := slices.Index(currentAllowedIPs, normalizePrefix(allowedIP)) + if idx < 0 { + return nil } + newAllowedIPs := slices.Delete(currentAllowedIPs, idx, idx+1) peer := wgtypes.PeerConfig{ PublicKey: peerKeyParsed, UpdateOnly: true, ReplaceAllowedIPs: true, - AllowedIPs: newAllowedIPs, + AllowedIPs: prefixesToIPNets(newAllowedIPs), } config := wgtypes.Config{ Peers: []wgtypes.PeerConfig{peer}, } - err = c.configure(config) - if err != nil { + if err := c.configure(config); err != nil { return fmt.Errorf("remove allowed IP %s on interface %s: %w", allowedIP, c.deviceName, err) } + + c.allowedIPs.set(peerKeyParsed, newAllowedIPs) return nil } -func (c *KernelConfigurer) getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) { - wg, err := wgctrl.New() - if err != nil { - return wgtypes.Peer{}, fmt.Errorf("wgctl: %w", err) +// peerAllowedIPs returns the allowed IPs configured for a peer, reading them from the device +// only for a peer the store has not seen. Dumping the device costs a netlink round trip +// proportional to the whole network map, and this runs on every relay and ICE transition. +func (c *KernelConfigurer) peerAllowedIPs(peerKey wgtypes.Key) ([]netip.Prefix, error) { + if prefixes, ok := c.allowedIPs.get(peerKey); ok { + return prefixes, nil } - defer func() { - err = wg.Close() - if err != nil { - log.Errorf("Got error while closing wgctl: %v", err) - } - }() - wgDevice, err := wg.Device(ifaceName) + existingPeer, err := c.getPeer(peerKey) if err != nil { - return wgtypes.Peer{}, fmt.Errorf("get device %s: %w", ifaceName, err) + return nil, fmt.Errorf("get peer: %w", err) + } + + prefixes := ipNetsToPrefixes(existingPeer.AllowedIPs) + c.allowedIPs.set(peerKey, prefixes) + return prefixes, nil +} + +// getPeer scans the device for one peer. wgtypes.Key is an array, so the comparison is a +// plain equality: Key.String would base64 encode into a fresh allocation for every peer. +func (c *KernelConfigurer) getPeer(peerPubKey wgtypes.Key) (wgtypes.Peer, error) { + wgDevice, err := c.device() + if err != nil { + return wgtypes.Peer{}, err } for _, peer := range wgDevice.Peers { - if peer.PublicKey.String() == peerPubKey { + if peer.PublicKey == peerPubKey { return peer, nil } } @@ -239,37 +290,73 @@ func (c *KernelConfigurer) getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, } func (c *KernelConfigurer) configure(config wgtypes.Config) error { - wg, err := wgctrl.New() + c.clientMu.Lock() + defer c.clientMu.Unlock() + + client, err := c.clientLocked() if err != nil { return err } - defer func() { - if err := wg.Close(); err != nil { - log.Errorf("Failed to close wgctrl client: %v", err) - } - }() - - return wg.ConfigureDevice(c.deviceName, config) + if err := client.ConfigureDevice(c.deviceName, config); err != nil { + c.discardLocked() + return err + } + return nil } -func (c *KernelConfigurer) Close() { +func (c *KernelConfigurer) device() (*wgtypes.Device, error) { + c.clientMu.Lock() + defer c.clientMu.Unlock() + + client, err := c.clientLocked() + if err != nil { + return nil, err + } + wgDevice, err := client.Device(c.deviceName) + if err != nil { + c.discardLocked() + return nil, fmt.Errorf("get device %s: %w", c.deviceName, err) + } + return wgDevice, nil } -func (c *KernelConfigurer) FullStats() (*Stats, error) { - wg, err := wgctrl.New() +func (c *KernelConfigurer) clientLocked() (wgClient, error) { + if c.closed { + return nil, errConfigurerClosed + } + if c.client != nil { + return c.client, nil + } + client, err := c.newWgctrlFn() if err != nil { return nil, fmt.Errorf("wgctl: %w", err) } - defer func() { - err = wg.Close() - if err != nil { - log.Errorf("Got error while closing wgctl: %v", err) - } - }() + c.client = client + return client, nil +} - wgDevice, err := wg.Device(c.deviceName) +func (c *KernelConfigurer) discardLocked() { + if c.client == nil { + return + } + if err := c.client.Close(); err != nil { + log.Warnf("failed to close wgctrl client: %v", err) + } + c.client = nil +} + +func (c *KernelConfigurer) Close() { + c.clientMu.Lock() + defer c.clientMu.Unlock() + + c.closed = true + c.discardLocked() +} + +func (c *KernelConfigurer) FullStats() (*Stats, error) { + wgDevice, err := c.device() if err != nil { - return nil, fmt.Errorf("get device %s: %w", c.deviceName, err) + return nil, err } fullStats := &Stats{ DeviceName: wgDevice.Name, @@ -305,23 +392,12 @@ func (c *KernelConfigurer) LastActivities() map[string]monotime.Time { } func (c *KernelConfigurer) fetchStats() (map[string]WGStats, error) { + wgDevice, err := c.device() + if err != nil { + return nil, err + } + stats := make(map[string]WGStats) - wg, err := wgctrl.New() - if err != nil { - return nil, fmt.Errorf("wgctl: %w", err) - } - defer func() { - err = wg.Close() - if err != nil { - log.Errorf("Got error while closing wgctl: %v", err) - } - }() - - wgDevice, err := wg.Device(c.deviceName) - if err != nil { - return nil, fmt.Errorf("get device %s: %w", c.deviceName, err) - } - for _, peer := range wgDevice.Peers { stats[peer.PublicKey.String()] = WGStats{ LastHandshake: peer.LastHandshakeTime, @@ -331,3 +407,13 @@ func (c *KernelConfigurer) fetchStats() (map[string]WGStats, error) { } return stats, nil } + +func buildPresharedKeyConfig(peerKey wgtypes.Key, psk wgtypes.Key, updateOnly bool) wgtypes.Config { + return wgtypes.Config{ + Peers: []wgtypes.PeerConfig{{ + PublicKey: peerKey, + PresharedKey: &psk, + UpdateOnly: updateOnly, + }}, + } +} diff --git a/client/iface/configurer/kernel_unix_test.go b/client/iface/configurer/kernel_unix_test.go new file mode 100644 index 000000000..8daa15a25 --- /dev/null +++ b/client/iface/configurer/kernel_unix_test.go @@ -0,0 +1,167 @@ +//go:build (linux && !android) || freebsd + +package configurer + +import ( + "errors" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +type fakeWGClient struct { + configureErr error + deviceErr error + calls int + closed bool +} + +type fakeWGClientFactory struct { + openErr error + clients []*fakeWGClient +} + +func (f *fakeWGClient) Device(string) (*wgtypes.Device, error) { + f.calls++ + if f.deviceErr != nil { + return nil, f.deviceErr + } + return &wgtypes.Device{}, nil +} + +func (f *fakeWGClient) ConfigureDevice(string, wgtypes.Config) error { + f.calls++ + return f.configureErr +} + +func (f *fakeWGClient) Close() error { + f.closed = true + return nil +} + +func (f *fakeWGClientFactory) open() (wgClient, error) { + if f.openErr != nil { + return nil, f.openErr + } + client := &fakeWGClient{} + f.clients = append(f.clients, client) + return client, nil +} + +func TestKernelConfigurer_ReusesClientAcrossRequests(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + require.NoError(t, c.RemovePeer(peerKey)) + require.NoError(t, c.RemovePeer(peerKey)) + _, err := c.FullStats() + require.NoError(t, err) + + require.Len(t, factory.clients, 1, "all requests should share one client") + assert.Equal(t, 3, factory.clients[0].calls, "every request should go through the shared client") + assert.False(t, factory.clients[0].closed, "a healthy client should stay open") +} + +func TestKernelConfigurer_ReopensClientAfterFailedConfigure(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + require.NoError(t, c.RemovePeer(peerKey)) + require.Len(t, factory.clients, 1) + + factory.clients[0].configureErr = errors.New("netlink down") + require.Error(t, c.RemovePeer(peerKey)) + assert.True(t, factory.clients[0].closed, "the failed client should be closed") + assert.Len(t, factory.clients, 1, "a failure must not open a client on its own") + + require.NoError(t, c.RemovePeer(peerKey)) + require.Len(t, factory.clients, 2, "the next request should open a fresh client") + assert.Equal(t, 1, factory.clients[1].calls, "the request should run on the fresh client") + assert.False(t, factory.clients[1].closed, "the fresh client should stay open") +} + +func TestKernelConfigurer_ReopensClientAfterFailedDeviceQuery(t *testing.T) { + c, factory := newFakeKernelConfigurer() + + _, err := c.FullStats() + require.NoError(t, err) + require.Len(t, factory.clients, 1) + + factory.clients[0].deviceErr = errors.New("netlink down") + _, err = c.FullStats() + require.Error(t, err) + assert.True(t, factory.clients[0].closed, "the failed client should be closed") + + _, err = c.FullStats() + require.NoError(t, err) + assert.Len(t, factory.clients, 2, "the next request should open a fresh client") +} + +func TestKernelConfigurer_RetriesOpenOnNextRequest(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + factory.openErr = errors.New("no netlink") + require.Error(t, c.RemovePeer(peerKey)) + assert.Empty(t, factory.clients, "a failed open leaves no client behind") + + factory.openErr = nil + require.NoError(t, c.RemovePeer(peerKey)) + assert.Len(t, factory.clients, 1, "the next request should open the client") +} + +func TestKernelConfigurer_CloseRejectsFurtherRequests(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + require.NoError(t, c.RemovePeer(peerKey)) + require.Len(t, factory.clients, 1) + + c.Close() + c.Close() + assert.True(t, factory.clients[0].closed, "Close should release the client") + + require.ErrorIs(t, c.RemovePeer(peerKey), errConfigurerClosed) + _, err := c.FullStats() + require.ErrorIs(t, err, errConfigurerClosed) + assert.Len(t, factory.clients, 1, "a closed configurer must not open a client") +} + +func TestKernelConfigurer_ConcurrentRequestsShareOneClient(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + const workers = 16 + errs := make(chan error, workers) + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + errs <- c.RemovePeer(peerKey) + }() + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + + require.Len(t, factory.clients, 1, "concurrent requests should share one client") + assert.Equal(t, workers, factory.clients[0].calls, "every request should reach the shared client") +} + +func newFakeKernelConfigurer() (*KernelConfigurer, *fakeWGClientFactory) { + factory := &fakeWGClientFactory{} + return newKernelConfigurer("wt0", factory.open), factory +} + +func testPeerKey(t *testing.T) string { + t.Helper() + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + return key.PublicKey().String() +} diff --git a/client/iface/configurer/usp.go b/client/iface/configurer/usp.go index 0a25c55bc..1b0965f0d 100644 --- a/client/iface/configurer/usp.go +++ b/client/iface/configurer/usp.go @@ -1,18 +1,16 @@ package configurer import ( - "encoding/base64" - "encoding/hex" "fmt" + "math" "net" "net/netip" "os" "runtime" - "strconv" - "strings" "time" log "github.com/sirupsen/logrus" + wgconn "golang.zx2c4.com/wireguard/conn" "golang.zx2c4.com/wireguard/device" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" @@ -21,19 +19,6 @@ import ( "github.com/netbirdio/netbird/monotime" ) -const ( - privateKey = "private_key" - ipcKeyLastHandshakeTimeSec = "last_handshake_time_sec" - ipcKeyTxBytes = "tx_bytes" - ipcKeyRxBytes = "rx_bytes" - allowedIP = "allowed_ip" - endpoint = "endpoint" - fwmark = "fwmark" - listenPort = "listen_port" - publicKey = "public_key" - presharedKey = "preshared_key" -) - var ErrAllowedIPNotFound = fmt.Errorf("allowed IP not found") type WGUSPConfigurer struct { @@ -45,17 +30,14 @@ type WGUSPConfigurer struct { uapiListener net.Listener } +// NewUSPConfigurer creates a userspace configurer and starts its UAPI listener. func NewUSPConfigurer(device *device.Device, deviceName string, activityRecorder *bind.ActivityRecorder) *WGUSPConfigurer { - wgCfg := &WGUSPConfigurer{ - device: device, - deviceName: deviceName, - activityRecorder: activityRecorder, - } - wgCfg.statsCache = newStatsCache(statsCacheTTL, wgCfg.fetchStats) + wgCfg := NewUSPConfigurerNoUAPI(device, deviceName, activityRecorder) wgCfg.startUAPI() return wgCfg } +// NewUSPConfigurerNoUAPI creates a userspace configurer without a UAPI listener. func NewUSPConfigurerNoUAPI(device *device.Device, deviceName string, activityRecorder *bind.ActivityRecorder) *WGUSPConfigurer { wgCfg := &WGUSPConfigurer{ device: device, @@ -66,21 +48,28 @@ func NewUSPConfigurerNoUAPI(device *device.Device, deviceName string, activityRe return wgCfg } +// ConfigureInterface sets the device key, port and firewall mark, replacing all peers. func (c *WGUSPConfigurer) ConfigureInterface(privateKey string, port int) error { log.Debugf("adding Wireguard private key") key, err := wgtypes.ParseKey(privateKey) if err != nil { return err } - fwmark := getFwmark() - config := wgtypes.Config{ - PrivateKey: &key, - ReplacePeers: true, - FirewallMark: &fwmark, - ListenPort: &port, + if port < 0 || port > math.MaxUint16 { + return fmt.Errorf("invalid listen port %d", port) } - return c.device.IpcSet(toWgUserspaceString(config)) + if err := c.device.SetPrivateKey(device.NoisePrivateKey(key)); err != nil { + return fmt.Errorf("set private key: %w", err) + } + if err := c.device.SetListenPort(uint16(port)); err != nil { + return fmt.Errorf("set listen port %d: %w", port, err) + } + c.device.RemoveAllPeers() + if err := c.device.BindSetMark(uint32(getFwmark())); err != nil { + return fmt.Errorf("set fwmark: %w", err) + } + return nil } // SetPresharedKey sets the preshared key for a peer. @@ -91,231 +80,178 @@ func (c *WGUSPConfigurer) SetPresharedKey(peerKey string, psk wgtypes.Key, updat return err } - cfg := buildPresharedKeyConfig(parsedPeerKey, psk, updateOnly) - return c.device.IpcSet(toWgUserspaceString(cfg)) + presharedKey := device.NoisePresharedKey(psk) + return c.device.ConfigurePeer(device.NoisePublicKey(parsedPeerKey), device.PeerConfig{ + PresharedKey: &presharedKey, + UpdateOnly: updateOnly, + }) } +// UpdatePeer creates or updates a peer, merging allowed IPs with its existing set. func (c *WGUSPConfigurer) UpdatePeer(peerKey string, allowedIps []netip.Prefix, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return err } - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - ReplaceAllowedIPs: false, - // don't replace allowed ips, wg will handle duplicated peer IP - AllowedIPs: prefixesToIPNets(allowedIps), - PersistentKeepaliveInterval: &keepAlive, - PresharedKey: preSharedKey, - Endpoint: endpoint, + + prefixes, err := validPrefixes(allowedIps) + if err != nil { + return err } - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, + cfg := device.PeerConfig{ + PersistentKeepalive: &keepAlive, + AllowedIPs: prefixes, } - if ipcErr := c.device.IpcSet(toWgUserspaceString(config)); ipcErr != nil { - return ipcErr + var addrPort netip.AddrPort + if endpoint != nil { + addr, ok := netip.AddrFromSlice(endpoint.IP) + if !ok { + return fmt.Errorf("parse endpoint address %v", endpoint.IP) + } + addr = addr.Unmap() + addrPort = netip.AddrPortFrom(addr, uint16(endpoint.Port)) + cfg.Endpoint = &bind.Endpoint{AddrPort: netip.AddrPortFrom(addr.WithZone(endpoint.Zone), addrPort.Port())} + } + + if preSharedKey != nil { + psk := device.NoisePresharedKey(*preSharedKey) + cfg.PresharedKey = &psk + } + + if err := c.device.ConfigurePeer(device.NoisePublicKey(peerKeyParsed), cfg); err != nil { + return fmt.Errorf("configure peer: %w", err) } if endpoint != nil { - addr, err := netip.ParseAddr(endpoint.IP.String()) - if err != nil { - return fmt.Errorf("failed to parse endpoint address: %w", err) - } - addrPort := netip.AddrPortFrom(addr.Unmap(), uint16(endpoint.Port)) c.activityRecorder.UpsertAddress(peerKey, addrPort) } return nil } +// RemoveEndpointAddress clears the endpoint of a peer while keeping it configured. +// The session is reset and the keepalive is switched off until the next endpoint arrives. func (c *WGUSPConfigurer) RemoveEndpointAddress(peerKey string) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return fmt.Errorf("parse peer key: %w", err) } - ipcStr, err := c.device.IpcGet() - if err != nil { - return fmt.Errorf("get IPC config: %w", err) + peer := c.device.LookupPeer(device.NoisePublicKey(peerKeyParsed)) + if peer == nil { + return ErrPeerNotFound } - // Parse current status to get allowed IPs for the peer - stats, err := parseStatus(c.deviceName, ipcStr) - if err != nil { - return fmt.Errorf("parse IPC config: %w", err) - } + peer.ClearEndpoint() + peer.ZeroAndFlushAll() - var allowedIPs []net.IPNet - found := false - for _, peer := range stats.Peers { - if peer.PublicKey == peerKey { - allowedIPs = peer.AllowedIPs - found = true - break - } + var noKeepAlive time.Duration + if err := c.device.ConfigurePeer(device.NoisePublicKey(peerKeyParsed), device.PeerConfig{ + PersistentKeepalive: &noKeepAlive, + UpdateOnly: true, + }); err != nil { + return fmt.Errorf("disable keepalive: %w", err) } - if !found { - return fmt.Errorf("peer %s not found", peerKey) - } - - // remove the peer from the WireGuard configuration - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - Remove: true, - } - - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, - } - if ipcErr := c.device.IpcSet(toWgUserspaceString(config)); ipcErr != nil { - return fmt.Errorf("failed to remove peer: %s", ipcErr) - } - - // Build the peer config - peer = wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - ReplaceAllowedIPs: true, - AllowedIPs: allowedIPs, - } - - config = wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, - } - - if err := c.device.IpcSet(toWgUserspaceString(config)); err != nil { - return fmt.Errorf("remove endpoint address: %w", err) - } - return nil } +// RemovePeer removes a peer and clears its activity record. func (c *WGUSPConfigurer) RemovePeer(peerKey string) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return err } - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - Remove: true, - } - - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, - } - ipcErr := c.device.IpcSet(toWgUserspaceString(config)) - + c.device.RemovePeer(device.NoisePublicKey(peerKeyParsed)) c.activityRecorder.Remove(peerKey) - return ipcErr + return nil } +// AddAllowedIP adds a prefix to an existing peer; an absent peer is a silent no-op. func (c *WGUSPConfigurer) AddAllowedIP(peerKey string, allowedIP netip.Prefix) error { - ipNet := net.IPNet{ - IP: allowedIP.Addr().AsSlice(), - Mask: net.CIDRMask(allowedIP.Bits(), allowedIP.Addr().BitLen()), - } - peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return err } - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - UpdateOnly: true, - ReplaceAllowedIPs: false, - AllowedIPs: []net.IPNet{ipNet}, + if !allowedIP.IsValid() { + return fmt.Errorf("invalid allowed IP %v", allowedIP) } - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, + peer := c.device.LookupPeer(device.NoisePublicKey(peerKeyParsed)) + if peer == nil { + return nil } - return c.device.IpcSet(toWgUserspaceString(config)) + peer.AddAllowedIP(normalizePrefix(allowedIP)) + return nil } +// RemoveAllowedIP removes a prefix while preserving the peer's other allowed IPs. +// It returns ErrAllowedIPNotFound if the prefix is not assigned to the peer. func (c *WGUSPConfigurer) RemoveAllowedIP(peerKey string, allowedIP netip.Prefix) error { - ipc, err := c.device.IpcGet() - if err != nil { - return err - } - peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { - return err + return fmt.Errorf("parse peer key: %w", err) } - hexKey := hex.EncodeToString(peerKeyParsed[:]) - - lines := strings.Split(ipc, "\n") - - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - UpdateOnly: true, - ReplaceAllowedIPs: true, - AllowedIPs: []net.IPNet{}, - } - - foundPeer := false - removedAllowedIP := false - ip := allowedIP.String() - - for _, line := range lines { - line = strings.TrimSpace(line) - - // If we're within the details of the found peer and encounter another public key, - // this means we're starting another peer's details. So, reset the flag. - if strings.HasPrefix(line, "public_key=") && foundPeer { - foundPeer = false - } - - // Identify the peer with the specific public key - if line == fmt.Sprintf("public_key=%s", hexKey) { - foundPeer = true - } - - // If we're within the details of the found peer and find the specific allowed IP, skip this line - if foundPeer && line == "allowed_ip="+ip { - removedAllowedIP = true - continue - } - - // Append the line to the output string - if foundPeer && strings.HasPrefix(line, "allowed_ip=") { - allowedIPStr := strings.TrimPrefix(line, "allowed_ip=") - _, ipNet, err := net.ParseCIDR(allowedIPStr) - if err != nil { - return err - } - peer.AllowedIPs = append(peer.AllowedIPs, *ipNet) - } - } - - if !removedAllowedIP { + if !allowedIP.IsValid() { return ErrAllowedIPNotFound } - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, + + peer := c.device.LookupPeer(device.NoisePublicKey(peerKeyParsed)) + if peer == nil { + return ErrPeerNotFound } - return c.device.IpcSet(toWgUserspaceString(config)) + + if !peer.RemoveAllowedIP(normalizePrefix(allowedIP)) { + return ErrAllowedIPNotFound + } + return nil } func (c *WGUSPConfigurer) FullStats() (*Stats, error) { - ipcStr, err := c.device.IpcGet() - if err != nil { - return nil, fmt.Errorf("IpcGet failed: %w", err) + peers := c.device.Peers() + stats := &Stats{ + DeviceName: c.deviceName, + PublicKey: wgtypes.Key(c.device.PublicKey()).String(), + ListenPort: int(c.device.ListenPort()), + FWMark: int(c.device.FirewallMark()), + Peers: make([]Peer, 0, len(peers)), } - - return parseStatus(c.deviceName, ipcStr) + for _, peer := range peers { + stats.Peers = append(stats.Peers, peerStats(peer)) + } + return stats, nil } func (c *WGUSPConfigurer) LastActivities() map[string]monotime.Time { return c.activityRecorder.GetLastActivities() } +func (c *WGUSPConfigurer) GetStats() (map[string]WGStats, error) { + return c.statsCache.get() +} + +func (c *WGUSPConfigurer) Close() { + if c.uapiListener != nil { + err := c.uapiListener.Close() + if err != nil { + log.Errorf("failed to close uapi listener: %v", err) + } + } + + if runtime.GOOS == "linux" { + sockPath := "/var/run/wireguard/" + c.deviceName + ".sock" + if _, statErr := os.Stat(sockPath); statErr == nil { + _ = os.Remove(sockPath) + } + } +} + // startUAPI starts the UAPI listener for managing the WireGuard interface via external tool -func (t *WGUSPConfigurer) startUAPI() { +func (c *WGUSPConfigurer) startUAPI() { var err error - t.uapiListener, err = openUAPI(t.deviceName) + c.uapiListener, err = openUAPI(c.deviceName) if err != nil { log.Errorf("failed to open uapi listener: %v", err) return @@ -329,315 +265,66 @@ func (t *WGUSPConfigurer) startUAPI() { return } go func() { - t.device.IpcHandle(uapiConn) + c.device.IpcHandle(uapiConn) }() } - }(t.uapiListener) + }(c.uapiListener) } -func (t *WGUSPConfigurer) Close() { - if t.uapiListener != nil { - err := t.uapiListener.Close() - if err != nil { - log.Errorf("failed to close uapi listener: %v", err) +func (c *WGUSPConfigurer) fetchStats() (map[string]WGStats, error) { + peers := c.device.Peers() + stats := make(map[string]WGStats, len(peers)) + for _, peer := range peers { + stats[wgtypes.Key(peer.PublicKey()).String()] = WGStats{ + LastHandshake: peer.LastHandshake(), + TxBytes: int64(peer.TxBytes()), + RxBytes: int64(peer.RxBytes()), } } - - if runtime.GOOS == "linux" { - sockPath := "/var/run/wireguard/" + t.deviceName + ".sock" - if _, statErr := os.Stat(sockPath); statErr == nil { - _ = os.Remove(sockPath) - } - } -} - -func (t *WGUSPConfigurer) GetStats() (map[string]WGStats, error) { - return t.statsCache.get() -} - -func (t *WGUSPConfigurer) fetchStats() (map[string]WGStats, error) { - ipc, err := t.device.IpcGet() - if err != nil { - return nil, fmt.Errorf("ipc get: %w", err) - } - - return parseTransfers(ipc) -} - -func parseTransfers(ipc string) (map[string]WGStats, error) { - stats := make(map[string]WGStats) - var ( - currentKey string - currentStats WGStats - hasPeer bool - ) - lines := strings.Split(ipc, "\n") - for _, line := range lines { - line = strings.TrimSpace(line) - - // If we're within the details of the found peer and encounter another public key, - // this means we're starting another peer's details. So, stop. - if strings.HasPrefix(line, "public_key=") { - peerID := strings.TrimPrefix(line, "public_key=") - h, err := hex.DecodeString(peerID) - if err != nil { - return nil, fmt.Errorf("decode peerID: %w", err) - } - currentKey = base64.StdEncoding.EncodeToString(h) - currentStats = WGStats{} // Reset stats for the new peer - hasPeer = true - stats[currentKey] = currentStats - continue - } - - if !hasPeer { - continue - } - - key := strings.SplitN(line, "=", 2) - if len(key) != 2 { - continue - } - switch key[0] { - case ipcKeyLastHandshakeTimeSec: - hs, err := toLastHandshake(key[1]) - if err != nil { - return nil, err - } - currentStats.LastHandshake = hs - stats[currentKey] = currentStats - case ipcKeyRxBytes: - rxBytes, err := toBytes(key[1]) - if err != nil { - return nil, fmt.Errorf("parse rx_bytes: %w", err) - } - currentStats.RxBytes = rxBytes - stats[currentKey] = currentStats - case ipcKeyTxBytes: - TxBytes, err := toBytes(key[1]) - if err != nil { - return nil, fmt.Errorf("parse tx_bytes: %w", err) - } - currentStats.TxBytes = TxBytes - stats[currentKey] = currentStats - } - } - return stats, nil } -func toWgUserspaceString(wgCfg wgtypes.Config) string { - var sb strings.Builder - if wgCfg.PrivateKey != nil { - hexKey := hex.EncodeToString(wgCfg.PrivateKey[:]) - sb.WriteString(fmt.Sprintf("private_key=%s\n", hexKey)) +func peerStats(peer *device.Peer) Peer { + stats := Peer{ + PublicKey: wgtypes.Key(peer.PublicKey()).String(), + AllowedIPs: prefixesToIPNets(peer.AllowedIPs()), + TxBytes: int64(peer.TxBytes()), + RxBytes: int64(peer.RxBytes()), + LastHandshake: peer.LastHandshake(), + PresharedKey: [32]byte(peer.PresharedKey()), } - - if wgCfg.ListenPort != nil { - sb.WriteString(fmt.Sprintf("listen_port=%d\n", *wgCfg.ListenPort)) + if addrPort, ok := endpointAddrPort(peer.Endpoint()); ok { + stats.Endpoint = net.UDPAddr{IP: addrPort.Addr().AsSlice(), Port: int(addrPort.Port())} } - - if wgCfg.ReplacePeers { - sb.WriteString("replace_peers=true\n") - } - - if wgCfg.FirewallMark != nil { - sb.WriteString(fmt.Sprintf("fwmark=%d\n", *wgCfg.FirewallMark)) - } - - for _, p := range wgCfg.Peers { - hexKey := hex.EncodeToString(p.PublicKey[:]) - sb.WriteString(fmt.Sprintf("public_key=%s\n", hexKey)) - - if p.Remove { - sb.WriteString("remove=true\n") - } - - if p.UpdateOnly { - sb.WriteString("update_only=true\n") - } - - if p.PresharedKey != nil { - preSharedHexKey := hex.EncodeToString(p.PresharedKey[:]) - sb.WriteString(fmt.Sprintf("preshared_key=%s\n", preSharedHexKey)) - } - - if p.Endpoint != nil { - sb.WriteString(fmt.Sprintf("endpoint=%s\n", p.Endpoint.String())) - } - - if p.PersistentKeepaliveInterval != nil { - sb.WriteString(fmt.Sprintf("persistent_keepalive_interval=%d\n", int(p.PersistentKeepaliveInterval.Seconds()))) - } - - if p.ReplaceAllowedIPs { - sb.WriteString("replace_allowed_ips=true\n") - } - - for _, aip := range p.AllowedIPs { - sb.WriteString(fmt.Sprintf("allowed_ip=%s\n", aip.String())) - } - } - return sb.String() + return stats } -func toLastHandshake(stringVar string) (time.Time, error) { - sec, err := strconv.ParseInt(stringVar, 10, 64) +func endpointAddrPort(endpoint wgconn.Endpoint) (netip.AddrPort, bool) { + if endpoint == nil { + return netip.AddrPort{}, false + } + if ep, ok := endpoint.(*bind.Endpoint); ok { + return ep.AddrPort, true + } + addrPort, err := netip.ParseAddrPort(endpoint.DstToString()) if err != nil { - return time.Time{}, fmt.Errorf("parse handshake sec: %w", err) + return netip.AddrPort{}, false } - - // If sec is 0 (Unix epoch), return zero time instead - // This indicates no handshake has occurred - if sec == 0 { - return time.Time{}, nil - } - - return time.Unix(sec, 0), nil + return addrPort, true } -func toBytes(s string) (int64, error) { - return strconv.ParseInt(s, 10, 64) +func validPrefixes(prefixes []netip.Prefix) ([]netip.Prefix, error) { + for _, prefix := range prefixes { + if !prefix.IsValid() { + return nil, fmt.Errorf("invalid allowed IP %v", prefix) + } + } + return normalizePrefixes(prefixes), nil } func getFwmark() int { if nbnet.AdvancedRouting() && runtime.GOOS == "linux" { - return nbnet.ControlPlaneMark + return int(nbnet.ControlPlaneMark) } return 0 } - -func hexToWireguardKey(hexKey string) (wgtypes.Key, error) { - // Decode hex string to bytes - keyBytes, err := hex.DecodeString(hexKey) - if err != nil { - return wgtypes.Key{}, fmt.Errorf("failed to decode hex key: %w", err) - } - - // Check if we have the right number of bytes (WireGuard keys are 32 bytes) - if len(keyBytes) != 32 { - return wgtypes.Key{}, fmt.Errorf("invalid key length: expected 32 bytes, got %d", len(keyBytes)) - } - - // Convert to wgtypes.Key - var key wgtypes.Key - copy(key[:], keyBytes) - - return key, nil -} - -func parseStatus(deviceName, ipcStr string) (*Stats, error) { - stats := &Stats{DeviceName: deviceName} - var currentPeer *Peer - for _, line := range strings.Split(strings.TrimSpace(ipcStr), "\n") { - if line == "" { - continue - } - parts := strings.SplitN(line, "=", 2) - if len(parts) != 2 { - continue - } - key := parts[0] - val := parts[1] - - switch key { - case privateKey: - key, err := hexToWireguardKey(val) - if err != nil { - log.Errorf("failed to parse private key: %v", err) - continue - } - stats.PublicKey = key.PublicKey().String() - case publicKey: - // Save previous peer - if currentPeer != nil { - stats.Peers = append(stats.Peers, *currentPeer) - } - key, err := hexToWireguardKey(val) - if err != nil { - log.Errorf("failed to parse public key: %v", err) - continue - } - currentPeer = &Peer{ - PublicKey: key.String(), - } - case listenPort: - if port, err := strconv.Atoi(val); err == nil { - stats.ListenPort = port - } - case fwmark: - if fwmark, err := strconv.Atoi(val); err == nil { - stats.FWMark = fwmark - } - case endpoint: - if currentPeer == nil { - continue - } - - host, portStr, err := net.SplitHostPort(val) - if err != nil { - log.Errorf("failed to parse endpoint: %v", err) - continue - } - port, err := strconv.Atoi(portStr) - if err != nil { - log.Errorf("failed to parse endpoint port: %v", err) - continue - } - currentPeer.Endpoint = net.UDPAddr{ - IP: net.ParseIP(host), - Port: port, - } - case allowedIP: - if currentPeer == nil { - continue - } - _, ipnet, err := net.ParseCIDR(val) - if err == nil { - currentPeer.AllowedIPs = append(currentPeer.AllowedIPs, *ipnet) - } - case ipcKeyTxBytes: - if currentPeer == nil { - continue - } - rxBytes, err := toBytes(val) - if err != nil { - continue - } - currentPeer.TxBytes = rxBytes - case ipcKeyRxBytes: - if currentPeer == nil { - continue - } - rxBytes, err := toBytes(val) - if err != nil { - continue - } - currentPeer.RxBytes = rxBytes - - case ipcKeyLastHandshakeTimeSec: - if currentPeer == nil { - continue - } - - ts, err := toLastHandshake(val) - if err != nil { - continue - } - currentPeer.LastHandshake = ts - case presharedKey: - if currentPeer == nil { - continue - } - if val != "" && val != "0000000000000000000000000000000000000000000000000000000000000000" { - if pskKey, err := hexToWireguardKey(val); err == nil { - currentPeer.PresharedKey = [32]byte(pskKey) - } - } - } - } - if currentPeer != nil { - stats.Peers = append(stats.Peers, *currentPeer) - } - return stats, nil -} diff --git a/client/iface/configurer/usp_allowedips_test.go b/client/iface/configurer/usp_allowedips_test.go new file mode 100644 index 000000000..effb360e9 --- /dev/null +++ b/client/iface/configurer/usp_allowedips_test.go @@ -0,0 +1,252 @@ +package configurer + +import ( + "net" + "net/netip" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + wgconn "golang.zx2c4.com/wireguard/conn" + wgdevice "golang.zx2c4.com/wireguard/device" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/client/iface/bind" +) + +// newTestUSPConfigurer builds a configurer over a real wireguard-go device backed by an +// in-memory TUN. The device stays down, so no socket is opened and no privileges are needed. +func newTestUSPConfigurer(t *testing.T) *WGUSPConfigurer { + t.Helper() + + dev := wgdevice.NewDevice(newIdleTUN(), wgconn.NewDefaultBind(), wgdevice.NewLogger(wgdevice.LogLevelSilent, "")) + t.Cleanup(dev.Close) + + c := NewUSPConfigurerNoUAPI(dev, "wgtest0", bind.NewActivityRecorder()) + + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate device private key") + require.NoError(t, c.ConfigureInterface(key.String(), 0), "configure test device") + + return c +} + +// seedPeers adds count peers, each with a /32 overlay address, and returns their public keys. +func seedPeers(t *testing.T, c *WGUSPConfigurer, count int) []string { + t.Helper() + + keys := make([]string, 0, count) + for i := 0; i < count; i++ { + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + pub := priv.PublicKey().String() + + addr := netip.PrefixFrom(netip.AddrFrom4([4]byte{100, 64, byte(i >> 8), byte(i)}), 32) + require.NoError(t, c.UpdatePeer(pub, []netip.Prefix{addr}, 25*time.Second, nil, nil), "add peer") + keys = append(keys, pub) + } + return keys +} + +func peerAllowedIPs(t *testing.T, c *WGUSPConfigurer, peerKey string) []string { + t.Helper() + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + + for _, p := range stats.Peers { + if p.PublicKey != peerKey { + continue + } + got := make([]string, 0, len(p.AllowedIPs)) + for _, ipNet := range p.AllowedIPs { + got = append(got, ipNet.String()) + } + return got + } + t.Fatalf("peer %s not found on device", peerKey) + return nil +} + +// TestRemoveEndpointAddressPreservesRoutedAllowedIPs covers the prefixes the route manager +// attaches to a routing peer through AddAllowedIP. Those are not known to the peer.Conn that +// triggers the endpoint removal, so dropping them here would silently blackhole every route +// behind that peer on each relay or ICE disconnect. +func TestRemoveEndpointAddressPreservesRoutedAllowedIPs(t *testing.T) { + c := newTestUSPConfigurer(t) + peerKey := seedPeers(t, c, 3)[1] + + routed := []netip.Prefix{ + netip.MustParsePrefix("10.20.0.0/16"), + netip.MustParsePrefix("192.168.7.0/24"), + } + for _, prefix := range routed { + require.NoError(t, c.AddAllowedIP(peerKey, prefix), "add routed prefix") + } + + before := peerAllowedIPs(t, c, peerKey) + require.Len(t, before, 3, "peer should hold its overlay address plus both routed prefixes") + + require.NoError(t, c.RemoveEndpointAddress(peerKey), "remove endpoint address") + + assert.ElementsMatch(t, before, peerAllowedIPs(t, c, peerKey), + "allowed IPs must survive the endpoint removal unchanged") +} + +// TestRemoveEndpointAddressDoesNotScaleWithPeerCount is the regression guard for the actual +// defect: clearing one peer's endpoint used to dump and parse the whole device, so its cost +// grew with the size of the network map. On a routing peer with thousands of peers that dump +// runs on every relay and ICE transition, under the interface lock. +func TestRemoveEndpointAddressDoesNotScaleWithPeerCount(t *testing.T) { + measure := func(peerCount int) float64 { + c := newTestUSPConfigurer(t) + peerKey := seedPeers(t, c, peerCount)[peerCount/2] + + return testing.AllocsPerRun(5, func() { + require.NoError(t, c.RemoveEndpointAddress(peerKey), "remove endpoint address") + }) + } + + small := measure(64) + large := measure(1024) + + assert.Less(t, large, small*2, + "clearing one endpoint allocated %.0f objects with 1024 peers against %.0f with 64: the cost still scales with the peer count", + large, small) +} + +func TestRemoveAllowedIPKeepsTheOtherPrefixes(t *testing.T) { + c := newTestUSPConfigurer(t) + peerKey := seedPeers(t, c, 3)[0] + routed := netip.MustParsePrefix("10.20.0.0/16") + require.NoError(t, c.AddAllowedIP(peerKey, routed), "add routed prefix") + require.NoError(t, c.AddAllowedIP(peerKey, netip.MustParsePrefix("192.168.7.0/24")), "add routed prefix") + + require.NoError(t, c.RemoveAllowedIP(peerKey, routed), "remove routed prefix") + + assert.ElementsMatch(t, []string{"100.64.0.0/32", "192.168.7.0/24"}, peerAllowedIPs(t, c, peerKey), + "only the removed prefix should be gone") + + assert.ErrorIs(t, c.RemoveAllowedIP(peerKey, routed), ErrAllowedIPNotFound, + "removing a prefix that is no longer configured must be reported") +} + +// TestAddAllowedIPOnAbsentPeerDoesNotResurrectIt covers the lazy connection window documented +// in #6863: AddAllowedIP is update-only, a silent no-op when the peer is absent, and clearing +// the endpoint of an absent peer must not create one either. +func TestAddAllowedIPOnAbsentPeerDoesNotResurrectIt(t *testing.T) { + c := newTestUSPConfigurer(t) + seedPeers(t, c, 2) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + absent := priv.PublicKey().String() + + require.NoError(t, c.AddAllowedIP(absent, netip.MustParsePrefix("10.20.0.0/16")), + "update-only add on an absent peer is a silent no-op") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + require.Len(t, stats.Peers, 2, "the absent peer must not have been created by AddAllowedIP") + + assert.ErrorIs(t, c.RemoveEndpointAddress(absent), ErrPeerNotFound, + "clearing the endpoint of a peer the device does not have must fail") + + stats, err = c.FullStats() + require.NoError(t, err, "read device stats") + assert.Len(t, stats.Peers, 2, "no peer may be created while clearing an endpoint") +} + +// TestRemoveEndpointAddressDoesNotStealAPrefixFromAnotherPeer covers WireGuard's rule that an +// allowed IP belongs to exactly one peer: configuring a prefix on a peer takes it away from +// whichever peer held it before. UpdatePeer relies on that rule rather than removing the prefix +// from the previous holder itself, so a prefix handed over between peers must not come back. +func TestRemoveEndpointAddressDoesNotStealAPrefixFromAnotherPeer(t *testing.T) { + c := newTestUSPConfigurer(t) + keys := seedPeers(t, c, 2) + peerA, peerB := keys[0], keys[1] + routed := netip.MustParsePrefix("10.20.0.0/16") + + require.NoError(t, c.AddAllowedIP(peerA, routed), "give the prefix to A") + require.Contains(t, peerAllowedIPs(t, c, peerA), routed.String(), "A must hold the prefix") + + // The route moves to B. The device takes it away from A on its own. + require.NoError(t, c.AddAllowedIP(peerB, routed), "hand the prefix over to B") + require.Contains(t, peerAllowedIPs(t, c, peerB), routed.String(), "B must hold the prefix") + require.NotContains(t, peerAllowedIPs(t, c, peerA), routed.String(), "the device must have taken it from A") + + require.NoError(t, c.RemoveEndpointAddress(peerA), "clear A's endpoint") + + assert.NotContains(t, peerAllowedIPs(t, c, peerA), routed.String(), + "clearing A's endpoint must not take the prefix back from B") + assert.Contains(t, peerAllowedIPs(t, c, peerB), routed.String(), + "B must still hold the prefix") +} + +// TestPresharedKeyCreatedPeerTakesPartInPrefixHandover covers a peer created by a preshared +// key write rather than by a peer update. Rosenpass applies a peer's first key without +// updateOnly, which creates the peer on the device, so a store that ignored that operation +// would treat the peer as unknown and would not account for a prefix later handed over to it. +func TestPresharedKeyCreatedPeerTakesPartInPrefixHandover(t *testing.T) { + c := newTestUSPConfigurer(t) + peerA := seedPeers(t, c, 1)[0] + routed := netip.MustParsePrefix("10.20.0.0/16") + require.NoError(t, c.AddAllowedIP(peerA, routed), "give the prefix to A") + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerB := priv.PublicKey().String() + + psk, err := wgtypes.GenerateKey() + require.NoError(t, err, "generate preshared key") + require.NoError(t, c.SetPresharedKey(peerB, psk, false), "a first key creates the peer") + + require.NoError(t, c.AddAllowedIP(peerB, routed), "hand the prefix over to B") + require.Contains(t, peerAllowedIPs(t, c, peerB), routed.String(), "B must hold the prefix") + + require.NoError(t, c.RemoveEndpointAddress(peerA), "clear A's endpoint") + + assert.NotContains(t, peerAllowedIPs(t, c, peerA), routed.String(), + "clearing A's endpoint must not take the prefix back from B") + assert.Contains(t, peerAllowedIPs(t, c, peerB), routed.String(), "B must still hold the prefix") +} + +// TestUpdatePeerDoesNotWidenAMappedPrefixOnTheDevice is the end to end form of the +// conversion: a v4-mapped prefix must not reach the device as a zero length allowed IP, +// which would route every v4 address to that peer. +func TestUpdatePeerDoesNotWidenAMappedPrefixOnTheDevice(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + + mapped := netip.MustParsePrefix("::ffff:10.1.2.3/112") + require.NoError(t, c.UpdatePeer(peerKey, []netip.Prefix{mapped}, 25*time.Second, nil, nil), "add peer") + + onDevice := peerAllowedIPs(t, c, peerKey) + assert.NotContains(t, onDevice, "0.0.0.0/0", "the device must not be given a catch-all allowed IP") + assert.Equal(t, []string{"10.1.0.0/16"}, onDevice, "the device holds the normalized prefix") +} + +// TestUpdatePeerWithAnUnusableEndpointTouchesNothing pins the ordering: the endpoint is +// parsed before the device is configured, so a failure cannot leave a half-configured peer +// on the device. +func TestUpdatePeerWithAnUnusableEndpointTouchesNothing(t *testing.T) { + c := newTestUSPConfigurer(t) + seedPeers(t, c, 2) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + + // A three byte address has no textual form netip can parse back. + endpoint := &net.UDPAddr{IP: net.IP{1, 2, 3}, Port: 51820} + require.Error(t, c.UpdatePeer(peerKey, []netip.Prefix{netip.MustParsePrefix("10.30.0.0/16")}, + 25*time.Second, endpoint, nil), "an unusable endpoint must fail the update") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + assert.Len(t, stats.Peers, 2, "the peer must not have reached the device") +} diff --git a/client/iface/configurer/usp_test.go b/client/iface/configurer/usp_test.go index e32491c54..48b5ad1f0 100644 --- a/client/iface/configurer/usp_test.go +++ b/client/iface/configurer/usp_test.go @@ -2,88 +2,222 @@ package configurer import ( "encoding/hex" + "net" + "net/netip" + "os" + "strings" "testing" + "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + wgdevice "golang.zx2c4.com/wireguard/device" + "golang.zx2c4.com/wireguard/tun" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) -var ipcFixture = ` -private_key=e84b5a6d2717c1003a13b431570353dbaca9146cf150c5f8575680feba52027a -listen_port=12912 -public_key=b85996fecc9c7f1fc6d2572a76eda11d59bcd20be8e543b15ce4bd85a8e75a33 -preshared_key=188515093e952f5f22e865cef3012e72f8b5f0b598ac0309d5dacce3b70fcf52 -allowed_ip=192.168.4.4/32 -endpoint=[abcd:23::33%2]:51820 -public_key=58402e695ba1772b1cc9309755f043251ea77fdcf10fbe63989ceb7e19321376 -tx_bytes=38333 -rx_bytes=2224 -allowed_ip=192.168.4.6/32 -persistent_keepalive_interval=111 -endpoint=182.122.22.19:3233 -public_key=662e14fd594556f522604703340351258903b64f35553763f19426ab2a515c58 -endpoint=5.152.198.39:51820 -allowed_ip=192.168.4.10/32 -allowed_ip=192.168.4.11/32 -tx_bytes=1212111 -rx_bytes=1929999999 -protocol_version=1 -errno=0 +// idleTUN never reports an up event, so the wireguard-go device stays down: no socket is +// opened, no peer goroutine starts, and the configuration state is all the tests observe. +type idleTUN struct { + events chan tun.Event + closed chan struct{} +} -` +func newIdleTUN() *idleTUN { + return &idleTUN{events: make(chan tun.Event), closed: make(chan struct{})} +} -func Test_parseTransfers(t *testing.T) { - tests := []struct { - name string - peerKey string - want WGStats - }{ - { - name: "single", - peerKey: "b85996fecc9c7f1fc6d2572a76eda11d59bcd20be8e543b15ce4bd85a8e75a33", - want: WGStats{ - TxBytes: 0, - RxBytes: 0, - }, - }, - { - name: "multiple", - peerKey: "58402e695ba1772b1cc9309755f043251ea77fdcf10fbe63989ceb7e19321376", - want: WGStats{ - TxBytes: 38333, - RxBytes: 2224, - }, - }, - { - name: "lastpeer", - peerKey: "662e14fd594556f522604703340351258903b64f35553763f19426ab2a515c58", - want: WGStats{ - TxBytes: 1212111, - RxBytes: 1929999999, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - res, err := hex.DecodeString(tt.peerKey) - require.NoError(t, err) +func (t *idleTUN) File() *os.File { return nil } - key, err := wgtypes.NewKey(res) - require.NoError(t, err) +func (t *idleTUN) Read([][]byte, []int, int) (int, error) { + <-t.closed + return 0, os.ErrClosed +} - stats, err := parseTransfers(ipcFixture) - if err != nil { - require.NoError(t, err) - return - } +func (t *idleTUN) Write(bufs [][]byte, _ int) (int, error) { return len(bufs), nil } - stat, ok := stats[key.String()] - if !ok { - require.True(t, ok) - return - } +func (t *idleTUN) MTU() (int, error) { return 1420, nil } - require.Equal(t, tt.want, stat) - }) +func (t *idleTUN) Name() (string, error) { return "idle", nil } + +func (t *idleTUN) Events() <-chan tun.Event { return t.events } + +func (t *idleTUN) Close() error { + close(t.closed) + close(t.events) + return nil +} + +func (t *idleTUN) BatchSize() int { return 1 } + +func TestConfigureInterfaceAppliesKeyAndPortAndReplacesPeers(t *testing.T) { + c := newTestUSPConfigurer(t) + seedPeers(t, c, 2) + + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate device private key") + require.NoError(t, c.ConfigureInterface(key.String(), 51820), "reconfigure the interface") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + assert.Equal(t, key.PublicKey().String(), stats.PublicKey, "device public key") + assert.Equal(t, 51820, stats.ListenPort, "listen port") + assert.Empty(t, stats.Peers, "configuring the interface replaces every peer") +} + +func TestConfigureInterfaceRejectsAnInvalidPort(t *testing.T) { + c := newTestUSPConfigurer(t) + + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate device private key") + assert.Error(t, c.ConfigureInterface(key.String(), 70000), "a port above 65535 must be rejected") +} + +func TestGetStatsIsKeyedByPeerKey(t *testing.T) { + c := newTestUSPConfigurer(t) + keys := seedPeers(t, c, 3) + + stats, err := c.GetStats() + require.NoError(t, err, "read stats") + require.Len(t, stats, 3, "one entry per peer") + for _, key := range keys { + entry, ok := stats[key] + require.True(t, ok, "peer %s must be reported", key) + assert.Zero(t, entry.TxBytes, "no traffic was sent") + assert.Zero(t, entry.RxBytes, "no traffic was received") + assert.True(t, entry.LastHandshake.IsZero(), "no handshake happened") } } + +func TestFullStatsMatchesTheDeviceDump(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + psk, err := wgtypes.GenerateKey() + require.NoError(t, err, "generate preshared key") + endpoint := &net.UDPAddr{IP: net.IPv4(192, 0, 2, 10), Port: 51820} + prefixes := []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32"), netip.MustParsePrefix("10.20.0.0/16")} + require.NoError(t, c.UpdatePeer(peerKey, prefixes, 25*time.Second, endpoint, &psk), "add peer") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + require.Len(t, stats.Peers, 1, "one peer on the device") + peer := stats.Peers[0] + assert.Equal(t, peerKey, peer.PublicKey, "peer key") + assert.Equal(t, "192.0.2.10:51820", peer.Endpoint.String(), "endpoint") + assert.Equal(t, [32]byte(psk), peer.PresharedKey, "preshared key") + assert.Equal(t, []string{"100.64.0.1/32", "10.20.0.0/16"}, ipNetStrings(peer.AllowedIPs), "allowed IPs") + + block := uapiPeerBlock(t, c, peerKey) + assert.Contains(t, block, "endpoint=192.0.2.10:51820", "the dump must show the endpoint") + assert.Contains(t, block, "preshared_key="+hex.EncodeToString(psk[:]), "the dump must show the preshared key") + assert.Contains(t, block, "persistent_keepalive_interval=25", "the dump must show the keepalive") + assert.Contains(t, block, "allowed_ip=100.64.0.1/32", "the dump must show the first prefix") + assert.Contains(t, block, "allowed_ip=10.20.0.0/16", "the dump must show the second prefix") +} + +func TestUpdatePeerKeepsTheEndpointZone(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + // The idle device opens no socket, so the zone name does not have to exist on the host. + endpoint := &net.UDPAddr{IP: net.ParseIP("fe80::1"), Port: 51820, Zone: "eth0"} + prefixes := []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")} + require.NoError(t, c.UpdatePeer(peerKey, prefixes, 25*time.Second, endpoint, nil), "add peer") + + assert.Contains(t, uapiPeerBlock(t, c, peerKey), "endpoint=[fe80::1%eth0]:51820", + "a link-local endpoint must keep its zone") +} + +func TestRemoveEndpointAddressKeepsThePeerAndStopsTheKeepalive(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + endpoint := &net.UDPAddr{IP: net.IPv4(192, 0, 2, 10), Port: 51820} + prefixes := []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")} + require.NoError(t, c.UpdatePeer(peerKey, prefixes, 25*time.Second, endpoint, nil), "add peer") + before := c.device.LookupPeer(wgdevice.NoisePublicKey(priv.PublicKey())) + require.NotNil(t, before, "the peer must be on the device") + + require.NoError(t, c.RemoveEndpointAddress(peerKey), "remove endpoint address") + + assert.Same(t, before, c.device.LookupPeer(wgdevice.NoisePublicKey(priv.PublicKey())), + "the peer object must survive the endpoint removal") + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + require.Len(t, stats.Peers, 1, "the peer stays configured") + assert.Nil(t, stats.Peers[0].Endpoint.IP, "the endpoint must be gone") + assert.Equal(t, []string{"100.64.0.1/32"}, ipNetStrings(stats.Peers[0].AllowedIPs), "allowed IPs survive") + assert.Contains(t, uapiPeerBlock(t, c, peerKey), "persistent_keepalive_interval=0", + "an endpoint-less peer must not keep sending keepalives") + + require.NoError(t, c.UpdatePeer(peerKey, prefixes, 25*time.Second, endpoint, nil), "restore the endpoint") + block := uapiPeerBlock(t, c, peerKey) + assert.Contains(t, block, "endpoint=192.0.2.10:51820", "the endpoint must be back") + assert.Contains(t, block, "persistent_keepalive_interval=25", "the keepalive must be back") +} + +func TestUpdatePeerRejectsAnInvalidPrefix(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + + require.Error(t, c.UpdatePeer(peerKey, []netip.Prefix{{}}, 25*time.Second, nil, nil), "an invalid prefix must be rejected") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + assert.Empty(t, stats.Peers, "the peer must not have reached the device") +} + +func TestRemoveAllowedIPOnAbsentPeerReportsThePeer(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + + assert.ErrorIs(t, c.RemoveAllowedIP(priv.PublicKey().String(), netip.MustParsePrefix("10.20.0.0/16")), ErrPeerNotFound, + "removing a prefix from a peer the device does not have must report the peer") +} + +func ipNetStrings(ipNets []net.IPNet) []string { + out := make([]string, 0, len(ipNets)) + for _, ipNet := range ipNets { + out = append(out, ipNet.String()) + } + return out +} + +// uapiPeerBlock reads the device back through the UAPI dump, so a typed write is checked +// against an independent view of the device. +func uapiPeerBlock(t *testing.T, c *WGUSPConfigurer, peerKey string) []string { + t.Helper() + + parsed, err := wgtypes.ParseKey(peerKey) + require.NoError(t, err, "parse peer key") + wanted := "public_key=" + hex.EncodeToString(parsed[:]) + + dump, err := c.device.IpcGet() + require.NoError(t, err, "dump device") + + var block []string + inBlock := false + for _, line := range strings.Split(strings.TrimSpace(dump), "\n") { + if strings.HasPrefix(line, "public_key=") { + inBlock = line == wanted + continue + } + if inBlock { + block = append(block, line) + } + } + require.NotEmpty(t, block, "peer %s must appear in the dump", peerKey) + return block +} diff --git a/client/iface/device/device_android.go b/client/iface/device/device_android.go index cbe88c10c..0ed1299ae 100644 --- a/client/iface/device/device_android.go +++ b/client/iface/device/device_android.go @@ -63,7 +63,12 @@ func (t *WGTunDevice) Create(routes []string, dns string, searchDomains []string searchDomainsToString = "" } - fd, err := t.tunAdapter.ConfigureInterface(t.address.String(), t.address.IPv6String(), int(t.mtu), dns, searchDomainsToString, routesString) + ipv6Host := "" + if t.address.HasIPv6() { + ipv6Host = t.address.IPv6HostPrefix().String() + } + + fd, err := t.tunAdapter.ConfigureInterface(t.address.HostPrefix().String(), ipv6Host, int(t.mtu), dns, searchDomainsToString, routesString) if err != nil { log.Errorf("failed to create Android interface: %s", err) return nil, err diff --git a/client/iface/device/device_filter_test.go b/client/iface/device/device_filter_test.go index 0d86c9323..a75ef90f9 100644 --- a/client/iface/device/device_filter_test.go +++ b/client/iface/device/device_filter_test.go @@ -4,7 +4,7 @@ import ( "net" "testing" - "github.com/golang/mock/gomock" + "go.uber.org/mock/gomock" "github.com/google/gopacket" "github.com/google/gopacket/layers" diff --git a/client/iface/device/device_kernel_unix.go b/client/iface/device/device_kernel_unix.go index 3c429fb96..ee965dd24 100644 --- a/client/iface/device/device_kernel_unix.go +++ b/client/iface/device/device_kernel_unix.go @@ -32,6 +32,7 @@ type TunKernelDevice struct { link *wgLink udpMuxConn net.PacketConn udpMux *udpmux.UniversalUDPMuxDefault + configurer WGConfigurer } func NewKernelDevice(name string, address wgaddr.Address, wgPort int, key string, mtu uint16, transportNet transport.Net) *TunKernelDevice { @@ -68,13 +69,18 @@ func (t *TunKernelDevice) Create() (WGConfigurer, error) { return nil, fmt.Errorf("set mtu: %w", err) } - configurer := configurer.NewKernelConfigurer(t.name) + cfg := configurer.NewKernelConfigurer(t.name) - if err := configurer.ConfigureInterface(t.key, t.wgPort); err != nil { + if err := cfg.ConfigureInterface(t.key, t.wgPort); err != nil { + cfg.Close() return nil, fmt.Errorf("error configuring interface: %s", err) } - return configurer, nil + if t.configurer != nil { + t.configurer.Close() + } + t.configurer = cfg + return cfg, nil } func (t *TunKernelDevice) Up() (*udpmux.UniversalUDPMuxDefault, error) { @@ -132,6 +138,10 @@ func (t *TunKernelDevice) Close() error { closErr = err } + if t.configurer != nil { + t.configurer.Close() + } + if t.udpMux != nil { if err := t.udpMux.Close(); err != nil { log.Debugf("failed to close udp mux: %s", err) diff --git a/client/iface/iface.go b/client/iface/iface.go index 247f421a2..f6006fa87 100644 --- a/client/iface/iface.go +++ b/client/iface/iface.go @@ -51,7 +51,6 @@ func ValidateMTU(mtu uint16) error { type wgProxyFactory interface { GetProxy() wgproxy.Proxy - GetProxyPort() uint16 Free() error } @@ -81,12 +80,6 @@ func (w *WGIface) GetProxy() wgproxy.Proxy { return w.wgProxyFactory.GetProxy() } -// GetProxyPort returns the proxy port used by the WireGuard proxy. -// Returns 0 if no proxy port is used (e.g., for userspace WireGuard). -func (w *WGIface) GetProxyPort() uint16 { - return w.wgProxyFactory.GetProxyPort() -} - // GetBind returns the EndpointManager userspace bind mode. func (w *WGIface) GetBind() device.EndpointManager { w.mu.Lock() diff --git a/client/iface/iface_close_test.go b/client/iface/iface_close_test.go index 171e15d0a..ecbbf4866 100644 --- a/client/iface/iface_close_test.go +++ b/client/iface/iface_close_test.go @@ -12,6 +12,7 @@ import ( "golang.zx2c4.com/wireguard/tun/netstack" "github.com/netbirdio/netbird/client/iface/device" + nbnetstack "github.com/netbirdio/netbird/client/iface/netstack" "github.com/netbirdio/netbird/client/iface/udpmux" "github.com/netbirdio/netbird/client/iface/wgaddr" "github.com/netbirdio/netbird/client/iface/wgproxy" @@ -34,14 +35,14 @@ func (f *fakeTunDevice) Create() (device.WGConfigurer, error) { func (f *fakeTunDevice) Up() (*udpmux.UniversalUDPMuxDefault, error) { return nil, errors.New("not implemented") } -func (f *fakeTunDevice) UpdateAddr(wgaddr.Address) error { return nil } -func (f *fakeTunDevice) WgAddress() wgaddr.Address { return wgaddr.Address{} } -func (f *fakeTunDevice) MTU() uint16 { return DefaultMTU } -func (f *fakeTunDevice) DeviceName() string { return "nb-close-test" } +func (f *fakeTunDevice) UpdateAddr(wgaddr.Address) error { return nil } +func (f *fakeTunDevice) WgAddress() wgaddr.Address { return wgaddr.Address{} } +func (f *fakeTunDevice) MTU() uint16 { return DefaultMTU } +func (f *fakeTunDevice) DeviceName() string { return "nb-close-test" } func (f *fakeTunDevice) FilteredDevice() *device.FilteredDevice { return nil } -func (f *fakeTunDevice) Device() *wgdevice.Device { return nil } -func (f *fakeTunDevice) GetNet() *netstack.Net { return nil } -func (f *fakeTunDevice) GetICEBind() device.EndpointManager { return nil } +func (f *fakeTunDevice) Device() *wgdevice.Device { return nil } +func (f *fakeTunDevice) GetNet() *netstack.Net { return nil } +func (f *fakeTunDevice) GetICEBind() device.EndpointManager { return nil } func (f *fakeTunDevice) Close() error { close(f.closeStarted) @@ -52,7 +53,6 @@ func (f *fakeTunDevice) Close() error { type fakeProxyFactory struct{} func (fakeProxyFactory) GetProxy() wgproxy.Proxy { return nil } -func (fakeProxyFactory) GetProxyPort() uint16 { return 0 } func (fakeProxyFactory) Free() error { return nil } // TestWGIface_CloseReleasesMutexBeforeTunClose guards against a deadlock @@ -63,6 +63,10 @@ func (fakeProxyFactory) Free() error { return nil } // WGIface.GetDevice() which needs the same mutex. The fix is to drop // the lock before tun.Close() returns control. func TestWGIface_CloseReleasesMutexBeforeTunClose(t *testing.T) { + // Netstack mode skips the platform interface cleanup in Close, so only + // the mutex and tun close behavior is exercised. + t.Setenv(nbnetstack.EnvUseNetstackMode, "true") + tun := &fakeTunDevice{ closeStarted: make(chan struct{}), unblockClose: make(chan struct{}), @@ -106,7 +110,10 @@ func TestWGIface_CloseReleasesMutexBeforeTunClose(t *testing.T) { close(tun.unblockClose) select { - case <-closeDone: + case err := <-closeDone: + if err != nil { + t.Errorf("WGIface.Close() returned an error: %v", err) + } case <-time.After(2 * time.Second): t.Fatal("WGIface.Close() never returned after the tun was unblocked") } diff --git a/client/iface/iface_destroy_windows.go b/client/iface/iface_destroy_windows.go index 0bfa4e211..54c0014c4 100644 --- a/client/iface/iface_destroy_windows.go +++ b/client/iface/iface_destroy_windows.go @@ -6,27 +6,14 @@ import ( "fmt" "os/exec" - log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/client/internal/wincmd" ) func (w *WGIface) Destroy() error { - netshCmd := GetSystem32Command("netsh") + netshCmd := wincmd.System32("netsh") out, err := exec.Command(netshCmd, "interface", "set", "interface", w.Name(), "admin=disable").CombinedOutput() if err != nil { return fmt.Errorf("failed to remove interface %s: %w - %s", w.Name(), err, out) } return nil } - -// GetSystem32Command checks if a command can be found in the system path and returns it. In case it can't find it -// in the path it will return the full path of a command assuming C:\windows\system32 as the base path. -func GetSystem32Command(command string) string { - _, err := exec.LookPath(command) - if err == nil { - return command - } - - log.Tracef("Command %s not found in PATH, using C:\\windows\\system32\\%s.exe path", command, command) - - return "C:\\windows\\system32\\" + command + ".exe" -} diff --git a/client/iface/iface_test.go b/client/iface/iface_test.go index 8ff2bbb54..690a0f66f 100644 --- a/client/iface/iface_test.go +++ b/client/iface/iface_test.go @@ -40,14 +40,18 @@ func init() { peerPubKey = peerPrivateKey.PublicKey().String() } +// testIFaceBlackList mirrors the prefixes profilemanager.DefaultInterfaceBlacklist +// carries for the overlay interface. These tests create their own utun device, and +// stdnet's filter probes with wgctrl every interface it is not told to skip, which +// on a userspace WireGuard platform reaches the UAPI socket of this same process. +// Declared here rather than imported because profilemanager imports this package. +var testIFaceBlackList = []string{"wt", "utun", "tun0"} + func TestWGIface_UpdateAddr(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+4) addr := "100.64.0.1/8" wgPort := 33100 - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -127,10 +131,7 @@ func getIfaceAddrs(ifaceName string) ([]net.Addr, error) { func Test_CreateInterface(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+1) wgIP := "10.99.99.1/32" - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, Address: wgaddr.MustParseWGAddress(wgIP), @@ -170,10 +171,7 @@ func Test_Close(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+2) wgIP := "10.99.99.2/32" wgPort := 33100 - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -215,10 +213,7 @@ func TestRecreation(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+2) wgIP := "10.99.99.2/32" wgPort := 33100 - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -288,10 +283,7 @@ func Test_ConfigureInterface(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+3) wgIP := "10.99.99.5/30" wgPort := 33100 - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, Address: wgaddr.MustParseWGAddress(wgIP), @@ -343,10 +335,7 @@ func Test_ConfigureInterface(t *testing.T) { func Test_UpdatePeer(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+4) wgIP := "10.99.99.9/30" - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -413,10 +402,7 @@ func Test_UpdatePeer(t *testing.T) { func Test_RemovePeer(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+4) wgIP := "10.99.99.13/30" - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -464,6 +450,8 @@ func Test_RemovePeer(t *testing.T) { } func Test_ConnectPeers(t *testing.T) { + t.Setenv("NB_DISABLE_EBPF_WG_PROXY", "true") + peer1ifaceName := fmt.Sprintf("utun%d", WgIntNumber+400) peer1wgIP := netip.MustParsePrefix("10.99.99.17/30") peer1Key, _ := wgtypes.GeneratePrivateKey() @@ -475,10 +463,7 @@ func Test_ConnectPeers(t *testing.T) { peer2wgPort := 33200 keepAlive := 1 * time.Second - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) guid := fmt.Sprintf("{%s}", uuid.New().String()) device.CustomWindowsGUIDString = strings.ToLower(guid) @@ -505,12 +490,8 @@ func Test_ConnectPeers(t *testing.T) { t.Fatal(err) } - localIP, err := getLocalIP() - if err != nil { - t.Fatal(err) - } - - peer1endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", localIP, peer1wgPort)) + localIP1 := "127.0.0.1" + peer1endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", localIP1, peer1wgPort)) if err != nil { t.Fatal(err) } @@ -518,10 +499,7 @@ func Test_ConnectPeers(t *testing.T) { guid = fmt.Sprintf("{%s}", uuid.New().String()) device.CustomWindowsGUIDString = strings.ToLower(guid) - newNet, err = stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet = stdnet.NewNet(context.Background(), testIFaceBlackList, nil) optsPeer2 := WGIFaceOpts{ IFaceName: peer2ifaceName, @@ -546,7 +524,8 @@ func Test_ConnectPeers(t *testing.T) { t.Fatal(err) } - peer2endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", localIP, peer2wgPort)) + localIP2 := "127.0.0.1" + peer2endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", localIP2, peer2wgPort)) if err != nil { t.Fatal(err) } @@ -569,17 +548,20 @@ func Test_ConnectPeers(t *testing.T) { if err != nil { t.Fatal(err) } - // todo: investigate why in some tests execution we need 30s + // On Linux with the kernel module both peers are kernel devices, elsewhere + // they run on wireguard-go. A tight busy-loop here would starve the + // wireguard-go goroutines that process the handshake, so poll on a ticker + // instead and yield the CPU between checks. WireGuard also only retries a + // lost handshake initiation every REKEY_TIMEOUT (5s), which is why the + // overall wait can occasionally stretch to tens of seconds. Each side sends + // its first initiation when its peer is configured, and the first one leaves + // before the other device knows the peer, so that one is always wasted. timeout := 30 * time.Second timeoutChannel := time.After(timeout) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() for { - select { - case <-timeoutChannel: - t.Fatalf("waiting for peer handshake timeout after %s", timeout.String()) - default: - } - peer, gpErr := getPeer(peer1ifaceName, peer2Key.PublicKey().String()) if gpErr != nil { t.Fatal(gpErr) @@ -588,10 +570,29 @@ func Test_ConnectPeers(t *testing.T) { t.Log("peers successfully handshake") break } + + select { + case <-timeoutChannel: + // The counters tell whether initiations were sent at all, whether they + // arrived, and whether only one direction is working. + t.Fatalf("waiting for peer handshake timeout after %s\n%s\n%s", timeout.String(), + describePeer(peer1ifaceName, peer2Key.PublicKey().String()), + describePeer(peer2ifaceName, peer1Key.PublicKey().String())) + case <-ticker.C: + } } } +func describePeer(ifaceName, peerPubKey string) string { + peer, err := getPeer(ifaceName, peerPubKey) + if err != nil { + return fmt.Sprintf("%s: peer %s: %v", ifaceName, peerPubKey, err) + } + return fmt.Sprintf("%s: peer %s endpoint=%v tx=%d rx=%d last_handshake=%v", + ifaceName, peerPubKey, peer.Endpoint, peer.TransmitBytes, peer.ReceiveBytes, peer.LastHandshakeTime) +} + func getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) { wg, err := wgctrl.New() if err != nil { @@ -615,28 +616,3 @@ func getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) { } return wgtypes.Peer{}, fmt.Errorf("peer not found") } - -func getLocalIP() (string, error) { - // Get all interfaces - addrs, err := net.InterfaceAddrs() - if err != nil { - return "", err - } - - for _, addr := range addrs { - ipNet, ok := addr.(*net.IPNet) - if !ok { - continue - } - if ipNet.IP.IsLoopback() { - continue - } - - if ipNet.IP.To4() == nil { - continue - } - return ipNet.IP.String(), nil - } - - return "", fmt.Errorf("no local IP found") -} diff --git a/client/iface/mocks/filter.go b/client/iface/mocks/filter.go index 5ae98039c..ff3dd0c8a 100644 --- a/client/iface/mocks/filter.go +++ b/client/iface/mocks/filter.go @@ -8,7 +8,7 @@ import ( "net/netip" reflect "reflect" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // MockPacketFilter is a mock of PacketFilter interface. diff --git a/client/iface/mocks/tun.go b/client/iface/mocks/tun.go index 677c82b0b..519ee6005 100644 --- a/client/iface/mocks/tun.go +++ b/client/iface/mocks/tun.go @@ -8,7 +8,7 @@ import ( os "os" reflect "reflect" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" tun "golang.zx2c4.com/wireguard/tun" ) diff --git a/client/iface/netstack/env.go b/client/iface/netstack/env.go index dd8cf29a3..b069301c1 100644 --- a/client/iface/netstack/env.go +++ b/client/iface/netstack/env.go @@ -3,14 +3,31 @@ package netstack import ( - "fmt" + "net" "os" "strconv" log "github.com/sirupsen/logrus" ) -const EnvUseNetstackMode = "NB_USE_NETSTACK_MODE" +const ( + EnvUseNetstackMode = "NB_USE_NETSTACK_MODE" + + // EnvSocks5ListenerPort overrides the port the SOCKS5 proxy listens on. + EnvSocks5ListenerPort = "NB_SOCKS5_LISTENER_PORT" + + // EnvSocks5ListenerAddress overrides the host/IP the SOCKS5 proxy binds to. + // The proxy is a bridge for local host applications into the userspace + // WireGuard netstack, so it binds to loopback by default. Override this only + // when the proxy must be reachable from other hosts (e.g. a container + // gateway); doing so exposes an unauthenticated SOCKS5 proxy on that + // address. + EnvSocks5ListenerAddress = "NB_SOCKS5_LISTENER_ADDRESS" + + // defaultSocks5Host is the loopback address the SOCKS5 proxy binds to unless + // overridden via EnvSocks5ListenerAddress. + defaultSocks5Host = "127.0.0.1" +) // IsEnabled todo: move these function to cmd layer func IsEnabled() bool { @@ -18,24 +35,40 @@ func IsEnabled() bool { } func ListenAddr() string { - sPort := os.Getenv("NB_SOCKS5_LISTENER_PORT") + return net.JoinHostPort(listenHost(), strconv.Itoa(listenPort())) +} + +// listenHost returns the host/IP the SOCKS5 proxy binds to. It defaults to +// loopback and only honors EnvSocks5ListenerAddress when it holds a valid IP. +func listenHost() string { + addr := os.Getenv(EnvSocks5ListenerAddress) + if addr == "" { + return defaultSocks5Host + } + if net.ParseIP(addr) == nil { + log.Warnf("invalid socks5 listener address %q, falling back to default: %s", addr, defaultSocks5Host) + return defaultSocks5Host + } + return addr +} + +// listenPort returns the port the SOCKS5 proxy binds to, defaulting to +// DefaultSocks5Port when EnvSocks5ListenerPort is unset or invalid. +func listenPort() int { + sPort := os.Getenv(EnvSocks5ListenerPort) if sPort == "" { - return listenAddr(DefaultSocks5Port) + return DefaultSocks5Port } port, err := strconv.Atoi(sPort) if err != nil { log.Warnf("invalid socks5 listener port, unable to convert it to int, falling back to default: %d", DefaultSocks5Port) - return listenAddr(DefaultSocks5Port) + return DefaultSocks5Port } if port < 1 || port > 65535 { log.Warnf("invalid socks5 listener port, it should be in the range 1-65535, falling back to default: %d", DefaultSocks5Port) - return listenAddr(DefaultSocks5Port) + return DefaultSocks5Port } - return listenAddr(port) -} - -func listenAddr(port int) string { - return fmt.Sprintf("0.0.0.0:%d", port) + return port } diff --git a/client/iface/netstack/env_test.go b/client/iface/netstack/env_test.go new file mode 100644 index 000000000..1083435a4 --- /dev/null +++ b/client/iface/netstack/env_test.go @@ -0,0 +1,63 @@ +//go:build !js + +package netstack + +import ( + "net" + "strconv" + "testing" +) + +func TestListenAddr_DefaultsToLoopback(t *testing.T) { + // No env overrides: must bind loopback, never all interfaces. + got := ListenAddr() + want := net.JoinHostPort("127.0.0.1", strconv.Itoa(DefaultSocks5Port)) + if got != want { + t.Fatalf("ListenAddr() = %q, want %q", got, want) + } +} + +func TestListenAddr_AddressOverride(t *testing.T) { + tests := []struct { + name string + env string + want string + }{ + {name: "valid override honored", env: "0.0.0.0", want: "0.0.0.0"}, + {name: "valid specific ip honored", env: "10.0.0.5", want: "10.0.0.5"}, + {name: "ipv6 loopback bracketed", env: "::1", want: "::1"}, + {name: "invalid falls back to loopback", env: "not-an-ip", want: "127.0.0.1"}, + {name: "empty falls back to loopback", env: "", want: "127.0.0.1"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(EnvSocks5ListenerAddress, tc.env) + want := net.JoinHostPort(tc.want, strconv.Itoa(DefaultSocks5Port)) + if got := ListenAddr(); got != want { + t.Fatalf("ListenAddr() = %q, want %q", got, want) + } + }) + } +} + +func TestListenAddr_PortOverride(t *testing.T) { + tests := []struct { + name string + env string + want int + }{ + {name: "valid port honored", env: "1081", want: 1081}, + {name: "non-numeric falls back", env: "abc", want: DefaultSocks5Port}, + {name: "out of range falls back", env: "70000", want: DefaultSocks5Port}, + {name: "zero falls back", env: "0", want: DefaultSocks5Port}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(EnvSocks5ListenerPort, tc.env) + want := net.JoinHostPort("127.0.0.1", strconv.Itoa(tc.want)) + if got := ListenAddr(); got != want { + t.Fatalf("ListenAddr() = %q, want %q", got, want) + } + }) + } +} diff --git a/client/iface/udpmux/mux.go b/client/iface/udpmux/mux.go index c5d2de4a5..2f7287b53 100644 --- a/client/iface/udpmux/mux.go +++ b/client/iface/udpmux/mux.go @@ -47,6 +47,7 @@ type SingleSocketUDPMux struct { // for UDP connection listen at unspecified address localAddrsForUnspecified []net.Addr + unspecified bool } const maxAddrSize = 512 @@ -157,6 +158,9 @@ func NewSingleSocketUDPMux(params Params) *SingleSocketUDPMux { if params.Logger == nil { params.Logger = getLogger() } + if params.Net == nil { + params.Net = stdnet.NewNet(context.Background(), nil, nil) + } mux := &SingleSocketUDPMux{ addressMap: map[string][]*udpMuxedConn{}, @@ -165,6 +169,7 @@ func NewSingleSocketUDPMux(params Params) *SingleSocketUDPMux { connsIPv6: make(map[string]*udpMuxedConn), candidateConnMap: make(map[string]*udpMuxedConn), closedChan: make(chan struct{}, 1), + unspecified: isUnspecified(params.UDPConn.LocalAddr()), pool: &sync.Pool{ New: func() interface{} { // big enough buffer to fit both packet and address @@ -173,7 +178,6 @@ func NewSingleSocketUDPMux(params Params) *SingleSocketUDPMux { }, } - mux.updateLocalAddresses() return mux } @@ -199,13 +203,6 @@ func (m *SingleSocketUDPMux) updateLocalAddresses() { m.params.Logger.Errorf("LocalAddr expected IPV4 or IPV6, got %T", m.params.UDPConn.LocalAddr()) } if len(networks) > 0 { - if m.params.Net == nil { - var err error - if m.params.Net, err = stdnet.NewNet(context.Background(), nil); err != nil { - m.params.Logger.Errorf("failed to get create network: %v", err) - } - } - ips, err := localInterfaces(m.params.Net, m.params.InterfaceFilter, nil, networks, true) if err == nil { for _, ip := range ips { @@ -244,10 +241,7 @@ func (m *SingleSocketUDPMux) GetListenAddresses() []net.Addr { // creates the connection if an existing one can't be found func (m *SingleSocketUDPMux) GetConn(ufrag string, addr net.Addr, candidateID string) (net.PacketConn, error) { // don't check addr for mux using unspecified address - m.mu.Lock() - lenLocalAddrs := len(m.localAddrsForUnspecified) - m.mu.Unlock() - if lenLocalAddrs == 0 && m.params.UDPConn.LocalAddr().String() != addr.String() { + if !m.unspecified && m.params.UDPConn.LocalAddr().String() != addr.String() { return nil, fmt.Errorf("invalid address %s", addr.String()) } @@ -511,3 +505,8 @@ func getLogger() logging.LeveledLogger { //fac.Writer = log.StandardLogger().Writer() return fac.NewLogger("ice") } + +func isUnspecified(addr net.Addr) bool { + udpAddr, ok := addr.(*net.UDPAddr) + return ok && udpAddr.IP.IsUnspecified() +} diff --git a/client/iface/udpmux/mux_test.go b/client/iface/udpmux/mux_test.go new file mode 100644 index 000000000..2e2770282 --- /dev/null +++ b/client/iface/udpmux/mux_test.go @@ -0,0 +1,99 @@ +package udpmux + +import ( + "fmt" + "net" + "sync" + "sync/atomic" + "testing" + + "github.com/pion/transport/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type countingNet struct { + transport.Net + calls atomic.Int64 + ifaces []*transport.Interface +} + +func (n *countingNet) Interfaces() ([]*transport.Interface, error) { + n.calls.Add(1) + return n.ifaces, nil +} + +func TestNewSingleSocketUDPMux_DoesNotDiscoverInterfaces(t *testing.T) { + conn := listenUDP(t, net.IPv4zero) + n := &countingNet{ifaces: []*transport.Interface{testInterface("eth0", net.IPv4(192, 168, 1, 10))}} + + mux := NewSingleSocketUDPMux(Params{UDPConn: conn, Net: n}) + t.Cleanup(func() { _ = mux.Close() }) + + assert.Equal(t, int64(0), n.calls.Load(), "constructing the mux must not enumerate interfaces") + + addrs := mux.GetListenAddresses() + assert.Equal(t, int64(1), n.calls.Load(), "listen addresses are discovered when they are asked for") + + port := conn.LocalAddr().(*net.UDPAddr).Port + require.Len(t, addrs, 1, "one address per discovered interface IP") + assert.Equal(t, fmt.Sprintf("192.168.1.10:%d", port), addrs[0].String(), "the discovered IP is paired with the socket port") +} + +func TestSingleSocketUDPMux_GetConnWithoutListenAddresses(t *testing.T) { + t.Run("unspecified socket accepts any address", func(t *testing.T) { + conn := listenUDP(t, net.IPv4zero) + n := &countingNet{} + mux := NewSingleSocketUDPMux(Params{UDPConn: conn, Net: n}) + t.Cleanup(func() { _ = mux.Close() }) + + port := conn.LocalAddr().(*net.UDPAddr).Port + pc, err := mux.GetConn("ufrag", &net.UDPAddr{IP: net.IPv4(192, 168, 1, 10), Port: port}, "candidate") + require.NoError(t, err) + assert.NotNil(t, pc, "a connection is created for a local address of an unspecified socket") + assert.Equal(t, int64(0), n.calls.Load(), "GetConn must not depend on a prior discovery") + }) + + t.Run("specific socket rejects a foreign address", func(t *testing.T) { + conn := listenUDP(t, net.IPv4(127, 0, 0, 1)) + mux := NewSingleSocketUDPMux(Params{UDPConn: conn}) + t.Cleanup(func() { _ = mux.Close() }) + + _, err := mux.GetConn("ufrag", &net.UDPAddr{IP: net.IPv4(192, 168, 1, 10), Port: 1234}, "candidate") + assert.Error(t, err, "an address the socket is not bound to must be rejected") + + pc, err := mux.GetConn("ufrag", conn.LocalAddr(), "candidate") + require.NoError(t, err) + assert.NotNil(t, pc, "the bound address is accepted") + }) +} + +func listenUDP(t *testing.T, ip net.IP) *net.UDPConn { + t.Helper() + conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: ip, Port: 0}) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + return conn +} + +func testInterface(name string, ip net.IP) *transport.Interface { + ifc := transport.NewInterface(net.Interface{Index: 1, MTU: 1500, Name: name, Flags: net.FlagUp | net.FlagMulticast}) + ifc.AddAddress(&net.IPNet{IP: ip, Mask: net.CIDRMask(24, 32)}) + return ifc +} + +func TestSingleSocketUDPMux_ConcurrentListenAddressesWithDefaultNet(t *testing.T) { + conn := listenUDP(t, net.IPv4zero) + mux := NewSingleSocketUDPMux(Params{UDPConn: conn}) + t.Cleanup(func() { _ = mux.Close() }) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + assert.NotEmpty(t, mux.GetListenAddresses(), "every caller gets at least the socket address") + }() + } + wg.Wait() +} diff --git a/client/iface/wgaddr/address.go b/client/iface/wgaddr/address.go index 43d1ec9aa..148e724f4 100644 --- a/client/iface/wgaddr/address.go +++ b/client/iface/wgaddr/address.go @@ -59,6 +59,19 @@ func (addr Address) IPv6Prefix() netip.Prefix { return netip.PrefixFrom(addr.IPv6, addr.IPv6Net.Bits()) } +// HostPrefix returns the v4 address as a single-host prefix. +func (addr Address) HostPrefix() netip.Prefix { + return netip.PrefixFrom(addr.IP, addr.IP.BitLen()) +} + +// IPv6HostPrefix returns the v6 address as a single-host prefix, or an invalid prefix when no v6 overlay address is assigned. +func (addr Address) IPv6HostPrefix() netip.Prefix { + if !addr.HasIPv6() { + return netip.Prefix{} + } + return netip.PrefixFrom(addr.IPv6, addr.IPv6.BitLen()) +} + // SetIPv6FromCompact decodes a compact prefix (5 or 17 bytes) and sets the IPv6 fields. // Returns an error if the bytes are invalid. A nil or empty input is a no-op. // diff --git a/client/iface/wgaddr/address_test.go b/client/iface/wgaddr/address_test.go new file mode 100644 index 000000000..61478b24c --- /dev/null +++ b/client/iface/wgaddr/address_test.go @@ -0,0 +1,24 @@ +package wgaddr + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAddress_HostPrefix(t *testing.T) { + addr := MustParseWGAddress("100.91.96.107/16") + + assert.Equal(t, netip.MustParsePrefix("100.91.96.107/32"), addr.HostPrefix(), "v4 host prefix must be a single host") + assert.Equal(t, netip.MustParsePrefix("100.91.0.0/16"), addr.Network, "network must keep the overlay prefix length") + assert.False(t, addr.IPv6HostPrefix().IsValid(), "no v6 overlay means no v6 host prefix") +} + +func TestAddress_IPv6HostPrefix(t *testing.T) { + addr := MustParseWGAddress("100.91.96.107/16") + addr.IPv6 = netip.MustParseAddr("fd00:1234::1") + addr.IPv6Net = netip.MustParsePrefix("fd00:1234::/64") + + assert.Equal(t, netip.MustParsePrefix("fd00:1234::1/128"), addr.IPv6HostPrefix(), "v6 host prefix must be a single host") +} diff --git a/client/iface/wgproxy/bind/proxy.go b/client/iface/wgproxy/bind/proxy.go index be690ed4f..fcaee15c7 100644 --- a/client/iface/wgproxy/bind/proxy.go +++ b/client/iface/wgproxy/bind/proxy.go @@ -53,15 +53,15 @@ func NewProxyBind(bind Bind, mtu uint16) *ProxyBind { return p } -// AddTurnConn adds a new connection to the bind. +// AddRelayedConn adds a new connection to the bind. // endpoint is the NetBird address of the remote peer. The SetEndpoint return with the address what will be used in the // WireGuard configuration. // // Parameters: // - ctx: Context is used for proxyToLocal to avoid unnecessary error messages // - nbAddr: The NetBird UDP address of the remote peer, it required to generate fake address -// - remoteConn: The established TURN connection to the remote peer -func (p *ProxyBind) AddTurnConn(ctx context.Context, nbAddr *net.UDPAddr, remoteConn net.Conn) error { +// - remoteConn: The established relayed connection to the remote peer +func (p *ProxyBind) AddRelayedConn(ctx context.Context, nbAddr *net.UDPAddr, remoteConn net.Conn) error { fakeNetIP, err := fakeAddress(nbAddr) if err != nil { return err diff --git a/client/iface/wgproxy/ebpf/portlookup.go b/client/iface/wgproxy/ebpf/portlookup.go deleted file mode 100644 index fce8f1507..000000000 --- a/client/iface/wgproxy/ebpf/portlookup.go +++ /dev/null @@ -1,32 +0,0 @@ -package ebpf - -import ( - "fmt" - "net" -) - -var ( - portRangeStart = 3128 - portRangeEnd = portRangeStart + 100 -) - -type portLookup struct { -} - -func (pl portLookup) searchFreePort() (int, error) { - for i := portRangeStart; i <= portRangeEnd; i++ { - if pl.tryToBind(i) == nil { - return i, nil - } - } - return 0, fmt.Errorf("failed to bind free port for eBPF proxy") -} - -func (pl portLookup) tryToBind(port int) error { - l, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port)) - if err != nil { - return err - } - _ = l.Close() - return nil -} diff --git a/client/iface/wgproxy/ebpf/portlookup_test.go b/client/iface/wgproxy/ebpf/portlookup_test.go deleted file mode 100644 index a2e92fc79..000000000 --- a/client/iface/wgproxy/ebpf/portlookup_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package ebpf - -import ( - "fmt" - "net" - "testing" -) - -func Test_portLookup_searchFreePort(t *testing.T) { - pl := portLookup{} - _, err := pl.searchFreePort() - if err != nil { - t.Fatal(err) - } -} - -func Test_portLookup_on_allocated(t *testing.T) { - pl := portLookup{} - - portRangeStart = 4128 - portRangeEnd = portRangeStart + 100 - - allocatedPort, err := allocatePort(portRangeStart) - if err != nil { - t.Fatal(err) - } - defer allocatedPort.Close() - - fp, err := pl.searchFreePort() - if err != nil { - t.Fatal(err) - } - - if fp != (portRangeStart + 1) { - t.Errorf("invalid free port, expected: %d, got: %d", portRangeStart+1, fp) - } -} - -func allocatePort(port int) (net.PacketConn, error) { - c, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port)) - if err != nil { - return nil, err - } - return c, err -} diff --git a/client/iface/wgproxy/ebpf/proxy.go b/client/iface/wgproxy/ebpf/proxy.go deleted file mode 100644 index 1b1a8ce1c..000000000 --- a/client/iface/wgproxy/ebpf/proxy.go +++ /dev/null @@ -1,243 +0,0 @@ -//go:build linux && !android - -package ebpf - -import ( - "context" - "fmt" - "net" - "sync" - - "github.com/hashicorp/go-multierror" - "github.com/pion/transport/v3" - log "github.com/sirupsen/logrus" - - nberrors "github.com/netbirdio/netbird/client/errors" - "github.com/netbirdio/netbird/client/iface/bufsize" - "github.com/netbirdio/netbird/client/iface/wgproxy/rawsocket" - "github.com/netbirdio/netbird/client/internal/ebpf" - ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager" - nbnet "github.com/netbirdio/netbird/client/net" -) - -const ( - loopbackAddr = "127.0.0.1" -) - -// WGEBPFProxy definition for proxy with EBPF support -type WGEBPFProxy struct { - localWGListenPort int - proxyPort int - mtu uint16 - - ebpfManager ebpfMgr.Manager - turnConnStore map[uint16]net.Conn - turnConnMutex sync.Mutex - - lastUsedPort uint16 - rawConnIPv4 net.PacketConn - rawConnIPv6 net.PacketConn - conn transport.UDPConn - - ctx context.Context - ctxCancel context.CancelFunc -} - -// NewWGEBPFProxy create new WGEBPFProxy instance -func NewWGEBPFProxy(wgPort int, mtu uint16) *WGEBPFProxy { - log.Debugf("instantiate ebpf proxy") - wgProxy := &WGEBPFProxy{ - localWGListenPort: wgPort, - mtu: mtu, - ebpfManager: ebpf.GetEbpfManagerInstance(), - turnConnStore: make(map[uint16]net.Conn), - } - return wgProxy -} - -// Listen load ebpf program and listen the proxy -func (p *WGEBPFProxy) Listen() error { - pl := portLookup{} - proxyPort, err := pl.searchFreePort() - if err != nil { - return err - } - p.proxyPort = proxyPort - - // Prepare IPv4 raw socket (required) - p.rawConnIPv4, err = rawsocket.PrepareSenderRawSocketIPv4() - if err != nil { - return err - } - - // Prepare IPv6 raw socket (optional) - p.rawConnIPv6, err = rawsocket.PrepareSenderRawSocketIPv6() - if err != nil { - log.Warnf("failed to prepare IPv6 raw socket, continuing with IPv4 only: %v", err) - } - - err = p.ebpfManager.LoadWgProxy(proxyPort, p.localWGListenPort) - if err != nil { - if closeErr := p.rawConnIPv4.Close(); closeErr != nil { - log.Warnf("failed to close IPv4 raw socket: %v", closeErr) - } - if p.rawConnIPv6 != nil { - if closeErr := p.rawConnIPv6.Close(); closeErr != nil { - log.Warnf("failed to close IPv6 raw socket: %v", closeErr) - } - } - return err - } - - addr := net.UDPAddr{ - Port: proxyPort, - IP: net.ParseIP(loopbackAddr), - } - - p.ctx, p.ctxCancel = context.WithCancel(context.Background()) - - conn, err := nbnet.ListenUDP("udp", &addr) - if err != nil { - if cErr := p.Free(); cErr != nil { - log.Errorf("Failed to close the wgproxy: %s", cErr) - } - return err - } - p.conn = conn - - go p.proxyToRemote() - log.Infof("local wg proxy listening on: %d", proxyPort) - return nil -} - -// AddTurnConn add new turn connection for the proxy -func (p *WGEBPFProxy) AddTurnConn(turnConn net.Conn) (*net.UDPAddr, error) { - wgEndpointPort, err := p.storeTurnConn(turnConn) - if err != nil { - return nil, err - } - - log.Infof("turn conn added to wg proxy store: %s, endpoint port: :%d", turnConn.RemoteAddr(), wgEndpointPort) - - wgEndpoint := &net.UDPAddr{ - IP: net.ParseIP(loopbackAddr), - Port: int(wgEndpointPort), - } - return wgEndpoint, nil -} - -// Free resources except the remoteConns will be keep open. -func (p *WGEBPFProxy) Free() error { - log.Debugf("free up ebpf wg proxy") - if p.ctx != nil && p.ctx.Err() != nil { - //nolint - return nil - } - - p.ctxCancel() - - var result *multierror.Error - if p.conn != nil { - if err := p.conn.Close(); err != nil { - result = multierror.Append(result, err) - } - } - - if err := p.ebpfManager.FreeWGProxy(); err != nil { - result = multierror.Append(result, err) - } - - if p.rawConnIPv4 != nil { - if err := p.rawConnIPv4.Close(); err != nil { - result = multierror.Append(result, err) - } - } - - if p.rawConnIPv6 != nil { - if err := p.rawConnIPv6.Close(); err != nil { - result = multierror.Append(result, err) - } - } - return nberrors.FormatErrorOrNil(result) -} - -// GetProxyPort returns the proxy listening port. -func (p *WGEBPFProxy) GetProxyPort() uint16 { - return uint16(p.proxyPort) -} - -// proxyToRemote read messages from local WireGuard interface and forward it to remote conn -// From this go routine has only one instance. -func (p *WGEBPFProxy) proxyToRemote() { - buf := make([]byte, p.mtu+bufsize.WGBufferOverhead) - for p.ctx.Err() == nil { - if err := p.readAndForwardPacket(buf); err != nil { - if p.ctx.Err() != nil { - return - } - log.Errorf("failed to proxy packet to remote conn: %s", err) - } - } -} - -func (p *WGEBPFProxy) readAndForwardPacket(buf []byte) error { - n, addr, err := p.conn.ReadFromUDP(buf) - if err != nil { - return fmt.Errorf("failed to read UDP packet from WG: %w", err) - } - - p.turnConnMutex.Lock() - conn, ok := p.turnConnStore[uint16(addr.Port)] - p.turnConnMutex.Unlock() - if !ok { - if p.ctx.Err() == nil { - log.Debugf("turn conn not found by port because conn already has been closed: %d", addr.Port) - } - return nil - } - - if _, err := conn.Write(buf[:n]); err != nil { - return fmt.Errorf("failed to forward local WG packet (%d) to remote turn conn: %w", addr.Port, err) - } - return nil -} - -func (p *WGEBPFProxy) storeTurnConn(turnConn net.Conn) (uint16, error) { - p.turnConnMutex.Lock() - defer p.turnConnMutex.Unlock() - - np, err := p.nextFreePort() - if err != nil { - return np, err - } - p.turnConnStore[np] = turnConn - return np, nil -} - -func (p *WGEBPFProxy) removeTurnConn(turnConnID uint16) { - p.turnConnMutex.Lock() - defer p.turnConnMutex.Unlock() - - _, ok := p.turnConnStore[turnConnID] - if ok { - log.Debugf("remove turn conn from store by port: %d", turnConnID) - } - delete(p.turnConnStore, turnConnID) -} - -func (p *WGEBPFProxy) nextFreePort() (uint16, error) { - if len(p.turnConnStore) == 65535 { - return 0, fmt.Errorf("reached maximum turn connection numbers") - } -generatePort: - if p.lastUsedPort == 65535 { - p.lastUsedPort = 1 - } else { - p.lastUsedPort++ - } - - if _, ok := p.turnConnStore[p.lastUsedPort]; ok { - goto generatePort - } - return p.lastUsedPort, nil -} diff --git a/client/iface/wgproxy/ebpf/proxy_test.go b/client/iface/wgproxy/ebpf/proxy_test.go deleted file mode 100644 index 3ec4f0eba..000000000 --- a/client/iface/wgproxy/ebpf/proxy_test.go +++ /dev/null @@ -1,56 +0,0 @@ -//go:build linux && !android - -package ebpf - -import ( - "testing" -) - -func TestWGEBPFProxy_connStore(t *testing.T) { - wgProxy := NewWGEBPFProxy(1, 1280) - - p, _ := wgProxy.storeTurnConn(nil) - if p != 1 { - t.Errorf("invalid initial port: %d", wgProxy.lastUsedPort) - } - - numOfConns := 10 - for i := 0; i < numOfConns; i++ { - p, _ = wgProxy.storeTurnConn(nil) - } - if p != uint16(numOfConns)+1 { - t.Errorf("invalid last used port: %d, expected: %d", p, numOfConns+1) - } - if len(wgProxy.turnConnStore) != numOfConns+1 { - t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.turnConnStore), numOfConns+1) - } -} - -func TestWGEBPFProxy_portCalculation_overflow(t *testing.T) { - wgProxy := NewWGEBPFProxy(1, 1280) - - _, _ = wgProxy.storeTurnConn(nil) - wgProxy.lastUsedPort = 65535 - p, _ := wgProxy.storeTurnConn(nil) - - if len(wgProxy.turnConnStore) != 2 { - t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.turnConnStore), 2) - } - - if p != 2 { - t.Errorf("invalid last used port: %d, expected: %d", p, 2) - } -} - -func TestWGEBPFProxy_portCalculation_maxConn(t *testing.T) { - wgProxy := NewWGEBPFProxy(1, 1280) - - for i := 0; i < 65535; i++ { - _, _ = wgProxy.storeTurnConn(nil) - } - - _, err := wgProxy.storeTurnConn(nil) - if err == nil { - t.Errorf("invalid turn conn store calculation") - } -} diff --git a/client/iface/wgproxy/factory_kernel.go b/client/iface/wgproxy/factory_kernel.go index 7821df3de..0b2329b96 100644 --- a/client/iface/wgproxy/factory_kernel.go +++ b/client/iface/wgproxy/factory_kernel.go @@ -8,11 +8,13 @@ import ( log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/client/iface/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/iface/wgproxy/loopback" udpProxy "github.com/netbirdio/netbird/client/iface/wgproxy/udp" ) const ( + envDisableKernelWGProxy = "NB_DISABLE_KERNEL_WG_PROXY" + // envDisableEBPFWGProxy is a deprecated alias for envDisableKernelWGProxy. envDisableEBPFWGProxy = "NB_DISABLE_EBPF_WG_PROXY" ) @@ -20,7 +22,7 @@ type KernelFactory struct { wgPort int mtu uint16 - ebpfProxy *ebpf.WGEBPFProxy + loopbackProxy *loopback.Proxy } func NewKernelFactory(wgPort int, mtu uint16) *KernelFactory { @@ -29,55 +31,56 @@ func NewKernelFactory(wgPort int, mtu uint16) *KernelFactory { mtu: mtu, } - if isEBPFDisabled() { + if isKernelProxyDisabled() { log.Infof("WireGuard Proxy Factory will produce UDP proxy") - log.Infof("eBPF WireGuard proxy is disabled via %s environment variable", envDisableEBPFWGProxy) return f } - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, mtu) - if err := ebpfProxy.Listen(); err != nil { + loopbackProxy := loopback.NewProxy(wgPort, mtu) + if err := loopbackProxy.Listen(); err != nil { log.Infof("WireGuard Proxy Factory will produce UDP proxy") - log.Warnf("failed to initialize ebpf proxy, fallback to user space proxy: %s", err) + log.Warnf("failed to initialize loopback proxy, fallback to user space proxy: %s", err) return f } - log.Infof("WireGuard Proxy Factory will produce eBPF proxy") - f.ebpfProxy = ebpfProxy + log.Infof("WireGuard Proxy Factory will produce loopback proxy") + f.loopbackProxy = loopbackProxy return f } func (w *KernelFactory) GetProxy() Proxy { - if w.ebpfProxy == nil { + if w.loopbackProxy == nil { return udpProxy.NewWGUDPProxy(w.wgPort, w.mtu) } - return ebpf.NewProxyWrapper(w.ebpfProxy) -} - -// GetProxyPort returns the eBPF proxy port, or 0 if eBPF is not active. -func (w *KernelFactory) GetProxyPort() uint16 { - if w.ebpfProxy == nil { - return 0 - } - return w.ebpfProxy.GetProxyPort() + return loopback.NewProxyWrapper(w.loopbackProxy) } func (w *KernelFactory) Free() error { - if w.ebpfProxy == nil { + if w.loopbackProxy == nil { return nil } - return w.ebpfProxy.Free() + return w.loopbackProxy.Free() } -func isEBPFDisabled() bool { - val := os.Getenv(envDisableEBPFWGProxy) +func isKernelProxyDisabled() bool { + env := envDisableKernelWGProxy + val := os.Getenv(env) + if val == "" { + env = envDisableEBPFWGProxy + val = os.Getenv(env) + } if val == "" { return false } + disabled, err := strconv.ParseBool(val) if err != nil { - log.Warnf("failed to parse %s: %v", envDisableEBPFWGProxy, err) + log.Warnf("failed to parse %s: %v", env, err) return false } + + if disabled { + log.Infof("kernel WireGuard proxy is disabled via %s", env) + } return disabled } diff --git a/client/iface/wgproxy/factory_usp.go b/client/iface/wgproxy/factory_usp.go index bbd67e076..a1b1c34d7 100644 --- a/client/iface/wgproxy/factory_usp.go +++ b/client/iface/wgproxy/factory_usp.go @@ -24,11 +24,6 @@ func (w *USPFactory) GetProxy() Proxy { return proxyBind.NewProxyBind(w.bind, w.mtu) } -// GetProxyPort returns 0 as userspace WireGuard doesn't use a separate proxy port. -func (w *USPFactory) GetProxyPort() uint16 { - return 0 -} - func (w *USPFactory) Free() error { return nil } diff --git a/client/iface/wgproxy/loopback/addr.go b/client/iface/wgproxy/loopback/addr.go new file mode 100644 index 000000000..52feee295 --- /dev/null +++ b/client/iface/wgproxy/loopback/addr.go @@ -0,0 +1,70 @@ +//go:build linux && !android + +package loopback + +import ( + "fmt" + "net/netip" +) + +// Peer endpoints live in the upper half of 127.0.0.0/8. Everything in that +// range is delivered to the loopback device without any address or route being +// configured, and staying out of 127.0.0.0/9 keeps well-known squatters such as +// 127.0.0.53 (systemd-resolved) and 127.0.1.1 out of the way. +const ( + addrRangeBase uint32 = 0x7f800000 // 127.128.0.0 + addrRangeSize uint32 = 1 << 23 // /9 + addrRangePrefix = "127.128.0.0/9" +) + +// allocator hands out one loopback address per relayed connection. The address +// is the peer's identity: WireGuard sends to it, and the proxy recovers which +// peer a packet belongs to from the destination address. +type allocator struct { + cursor uint32 +} + +// next returns the first free address at or after the cursor, wrapping once. +// inUse reports whether an address is already handed out. +func (a *allocator) next(inUse func(netip.Addr) bool) (netip.Addr, error) { + for i := uint32(0); i < addrRangeSize; i++ { + a.cursor = (a.cursor + 1) % addrRangeSize + addr := addrFromOffset(a.cursor) + if !addr.IsValid() { + continue + } + if inUse(addr) { + continue + } + return addr, nil + } + return netip.Addr{}, fmt.Errorf("no free endpoint address in %s", addrRangePrefix) +} + +// addrFromOffset maps an offset in the range to an address, skipping the .0 and +// .255 hosts. They are unremarkable on loopback, but tools and firewall rules +// tend to treat them as network and broadcast addresses. +func addrFromOffset(offset uint32) netip.Addr { + last := offset & 0xff + if last == 0 || last == 0xff { + return netip.Addr{} + } + + v := addrRangeBase + offset + return netip.AddrFrom4([4]byte{ + byte(v >> 24), + byte(v >> 16), + byte(v >> 8), + byte(v), + }) +} + +// inRange reports whether addr is one this proxy could have handed out. +func inRange(addr netip.Addr) bool { + if !addr.Is4() { + return false + } + b := addr.As4() + v := uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]) + return v >= addrRangeBase && v < addrRangeBase+addrRangeSize && b[3] != 0 && b[3] != 0xff +} diff --git a/client/iface/wgproxy/loopback/addr_test.go b/client/iface/wgproxy/loopback/addr_test.go new file mode 100644 index 000000000..3755b7269 --- /dev/null +++ b/client/iface/wgproxy/loopback/addr_test.go @@ -0,0 +1,114 @@ +//go:build linux && !android + +package loopback + +import ( + "net/netip" + "testing" +) + +func TestAllocatorHandsOutDistinctAddresses(t *testing.T) { + var a allocator + taken := make(map[netip.Addr]bool) + + for i := 0; i < 1000; i++ { + addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] }) + if err != nil { + t.Fatalf("allocate %d: %v", i, err) + } + if taken[addr] { + t.Fatalf("address %s handed out twice", addr) + } + if !inRange(addr) { + t.Fatalf("address %s outside %s", addr, addrRangePrefix) + } + taken[addr] = true + } +} + +func TestAllocatorSkipsNetworkAndBroadcastHosts(t *testing.T) { + var a allocator + taken := make(map[netip.Addr]bool) + + // enough allocations to walk past a .255/.0 boundary + for i := 0; i < 600; i++ { + addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] }) + if err != nil { + t.Fatalf("allocate %d: %v", i, err) + } + last := addr.As4()[3] + if last == 0 || last == 255 { + t.Fatalf("address %s ends in .%d", addr, last) + } + taken[addr] = true + } +} + +func TestAllocatorReusesReleasedAddresses(t *testing.T) { + var a allocator + taken := make(map[netip.Addr]bool) + inUse := func(candidate netip.Addr) bool { return taken[candidate] } + alloc := func() netip.Addr { + t.Helper() + addr, err := a.next(inUse) + if err != nil { + t.Fatalf("allocate: %v", err) + } + taken[addr] = true + return addr + } + + first := alloc() + second := alloc() + delete(taken, first) + + // The cursor only moves forward, so a released address comes back after a + // wrap. Park the cursor near the end of the range instead of allocating + // 2^23 addresses: the next call takes the last usable address, and the one + // after that wraps past the skipped .255 and .0 hosts to the released one. + a.cursor = addrRangeSize - 3 + last := alloc() + if want := netip.MustParseAddr("127.255.255.254"); last != want { + t.Fatalf("expected the last usable address %s before the wrap, got %s", want, last) + } + + if reused := alloc(); reused != first { + t.Fatalf("expected the released address %s after the wrap, got %s", first, reused) + } + + // second is still held, so the allocator must step over it. + if next := alloc(); next == second { + t.Fatalf("allocator handed out %s while it was still in use", second) + } +} + +func TestInRange(t *testing.T) { + tests := []struct { + addr string + want bool + }{ + {"127.128.0.1", true}, + {"127.255.255.254", true}, + {"127.128.0.0", false}, // network host, never handed out + {"127.128.5.255", false}, // broadcast host, never handed out + {"127.127.255.255", false}, // below the range, where 127.0.0.53 and friends live + {"127.0.0.1", false}, + {"127.0.0.53", false}, + {"127.0.1.1", false}, + {"128.0.0.1", false}, + {"10.0.0.1", false}, + } + + for _, tc := range tests { + addr := netip.MustParseAddr(tc.addr) + if got := inRange(addr); got != tc.want { + t.Errorf("inRange(%s) = %v, want %v", tc.addr, got, tc.want) + } + } +} + +func TestInRangeIgnoresIPv6(t *testing.T) { + if inRange(netip.MustParseAddr("::1")) { + t.Error("inRange(::1) = true, want false") + } +} diff --git a/client/iface/wgproxy/loopback/proxy.go b/client/iface/wgproxy/loopback/proxy.go new file mode 100644 index 000000000..f9364c766 --- /dev/null +++ b/client/iface/wgproxy/loopback/proxy.go @@ -0,0 +1,291 @@ +//go:build linux && !android + +package loopback + +import ( + "context" + "fmt" + "net" + "net/netip" + "sync" + "syscall" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + "golang.org/x/net/ipv4" + "golang.org/x/sys/unix" + + nberrors "github.com/netbirdio/netbird/client/errors" + "github.com/netbirdio/netbird/client/iface/bufsize" + "github.com/netbirdio/netbird/client/iface/wgproxy/rawsocket" +) + +const ( + loopbackDevice = "lo" + + portRangeStart = 3128 + portRangeEnd = portRangeStart + 100 +) + +// Proxy forwards packets between relayed connections and a local kernel +// WireGuard instance. Every relayed peer gets its own loopback address as its +// WireGuard endpoint, so a single socket serves all of them: the destination +// address of an incoming packet identifies the peer. +type Proxy struct { + localWGListenPort int + mtu uint16 + proxyPort int + + conn *net.UDPConn + packetConn *ipv4.PacketConn + loIndex int + rawConnIPv4 net.PacketConn + rawConnIPv6 net.PacketConn + + relayedConnMutex sync.Mutex + relayedConnStore map[netip.Addr]net.Conn + addrs allocator + + ctx context.Context + ctxCancel context.CancelFunc +} + +// NewProxy creates a proxy for the WireGuard instance listening on wgPort. +func NewProxy(wgPort int, mtu uint16) *Proxy { + log.Debugf("instantiate loopback wg proxy") + return &Proxy{ + localWGListenPort: wgPort, + mtu: mtu, + relayedConnStore: make(map[netip.Addr]net.Conn), + } +} + +// Listen opens the shared socket and starts forwarding WireGuard packets to the +// relayed connections. +func (p *Proxy) Listen() error { + rawConnIPv4, err := rawsocket.PrepareSenderRawSocketIPv4() + if err != nil { + return fmt.Errorf("prepare IPv4 raw socket: %w", err) + } + p.rawConnIPv4 = rawConnIPv4 + + p.rawConnIPv6, err = rawsocket.PrepareSenderRawSocketIPv6() + if err != nil { + log.Warnf("failed to prepare IPv6 raw socket, continuing with IPv4 only: %v", err) + } + + loopback, err := net.InterfaceByName(loopbackDevice) + if err != nil { + if freeErr := p.Free(); freeErr != nil { + log.Errorf("failed to free the wgproxy: %s", freeErr) + } + return fmt.Errorf("look up %s: %w", loopbackDevice, err) + } + p.loIndex = loopback.Index + + if err := p.listen(); err != nil { + if freeErr := p.Free(); freeErr != nil { + log.Errorf("failed to free the wgproxy: %s", freeErr) + } + return err + } + + p.ctx, p.ctxCancel = context.WithCancel(context.Background()) + + go p.proxyToRemote() + log.Infof("local wg proxy listening on %s:%d", addrRangePrefix, p.proxyPort) + return nil +} + +// listen binds the shared socket on the first free port of the range. The bind +// has to be a wildcard one to receive every peer address in the range, so it is +// restricted to the loopback device: without that the port would be reachable +// on every interface. +func (p *Proxy) listen() error { + var lastErr error + for port := portRangeStart; port <= portRangeEnd; port++ { + err := p.listenOn(port) + if err == nil { + p.proxyPort = port + return nil + } + lastErr = err + } + return fmt.Errorf("bind proxy port in range %d-%d: %w", portRangeStart, portRangeEnd, lastErr) +} + +func (p *Proxy) listenOn(proxyPort int) error { + lc := net.ListenConfig{ + Control: func(_, _ string, c syscall.RawConn) error { + var sockErr error + if err := c.Control(func(fd uintptr) { + if err := unix.SetsockoptString(int(fd), unix.SOL_SOCKET, unix.SO_BINDTODEVICE, loopbackDevice); err != nil { + sockErr = fmt.Errorf("bind to %s: %w", loopbackDevice, err) + return + } + }); err != nil { + return fmt.Errorf("control socket: %w", err) + } + return sockErr + }, + } + + conn, err := lc.ListenPacket(context.Background(), "udp4", fmt.Sprintf(":%d", proxyPort)) + if err != nil { + return fmt.Errorf("listen on :%d: %w", proxyPort, err) + } + + udpConn, ok := conn.(*net.UDPConn) + if !ok { + if closeErr := conn.Close(); closeErr != nil { + log.Errorf("failed to close proxy conn: %s", closeErr) + } + return fmt.Errorf("unexpected conn type %T", conn) + } + + packetConn := ipv4.NewPacketConn(udpConn) + // the destination address carries the peer identity, the interface index is + // checked on receive as a second line of defense behind SO_BINDTODEVICE + if err := packetConn.SetControlMessage(ipv4.FlagDst|ipv4.FlagInterface, true); err != nil { + if closeErr := udpConn.Close(); closeErr != nil { + log.Errorf("failed to close proxy conn: %s", closeErr) + } + return fmt.Errorf("request destination address: %w", err) + } + + p.conn = udpConn + p.packetConn = packetConn + return nil +} + +// AddRelayedConn assigns an endpoint address to the relayed connection and +// returns the address WireGuard should send to, along with the key the +// connection is stored under. +func (p *Proxy) AddRelayedConn(relayedConn net.Conn) (*net.UDPAddr, netip.Addr, error) { + addr, err := p.storeRelayedConn(relayedConn) + if err != nil { + return nil, netip.Addr{}, err + } + + log.Infof("relayed conn added to wg proxy store: %s, endpoint address: %s", relayedConn.RemoteAddr(), addr) + + return &net.UDPAddr{ + IP: addr.AsSlice(), + Port: p.proxyPort, + }, addr, nil +} + +// Free releases the proxy resources. The relayed connections are left open. +func (p *Proxy) Free() error { + log.Debugf("free up loopback wg proxy") + if p.ctx != nil && p.ctx.Err() != nil { + //nolint + return nil + } + + if p.ctxCancel != nil { + p.ctxCancel() + } + + var result *multierror.Error + if p.conn != nil { + if err := p.conn.Close(); err != nil { + result = multierror.Append(result, err) + } + } + + if p.rawConnIPv4 != nil { + if err := p.rawConnIPv4.Close(); err != nil { + result = multierror.Append(result, err) + } + } + + if p.rawConnIPv6 != nil { + if err := p.rawConnIPv6.Close(); err != nil { + result = multierror.Append(result, err) + } + } + return nberrors.FormatErrorOrNil(result) +} + +// proxyToRemote reads packets from the local WireGuard instance and forwards +// them to the relayed connection the destination address belongs to. +func (p *Proxy) proxyToRemote() { + buf := make([]byte, p.mtu+bufsize.WGBufferOverhead) + for p.ctx.Err() == nil { + if err := p.readAndForwardPacket(buf); err != nil { + if p.ctx.Err() != nil { + return + } + log.Errorf("failed to proxy packet to remote conn: %s", err) + } + } +} + +func (p *Proxy) readAndForwardPacket(buf []byte) error { + n, cm, _, err := p.packetConn.ReadFrom(buf) + if err != nil { + return fmt.Errorf("read UDP packet from WG: %w", err) + } + + if cm == nil { + return fmt.Errorf("no control message on packet") + } + + if cm.IfIndex != p.loIndex { + log.Tracef("dropping packet received on interface %d instead of %s", cm.IfIndex, loopbackDevice) + return nil + } + + dst, ok := netip.AddrFromSlice(cm.Dst.To4()) + if !ok || !inRange(dst) { + log.Tracef("dropping packet for unexpected destination %s", cm.Dst) + return nil + } + + p.relayedConnMutex.Lock() + conn, ok := p.relayedConnStore[dst] + p.relayedConnMutex.Unlock() + if !ok { + if p.ctx.Err() == nil { + log.Debugf("relayed conn not found by address because conn already has been closed: %s", dst) + } + return nil + } + + if _, err := conn.Write(buf[:n]); err != nil { + return fmt.Errorf("forward local WG packet (%s) to remote relayed conn: %w", dst, err) + } + return nil +} + +func (p *Proxy) storeRelayedConn(relayedConn net.Conn) (netip.Addr, error) { + p.relayedConnMutex.Lock() + defer p.relayedConnMutex.Unlock() + + addr, err := p.addrs.next(func(a netip.Addr) bool { + _, ok := p.relayedConnStore[a] + return ok + }) + if err != nil { + return netip.Addr{}, err + } + + p.relayedConnStore[addr] = relayedConn + return addr, nil +} + +// removeRelayedConn releases an endpoint address. It only removes the entry +// while it still belongs to relayedConn, so a late release cannot take an +// address away from the peer it was handed to next. +func (p *Proxy) removeRelayedConn(addr netip.Addr, relayedConn net.Conn) { + p.relayedConnMutex.Lock() + defer p.relayedConnMutex.Unlock() + + if stored, ok := p.relayedConnStore[addr]; !ok || stored != relayedConn { + return + } + + log.Debugf("remove relayed conn from store by address: %s", addr) + delete(p.relayedConnStore, addr) +} diff --git a/client/iface/wgproxy/loopback/proxy_privileged_test.go b/client/iface/wgproxy/loopback/proxy_privileged_test.go new file mode 100644 index 000000000..6314fe2de --- /dev/null +++ b/client/iface/wgproxy/loopback/proxy_privileged_test.go @@ -0,0 +1,196 @@ +//go:build linux && !android && privileged + +package loopback + +import ( + "context" + "net" + "strconv" + "testing" + "time" +) + +const testWGPort = 51862 + +// relayEnd stands in for a relayed connection: the proxy writes what it read +// from WireGuard into it, and the test reads it back out here. +func relayEnd(t *testing.T) (proxySide net.Conn, testSide *net.UDPConn) { + t.Helper() + + testSide, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")}) + if err != nil { + t.Fatalf("relay listener: %v", err) + } + t.Cleanup(func() { + if err := testSide.Close(); err != nil { + t.Logf("close relay listener: %v", err) + } + }) + + proxySide, err = net.Dial("udp", testSide.LocalAddr().String()) + if err != nil { + t.Fatalf("relay conn: %v", err) + } + t.Cleanup(func() { + if err := proxySide.Close(); err != nil { + t.Logf("close relay conn: %v", err) + } + }) + + return proxySide, testSide +} + +// TestProxyDemuxesByDestinationAddress is the core of the design: one socket +// serves every peer, and the destination address decides which relayed +// connection a WireGuard packet belongs to. +func TestProxyDemuxesByDestinationAddress(t *testing.T) { + proxy := NewProxy(testWGPort, 1280) + if err := proxy.Listen(); err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { + if err := proxy.Free(); err != nil { + t.Errorf("free proxy: %v", err) + } + }() + + const peers = 3 + endpoints := make([]*net.UDPAddr, 0, peers) + readers := make([]*net.UDPConn, 0, peers) + for i := 0; i < peers; i++ { + proxySide, testSide := relayEnd(t) + endpoint, _, err := proxy.AddRelayedConn(proxySide) + if err != nil { + t.Fatalf("add relayed conn %d: %v", i, err) + } + if endpoint.Port != proxy.proxyPort { + t.Errorf("peer %d endpoint port = %d, want the shared proxy port %d", i, endpoint.Port, proxy.proxyPort) + } + endpoints = append(endpoints, endpoint) + readers = append(readers, testSide) + } + + // every peer must have its own address, otherwise they are indistinguishable + seen := make(map[string]bool, peers) + for i, endpoint := range endpoints { + if seen[endpoint.IP.String()] { + t.Fatalf("peer %d reuses endpoint address %s", i, endpoint.IP) + } + seen[endpoint.IP.String()] = true + } + + wgSock, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")}) + if err != nil { + t.Fatalf("wg socket: %v", err) + } + defer func() { + if err := wgSock.Close(); err != nil { + t.Logf("close wg socket: %v", err) + } + }() + + for i, endpoint := range endpoints { + payload := []byte{byte(i), 'p', 'k', 't'} + if _, err := wgSock.WriteTo(payload, endpoint); err != nil { + t.Fatalf("write to peer %d endpoint %s: %v", i, endpoint, err) + } + + buf := make([]byte, 1500) + if err := readers[i].SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set read deadline: %v", err) + } + n, _, err := readers[i].ReadFrom(buf) + if err != nil { + t.Fatalf("peer %d did not receive its packet: %v", i, err) + } + if string(buf[:n]) != string(payload) { + t.Errorf("peer %d got %q, want %q", i, buf[:n], payload) + } + + // no other peer may see it + for j, other := range readers { + if j == i { + continue + } + if err := other.SetReadDeadline(time.Now().Add(100 * time.Millisecond)); err != nil { + t.Fatalf("set read deadline: %v", err) + } + if _, _, err := other.ReadFrom(buf); err == nil { + t.Errorf("packet for peer %d also delivered to peer %d", i, j) + } + } + } +} + +// TestProxyDropsPacketsOutsideTheRange guards the wildcard bind: anything that +// is not addressed to a handed-out endpoint must not reach a relayed peer. +func TestProxyDropsPacketsOutsideTheRange(t *testing.T) { + proxy := NewProxy(testWGPort+1, 1280) + if err := proxy.Listen(); err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { + if err := proxy.Free(); err != nil { + t.Errorf("free proxy: %v", err) + } + }() + + proxySide, testSide := relayEnd(t) + if _, _, err := proxy.AddRelayedConn(proxySide); err != nil { + t.Fatalf("add relayed conn: %v", err) + } + + sender, err := net.Dial("udp", net.JoinHostPort("127.0.0.1", strconv.Itoa(proxy.proxyPort))) + if err != nil { + t.Fatalf("sender: %v", err) + } + defer func() { + if err := sender.Close(); err != nil { + t.Logf("close sender: %v", err) + } + }() + + if _, err := sender.Write([]byte("stray")); err != nil { + t.Fatalf("write stray packet: %v", err) + } + + buf := make([]byte, 1500) + if err := testSide.SetReadDeadline(time.Now().Add(500 * time.Millisecond)); err != nil { + t.Fatalf("set read deadline: %v", err) + } + if _, _, err := testSide.ReadFrom(buf); err == nil { + t.Error("packet addressed to 127.0.0.1 was forwarded to a relayed peer") + } +} + +// A wrapper that is closed before it starts forwarding still has to give its +// endpoint address back, otherwise the range leaks an address per attempt. +func TestClosingBeforeWorkReleasesTheAddress(t *testing.T) { + proxy := NewProxy(testWGPort+2, 1280) + if err := proxy.Listen(); err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { + if err := proxy.Free(); err != nil { + t.Errorf("free proxy: %v", err) + } + }() + + proxySide, _ := relayEnd(t) + wrapper := NewProxyWrapper(proxy) + if err := wrapper.AddRelayedConn(context.Background(), nil, proxySide); err != nil { + t.Fatalf("add relayed conn: %v", err) + } + + if got := len(proxy.relayedConnStore); got != 1 { + t.Fatalf("store holds %d entries after adding one conn, want 1", got) + } + + if err := wrapper.CloseConn(); err != nil { + t.Fatalf("close conn: %v", err) + } + + if got := len(proxy.relayedConnStore); got != 0 { + t.Errorf("store holds %d entries after close, want 0", got) + } +} diff --git a/client/iface/wgproxy/ebpf/wrapper.go b/client/iface/wgproxy/loopback/wrapper.go similarity index 81% rename from client/iface/wgproxy/ebpf/wrapper.go rename to client/iface/wgproxy/loopback/wrapper.go index a6156a661..a9cb1ab59 100644 --- a/client/iface/wgproxy/ebpf/wrapper.go +++ b/client/iface/wgproxy/loopback/wrapper.go @@ -1,6 +1,6 @@ //go:build linux && !android -package ebpf +package loopback import ( "context" @@ -8,6 +8,7 @@ import ( "fmt" "io" "net" + "net/netip" "sync" "github.com/google/gopacket" @@ -95,13 +96,14 @@ func NewPacketHeaders(localWGListenPort int, endpoint *net.UDPAddr) (*PacketHead // ProxyWrapper help to keep the remoteConn instance for net.Conn.Close function call type ProxyWrapper struct { - wgeBPFProxy *WGEBPFProxy + proxy *Proxy remoteConn net.Conn ctx context.Context cancel context.CancelFunc wgRelayedEndpointAddr *net.UDPAddr + peerAddr netip.Addr headers *PacketHeaders headerCurrentUsed *PacketHeaders rawConn net.PacketConn @@ -113,36 +115,44 @@ type ProxyWrapper struct { closeListener *listener.CloseListener } -func NewProxyWrapper(proxy *WGEBPFProxy) *ProxyWrapper { +func NewProxyWrapper(proxy *Proxy) *ProxyWrapper { return &ProxyWrapper{ - wgeBPFProxy: proxy, + proxy: proxy, pausedCond: sync.NewCond(&sync.Mutex{}), closeListener: listener.NewCloseListener(), } } -func (p *ProxyWrapper) AddTurnConn(ctx context.Context, _ *net.UDPAddr, remoteConn net.Conn) error { - addr, err := p.wgeBPFProxy.AddTurnConn(remoteConn) +func (p *ProxyWrapper) AddRelayedConn(ctx context.Context, _ *net.UDPAddr, remoteConn net.Conn) error { + addr, peerAddr, err := p.proxy.AddRelayedConn(remoteConn) if err != nil { - return fmt.Errorf("add turn conn: %w", err) + return fmt.Errorf("add relayed conn: %w", err) } - headers, err := NewPacketHeaders(p.wgeBPFProxy.localWGListenPort, addr) + // the endpoint address is otherwise only released by the forwarding + // goroutine, which never starts when the setup below fails + release := func() { p.proxy.removeRelayedConn(peerAddr, remoteConn) } + + headers, err := NewPacketHeaders(p.proxy.localWGListenPort, addr) if err != nil { + release() return fmt.Errorf("create packet sender: %w", err) } // Check if required raw connection is available - if !headers.isIPv4 && p.wgeBPFProxy.rawConnIPv6 == nil { + if !headers.isIPv4 && p.proxy.rawConnIPv6 == nil { + release() return errIPv6ConnNotAvailable } - if headers.isIPv4 && p.wgeBPFProxy.rawConnIPv4 == nil { + if headers.isIPv4 && p.proxy.rawConnIPv4 == nil { + release() return errIPv4ConnNotAvailable } p.remoteConn = remoteConn p.ctx, p.cancel = context.WithCancel(ctx) p.wgRelayedEndpointAddr = addr + p.peerAddr = peerAddr p.headers = headers p.rawConn = p.selectRawConn(headers) return nil @@ -193,18 +203,18 @@ func (p *ProxyWrapper) RedirectAs(endpoint *net.UDPAddr) { return } - header, err := NewPacketHeaders(p.wgeBPFProxy.localWGListenPort, endpoint) + header, err := NewPacketHeaders(p.proxy.localWGListenPort, endpoint) if err != nil { log.Errorf("failed to create packet headers: %s", err) return } // Check if required raw connection is available - if !header.isIPv4 && p.wgeBPFProxy.rawConnIPv6 == nil { + if !header.isIPv4 && p.proxy.rawConnIPv6 == nil { log.Error(errIPv6ConnNotAvailable) return } - if header.isIPv4 && p.wgeBPFProxy.rawConnIPv4 == nil { + if header.isIPv4 && p.proxy.rawConnIPv4 == nil { log.Error(errIPv4ConnNotAvailable) return } @@ -240,6 +250,10 @@ func (p *ProxyWrapper) CloseConn() error { p.closeListener.SetCloseListener(nil) + // releases the endpoint address for a wrapper that was never started, and + // is a no-op once the forwarding goroutine has released it + p.proxy.removeRelayedConn(p.peerAddr, p.remoteConn) + p.pausedCond.L.Lock() p.paused = false p.pausedCond.Signal() @@ -252,9 +266,9 @@ func (p *ProxyWrapper) CloseConn() error { } func (p *ProxyWrapper) proxyToLocal(ctx context.Context) { - defer p.wgeBPFProxy.removeTurnConn(uint16(p.wgRelayedEndpointAddr.Port)) + defer p.proxy.removeRelayedConn(p.peerAddr, p.remoteConn) - buf := make([]byte, p.wgeBPFProxy.mtu+bufsize.WGBufferOverhead) + buf := make([]byte, p.proxy.mtu+bufsize.WGBufferOverhead) for { n, err := p.readFromRemote(ctx, buf) if err != nil { @@ -273,7 +287,7 @@ func (p *ProxyWrapper) proxyToLocal(ctx context.Context) { if ctx.Err() != nil { return } - log.Errorf("failed to write out turn pkg to local conn: %v", err) + log.Errorf("failed to write out relayed pkg to local conn: %v", err) } } } @@ -286,7 +300,7 @@ func (p *ProxyWrapper) readFromRemote(ctx context.Context, buf []byte) (int, err } p.closeListener.Notify() if !errors.Is(err, io.EOF) { - log.Errorf("failed to read from turn conn (endpoint: :%d): %s", p.wgRelayedEndpointAddr.Port, err) + log.Errorf("failed to read from relayed conn (endpoint: %s): %s", p.wgRelayedEndpointAddr, err) } return 0, err } @@ -314,7 +328,7 @@ func (p *ProxyWrapper) sendPkg(data []byte, header *PacketHeaders) error { func (p *ProxyWrapper) selectRawConn(header *PacketHeaders) net.PacketConn { if header.isIPv4 { - return p.wgeBPFProxy.rawConnIPv4 + return p.proxy.rawConnIPv4 } - return p.wgeBPFProxy.rawConnIPv6 + return p.proxy.rawConnIPv6 } diff --git a/client/iface/wgproxy/proxy.go b/client/iface/wgproxy/proxy.go index 40346bc15..b0033bffa 100644 --- a/client/iface/wgproxy/proxy.go +++ b/client/iface/wgproxy/proxy.go @@ -7,7 +7,7 @@ import ( // Proxy is a transfer layer between the relayed connection and the WireGuard type Proxy interface { - AddTurnConn(ctx context.Context, endpoint *net.UDPAddr, remoteConn net.Conn) error + AddRelayedConn(ctx context.Context, endpoint *net.UDPAddr, remoteConn net.Conn) error EndpointAddr() *net.UDPAddr // EndpointAddr returns the address of the WireGuard peer endpoint Work() // Work start or resume the proxy Pause() // Pause to forward the packages from remote connection to WireGuard. The opposite way still works. diff --git a/client/iface/wgproxy/proxy_linux_test.go b/client/iface/wgproxy/proxy_linux_test.go index e34dd3b6b..88d4588a5 100644 --- a/client/iface/wgproxy/proxy_linux_test.go +++ b/client/iface/wgproxy/proxy_linux_test.go @@ -9,25 +9,25 @@ import ( "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/wgaddr" bindproxy "github.com/netbirdio/netbird/client/iface/wgproxy/bind" - "github.com/netbirdio/netbird/client/iface/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/iface/wgproxy/loopback" "github.com/netbirdio/netbird/client/iface/wgproxy/udp" ) func seedProxies() ([]proxyInstance, error) { pl := make([]proxyInstance, 0) - ebpfProxy := ebpf.NewWGEBPFProxy(51831, 1280) - if err := ebpfProxy.Listen(); err != nil { - return nil, fmt.Errorf("failed to initialize ebpf proxy: %s", err) + loopbackProxy := loopback.NewProxy(51831, 1280) + if err := loopbackProxy.Listen(); err != nil { + return nil, fmt.Errorf("failed to initialize loopback proxy: %s", err) } - pEbpf := proxyInstance{ - name: "ebpf kernel proxy", - proxy: ebpf.NewProxyWrapper(ebpfProxy), + pLoopback := proxyInstance{ + name: "loopback kernel proxy", + proxy: loopback.NewProxyWrapper(loopbackProxy), wgPort: 51831, - closeFn: ebpfProxy.Free, + closeFn: loopbackProxy.Free, } - pl = append(pl, pEbpf) + pl = append(pl, pLoopback) pUDP := proxyInstance{ name: "udp kernel proxy", @@ -42,18 +42,18 @@ func seedProxies() ([]proxyInstance, error) { func seedProxyForProxyCloseByRemoteConn() ([]proxyInstance, error) { pl := make([]proxyInstance, 0) - ebpfProxy := ebpf.NewWGEBPFProxy(51831, 1280) - if err := ebpfProxy.Listen(); err != nil { - return nil, fmt.Errorf("failed to initialize ebpf proxy: %s", err) + loopbackProxy := loopback.NewProxy(51831, 1280) + if err := loopbackProxy.Listen(); err != nil { + return nil, fmt.Errorf("failed to initialize loopback proxy: %s", err) } - pEbpf := proxyInstance{ - name: "ebpf kernel proxy", - proxy: ebpf.NewProxyWrapper(ebpfProxy), + pLoopback := proxyInstance{ + name: "loopback kernel proxy", + proxy: loopback.NewProxyWrapper(loopbackProxy), wgPort: 51831, - closeFn: ebpfProxy.Free, + closeFn: loopbackProxy.Free, } - pl = append(pl, pEbpf) + pl = append(pl, pLoopback) pUDP := proxyInstance{ name: "udp kernel proxy", diff --git a/client/iface/wgproxy/proxy_test.go b/client/iface/wgproxy/proxy_test.go index 1aeab66b7..d86cdbe80 100644 --- a/client/iface/wgproxy/proxy_test.go +++ b/client/iface/wgproxy/proxy_test.go @@ -95,7 +95,7 @@ func TestProxyCloseByRemoteConn(t *testing.T) { t.Run(tt.name, func(t *testing.T) { addr, _ := net.ResolveUDPAddr("udp", "100.108.135.221:51892") relayedConn := newMockConn() - err := tt.proxy.AddTurnConn(ctx, addr, relayedConn) + err := tt.proxy.AddRelayedConn(ctx, addr, relayedConn) if err != nil { t.Errorf("error: %v", err) } @@ -157,7 +157,7 @@ func redirectTraffic(t *testing.T, proxy Proxy, wgPort int, endPointAddr *net.UD _ = relayedServer.Close() }() - if err := proxy.AddTurnConn(context.Background(), endPointAddr, relayedConn); err != nil { + if err := proxy.AddRelayedConn(context.Background(), endPointAddr, relayedConn); err != nil { t.Errorf("error: %v", err) } defer func() { diff --git a/client/iface/wgproxy/rawsocket/rawsocket.go b/client/iface/wgproxy/rawsocket/rawsocket.go index bc785b43a..37aaa160f 100644 --- a/client/iface/wgproxy/rawsocket/rawsocket.go +++ b/client/iface/wgproxy/rawsocket/rawsocket.go @@ -10,8 +10,6 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/sys/unix" - - nbnet "github.com/netbirdio/netbird/client/net" ) // PrepareSenderRawSocketIPv4 creates and configures a raw socket for sending IPv4 packets @@ -60,14 +58,12 @@ func prepareSenderRawSocket(family int, isIPv4 bool) (net.PacketConn, error) { return nil, fmt.Errorf("binding to lo interface failed: %w", err) } - // Set the fwmark on the socket. - err = nbnet.SetSocketOpt(fd) - if err != nil { - if closeErr := syscall.Close(fd); closeErr != nil { - log.Warnf("failed to close raw socket fd: %v", closeErr) - } - return nil, fmt.Errorf("setting fwmark failed: %w", err) - } + // The socket is bound to lo and only ever sends to the local WireGuard + // instance, a destination the local routing table resolves without help, so + // it carries no fwmark. Staying unmarked also keeps these packets out of + // third-party NAT rules that match on marks: such a rule rewriting the + // source would make WireGuard adopt the rewritten address as the peer + // endpoint. // Convert the file descriptor to a PacketConn. file := os.NewFile(uintptr(fd), fmt.Sprintf("fd %d", fd)) diff --git a/client/iface/wgproxy/rawsocket/rawsocket_privileged_test.go b/client/iface/wgproxy/rawsocket/rawsocket_privileged_test.go new file mode 100644 index 000000000..03748c6f9 --- /dev/null +++ b/client/iface/wgproxy/rawsocket/rawsocket_privileged_test.go @@ -0,0 +1,77 @@ +//go:build linux && !android && privileged + +package rawsocket + +import ( + "net" + "syscall" + "testing" + + "golang.org/x/sys/unix" + + nbnet "github.com/netbirdio/netbird/client/net" +) + +// The sender sockets must stay unmarked: a NAT rule matching on fwmark that +// rewrites the source of an injected packet makes WireGuard adopt the rewritten +// address as the peer endpoint. +func TestSenderRawSocketsCarryNoFwmark(t *testing.T) { + // the mark is only ever applied when advanced routing is available, so + // without it the assertion below would hold for the wrong reason + nbnet.Init() + if !nbnet.AdvancedRouting() { + t.Skip("advanced routing unsupported, the sockets carry no mark either way") + } + + tests := []struct { + name string + prepare func() (net.PacketConn, error) + // the proxy treats the IPv6 socket as optional, so a host without IPv6 + // is a reason to skip rather than to fail + optional bool + }{ + {name: "IPv4", prepare: PrepareSenderRawSocketIPv4}, + {name: "IPv6", prepare: PrepareSenderRawSocketIPv6, optional: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + conn, err := tc.prepare() + if err != nil { + if tc.optional { + t.Skipf("prepare raw socket: %v", err) + } + t.Fatalf("prepare raw socket: %v", err) + } + defer func() { + if err := conn.Close(); err != nil { + t.Logf("close raw socket: %v", err) + } + }() + + syscallConn, ok := conn.(syscall.Conn) + if !ok { + t.Fatalf("raw socket %T does not expose a syscall conn", conn) + } + raw, err := syscallConn.SyscallConn() + if err != nil { + t.Fatalf("syscall conn: %v", err) + } + + var mark int + var markErr error + if err := raw.Control(func(fd uintptr) { + mark, markErr = unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_MARK) + }); err != nil { + t.Fatalf("control: %v", err) + } + if markErr != nil { + t.Fatalf("get SO_MARK: %v", markErr) + } + + if mark != 0 { + t.Errorf("SO_MARK = %#x, want 0", mark) + } + }) + } +} diff --git a/client/iface/wgproxy/redirect_test.go b/client/iface/wgproxy/redirect_test.go index 135970838..47f571f2b 100644 --- a/client/iface/wgproxy/redirect_test.go +++ b/client/iface/wgproxy/redirect_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/netbirdio/netbird/client/iface/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/iface/wgproxy/loopback" "github.com/netbirdio/netbird/client/iface/wgproxy/udp" ) @@ -119,9 +119,9 @@ func testRedirectAs(t *testing.T, proxy Proxy, wgPort int, nbAddr, p2pEndpoint * } defer relayConn.Close() - // Add TURN connection to proxy - if err := proxy.AddTurnConn(ctx, nbAddr, relayConn); err != nil { - t.Fatalf("failed to add TURN connection: %v", err) + // Add relayed connection to proxy + if err := proxy.AddRelayedConn(ctx, nbAddr, relayConn); err != nil { + t.Fatalf("failed to add relayed connection: %v", err) } defer func() { if err := proxy.CloseConn(); err != nil { @@ -198,20 +198,20 @@ func testRedirectAs(t *testing.T, proxy Proxy, wgPort int, nbAddr, p2pEndpoint * } } -// TestRedirectAs_eBPF_IPv4 tests RedirectAs with eBPF proxy using IPv4 addresses -func TestRedirectAs_eBPF_IPv4(t *testing.T) { +// TestRedirectAs_Loopback_IPv4 tests RedirectAs with the loopback proxy using IPv4 addresses +func TestRedirectAs_Loopback_IPv4(t *testing.T) { wgPort := 51850 - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280) - if err := ebpfProxy.Listen(); err != nil { - t.Fatalf("failed to initialize ebpf proxy: %v", err) + loopbackProxy := loopback.NewProxy(wgPort, 1280) + if err := loopbackProxy.Listen(); err != nil { + t.Fatalf("failed to initialize loopback proxy: %v", err) } defer func() { - if err := ebpfProxy.Free(); err != nil { - t.Errorf("failed to free ebpf proxy: %v", err) + if err := loopbackProxy.Free(); err != nil { + t.Errorf("failed to free loopback proxy: %v", err) } }() - proxy := ebpf.NewProxyWrapper(ebpfProxy) + proxy := loopback.NewProxyWrapper(loopbackProxy) // NetBird UDP address of the remote peer nbAddr := &net.UDPAddr{ @@ -227,20 +227,20 @@ func TestRedirectAs_eBPF_IPv4(t *testing.T) { testRedirectAs(t, proxy, wgPort, nbAddr, p2pEndpoint) } -// TestRedirectAs_eBPF_IPv6 tests RedirectAs with eBPF proxy using IPv6 addresses -func TestRedirectAs_eBPF_IPv6(t *testing.T) { +// TestRedirectAs_Loopback_IPv6 tests RedirectAs with the loopback proxy using IPv6 addresses +func TestRedirectAs_Loopback_IPv6(t *testing.T) { wgPort := 51851 - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280) - if err := ebpfProxy.Listen(); err != nil { - t.Fatalf("failed to initialize ebpf proxy: %v", err) + loopbackProxy := loopback.NewProxy(wgPort, 1280) + if err := loopbackProxy.Listen(); err != nil { + t.Fatalf("failed to initialize loopback proxy: %v", err) } defer func() { - if err := ebpfProxy.Free(); err != nil { - t.Errorf("failed to free ebpf proxy: %v", err) + if err := loopbackProxy.Free(); err != nil { + t.Errorf("failed to free loopback proxy: %v", err) } }() - proxy := ebpf.NewProxyWrapper(ebpfProxy) + proxy := loopback.NewProxyWrapper(loopbackProxy) // NetBird UDP address of the remote peer nbAddr := &net.UDPAddr{ @@ -259,17 +259,17 @@ func TestRedirectAs_eBPF_IPv6(t *testing.T) { // TestRedirectAs_Multiple_Switches tests switching between multiple endpoints func TestRedirectAs_Multiple_Switches(t *testing.T) { wgPort := 51856 - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280) - if err := ebpfProxy.Listen(); err != nil { - t.Fatalf("failed to initialize ebpf proxy: %v", err) + loopbackProxy := loopback.NewProxy(wgPort, 1280) + if err := loopbackProxy.Listen(); err != nil { + t.Fatalf("failed to initialize loopback proxy: %v", err) } defer func() { - if err := ebpfProxy.Free(); err != nil { - t.Errorf("failed to free ebpf proxy: %v", err) + if err := loopbackProxy.Free(); err != nil { + t.Errorf("failed to free loopback proxy: %v", err) } }() - proxy := ebpf.NewProxyWrapper(ebpfProxy) + proxy := loopback.NewProxyWrapper(loopbackProxy) ctx := context.Background() @@ -304,8 +304,8 @@ func TestRedirectAs_Multiple_Switches(t *testing.T) { Port: 38746, } - if err := proxy.AddTurnConn(ctx, nbAddr, relayConn); err != nil { - t.Fatalf("failed to add TURN connection: %v", err) + if err := proxy.AddRelayedConn(ctx, nbAddr, relayConn); err != nil { + t.Fatalf("failed to add relayed connection: %v", err) } defer func() { if err := proxy.CloseConn(); err != nil { diff --git a/client/iface/wgproxy/udp/proxy.go b/client/iface/wgproxy/udp/proxy.go index 783843aba..a0895c8c7 100644 --- a/client/iface/wgproxy/udp/proxy.go +++ b/client/iface/wgproxy/udp/proxy.go @@ -51,12 +51,12 @@ func NewWGUDPProxy(wgPort int, mtu uint16) *WGUDPProxy { return p } -// AddTurnConn +// AddRelayedConn dials the local WireGuard port and stores the relayed connection. // The provided Context must be non-nil. If the context expires before // the connection is complete, an error is returned. Once successfully // connected, any expiration of the context will not affect the // connection. -func (p *WGUDPProxy) AddTurnConn(ctx context.Context, _ *net.UDPAddr, remoteConn net.Conn) error { +func (p *WGUDPProxy) AddRelayedConn(ctx context.Context, _ *net.UDPAddr, remoteConn net.Conn) error { dialer := net.Dialer{} localConn, err := dialer.DialContext(ctx, "udp", fmt.Sprintf(":%d", p.localWGListenPort)) if err != nil { diff --git a/client/installer.nsis b/client/installer.nsis index 63bff1c5b..304d38864 100644 --- a/client/installer.nsis +++ b/client/installer.nsis @@ -2,11 +2,11 @@ !define COMP_NAME "Netbird" !define WEB_SITE "Netbird.io" !define VERSION $%APPVER% -!define COPYRIGHT "Netbird Authors, 2022" +!define COPYRIGHT "Netbird Authors, 2022-2026" !define DESCRIPTION "Connect your devices into a secure WireGuard-based overlay network with SSO, MFA, and granular access controls." !define INSTALLER_NAME "netbird-installer.exe" !define MAIN_APP_EXE "Netbird" -!define ICON "ui\\assets\\netbird.ico" +!define ICON "ui\\build\\windows\\icon.ico" !define BANNER "ui\\build\\banner.bmp" !define LICENSE_DATA "..\\LICENSE" @@ -22,8 +22,6 @@ !define UI_REG_APP_PATH "Software\Microsoft\Windows\CurrentVersion\App Paths\${UI_APP_EXE}" !define UI_UNINSTALL_PATH "Software\Microsoft\Windows\CurrentVersion\Uninstall\${UI_APP_NAME}" -!define AUTOSTART_REG_KEY "Software\Microsoft\Windows\CurrentVersion\Run" - !define NETBIRD_DATA_DIR "$COMMONPROGRAMDATA\Netbird" Unicode True @@ -79,8 +77,6 @@ ShowInstDetails Show !insertmacro MUI_PAGE_DIRECTORY -Page custom AutostartPage AutostartPageLeave - !insertmacro MUI_PAGE_INSTFILES !insertmacro MUI_PAGE_FINISH @@ -97,40 +93,12 @@ UninstPage custom un.DeleteDataPage un.DeleteDataPageLeave !insertmacro MUI_LANGUAGE "English" -; Variables for autostart option -Var AutostartCheckbox -Var AutostartEnabled - ; Variables for uninstall data deletion option Var DeleteDataCheckbox Var DeleteDataEnabled ###################################################################### -; Function to create the autostart options page -Function AutostartPage - !insertmacro MUI_HEADER_TEXT "Startup Options" "Configure how ${APP_NAME} launches with Windows." - - nsDialogs::Create 1018 - Pop $0 - - ${If} $0 == error - Abort - ${EndIf} - - ${NSD_CreateCheckbox} 0 20u 100% 10u "Start ${APP_NAME} UI automatically when Windows starts" - Pop $AutostartCheckbox - ${NSD_Check} $AutostartCheckbox - StrCpy $AutostartEnabled "1" - - nsDialogs::Show -FunctionEnd - -; Function to handle leaving the autostart page -Function AutostartPageLeave - ${NSD_GetState} $AutostartCheckbox $AutostartEnabled -FunctionEnd - ; Function to create the uninstall data deletion page Function un.DeleteDataPage !insertmacro MUI_HEADER_TEXT "Uninstall Options" "Choose whether to delete ${APP_NAME} data." @@ -201,8 +169,6 @@ Pop $0 Function .onInit StrCpy $INSTDIR "${INSTALL_DIR}" -; Default autostart to enabled so silent installs (/S) match the interactive default -StrCpy $AutostartEnabled "1" ; Pre-0.70.1 installers ran without SetRegView, so their uninstall keys live ; in the 32-bit view. Fall back to it so upgrades still find them. @@ -260,18 +226,6 @@ WriteRegStr ${REG_ROOT} "${UNINSTALL_PATH}" "Publisher" "${COMP_NAME}" WriteRegStr ${REG_ROOT} "${UI_REG_APP_PATH}" "" "$INSTDIR\${UI_APP_EXE}" -; Create autostart registry entry based on checkbox -DetailPrint "Autostart enabled: $AutostartEnabled" -${If} $AutostartEnabled == "1" - WriteRegStr HKLM "${AUTOSTART_REG_KEY}" "${APP_NAME}" '"$INSTDIR\${UI_APP_EXE}.exe"' - DetailPrint "Added autostart registry entry: $INSTDIR\${UI_APP_EXE}.exe" -${Else} - DeleteRegValue HKLM "${AUTOSTART_REG_KEY}" "${APP_NAME}" - ; Legacy: pre-HKLM installs wrote to HKCU; clean that up too. - DeleteRegValue HKCU "${AUTOSTART_REG_KEY}" "${APP_NAME}" - DetailPrint "Autostart not enabled by user" -${EndIf} - EnVar::SetHKLM EnVar::AddValueEx "path" "$INSTDIR" @@ -280,6 +234,43 @@ CreateShortCut "$SMPROGRAMS\${APP_NAME}.lnk" "$INSTDIR\${UI_APP_EXE}" CreateShortCut "$DESKTOP\${APP_NAME}.lnk" "$INSTDIR\${UI_APP_EXE}" SectionEnd +# Install the Microsoft Edge WebView2 runtime if it isn't already present. +# Macro adapted from Wails3's NSIS template (wails_tools.nsh): a registry +# probe followed by a silent install of the embedded evergreen bootstrapper. +# The MicrosoftEdgeWebview2Setup.exe payload is staged next to this script +# by the sign-pipelines build step (`wails3 generate webview2bootstrapper`). +!macro nb.webview2runtime + SetRegView 64 + # Per-machine install marker — populated when the runtime ships with + # Edge or has been installed by an admin previously. + ReadRegStr $0 HKLM "SOFTWARE\WOW6432Node\Microsoft\EdgeUpdate\Clients\{F3017226-FE2A-4295-8BDF-00C3A9A7E4C5}" "pv" + ${If} $0 != "" + Goto webview2_ok + ${EndIf} + # Per-user fallback for HKCU installs. + ReadRegStr $0 HKCU "Software\Microsoft\EdgeUpdate\Clients\{F3017226-FE2A-4295-8BDF-00C3A9A7E4C5}" "pv" + ${If} $0 != "" + Goto webview2_ok + ${EndIf} + + SetDetailsPrint both + DetailPrint "Installing: WebView2 Runtime" + SetDetailsPrint listonly + + InitPluginsDir + CreateDirectory "$pluginsdir\webview2bootstrapper" + SetOutPath "$pluginsdir\webview2bootstrapper" + File "MicrosoftEdgeWebview2Setup.exe" + ExecWait '"$pluginsdir\webview2bootstrapper\MicrosoftEdgeWebview2Setup.exe" /silent /install' + + SetDetailsPrint both + webview2_ok: +!macroend + +Section -WebView2 + !insertmacro nb.webview2runtime +SectionEnd + Section -Post ExecWait '"$INSTDIR\${MAIN_APP_EXE}" service install' ExecWait '"$INSTDIR\${MAIN_APP_EXE}" service start' @@ -299,12 +290,6 @@ ExecWait '"$INSTDIR\${MAIN_APP_EXE}" service uninstall' DetailPrint "Terminating Netbird UI process..." ExecWait `taskkill /im ${UI_APP_EXE}.exe /f` -; Remove autostart registry entry -DetailPrint "Removing autostart registry entry if exists..." -DeleteRegValue HKLM "${AUTOSTART_REG_KEY}" "${APP_NAME}" -; Legacy: pre-HKLM installs wrote to HKCU; clean that up too. -DeleteRegValue HKCU "${AUTOSTART_REG_KEY}" "${APP_NAME}" - ; Handle data deletion based on checkbox DetailPrint "Checking if user requested data deletion..." ${If} $DeleteDataEnabled == "1" @@ -326,9 +311,9 @@ DetailPrint "Deleting application files..." Delete "$INSTDIR\${UI_APP_EXE}" Delete "$INSTDIR\${MAIN_APP_EXE}" Delete "$INSTDIR\wintun.dll" -!if ${ARCH} == "amd64" +# Legacy: pre-Wails installs shipped opengl32.dll (Mesa3D for Fyne); remove +# any leftover copy on uninstall so old upgrades don't leave it behind. Delete "$INSTDIR\opengl32.dll" -!endif DetailPrint "Removing application directory..." RmDir /r "$INSTDIR" diff --git a/client/internal/acl/dispatch_test.go b/client/internal/acl/dispatch_test.go new file mode 100644 index 000000000..be82e414f --- /dev/null +++ b/client/internal/acl/dispatch_test.go @@ -0,0 +1,190 @@ +package acl + +import ( + "net/netip" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/netbirdio/netbird/client/firewall" + fwmgr "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/wgaddr" + "github.com/netbirdio/netbird/client/internal/acl/mocks" + mgmProto "github.com/netbirdio/netbird/shared/management/proto" +) + +// TestNetworkZeroPrefixIsRoute guards the route-vs-peer dispatch +// invariant: the backends classify a rule as a peer rule purely by the +// absence of a destination (neither prefix nor set). A default route +// (0.0.0.0/0 or ::/0) is a valid prefix and must therefore classify as +// a route, not collapse into the peer path. +func TestNetworkZeroPrefixIsRoute(t *testing.T) { + for _, p := range []string{"0.0.0.0/0", "::/0", "10.0.0.0/8"} { + n := fwmgr.Network{Prefix: netip.MustParsePrefix(p)} + assert.True(t, n.IsPrefix(), "%s must report IsPrefix", p) + assert.True(t, n.IsPrefix() || n.IsSet(), "%s must classify as a route", p) + } + + // A zero-value Network is the only peer-rule shape. + var empty fwmgr.Network + assert.False(t, empty.IsPrefix(), "zero Network must not be a prefix") + assert.False(t, empty.IsSet(), "zero Network must not be a set") +} + +// TestDetermineDestinationAlwaysRoute verifies determineDestination +// never yields an empty Network for a valid route rule: every branch +// (static prefix, default route, dynamic with/without domains, with and +// without a local resolver) produces a destination that classifies as a +// route. If this regresses, a route rule would be dispatched down the +// peer path, which matches on source only. +func TestDetermineDestinationAlwaysRoute(t *testing.T) { + v4 := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")} + v6 := []netip.Prefix{netip.MustParsePrefix("2001:db8::/48")} + + cases := []struct { + name string + rule *mgmProto.RouteFirewallRule + resolver bool + sources []netip.Prefix + }{ + {"static prefix", &mgmProto.RouteFirewallRule{Destination: "192.168.0.0/16"}, false, v4}, + {"static default route", &mgmProto.RouteFirewallRule{Destination: "0.0.0.0/0"}, false, v4}, + {"dynamic with domains + resolver", &mgmProto.RouteFirewallRule{IsDynamic: true, Domains: []string{"example.com"}}, true, v4}, + {"dynamic no domains + resolver (v4)", &mgmProto.RouteFirewallRule{IsDynamic: true}, true, v4}, + {"dynamic no domains + resolver (v6)", &mgmProto.RouteFirewallRule{IsDynamic: true}, true, v6}, + {"dynamic + no local resolver (v4)", &mgmProto.RouteFirewallRule{IsDynamic: true}, false, v4}, + {"dynamic + no local resolver (v6)", &mgmProto.RouteFirewallRule{IsDynamic: true}, false, v6}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dest, err := determineDestination(tc.rule, tc.resolver, tc.sources) + require.NoError(t, err) + assert.True(t, dest.IsPrefix() || dest.IsSet(), + "destination must classify as a route, got empty Network") + }) + } +} + +// countingFirewall wraps a real firewall.Manager and counts filter-rule +// add/delete calls so a test can assert how many backing rules the acl +// manager actually creates and tears down. +type countingFirewall struct { + fwmgr.Manager + mu sync.Mutex + addCalls int + dels int + ruleIDs map[fwmgr.RuleID]struct{} +} + +// distinctRules returns the number of distinct backing rules the +// backend produced. Because the backend dedups identical content, +// repeated AddFilterRule calls for the same rule resolve to one id. +func (f *countingFirewall) distinctRules() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.ruleIDs) +} + +func (f *countingFirewall) AddFilterRule(id []byte, sources []netip.Prefix, destination fwmgr.Network, proto fwmgr.Protocol, sPort, dPort *fwmgr.Port, action fwmgr.Action) (fwmgr.Rule, error) { + rule, err := f.Manager.AddFilterRule(id, sources, destination, proto, sPort, dPort, action) + if err == nil { + f.mu.Lock() + f.addCalls++ + if f.ruleIDs == nil { + f.ruleIDs = make(map[fwmgr.RuleID]struct{}) + } + if rule != nil { + f.ruleIDs[rule.ID()] = struct{}{} + } + f.mu.Unlock() + } + return rule, err +} + +func (f *countingFirewall) DeleteFilterRule(r fwmgr.Rule) error { + err := f.Manager.DeleteFilterRule(r) + if err == nil { + f.mu.Lock() + f.dels++ + delete(f.ruleIDs, r.ID()) + f.mu.Unlock() + } + return err +} + +func newCountingACL(t *testing.T) (*DefaultManager, *countingFirewall, func()) { + t.Helper() + t.Setenv("NB_WG_KERNEL_DISABLED", "true") + t.Setenv(firewall.EnvForceUserspaceFirewall, "true") + + ctrl := gomock.NewController(t) + ifaceMock := mocks.NewMockIFaceMapper(ctrl) + ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes() + ifaceMock.EXPECT().SetFilter(gomock.Any()) + network := netip.MustParsePrefix("172.0.0.1/32") + ifaceMock.EXPECT().Name().Return("lo").AnyTimes() + ifaceMock.EXPECT().Address().Return(wgaddr.Address{IP: network.Addr(), Network: network}).AnyTimes() + ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes() + + realFW, err := firewall.NewFirewall(ifaceMock, nil, flowLogger, false, iface.DefaultMTU) + require.NoError(t, err) + + fw := &countingFirewall{Manager: realFW} + cleanup := func() { + require.NoError(t, realFW.Close(nil)) + ctrl.Finish() + } + return NewDefaultManager(fw), fw, cleanup +} + +// TestDuplicateContentPoliciesShareOneRule verifies the dedup contract +// the backends rely on: two policies that authorize an identical flow +// (same selector and sources) collapse to a single backing firewall +// rule, and that rule survives until BOTH policies are gone. This is +// why the backend can dedup on add without refcounting on delete: the +// acl manager's pair key matches the backend's content key, so add and +// delete stay balanced per content key across full-state reapplies. +func TestDuplicateContentPoliciesShareOneRule(t *testing.T) { + acl, fw, cleanup := newCountingACL(t) + defer cleanup() + + ruleA := &mgmProto.FirewallRule{ + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.1", //nolint:staticcheck + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + } + ruleB := &mgmProto.FirewallRule{ + PolicyID: []byte("policy-B"), + PeerIP: "10.0.0.1", //nolint:staticcheck + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + } + + // Both policies present: identical content collapses to one rule. + acl.ApplyFiltering(&mgmProto.NetworkMap{FirewallRules: []*mgmProto.FirewallRule{ruleA, ruleB}, FirewallRulesIsEmpty: false}, false) + assert.Equal(t, 1, fw.distinctRules(), "identical-content policies must produce one backing rule") + assert.Equal(t, 1, len(acl.peerRulesPairs), "one content key, one pair") + + // Drop policy A only: the shared rule is still authorized by B, so + // nothing is deleted. + acl.ApplyFiltering(&mgmProto.NetworkMap{FirewallRules: []*mgmProto.FirewallRule{ruleB}, FirewallRulesIsEmpty: false}, false) + assert.Equal(t, 1, fw.distinctRules(), "no new backing rule on reapply") + assert.Equal(t, 0, fw.dels, "rule must survive while any policy still authorizes it") + assert.Equal(t, 1, len(acl.peerRulesPairs)) + + // Drop policy B too: now the content key has no authorizer and the + // single backing rule is removed exactly once. + acl.ApplyFiltering(&mgmProto.NetworkMap{FirewallRules: nil, FirewallRulesIsEmpty: true}, false) + assert.Equal(t, 1, fw.dels, "rule removed once when last policy is gone") + assert.Equal(t, 0, len(acl.peerRulesPairs)) +} diff --git a/client/internal/acl/grouping_test.go b/client/internal/acl/grouping_test.go new file mode 100644 index 000000000..d6cf29b59 --- /dev/null +++ b/client/internal/acl/grouping_test.go @@ -0,0 +1,318 @@ +package acl + +import ( + "errors" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/netbirdio/netbird/client/firewall" + fwmgr "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/wgaddr" + "github.com/netbirdio/netbird/client/internal/acl/mocks" + mgmProto "github.com/netbirdio/netbird/shared/management/proto" + "github.com/netbirdio/netbird/shared/netiputil" +) + +// TestGroupPeerRulesPolicyIDSeparates verifies that two FirewallRules +// with identical selectors but different PolicyIDs do NOT get merged +// into one group, so each policy's sources merge under its own +// attribution id. (Identical-content groups may still dedup to one +// backing rule at the backend; see TestDuplicateContentPoliciesShareOneRule.) +func TestGroupPeerRulesPolicyIDSeparates(t *testing.T) { + rules := []*mgmProto.FirewallRule{ + { + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + { + PolicyID: []byte("policy-B"), + PeerIP: "10.0.0.1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + } + + groups, denyErr, err := groupPeerRules(rules) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 2, "rules with different PolicyIDs must produce separate groups") +} + +// TestGroupPeerRulesFamilySeparates verifies that v4 and v6 rules +// belonging to the same policy don't merge. +func TestGroupPeerRulesFamilySeparates(t *testing.T) { + rules := []*mgmProto.FirewallRule{ + { + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + { + PolicyID: []byte("policy-A"), + PeerIP: "2001:db8::1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + } + + groups, denyErr, err := groupPeerRules(rules) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 2, "rules of different families must produce separate groups") + + var sawV4, sawV6 bool + for _, g := range groups { + require.Len(t, g.sources, 1) + if g.sources[0].Addr().Is4() { + sawV4 = true + } + if g.sources[0].Addr().Is6() { + sawV6 = true + } + } + assert.True(t, sawV4 && sawV6) +} + +// TestGroupPeerRulesSplitsMixedFamilySingleRule verifies that a single +// FirewallRule carrying both v4 and v6 source prefixes is split into one +// group per family. Each backend keys a rule to a single family, so a +// group whose sources span families would mismatch the other family's +// sources. mgmt normally emits one rule per family; this guards against +// a mixed-family rule slipping through. +func TestGroupPeerRulesSplitsMixedFamilySingleRule(t *testing.T) { + srcs := [][]byte{ + netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.1")), + netiputil.EncodeAddr(netip.MustParseAddr("2001:db8::1")), + netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.2")), + netiputil.EncodeAddr(netip.MustParseAddr("2001:db8::2")), + } + rules := []*mgmProto.FirewallRule{ + { + PolicyID: []byte("policy-A"), + SourcePrefixes: srcs, + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + } + + groups, denyErr, err := groupPeerRules(rules) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 2, "mixed-family sources in one rule must split into two groups") + + for _, g := range groups { + require.Len(t, g.sources, 2) + v6 := prefixIsV6(g.sources[0]) + for _, s := range g.sources { + assert.Equal(t, v6, prefixIsV6(s), "every source in a group must share one family") + } + } +} + +// TestGroupPeerRulesMergesSameSelector verifies that rules sharing +// every distinguishing field (policy, family, direction, action, +// proto, port) collapse into a single multi-source group. +func TestGroupPeerRulesMergesSameSelector(t *testing.T) { + mk := func(peerIP string) *mgmProto.FirewallRule { + return &mgmProto.FirewallRule{ + PolicyID: []byte("policy-A"), + PeerIP: peerIP, //nolint:staticcheck + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + } + } + rules := []*mgmProto.FirewallRule{mk("10.0.0.1"), mk("10.0.0.2"), mk("10.0.0.3")} + + groups, denyErr, err := groupPeerRules(rules) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 1) + require.Len(t, groups[0].sources, 3) +} + +// TestGroupPeerRulesPortSeparates verifies that PortInfo is part of the +// selector key: rules differing only in port must not merge, and a +// single port must not merge with a range. A regression dropping the +// port from the key would collapse rules for different ports into one. +func TestGroupPeerRulesPortSeparates(t *testing.T) { + mkPort := func(peerIP string, port uint32) *mgmProto.FirewallRule { + return &mgmProto.FirewallRule{ + PolicyID: []byte("policy-A"), + PeerIP: peerIP, //nolint:staticcheck + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + PortInfo: &mgmProto.PortInfo{PortSelection: &mgmProto.PortInfo_Port{Port: port}}, + } + } + + groups, denyErr, err := groupPeerRules([]*mgmProto.FirewallRule{ + mkPort("10.0.0.1", 80), mkPort("10.0.0.2", 80), mkPort("10.0.0.3", 443), + }) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 2, "rules on different ports must not merge") + + rangeRule := &mgmProto.FirewallRule{ + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.4", //nolint:staticcheck + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + PortInfo: &mgmProto.PortInfo{PortSelection: &mgmProto.PortInfo_Range_{Range: &mgmProto.PortInfo_Range{Start: 80, End: 90}}}, + } + groups, denyErr, err = groupPeerRules([]*mgmProto.FirewallRule{mkPort("10.0.0.1", 80), rangeRule}) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 2, "a single port and a range must not merge") +} + +// TestGroupPeerRulesUsesSourcePrefixesWhenPresent verifies that the +// new sourcePrefixes wire field is consumed and produces a +// multi-source group in one shot (no client-side merging needed). +func TestGroupPeerRulesUsesSourcePrefixesWhenPresent(t *testing.T) { + srcs := [][]byte{ + netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.1")), + netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.2")), + netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.3")), + } + rules := []*mgmProto.FirewallRule{ + { + PolicyID: []byte("policy-A"), + SourcePrefixes: srcs, + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + } + + groups, denyErr, err := groupPeerRules(rules) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 1) + require.Len(t, groups[0].sources, 3) +} + +// TestGroupPeerRulesActionSeparates verifies the obvious: accept +// and drop rules with the same selector don't merge. +func TestGroupPeerRulesActionSeparates(t *testing.T) { + rules := []*mgmProto.FirewallRule{ + { + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_ACCEPT, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + { + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_DROP, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "443", + }, + } + + groups, denyErr, err := groupPeerRules(rules) + require.NoError(t, denyErr) + require.NoError(t, err) + require.Len(t, groups, 2) +} + +// failingDeleteFirewall wraps a real firewall.Manager and forces the +// next N DeleteFilterRule calls to fail. Used to verify that the acl +// manager retains rules whose deletion was rejected by the backend, +// so they get retried on the next ApplyFiltering pass instead of +// becoming orphans. +type failingDeleteFirewall struct { + fwmgr.Manager + failCount int +} + +func (f *failingDeleteFirewall) DeleteFilterRule(r fwmgr.Rule) error { + if f.failCount > 0 { + f.failCount-- + return errors.New("simulated delete failure") + } + return f.Manager.DeleteFilterRule(r) +} + +// TestApplyFilteringRetainsRulesOnDeleteFailure verifies that a +// transient DeleteFilterRule error doesn't make the acl manager +// forget about a rule. The rule must remain in peerRulesPairs so the +// next ApplyFiltering pass attempts the delete again. +func TestApplyFilteringRetainsRulesOnDeleteFailure(t *testing.T) { + t.Setenv("NB_WG_KERNEL_DISABLED", "true") + t.Setenv(firewall.EnvForceUserspaceFirewall, "true") + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ifaceMock := mocks.NewMockIFaceMapper(ctrl) + ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes() + ifaceMock.EXPECT().SetFilter(gomock.Any()) + network := netip.MustParsePrefix("172.0.0.1/32") + ifaceMock.EXPECT().Name().Return("lo").AnyTimes() + ifaceMock.EXPECT().Address().Return(wgaddr.Address{IP: network.Addr(), Network: network}).AnyTimes() + ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes() + + realFW, err := firewall.NewFirewall(ifaceMock, nil, flowLogger, false, iface.DefaultMTU) + require.NoError(t, err) + defer func() { require.NoError(t, realFW.Close(nil)) }() + + fw := &failingDeleteFirewall{Manager: realFW} + acl := NewDefaultManager(fw) + + // First pass: install a rule. + netmap1 := &mgmProto.NetworkMap{ + FirewallRules: []*mgmProto.FirewallRule{ + { + PolicyID: []byte("policy-A"), + PeerIP: "10.0.0.1", + Direction: mgmProto.RuleDirection_IN, + Action: mgmProto.RuleAction_DROP, + Protocol: mgmProto.RuleProtocol_TCP, + Port: "22", + }, + }, + FirewallRulesIsEmpty: false, + } + acl.ApplyFiltering(netmap1, false) + require.Equal(t, 1, len(acl.peerRulesPairs), "rule should be installed") + + // Second pass: remove the rule from the map. The backend will + // fail the delete; the acl manager must retain the rule. + fw.failCount = 1 + netmap2 := &mgmProto.NetworkMap{FirewallRules: nil, FirewallRulesIsEmpty: true} + acl.ApplyFiltering(netmap2, false) + require.Equal(t, 1, len(acl.peerRulesPairs), + "rule must be retained when DeleteFilterRule fails so it gets retried") + + // Third pass: same map, backend no longer fails. The rule + // should now succeed in being removed. + acl.ApplyFiltering(netmap2, false) + require.Equal(t, 0, len(acl.peerRulesPairs), "retry should succeed") +} diff --git a/client/internal/acl/id/id.go b/client/internal/acl/id/id.go index 23451453e..952403bbd 100644 --- a/client/internal/acl/id/id.go +++ b/client/internal/acl/id/id.go @@ -5,18 +5,18 @@ import ( "encoding/hex" "fmt" "net/netip" + "slices" "strconv" "github.com/netbirdio/netbird/client/firewall/manager" ) -type RuleID string +// RuleID aliases manager.RuleID so existing nbid.RuleID references +// keep working while the canonical type lives in the firewall package. +type RuleID = manager.RuleID -func (r RuleID) ID() string { - return string(r) -} - -func GenerateRouteRuleKey( +// GenerateRuleID returns a deterministic content hash identifying a filter rule. +func GenerateRuleID( sources []netip.Prefix, destination manager.Network, proto manager.Protocol, @@ -24,6 +24,7 @@ func GenerateRouteRuleKey( dPort *manager.Port, action manager.Action, ) RuleID { + sources = slices.Clone(sources) manager.SortPrefixes(sources) h := sha256.New() diff --git a/client/internal/acl/legacy_fallback_test.go b/client/internal/acl/legacy_fallback_test.go new file mode 100644 index 000000000..00ca013f3 --- /dev/null +++ b/client/internal/acl/legacy_fallback_test.go @@ -0,0 +1,75 @@ +package acl + +import ( + "net/netip" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/netbirdio/netbird/client/firewall" + fwmgr "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/wgaddr" + "github.com/netbirdio/netbird/client/internal/acl/mocks" + mgmProto "github.com/netbirdio/netbird/shared/management/proto" +) + +// sourcesRecordingFirewall wraps a real firewall.Manager and records +// the source prefixes of every AddFilterRule call. +type sourcesRecordingFirewall struct { + fwmgr.Manager + mu sync.Mutex + sources [][]netip.Prefix +} + +func (f *sourcesRecordingFirewall) AddFilterRule(id []byte, sources []netip.Prefix, destination fwmgr.Network, proto fwmgr.Protocol, sPort, dPort *fwmgr.Port, action fwmgr.Action) (fwmgr.Rule, error) { + f.mu.Lock() + f.sources = append(f.sources, sources) + f.mu.Unlock() + return f.Manager.AddFilterRule(id, sources, destination, proto, sPort, dPort, action) +} + +// TestLegacyManagementFallbackUsesMatchAnySources verifies the +// allow-all fallback for old management servers (empty FirewallRules +// without the FirewallRulesIsEmpty flag) reaches the firewall as /0 +// match-any sources. The fallback rule carries PeerIP 0.0.0.0; if that +// were converted to a host prefix (0.0.0.0/32) it would match nothing +// and all peer traffic would be dropped. +func TestLegacyManagementFallbackUsesMatchAnySources(t *testing.T) { + t.Setenv("NB_WG_KERNEL_DISABLED", "true") + t.Setenv(firewall.EnvForceUserspaceFirewall, "true") + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ifaceMock := mocks.NewMockIFaceMapper(ctrl) + ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes() + ifaceMock.EXPECT().SetFilter(gomock.Any()) + network := netip.MustParsePrefix("172.0.0.1/32") + ifaceMock.EXPECT().Name().Return("lo").AnyTimes() + ifaceMock.EXPECT().Address().Return(wgaddr.Address{IP: network.Addr(), Network: network}).AnyTimes() + ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes() + + realFW, err := firewall.NewFirewall(ifaceMock, nil, flowLogger, false, iface.DefaultMTU) + require.NoError(t, err) + defer func() { require.NoError(t, realFW.Close(nil)) }() + + fw := &sourcesRecordingFirewall{Manager: realFW} + acl := NewDefaultManager(fw) + + // Old management: no rules and no FirewallRulesIsEmpty flag. + acl.ApplyFiltering(&mgmProto.NetworkMap{FirewallRules: nil, FirewallRulesIsEmpty: false}, false) + + fw.mu.Lock() + defer fw.mu.Unlock() + require.NotEmpty(t, fw.sources, "legacy fallback must install at least one allow-all rule") + for _, sources := range fw.sources { + require.NotEmpty(t, sources) + for _, p := range sources { + assert.Equal(t, 0, p.Bits(), "legacy fallback source %s must be a /0 match-any prefix", p) + } + } +} diff --git a/client/internal/acl/manager.go b/client/internal/acl/manager.go index d9b179457..6c544b345 100644 --- a/client/internal/acl/manager.go +++ b/client/internal/acl/manager.go @@ -1,8 +1,6 @@ package acl import ( - "crypto/md5" - "encoding/hex" "errors" "fmt" "net/netip" @@ -24,6 +22,10 @@ import ( var ErrSourceRangesEmpty = errors.New("sources range is empty") +// ErrNoRuleReturned is returned when the firewall backend reports success +// from AddFilterRule but yields no rule to track. +var ErrNoRuleReturned = errors.New("backend returned no rule") + // Manager is a ACL rules manager type Manager interface { ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRouteFeatureFlag bool) @@ -32,19 +34,48 @@ type Manager interface { // DefaultManager uses firewall manager to handle type DefaultManager struct { firewall firewall.Manager - ipsetCounter int peerRulesPairs map[id.RuleID][]firewall.Rule - routeRules map[id.RuleID]struct{} + routeRules map[id.RuleID]firewall.Rule previousConfigHash uint64 hasAppliedConfig bool mutex sync.Mutex } +// peerRuleGroup collapses a set of single-source FirewallRules sharing +// the same selector into one multi-source rule to push to the backend. +type peerRuleGroup struct { + direction mgmProto.RuleDirection + action mgmProto.RuleAction + protocol mgmProto.RuleProtocol + port *mgmProto.PortInfo + // legacyPort is used only when PortInfo is empty (old management). + legacyPort string + policyID []byte + sources []netip.Prefix +} + +// peerRuleKey is the comparable selector that decides which single-source +// rules merge into one group. Rules with an equal key collapse into one +// multi-source backend rule. PortInfo is flattened into its scalar fields +// so the key compares by value; policyID keeps policies separate so two +// policies authorizing different peers don't merge under one attribution. +type peerRuleKey struct { + v6 bool + policyID string + direction mgmProto.RuleDirection + action mgmProto.RuleAction + protocol mgmProto.RuleProtocol + legacyPort string + port uint16 + rangeStart uint16 + rangeEnd uint16 +} + func NewDefaultManager(fm firewall.Manager) *DefaultManager { return &DefaultManager{ firewall: fm, peerRulesPairs: make(map[id.RuleID][]firewall.Rule), - routeRules: make(map[id.RuleID]struct{}), + routeRules: make(map[id.RuleID]firewall.Rule), } } @@ -88,11 +119,14 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRout time.Since(start), total) }() - d.applyPeerACLs(networkMap) + peerErr := d.applyPeerACLs(networkMap) + if peerErr != nil { + log.Errorf("apply peer ACLs: %v", peerErr) + } routeErr := d.applyRouteACLs(networkMap.RoutesFirewallRules, dnsRouteFeatureFlag) if routeErr != nil { - log.Errorf("Failed to apply route ACLs: %v", routeErr) + log.Errorf("apply route ACLs: %v", routeErr) } flushErr := d.firewall.Flush() @@ -104,7 +138,7 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRout // If applying or flushing failed, leave the previous hash untouched so the // next (possibly identical) update is not skipped and gets a chance to // reconcile the firewall state. - if err == nil && routeErr == nil && flushErr == nil { + if err == nil && peerErr == nil && routeErr == nil && flushErr == nil { d.previousConfigHash = hash d.hasAppliedConfig = true } else { @@ -116,11 +150,11 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRout // firewall state, so an identical hash means an identical resulting ruleset. func (d *DefaultManager) firewallConfigHash(networkMap *mgmProto.NetworkMap, dnsRouteFeatureFlag bool) (uint64, error) { return hashstructure.Hash(struct { - PeerRules []*mgmProto.FirewallRule - PeerRulesIsEmpty bool - RouteRules []*mgmProto.RouteFirewallRule - RouteRulesIsEmpty bool - DNSRouteFeatureFlag bool + PeerRules []*mgmProto.FirewallRule + PeerRulesIsEmpty bool + RouteRules []*mgmProto.RouteFirewallRule + RouteRulesIsEmpty bool + DNSRouteFeatureFlag bool }{ PeerRules: networkMap.GetFirewallRules(), PeerRulesIsEmpty: networkMap.GetFirewallRulesIsEmpty(), @@ -135,7 +169,7 @@ func (d *DefaultManager) firewallConfigHash(networkMap *mgmProto.NetworkMap, dns }) } -func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) { +func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) error { rules := networkMap.FirewallRules // if we got empty rules list but management not set networkMap.FirewallRulesIsEmpty flag @@ -144,13 +178,13 @@ func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) { log.Warn("this peer is connected to a NetBird Management service with an older version. Allowing all traffic from connected peers") rules = append(rules, &mgmProto.FirewallRule{ - PeerIP: "0.0.0.0", + PeerIP: "0.0.0.0", //nolint:staticcheck Direction: mgmProto.RuleDirection_IN, Action: mgmProto.RuleAction_ACCEPT, Protocol: mgmProto.RuleProtocol_ALL, }, &mgmProto.FirewallRule{ - PeerIP: "0.0.0.0", + PeerIP: "0.0.0.0", //nolint:staticcheck Direction: mgmProto.RuleDirection_OUT, Action: mgmProto.RuleAction_ACCEPT, Protocol: mgmProto.RuleProtocol_ALL, @@ -158,59 +192,167 @@ func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) { ) } - newRulePairs := make(map[id.RuleID][]firewall.Rule) - ipsetByRuleSelectors := make(map[string]string) + // Group incoming single-source rules from management by their + // (direction, action, proto, port) selector and merge sources. + // One call to the firewall backend per merged rule. + // A deny we cannot decode would leave its traffic unblocked, so skip + // the whole pass and keep existing rules until the next sync. + groups, denyErr, err := groupPeerRules(rules) + if denyErr != nil { + return fmt.Errorf("decode deny rule sources: %w", denyErr) + } - // TODO: deny rules should be fatal: if a deny rule fails to apply, we must - // roll back all allow rules to avoid a fail-open where allowed traffic bypasses - // the missing deny. Currently we accumulate errors and continue. + newRulePairs := make(map[id.RuleID][]firewall.Rule) var merr *multierror.Error - for _, r := range rules { - // if this rule is member of rule selection with more than DefaultIPsCountForSet - // it's IP address can be used in the ipset for firewall manager which supports it - selector := d.getRuleGroupingSelector(r) - ipsetName, ok := ipsetByRuleSelectors[selector] - if !ok { - d.ipsetCounter++ - ipsetName = fmt.Sprintf("nb%07d", d.ipsetCounter) - ipsetByRuleSelectors[selector] = ipsetName - } - pairID, rulePair, err := d.protoRuleToFirewallRule(r, ipsetName) - if err != nil { - merr = multierror.Append(merr, fmt.Errorf("apply firewall rule: %w", err)) + if err != nil { + merr = multierror.Append(merr, err) + } + + // Apply denies first. A deny that fails to install is a security + // failure (fail-open), so if any deny errors we roll back the + // denies we already installed in this pass and bail out without + // installing any accept. Pre-existing rules stay untouched until + // the next successful pass clears them. + denies, accepts := splitDenyAccept(groups) + if err := d.installPeerGroups(denies, newRulePairs, true); err != nil { + return fmt.Errorf("install deny rules: %w", err) + } + + if err := d.installPeerGroups(accepts, newRulePairs, false); err != nil { + merr = multierror.Append(merr, err) + } + + // Tear down rules that disappeared from the networkmap. Any rule + // the backend refuses to delete stays in our tracking so it gets + // retried on the next ApplyFiltering. Otherwise a transient + // delete failure would leak the rule in the firewall until the + // process exits. + for pairID, rules := range d.peerRulesPairs { + if _, ok := newRulePairs[pairID]; ok { continue } - if len(rulePair) > 0 { - d.peerRulesPairs[pairID] = rulePair - newRulePairs[pairID] = rulePair - } - } - - if merr != nil { - log.Errorf("failed to apply %d peer ACL rule(s): %v", merr.Len(), nberrors.FormatErrorOrNil(merr)) - } - - for pairID, rules := range d.peerRulesPairs { - if _, ok := newRulePairs[pairID]; !ok { - for _, rule := range rules { - if err := d.firewall.DeletePeerRule(rule); err != nil { - log.Errorf("failed to delete peer firewall rule: %v", err) - continue - } + var remaining []firewall.Rule + for _, rule := range rules { + if err := d.firewall.DeleteFilterRule(rule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete peer rule, will retry: %w", err)) + remaining = append(remaining, rule) } - delete(d.peerRulesPairs, pairID) + } + if len(remaining) > 0 { + newRulePairs[pairID] = remaining } } d.peerRulesPairs = newRulePairs + + return nberrors.FormatErrorOrNil(merr) +} + +// installPeerGroups applies each group and records the resulting rule +// pairs in newRulePairs. With atomic set (deny rules), a single failure +// rolls back every rule installed in this call and returns, leaving the +// firewall exactly as before: denies are fail-closed and must be applied +// all-or-nothing. With atomic unset (accept rules), failures are +// accumulated and the remaining groups still install, so one malformed +// allow cannot drop every other legitimate allow in the pass. +func (d *DefaultManager) installPeerGroups(groups []*peerRuleGroup, newRulePairs map[id.RuleID][]firewall.Rule, atomic bool) error { + var freshlyInstalled []id.RuleID + var merr *multierror.Error + for _, g := range groups { + pairID, rulePair, err := d.applyPeerGroup(g) + if err != nil { + if atomic { + d.rollbackInstalled(freshlyInstalled) + return fmt.Errorf("apply firewall rule: %w", err) + } + merr = multierror.Append(merr, fmt.Errorf("apply firewall rule: %w", err)) + continue + } + if len(rulePair) == 0 { + continue + } + if _, existed := d.peerRulesPairs[pairID]; !existed { + freshlyInstalled = append(freshlyInstalled, pairID) + } + d.peerRulesPairs[pairID] = rulePair + newRulePairs[pairID] = rulePair + } + return nberrors.FormatErrorOrNil(merr) +} + +func (d *DefaultManager) rollbackInstalled(pairIDs []id.RuleID) { + var merr *multierror.Error + for _, pairID := range pairIDs { + // Keep any rule the backend refuses to delete tracked so it is + // retried on the next ApplyFiltering instead of leaking in the + // firewall with no tracking left to remove it. + var remaining []firewall.Rule + for _, rule := range d.peerRulesPairs[pairID] { + if err := d.firewall.DeleteFilterRule(rule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("rule %s: %w", pairID, err)) + remaining = append(remaining, rule) + } + } + if len(remaining) > 0 { + d.peerRulesPairs[pairID] = remaining + } else { + delete(d.peerRulesPairs, pairID) + } + } + if err := nberrors.FormatErrorOrNil(merr); err != nil { + log.Errorf("rollback peer rules: %v", err) + } +} + +func (d *DefaultManager) applyPeerGroup(g *peerRuleGroup) (id.RuleID, []firewall.Rule, error) { + protocol, err := ConvertToFirewallProtocol(g.protocol) + if err != nil { + return "", nil, fmt.Errorf("skipping firewall rule: %w", err) + } + action, err := convertFirewallAction(g.action) + if err != nil { + return "", nil, fmt.Errorf("skipping firewall rule: %w", err) + } + port, err := resolveGroupPort(g) + if err != nil { + return "", nil, err + } + + var fwRule firewall.Rule + switch g.direction { + case mgmProto.RuleDirection_IN: + fwRule, err = d.firewall.AddFilterRule(g.policyID, g.sources, firewall.Network{}, protocol, nil, port, action) + case mgmProto.RuleDirection_OUT: + if d.firewall.IsStateful() { + return "", nil, nil + } + if shouldSkipInvertedRule(protocol, port) { + return "", nil, nil + } + fwRule, err = d.firewall.AddFilterRule(g.policyID, g.sources, firewall.Network{}, protocol, port, nil, action) + default: + return "", nil, errors.New("invalid direction") + } + + if err != nil { + return "", nil, fmt.Errorf("add firewall rule: %w", err) + } + if fwRule == nil { + return "", nil, fmt.Errorf("add firewall rule: %w", ErrNoRuleReturned) + } + + // Derive the pair id from the backend rule, like the route path: + // the backend dedups identical content, so two policies authorizing + // the same flow resolve to the same id and a single backing rule. + return fwRule.ID(), []firewall.Rule{fwRule}, nil } func (d *DefaultManager) applyRouteACLs(rules []*mgmProto.RouteFirewallRule, dynamicResolver bool) error { - newRouteRules := make(map[id.RuleID]struct{}, len(rules)) + newRouteRules := make(map[id.RuleID]firewall.Rule, len(rules)) var merr *multierror.Error - // Apply new rules - firewall manager will return existing rule ID if already present + // Apply new rules - firewall manager will return the existing rule if already present for _, rule := range rules { - id, err := d.applyRouteACL(rule, dynamicResolver) + addedRule, err := d.applyRouteACL(rule, dynamicResolver) if err != nil { if errors.Is(err, ErrSourceRangesEmpty) { log.Debugf("skipping empty sources rule with destination %s: %v", rule.Destination, err) @@ -219,16 +361,18 @@ func (d *DefaultManager) applyRouteACLs(rules []*mgmProto.RouteFirewallRule, dyn } continue } - newRouteRules[id] = struct{}{} + newRouteRules[addedRule.ID()] = addedRule } - // Clean up old firewall rules - for id := range d.routeRules { - if _, exists := newRouteRules[id]; !exists { - if err := d.firewall.DeleteRouteRule(id); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete route rule: %w", err)) - } - // implicitly deleted from the map + // Tear down old route rules; retain ones the backend refused so a + // transient failure doesn't leave orphaned rules in the firewall. + for ruleID, rule := range d.routeRules { + if _, exists := newRouteRules[ruleID]; exists { + continue + } + if err := d.firewall.DeleteFilterRule(rule); err != nil { + merr = multierror.Append(merr, fmt.Errorf("delete route rule, will retry: %w", err)) + newRouteRules[ruleID] = rule } } @@ -236,102 +380,202 @@ func (d *DefaultManager) applyRouteACLs(rules []*mgmProto.RouteFirewallRule, dyn return nberrors.FormatErrorOrNil(merr) } -func (d *DefaultManager) applyRouteACL(rule *mgmProto.RouteFirewallRule, dynamicResolver bool) (id.RuleID, error) { +func (d *DefaultManager) applyRouteACL(rule *mgmProto.RouteFirewallRule, dynamicResolver bool) (firewall.Rule, error) { if len(rule.SourceRanges) == 0 { - return "", ErrSourceRangesEmpty + return nil, ErrSourceRangesEmpty } var sources []netip.Prefix for _, sourceRange := range rule.SourceRanges { source, err := netip.ParsePrefix(sourceRange) if err != nil { - return "", fmt.Errorf("parse source range: %w", err) + return nil, fmt.Errorf("parse source range: %w", err) } - sources = append(sources, source) + sources = append(sources, firewall.UnmapPrefix(source)) } destination, err := determineDestination(rule, dynamicResolver, sources) if err != nil { - return "", fmt.Errorf("determine destination: %w", err) + return nil, fmt.Errorf("determine destination: %w", err) } - protocol, err := convertToFirewallProtocol(rule.Protocol) + protocol, err := ConvertToFirewallProtocol(rule.Protocol) if err != nil { - return "", fmt.Errorf("invalid protocol: %w", err) + return nil, fmt.Errorf("invalid protocol: %w", err) } action, err := convertFirewallAction(rule.Action) if err != nil { - return "", fmt.Errorf("invalid action: %w", err) + return nil, fmt.Errorf("invalid action: %w", err) } dPorts := convertPortInfo(rule.PortInfo) - addedRule, err := d.firewall.AddRouteFiltering(rule.PolicyID, sources, destination, protocol, nil, dPorts, action) + addedRule, err := d.firewall.AddFilterRule(rule.PolicyID, sources, destination, protocol, nil, dPorts, action) if err != nil { - return "", fmt.Errorf("add route rule: %w", err) + return nil, fmt.Errorf("add route rule: %w", err) + } + if addedRule == nil { + return nil, fmt.Errorf("add route rule: %w", ErrNoRuleReturned) } - return id.RuleID(addedRule.ID()), nil + return addedRule, nil } -func (d *DefaultManager) protoRuleToFirewallRule( - r *mgmProto.FirewallRule, - ipsetName string, -) (id.RuleID, []firewall.Rule, error) { - ip, err := extractRuleIP(r) - if err != nil { - return "", nil, err +// splitDenyAccept partitions groups by action so denies can be +// applied before accepts. Order within each bucket is preserved. +func splitDenyAccept(groups []*peerRuleGroup) (denies, accepts []*peerRuleGroup) { + for _, g := range groups { + if g.action == mgmProto.RuleAction_DROP { + denies = append(denies, g) + } else { + accepts = append(accepts, g) + } + } + return denies, accepts +} + +// groupPeerRules merges single-source rules sharing a selector into +// multi-source groups. It splits source-decode failures by action: +// denyErr is non-nil when a deny rule could not be decoded, which is a +// fail-open risk the caller must treat as fatal for the pass; err +// carries the tolerable accept-rule failures the caller can log and +// continue past. +func groupPeerRules(rules []*mgmProto.FirewallRule) (groups []*peerRuleGroup, denyErr error, err error) { + var denyMerr, acceptMerr *multierror.Error + byKey := make(map[peerRuleKey]*peerRuleGroup) + order := make([]peerRuleKey, 0) + + for _, r := range rules { + srcs, decErr := extractRuleSources(r) + if decErr != nil { + if r.Action == mgmProto.RuleAction_DROP { + denyMerr = multierror.Append(denyMerr, decErr) + } else { + acceptMerr = multierror.Append(acceptMerr, decErr) + } + continue + } + // A single FirewallRule normally carries one address family, but + // split by family defensively: each backend keys a rule to one + // family and would mismatch sources of the other, so a group's + // sources must never span families. + v4, v6 := splitPrefixesByFamily(srcs) + for _, sub := range []struct { + isV6 bool + sources []netip.Prefix + }{{false, v4}, {true, v6}} { + if len(sub.sources) == 0 { + continue + } + key := ruleGroupKey(r, sub.isV6) + g, ok := byKey[key] + if !ok { + g = &peerRuleGroup{ + direction: r.Direction, + action: r.Action, + protocol: r.Protocol, + port: r.PortInfo, + legacyPort: r.Port, + policyID: r.PolicyID, + } + byKey[key] = g + order = append(order, key) + } + g.sources = append(g.sources, sub.sources...) + } } - protocol, err := convertToFirewallProtocol(r.Protocol) - if err != nil { - return "", nil, fmt.Errorf("skipping firewall rule: %s", err) + out := make([]*peerRuleGroup, 0, len(order)) + for _, k := range order { + out = append(out, byKey[k]) + } + return out, nberrors.FormatErrorOrNil(denyMerr), nberrors.FormatErrorOrNil(acceptMerr) +} + +func prefixIsV6(p netip.Prefix) bool { + return p.Addr().Is6() && !p.Addr().Is4In6() +} + +// splitPrefixesByFamily partitions prefixes into IPv4 and IPv6 groups. +func splitPrefixesByFamily(prefixes []netip.Prefix) (v4, v6 []netip.Prefix) { + for _, p := range prefixes { + if prefixIsV6(p) { + v6 = append(v6, p) + } else { + v4 = append(v4, p) + } + } + return v4, v6 +} + +// ruleGroupKey builds the selector key for a rule. v6 must reflect the +// rule's source family: mgmt emits one rule per family and mixing them +// would break ICMP-variant selection in uspfilter. +func ruleGroupKey(r *mgmProto.FirewallRule, v6 bool) peerRuleKey { + k := peerRuleKey{ + v6: v6, + policyID: string(r.PolicyID), + direction: r.Direction, + action: r.Action, + protocol: r.Protocol, + legacyPort: r.Port, + } + if pi := r.PortInfo; pi != nil { + k.port = uint16(pi.GetPort()) + if rng := pi.GetRange(); rng != nil { + k.rangeStart = uint16(rng.GetStart()) + k.rangeEnd = uint16(rng.GetEnd()) + } + } + return k +} + +// extractRuleSources returns all source prefixes the rule applies to. +// New management populates sourcePrefixes; older management sets PeerIP. +func extractRuleSources(r *mgmProto.FirewallRule) ([]netip.Prefix, error) { + if len(r.SourcePrefixes) > 0 { + out := make([]netip.Prefix, 0, len(r.SourcePrefixes)) + for _, raw := range r.SourcePrefixes { + addr, err := netiputil.DecodeAddr(raw) + if err != nil { + return nil, fmt.Errorf("decode source prefix: %w", err) + } + out = append(out, netip.PrefixFrom(addr.Unmap(), addr.Unmap().BitLen())) + } + return out, nil } - action, err := convertFirewallAction(r.Action) + peerIP := r.PeerIP //nolint:staticcheck // PeerIP is the legacy source field for old management servers + addr, err := netip.ParseAddr(peerIP) if err != nil { - return "", nil, fmt.Errorf("skipping firewall rule: %s", err) + return nil, fmt.Errorf("parse peer IP %q: %w", peerIP, err) } + addr = addr.Unmap() + // An unspecified PeerIP means "any peer" (legacy management + // allow-all fallback); only a /0 prefix matches any source in the + // backends, a full-length prefix would match nothing. + if addr.IsUnspecified() { + return []netip.Prefix{netip.PrefixFrom(addr, 0)}, nil + } + return []netip.Prefix{netip.PrefixFrom(addr, addr.BitLen())}, nil +} - var port *firewall.Port - if !portInfoEmpty(r.PortInfo) { - port = convertPortInfo(r.PortInfo) - } else if r.Port != "" { - // old version of management, single port - value, err := strconv.Atoi(r.Port) +func resolveGroupPort(g *peerRuleGroup) (*firewall.Port, error) { + if !portInfoEmpty(g.port) { + return convertPortInfo(g.port), nil + } + if g.legacyPort != "" { + value, err := strconv.ParseUint(g.legacyPort, 10, 16) if err != nil { - return "", nil, fmt.Errorf("invalid port: %w", err) + return nil, fmt.Errorf("invalid port: %w", err) } - port = &firewall.Port{ + return &firewall.Port{ Values: []uint16{uint16(value)}, - } + }, nil } - - ruleID := d.getPeerRuleID(ip, protocol, int(r.Direction), port, action) - if rulesPair, ok := d.peerRulesPairs[ruleID]; ok { - return ruleID, rulesPair, nil - } - - var rules []firewall.Rule - switch r.Direction { - case mgmProto.RuleDirection_IN: - rules, err = d.addInRules(r.PolicyID, ip, protocol, port, action, ipsetName) - case mgmProto.RuleDirection_OUT: - if d.firewall.IsStateful() { - return "", nil, nil - } - // return traffic for outbound connections if firewall is stateless - rules, err = d.addOutRules(r.PolicyID, ip, protocol, port, action, ipsetName) - default: - return "", nil, fmt.Errorf("invalid direction, skipping firewall rule") - } - - if err != nil { - return "", nil, err - } - - return ruleID, rules, nil + // nolint:nilnil // a nil port legitimately means "no port restriction" + return nil, nil } func portInfoEmpty(portInfo *mgmProto.PortInfo) bool { @@ -350,85 +594,9 @@ func portInfoEmpty(portInfo *mgmProto.PortInfo) bool { } } -func (d *DefaultManager) addInRules( - id []byte, - ip netip.Addr, - protocol firewall.Protocol, - port *firewall.Port, - action firewall.Action, - ipsetName string, -) ([]firewall.Rule, error) { - rule, err := d.firewall.AddPeerFiltering(id, ip.AsSlice(), protocol, nil, port, action, ipsetName) - if err != nil { - return nil, fmt.Errorf("add firewall rule: %w", err) - } - - return rule, nil -} - -func (d *DefaultManager) addOutRules( - id []byte, - ip netip.Addr, - protocol firewall.Protocol, - port *firewall.Port, - action firewall.Action, - ipsetName string, -) ([]firewall.Rule, error) { - if shouldSkipInvertedRule(protocol, port) { - return nil, nil - } - - rule, err := d.firewall.AddPeerFiltering(id, ip.AsSlice(), protocol, port, nil, action, ipsetName) - if err != nil { - return nil, fmt.Errorf("add firewall rule: %w", err) - } - - return rule, nil -} - -// getPeerRuleID returns unique ID for the rule based on its parameters. -func (d *DefaultManager) getPeerRuleID( - ip netip.Addr, - proto firewall.Protocol, - direction int, - port *firewall.Port, - action firewall.Action, -) id.RuleID { - idStr := ip.String() + string(proto) + strconv.Itoa(direction) + strconv.Itoa(int(action)) - if port != nil { - idStr += port.String() - } - - return id.RuleID(hex.EncodeToString(md5.New().Sum([]byte(idStr)))) -} - -// getRuleGroupingSelector takes all rule properties except IP address to build selector -func (d *DefaultManager) getRuleGroupingSelector(rule *mgmProto.FirewallRule) string { - return fmt.Sprintf("%v:%v:%v:%s:%v", strconv.Itoa(int(rule.Direction)), rule.Action, rule.Protocol, rule.Port, rule.PortInfo) -} - - -// extractRuleIP extracts the peer IP from a firewall rule. -// If sourcePrefixes is populated (new management), decode the first entry and use its address. -// Otherwise fall back to the deprecated PeerIP string field (old management). -func extractRuleIP(r *mgmProto.FirewallRule) (netip.Addr, error) { - if len(r.SourcePrefixes) > 0 { - addr, err := netiputil.DecodeAddr(r.SourcePrefixes[0]) - if err != nil { - return netip.Addr{}, fmt.Errorf("decode source prefix: %w", err) - } - return addr.Unmap(), nil - } - - //nolint:staticcheck // PeerIP used for backward compatibility with old management - addr, err := netip.ParseAddr(r.PeerIP) - if err != nil { - return netip.Addr{}, fmt.Errorf("invalid IP address, skipping firewall rule") - } - return addr.Unmap(), nil -} - -func convertToFirewallProtocol(protocol mgmProto.RuleProtocol) (firewall.Protocol, error) { +// ConvertToFirewallProtocol maps a management rule protocol to the +// firewall protocol type. +func ConvertToFirewallProtocol(protocol mgmProto.RuleProtocol) (firewall.Protocol, error) { switch protocol { case mgmProto.RuleProtocol_TCP: return firewall.ProtocolTCP, nil diff --git a/client/internal/acl/manager_test.go b/client/internal/acl/manager_test.go index 968654ae9..19bef9058 100644 --- a/client/internal/acl/manager_test.go +++ b/client/internal/acl/manager_test.go @@ -5,11 +5,12 @@ import ( "net/netip" "testing" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "github.com/netbirdio/netbird/client/firewall" + fwmanager "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/iface" "github.com/netbirdio/netbird/client/iface/wgaddr" "github.com/netbirdio/netbird/client/internal/acl/mocks" @@ -77,9 +78,9 @@ func TestDefaultManager(t *testing.T) { }) t.Run("add extra rules", func(t *testing.T) { - existedPairs := map[string]struct{}{} + existedPairs := map[fwmanager.RuleID]struct{}{} for id := range acl.peerRulesPairs { - existedPairs[id.ID()] = struct{}{} + existedPairs[id] = struct{}{} } // remove first rule @@ -87,7 +88,7 @@ func TestDefaultManager(t *testing.T) { networkMap.FirewallRules = append( networkMap.FirewallRules, &mgmProto.FirewallRule{ - PeerIP: "10.93.0.3", + PeerIP: "10.93.0.3", //nolint:staticcheck Direction: mgmProto.RuleDirection_IN, Action: mgmProto.RuleAction_DROP, Protocol: mgmProto.RuleProtocol_ICMP, @@ -106,7 +107,7 @@ func TestDefaultManager(t *testing.T) { // check that old rule was removed previousCount := 0 for id := range acl.peerRulesPairs { - if _, ok := existedPairs[id.ID()]; ok { + if _, ok := existedPairs[id]; ok { previousCount++ } } @@ -556,12 +557,12 @@ func TestApplyFilteringSkipsUnchangedConfig(t *testing.T) { func buildNetworkMap(peerRules, routeRules int) *mgmProto.NetworkMap { nm := &mgmProto.NetworkMap{ - FirewallRulesIsEmpty: peerRules == 0, + FirewallRulesIsEmpty: peerRules == 0, RoutesFirewallRulesIsEmpty: routeRules == 0, } for i := range peerRules { nm.FirewallRules = append(nm.FirewallRules, &mgmProto.FirewallRule{ - PeerIP: fmt.Sprintf("10.%d.%d.%d", i>>16&0xff, i>>8&0xff, i&0xff), + PeerIP: fmt.Sprintf("10.%d.%d.%d", i>>16&0xff, i>>8&0xff, i&0xff), //nolint:staticcheck Direction: mgmProto.RuleDirection_IN, Action: mgmProto.RuleAction_ACCEPT, Protocol: mgmProto.RuleProtocol_TCP, diff --git a/client/internal/acl/mocks/iface_mapper.go b/client/internal/acl/mocks/iface_mapper.go index 95d5a2c58..f8cca1c2d 100644 --- a/client/internal/acl/mocks/iface_mapper.go +++ b/client/internal/acl/mocks/iface_mapper.go @@ -7,7 +7,7 @@ package mocks import ( reflect "reflect" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" wgdevice "golang.zx2c4.com/wireguard/device" "github.com/netbirdio/netbird/client/iface/device" diff --git a/client/internal/auth/account_match_test.go b/client/internal/auth/account_match_test.go new file mode 100644 index 000000000..b879cf9ab --- /dev/null +++ b/client/internal/auth/account_match_test.go @@ -0,0 +1,113 @@ +package auth + +import ( + "encoding/base64" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTokenInfoMatchesAccount(t *testing.T) { + tests := []struct { + name string + token TokenInfo + hint string + match bool + }{ + { + name: "same account", + token: TokenInfo{EmailClaim: "user@example.com"}, + hint: "user@example.com", + match: true, + }, + { + name: "different account", + token: TokenInfo{EmailClaim: "other@example.com"}, + hint: "user@example.com", + match: false, + }, + { + name: "case differences are the same account", + token: TokenInfo{EmailClaim: "User@Example.com"}, + hint: "user@example.com", + match: true, + }, + { + name: "no hint leaves the choice to the IdP", + token: TokenInfo{EmailClaim: "other@example.com"}, + hint: "", + match: true, + }, + { + name: "token without an email claim is not judged", + token: TokenInfo{EmailClaim: ""}, + hint: "user@example.com", + match: true, + }, + { + name: "name fallback does not trigger matching", + token: TokenInfo{Email: "Some One"}, + hint: "user@example.com", + match: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.match, tc.token.MatchesAccount(tc.hint)) + }) + } +} + +func TestParseEmailFromIDToken(t *testing.T) { + tests := []struct { + name string + claims map[string]interface{} + wantValue string + wantFromEmail bool + wantErr bool + }{ + { + name: "email claim", + claims: map[string]interface{}{"email": "user@example.com", "name": "Some One"}, + wantValue: "user@example.com", + wantFromEmail: true, + }, + { + name: "name fallback", + claims: map[string]interface{}{"name": "Some One"}, + wantValue: "Some One", + }, + { + name: "neither claim present", + claims: map[string]interface{}{"sub": "abc"}, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + value, fromEmailClaim, err := parseEmailFromIDToken(idTokenWithClaims(t, tc.claims)) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantValue, value) + assert.Equal(t, tc.wantFromEmail, fromEmailClaim) + }) + } +} + +func TestRetryFlowForAccountUnsupportedFlow(t *testing.T) { + assert.Nil(t, RetryFlowForAccount(&DeviceAuthorizationFlow{})) +} + +func idTokenWithClaims(t *testing.T, claims map[string]interface{}) string { + t.Helper() + payload, err := json.Marshal(claims) + require.NoError(t, err) + return "header." + base64.RawURLEncoding.EncodeToString(payload) + ".signature" +} diff --git a/client/internal/auth/auth.go b/client/internal/auth/auth.go index 850e0706d..7c156b5fd 100644 --- a/client/internal/auth/auth.go +++ b/client/internal/auth/auth.go @@ -3,6 +3,7 @@ package auth import ( "context" "net/url" + "strings" "sync" "time" @@ -21,6 +22,25 @@ import ( mgmProto "github.com/netbirdio/netbird/shared/management/proto" ) +// peerLoginExpiredMsg is the exact phrase the management server returns +// when a previously SSO-enrolled peer's login has expired. Sourced from +// shared/management/status/error.go (NewPeerLoginExpiredError). Matched +// by substring so a future server-side rewording that keeps the phrase +// still triggers the friendly fallback in Login(). +const peerLoginExpiredMsg = "peer login has expired" + +// errSetupKeyOnSSOExpiredPeer replaces the raw management error when the +// user runs `netbird login -k ` against a peer that was +// originally enrolled via SSO. Wrapped in a PermissionDenied gRPC status +// so callers' existing isPermissionDenied / isAuthError checks still +// classify it correctly (early-exit from retry backoff, StatusNeedsLogin +// in the server state machine). +var errSetupKeyOnSSOExpiredPeer = status.Error( + codes.PermissionDenied, + "this peer was originally enrolled via SSO and its session has expired. "+ + "Setup keys can only enrol new peers — run `netbird up` (interactive SSO) to re-login.", +) + // Auth manages authentication operations with the management server // It maintains a long-lived connection and automatically handles reconnection with backoff type Auth struct { @@ -83,7 +103,7 @@ func (a *Auth) IsSSOSupported(ctx context.Context) (bool, error) { err := a.withRetry(ctx, func(client *mgm.GrpcClient) error { // Try PKCE flow first - _, err := a.getPKCEFlow(client) + _, err := a.getPKCEFlow(client, false) if err == nil { supportsSSO = true return nil @@ -116,28 +136,43 @@ func (a *Auth) IsSSOSupported(ctx context.Context) (bool, error) { return supportsSSO, err } -// GetOAuthFlow returns an OAuth flow (PKCE or Device) using the existing management connection -// This avoids creating a new connection to the management server -func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool) (OAuthFlow, error) { +// GetOAuthFlow returns an OAuth flow (PKCE or Device) using the existing management connection. +// This avoids creating a new connection to the management server. +// +// sessionExtend marks the flow as renewing an existing peer's session rather than +// logging one in; the server needs it to rule out a silent authorization that the +// IdP could answer from another account. See PKCEAuthorizationFlowRequest. +func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool, sessionExtend bool, hint string) (OAuthFlow, error) { var flow OAuthFlow - var err error - err = a.withRetry(ctx, func(client *mgm.GrpcClient) error { + err := a.withRetry(ctx, func(client *mgm.GrpcClient) error { if forceDeviceAuth { - flow, err = a.getDeviceFlow(client) - return err + deviceFlow, err := a.getDeviceFlow(client) + if err != nil { + return err + } + deviceFlow.SetLoginHint(hint) + flow = deviceFlow + return nil } // Try PKCE flow first - flow, err = a.getPKCEFlow(client) + pkceFlow, err := a.getPKCEFlow(client, sessionExtend) if err != nil { // If PKCE not supported, try Device flow if s, ok := status.FromError(err); ok && (s.Code() == codes.NotFound || s.Code() == codes.Unimplemented) { - flow, err = a.getDeviceFlow(client) - return err + deviceFlow, err := a.getDeviceFlow(client) + if err != nil { + return err + } + deviceFlow.SetLoginHint(hint) + flow = deviceFlow + return nil } return err } + pkceFlow.SetLoginHint(hint) + flow = pkceFlow return nil }) @@ -184,6 +219,15 @@ func (a *Auth) Login(ctx context.Context, setupKey string, jwtToken string) (err log.Debugf("peer registration required") _, err = a.registerPeer(client, ctx, setupKey, jwtToken, pubSSHKey) if err != nil { + // The peer pub-key is already on file with the management + // server (originally enrolled via SSO) and the session has + // expired. The setup-key path can only enrol new peers, so + // retrying with -k will keep failing. Replace the raw mgm + // message with an actionable hint that tells the user to + // re-authenticate via SSO instead. + if setupKey != "" && jwtToken == "" && isPeerLoginExpired(err) { + err = errSetupKeyOnSSOExpiredPeer + } isAuthError = isPermissionDenied(err) return err } @@ -200,8 +244,8 @@ func (a *Auth) Login(ctx context.Context, setupKey string, jwtToken string) (err } // getPKCEFlow retrieves PKCE authorization flow configuration and creates a flow instance -func (a *Auth) getPKCEFlow(client *mgm.GrpcClient) (*PKCEAuthorizationFlow, error) { - protoFlow, err := client.GetPKCEAuthorizationFlow() +func (a *Auth) getPKCEFlow(client *mgm.GrpcClient, sessionExtend bool) (*PKCEAuthorizationFlow, error) { + protoFlow, err := client.GetPKCEAuthorizationFlow(sessionExtend) if err != nil { if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound { log.Warnf("server couldn't find pkce flow, contact admin: %v", err) @@ -322,11 +366,13 @@ func (a *Auth) setSystemInfoFlags(info *system.Info) { a.config.BlockLANAccess, a.config.BlockInbound, a.config.DisableIPv6, + a.config.SyncMessageVersion, a.config.EnableSSHRoot, a.config.EnableSSHSFTP, a.config.EnableSSHLocalPortForwarding, a.config.EnableSSHRemotePortForwarding, a.config.DisableSSHAuth, + a.config.RemoteJobsAllowed, ) } @@ -473,3 +519,16 @@ func isLoginNeeded(err error) bool { func isRegistrationNeeded(err error) bool { return isPermissionDenied(err) } + +// isPeerLoginExpired reports whether err is the management server's +// "peer login has expired" PermissionDenied response. Used by Login to +// detect the case where the caller passed a setup-key but the peer is +// actually an SSO-enrolled record whose session needs refreshing — the +// setup-key path cannot help there. +func isPeerLoginExpired(err error) bool { + if !isPermissionDenied(err) { + return false + } + s, _ := status.FromError(err) + return strings.Contains(s.Message(), peerLoginExpiredMsg) +} diff --git a/client/internal/auth/auth_test.go b/client/internal/auth/auth_test.go new file mode 100644 index 000000000..e393beccb --- /dev/null +++ b/client/internal/auth/auth_test.go @@ -0,0 +1,80 @@ +package auth + +import ( + "errors" + "strings" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestIsPeerLoginExpired(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + { + name: "nil", + err: nil, + want: false, + }, + { + name: "plain error (not a gRPC status)", + err: errors.New("network read: connection reset"), + want: false, + }, + { + name: "PermissionDenied with different message", + err: status.Error(codes.PermissionDenied, "user is blocked"), + want: false, + }, + { + name: "Unauthenticated with the expected phrase", + // Wrong status code — must still return false. + err: status.Error(codes.Unauthenticated, "peer login has expired, please log in once more"), + want: false, + }, + { + name: "exact server message", + err: status.Error(codes.PermissionDenied, "peer login has expired, please log in once more"), + want: true, + }, + { + name: "phrase as substring", + // Future-proofing: if mgm reworords but keeps the phrase, + // the friendly fallback must still kick in. + err: status.Error(codes.PermissionDenied, "session refused: peer login has expired (account=foo)"), + want: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isPeerLoginExpired(tc.err); got != tc.want { + t.Fatalf("isPeerLoginExpired(%v) = %v, want %v", tc.err, got, tc.want) + } + }) + } +} + +func TestErrSetupKeyOnSSOExpiredPeer(t *testing.T) { + // Sentinel must surface as PermissionDenied so the upstream + // isPermissionDenied / isAuthError checks classify it correctly + // (short-circuit retry backoff, set StatusNeedsLogin). + if !isPermissionDenied(errSetupKeyOnSSOExpiredPeer) { + t.Fatalf("errSetupKeyOnSSOExpiredPeer must be a PermissionDenied gRPC error") + } + + // Message must actually mention SSO and `netbird up` so it is + // actionable for the end user. Loose substring checks keep the + // test resilient to copy edits. + s, _ := status.FromError(errSetupKeyOnSSOExpiredPeer) + msg := strings.ToLower(s.Message()) + for _, want := range []string{"sso", "netbird up"} { + if !strings.Contains(msg, want) { + t.Errorf("sentinel message should contain %q, got %q", want, s.Message()) + } + } +} diff --git a/client/internal/auth/device_flow.go b/client/internal/auth/device_flow.go index e33765300..c2b8d34b8 100644 --- a/client/internal/auth/device_flow.go +++ b/client/internal/auth/device_flow.go @@ -259,12 +259,18 @@ func (d *DeviceAuthorizationFlow) WaitToken(ctx context.Context, info AuthFlowIn ticker := time.NewTicker(interval) defer ticker.Stop() + log.Infof("device flow: waiting for user authorization, polling token endpoint every %s, code expires in %s", interval, timeout) + + start := time.Now() + polls := 0 + for { select { case <-waitCtx.Done(): return TokenInfo{}, waitCtx.Err() case <-ticker.C: + polls++ tokenResponse, err := d.requestToken(info) if err != nil { return TokenInfo{}, fmt.Errorf("parsing token response failed with error: %v", err) @@ -272,10 +278,12 @@ func (d *DeviceAuthorizationFlow) WaitToken(ctx context.Context, info AuthFlowIn if tokenResponse.Error != "" { if tokenResponse.Error == "authorization_pending" { + log.Tracef("device flow: authorization still pending after poll %d", polls) continue } else if tokenResponse.Error == "slow_down" { interval += (3 * time.Second) ticker.Reset(interval) + log.Infof("device flow: IdP requested slow_down, polling interval increased to %s", interval) continue } @@ -291,11 +299,25 @@ func (d *DeviceAuthorizationFlow) WaitToken(ctx context.Context, info AuthFlowIn UseIDToken: d.providerConfig.UseIDToken, } - err = isValidAccessToken(tokenInfo.GetTokenToUse(), d.providerConfig.Audience) + err = validateTokenAudience(tokenInfo.GetTokenToUse(), d.providerConfig.Audience) if err != nil { return TokenInfo{}, fmt.Errorf("validate access token failed with error: %v", err) } + // Same as the PKCE flow: the account the token belongs to is what + // callers store to send back as the login_hint. Without it a client + // driven through the device flow — Android TV and tvOS — never binds + // an account to its profile and every later login goes out blind. + if email, fromEmailClaim, err := parseEmailFromIDToken(tokenInfo.IDToken); err != nil { + log.Warnf("failed to parse email from ID token: %v", err) + } else { + tokenInfo.Email = email + if fromEmailClaim { + tokenInfo.EmailClaim = email + } + } + + log.Infof("device flow: user authorization confirmed after %d polls in %s", polls, time.Since(start).Round(time.Second)) return tokenInfo, err } } diff --git a/client/internal/auth/oauth.go b/client/internal/auth/oauth.go index a50a2ce6f..54ecb5d86 100644 --- a/client/internal/auth/oauth.go +++ b/client/internal/auth/oauth.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "runtime" + "strings" log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" @@ -25,6 +26,14 @@ type HTTPClient interface { Do(req *http.Request) (*http.Response, error) } +// accountPromptForcer is implemented by the PKCE flow only. The device code +// flow has no equivalent: RFC 8628 defines no prompt parameter, and the user +// confirms the code on a page that shows which account signs in, so a silent +// wrong-account answer is not the failure mode there. +type accountPromptForcer interface { + ForceAccountPrompt() +} + // AuthFlowInfo holds information for the OAuth 2.0 authorization flow type AuthFlowInfo struct { //nolint:revive DeviceCode string `json:"device_code"` @@ -49,6 +58,23 @@ type TokenInfo struct { ExpiresIn int `json:"expires_in"` UseIDToken bool `json:"-"` Email string `json:"-"` + EmailClaim string `json:"-"` +} + +// MatchesAccount reports whether the token belongs to the account a profile is +// bound to. A hint the IdP could not have acted on — no hint stored, or a token +// that carried no email claim — is reported as a match: the check exists to catch a +// login answered from the wrong account, not to block one it cannot judge. +// +// The comparison is case-insensitive. Local-parts are case-sensitive per RFC +// 5321, but no IdP in practice issues two accounts differing only in case, and +// an IdP that echoes a differently-cased address would otherwise fail every +// login. +func (t TokenInfo) MatchesAccount(hint string) bool { + if hint == "" || t.EmailClaim == "" { + return true + } + return strings.EqualFold(t.EmailClaim, hint) } // GetTokenToUse returns either the access or id token based on UseIDToken field @@ -63,19 +89,22 @@ func shouldUseDeviceFlow(force bool, isUnixDesktopClient bool) bool { return force || (runtime.GOOS == "linux" || runtime.GOOS == "freebsd") && !isUnixDesktopClient } -// NewOAuthFlow initializes and returns the appropriate OAuth flow based on the management configuration +// NewOAuthFlow initializes and returns the appropriate OAuth flow based on the management configuration. // // It starts by initializing the PKCE.If this process fails, it resorts to the Device Code Flow, // and if that also fails, the authentication process is deemed unsuccessful // -// On Linux distros without desktop environment support, it only tries to initialize the Device Code Flow -// forceDeviceCodeFlow can be used to skip PKCE and go directly to Device Code Flow (e.g., for Android TV) -func NewOAuthFlow(ctx context.Context, config *profilemanager.Config, isUnixDesktopClient bool, forceDeviceCodeFlow bool, hint string) (OAuthFlow, error) { +// On Linux distros without desktop environment support, it only tries to initialize the Device Code Flow. +// forceDeviceCodeFlow can be used to skip PKCE and go directly to Device Code Flow (e.g., for Android TV). +// +// sessionExtend marks the flow as renewing an existing peer's session rather than +// logging one in. See PKCEAuthorizationFlowRequest for what the server makes of it. +func NewOAuthFlow(ctx context.Context, config *profilemanager.Config, isUnixDesktopClient bool, forceDeviceCodeFlow bool, hint string, sessionExtend bool) (OAuthFlow, error) { if shouldUseDeviceFlow(forceDeviceCodeFlow, isUnixDesktopClient) { return authenticateWithDeviceCodeFlow(ctx, config, hint) } - pkceFlow, err := authenticateWithPKCEFlow(ctx, config, hint) + pkceFlow, err := authenticateWithPKCEFlow(ctx, config, hint, sessionExtend) if err != nil { log.Debugf("failed to initialize pkce authentication with error: %v\n", err) log.Debug("falling back to device code flow") @@ -85,21 +114,19 @@ func NewOAuthFlow(ctx context.Context, config *profilemanager.Config, isUnixDesk } // authenticateWithPKCEFlow initializes the Proof Key for Code Exchange flow auth flow -func authenticateWithPKCEFlow(ctx context.Context, config *profilemanager.Config, hint string) (OAuthFlow, error) { +func authenticateWithPKCEFlow(ctx context.Context, config *profilemanager.Config, hint string, sessionExtend bool) (OAuthFlow, error) { authClient, err := NewAuth(ctx, config.PrivateKey, config.ManagementURL, config) if err != nil { return nil, fmt.Errorf("failed to create auth client: %v", err) } defer authClient.Close() - pkceFlowInfo, err := authClient.getPKCEFlow(authClient.client) + pkceFlowInfo, err := authClient.getPKCEFlow(authClient.client, sessionExtend) if err != nil { return nil, fmt.Errorf("getting pkce authorization flow info failed with error: %v", err) } - if hint != "" { - pkceFlowInfo.SetLoginHint(hint) - } + pkceFlowInfo.SetLoginHint(hint) return pkceFlowInfo, nil } @@ -127,9 +154,25 @@ func authenticateWithDeviceCodeFlow(ctx context.Context, config *profilemanager. } } - if hint != "" { - deviceFlowInfo.SetLoginHint(hint) - } + deviceFlowInfo.SetLoginHint(hint) return deviceFlowInfo, nil } + +// RetryFlowForAccount returns a flow that asks the IdP to re-authenticate, for +// a login answered with an account other than the one hinted. Returns nil when +// the flow cannot ask — the caller then proceeds with the token it has. +// +// Proceeding rather than failing is deliberate. The hint is an email that may +// simply have changed since it was stored, and refusing the login would lock a +// user out of their own profile over a rename. The retry gives the account a +// chance to be corrected; the server still rejects a token that does not own +// the peer. +func RetryFlowForAccount(flow OAuthFlow) OAuthFlow { + forcer, ok := flow.(accountPromptForcer) + if !ok { + return nil + } + forcer.ForceAccountPrompt() + return flow +} diff --git a/client/internal/auth/pending_flow.go b/client/internal/auth/pending_flow.go new file mode 100644 index 000000000..daeb18bc2 --- /dev/null +++ b/client/internal/auth/pending_flow.go @@ -0,0 +1,89 @@ +package auth + +import ( + "context" + "sync" + "time" +) + +// PendingFlow stores an in-progress OAuth flow between the RPC that +// initiates it (returns the verification URI to the UI) and the RPC +// that waits for the user to complete it. The flow handle, the +// device-code info, and the absolute expiry are kept together so the +// waiting RPC can validate the device code and reuse the same flow. +// +// PendingFlow is safe for concurrent use; callers must not access the +// stored fields directly. +type PendingFlow struct { + mu sync.Mutex + flow OAuthFlow + info AuthFlowInfo + expiresAt time.Time + waitCancel context.CancelFunc +} + +// NewPendingFlow returns an empty PendingFlow ready to be populated by Set. +func NewPendingFlow() *PendingFlow { + return &PendingFlow{} +} + +// Set stores the flow and its authorization info, computing the absolute +// expiry from info.ExpiresIn (seconds, as returned by the IdP). +func (p *PendingFlow) Set(flow OAuthFlow, info AuthFlowInfo) { + p.mu.Lock() + defer p.mu.Unlock() + p.flow = flow + p.info = info + p.expiresAt = time.Now().Add(time.Duration(info.ExpiresIn) * time.Second) +} + +// Get returns the stored flow, info, and whether a flow is currently +// pending. Returns (nil, zero, false) after Clear or before Set. +func (p *PendingFlow) Get() (OAuthFlow, AuthFlowInfo, bool) { + p.mu.Lock() + defer p.mu.Unlock() + if p.flow == nil { + return nil, AuthFlowInfo{}, false + } + return p.flow, p.info, true +} + +// ExpiresAt returns the absolute expiry of the pending flow. Returns +// the zero time when no flow is pending. +func (p *PendingFlow) ExpiresAt() time.Time { + p.mu.Lock() + defer p.mu.Unlock() + return p.expiresAt +} + +// SetWaitCancel records the cancel function for the goroutine currently +// blocked in WaitToken so a new RequestAuth can preempt it. +func (p *PendingFlow) SetWaitCancel(cancel context.CancelFunc) { + p.mu.Lock() + defer p.mu.Unlock() + p.waitCancel = cancel +} + +// CancelWait invokes and clears the stored wait-cancel, if any. Safe to +// call when no wait is in progress. +func (p *PendingFlow) CancelWait() { + p.mu.Lock() + cancel := p.waitCancel + p.waitCancel = nil + p.mu.Unlock() + if cancel != nil { + cancel() + } +} + +// Clear resets the pending flow to empty. Any stored wait-cancel is +// dropped without being invoked — call CancelWait first if the waiting +// goroutine must be stopped. +func (p *PendingFlow) Clear() { + p.mu.Lock() + defer p.mu.Unlock() + p.flow = nil + p.info = AuthFlowInfo{} + p.expiresAt = time.Time{} + p.waitCancel = nil +} diff --git a/client/internal/auth/pkce_flow.go b/client/internal/auth/pkce_flow.go index 84fa8a214..ebb0d4f1f 100644 --- a/client/internal/auth/pkce_flow.go +++ b/client/internal/auth/pkce_flow.go @@ -87,10 +87,11 @@ func validatePKCEConfig(config *PKCEAuthProviderConfig) error { // PKCEAuthorizationFlow implements the OAuthFlow interface for // the Authorization Code Flow with PKCE. type PKCEAuthorizationFlow struct { - providerConfig PKCEAuthProviderConfig - state string - codeVerifier string - oAuthConfig *oauth2.Config + providerConfig PKCEAuthProviderConfig + state string + codeVerifier string + oAuthConfig *oauth2.Config + forceAccountPrompt bool } // NewPKCEAuthorizationFlow returns new PKCE authorization code flow. @@ -153,11 +154,16 @@ func (p *PKCEAuthorizationFlow) RequestAuthInfo(ctx context.Context) (AuthFlowIn oauth2.SetAuthURLParam("code_challenge", codeChallenge), oauth2.SetAuthURLParam("audience", p.providerConfig.Audience), } + forceAccountPrompt := p.forceAccountPrompt + p.forceAccountPrompt = false + if !p.providerConfig.DisablePromptLogin { - switch p.providerConfig.LoginFlag { - case common.LoginFlagPromptLogin: + switch { + case forceAccountPrompt: params = append(params, oauth2.SetAuthURLParam("prompt", "login")) - case common.LoginFlagMaxAge0: + case p.providerConfig.LoginFlag == common.LoginFlagPromptLogin: + params = append(params, oauth2.SetAuthURLParam("prompt", "login")) + case p.providerConfig.LoginFlag == common.LoginFlagMaxAge0: params = append(params, oauth2.SetAuthURLParam("max_age", "0")) } } @@ -178,6 +184,20 @@ func (p *PKCEAuthorizationFlow) SetLoginHint(hint string) { p.providerConfig.LoginHint = hint } +// ForceAccountPrompt makes the next authorization request ask the IdP to +// re-authenticate instead of answering from the session it already holds. Used +// to retry a login that came back for an account other than the one hinted. +// +// The next RequestAuthInfo consumes the flag, so a flow that outlives its retry +// goes back to the configured behaviour instead of re-authenticating forever. +// +// DisablePromptLogin still wins: it is set for IdPs that break on prompt=login, +// where retrying with it would replace a wrong-account login with one that +// cannot complete at all. +func (p *PKCEAuthorizationFlow) ForceAccountPrompt() { + p.forceAccountPrompt = true +} + // WaitToken waits for the OAuth token in the PKCE Authorization Flow. // It starts an HTTP server to receive the OAuth token callback and waits for the token or an error. // Once the token is received, it is converted to TokenInfo and validated before returning. @@ -188,6 +208,8 @@ func (p *PKCEAuthorizationFlow) WaitToken(ctx context.Context, info AuthFlowInfo waitCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() + log.Infof("pkce flow: waiting for authorization callback on %s, timeout %s", p.oAuthConfig.RedirectURL, timeout) + tokenChan := make(chan *oauth2.Token, 1) errChan := make(chan error, 1) @@ -221,6 +243,7 @@ func (p *PKCEAuthorizationFlow) WaitToken(ctx context.Context, info AuthFlowInfo func (p *PKCEAuthorizationFlow) startServer(server *http.Server, tokenChan chan<- *oauth2.Token, errChan chan<- error) { mux := http.NewServeMux() mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { + log.Infof("pkce flow: received authorization callback from IdP") cert := p.providerConfig.ClientCertPair if cert != nil { tr := &http.Transport{ @@ -271,11 +294,18 @@ func (p *PKCEAuthorizationFlow) handleRequest(req *http.Request) (*oauth2.Token, return nil, fmt.Errorf("authentication failed: missing code") } - return p.oAuthConfig.Exchange( + exchangeStart := time.Now() + token, err := p.oAuthConfig.Exchange( req.Context(), code, oauth2.SetAuthURLParam("code_verifier", p.codeVerifier), ) + if err != nil { + return nil, err + } + + log.Infof("pkce flow: authorization code exchanged for token in %s", time.Since(exchangeStart).Round(time.Millisecond)) + return token, nil } func (p *PKCEAuthorizationFlow) parseOAuthToken(token *oauth2.Token) (TokenInfo, error) { @@ -296,48 +326,52 @@ func (p *PKCEAuthorizationFlow) parseOAuthToken(token *oauth2.Token) (TokenInfo, audience = p.providerConfig.ClientID } - if err := isValidAccessToken(tokenInfo.GetTokenToUse(), audience); err != nil { + if err := validateTokenAudience(tokenInfo.GetTokenToUse(), audience); err != nil { return TokenInfo{}, fmt.Errorf("authentication failed: invalid access token - %w", err) } - email, err := parseEmailFromIDToken(tokenInfo.IDToken) + email, fromEmailClaim, err := parseEmailFromIDToken(tokenInfo.IDToken) if err != nil { log.Warnf("failed to parse email from ID token: %v", err) } else { tokenInfo.Email = email + if fromEmailClaim { + tokenInfo.EmailClaim = email + } } return tokenInfo, nil } -func parseEmailFromIDToken(token string) (string, error) { +// parseEmailFromIDToken extracts the email (or name) claim from an ID token +// without verifying its signature. The value is best-effort: it prefills the +// login hint and is displayed. Account matching (see MatchesAccount) only uses +// it when it came from the email claim, which fromEmailClaim reports. It never +// grants anything — the authoritative identity is established server-side from +// the signature-verified token. +func parseEmailFromIDToken(token string) (value string, fromEmailClaim bool, err error) { parts := strings.Split(token, ".") if len(parts) < 2 { - return "", fmt.Errorf("invalid token format") + return "", false, fmt.Errorf("invalid token format") } data, err := base64.RawURLEncoding.DecodeString(parts[1]) if err != nil { - return "", fmt.Errorf("failed to decode payload: %w", err) + return "", false, fmt.Errorf("failed to decode payload: %w", err) } var claims map[string]interface{} if err := json.Unmarshal(data, &claims); err != nil { - return "", fmt.Errorf("json unmarshal error: %w", err) + return "", false, fmt.Errorf("json unmarshal error: %w", err) } - var email string - if emailValue, ok := claims["email"].(string); ok { - email = emailValue - } else { - val, ok := claims["name"].(string) - if ok { - email = val - } else { - return "", fmt.Errorf("email or name field not found in token payload") - } + if email, ok := claims["email"].(string); ok { + return email, true, nil + } + if name, ok := claims["name"].(string); ok { + return name, false, nil } - return email, nil + return "", false, fmt.Errorf("email or name field not found in token payload") } func createCodeChallenge(codeVerifier string) string { diff --git a/client/internal/auth/pkce_flow_test.go b/client/internal/auth/pkce_flow_test.go index c487c13df..ccbca10e9 100644 --- a/client/internal/auth/pkce_flow_test.go +++ b/client/internal/auth/pkce_flow_test.go @@ -76,6 +76,32 @@ func TestPromptLogin(t *testing.T) { } } +func TestForceAccountPromptAppliesOnlyToTheRetry(t *testing.T) { + config := PKCEAuthProviderConfig{ + ClientID: "test-client-id", + Audience: "test-audience", + TokenEndpoint: "https://test-token-endpoint.com/token", + Scope: "openid email profile", + AuthorizationEndpoint: "https://test-auth-endpoint.com/authorize", + RedirectURLs: []string{"http://127.0.0.1:33992/"}, + UseIDToken: true, + LoginFlag: mgm.LoginFlagNone, + } + pkce, err := NewPKCEAuthorizationFlow(config) + require.NoError(t, err) + + pkce.ForceAccountPrompt() + + retry, err := pkce.RequestAuthInfo(context.Background()) + require.NoError(t, err) + require.Contains(t, retry.VerificationURIComplete, "prompt=login") + + next, err := pkce.RequestAuthInfo(context.Background()) + require.NoError(t, err) + require.NotContains(t, next.VerificationURIComplete, "prompt=login", + "the forced prompt outlived the retry it was armed for") +} + func TestIsPortInExcludedRange(t *testing.T) { tests := []struct { name string diff --git a/client/internal/auth/sessionwatch/event.go b/client/internal/auth/sessionwatch/event.go new file mode 100644 index 000000000..3e55b26dd --- /dev/null +++ b/client/internal/auth/sessionwatch/event.go @@ -0,0 +1,82 @@ +package sessionwatch + +import ( + "strconv" + "time" +) + +// internal event kinds are no longer exposed: the watcher drives the Sink +// directly (NotifyStateChange on deadline change/clear, PublishEvent at +// each warning lead). Tests use a mock Sink to observe what the watcher +// emits. + +// Metadata keys attached by the daemon to session-warning SystemEvents. +// The UI tray reads these to build a locale-aware notification without +// relying on the daemon's locale-less UserMessage string, and to +// disambiguate the T-WarningLead notification from the T-FinalWarningLead +// fallback that auto-opens the SessionAboutToExpire dialog. +const ( + // MetaSessionWarning is set to "true" on both warning events (T-10 and + // T-2) so the UI can detect a session-warning SystemEvent without + // matching on the message text. Use MetaSessionFinal to distinguish + // the two. + MetaSessionWarning = "session_warning" + // MetaSessionFinal is set to "true" on the T-FinalWarningLead event + // only. Consumers that need to auto-open the SessionAboutToExpire + // dialog gate on this; T-WarningLead events leave the field unset. + MetaSessionFinal = "session_final_warning" + // MetaSessionExpiresAt carries the absolute UTC deadline encoded with + // FormatExpiresAt; consumers must decode with ParseExpiresAt so a + // future format change stays a single edit. + MetaSessionExpiresAt = "session_expires_at" + // MetaSessionLeadMinutes carries the lead in whole minutes (WarningLead + // for the T-10 event, FinalWarningLead for the T-2 event) so the UI + // can show "expires in ~N minutes" without hardcoding either constant. + MetaSessionLeadMinutes = "lead_minutes" + // MetaSessionDeadlineRejected is attached to the ERROR/AUTHENTICATION + // SystemEvent the daemon emits when it discards a deadline from the + // management server (pre-epoch, too far in the future, or past the + // clock-skew tolerance). The value is the rejection reason string. + // userMessage is left empty; the UI detects the event via this key + // and builds a localized notification — same pattern as the session + // warnings above. + MetaSessionDeadlineRejected = "session_deadline_rejected" +) + +// expiresAtLayout is the wire format used for MetaSessionExpiresAt. +// Producer and consumers both go through FormatExpiresAt/ParseExpiresAt +// so this layout stays a single source of truth. +const expiresAtLayout = time.RFC3339 + +// FormatExpiresAt encodes a deadline for MetaSessionExpiresAt. Always +// emits UTC so a consumer in another timezone reads the same wall-clock +// deadline. +func FormatExpiresAt(t time.Time) string { + return t.UTC().Format(expiresAtLayout) +} + +// ParseExpiresAt decodes the MetaSessionExpiresAt value back to a UTC +// time. Returns an error when the field is empty or malformed; the +// caller decides whether to fall back (zero value) or propagate. +func ParseExpiresAt(s string) (time.Time, error) { + t, err := time.Parse(expiresAtLayout, s) + if err != nil { + return time.Time{}, err + } + return t.UTC(), nil +} + +// FormatLeadMinutes encodes a lead duration for MetaSessionLeadMinutes +// as the integer count of whole minutes. Sub-minute residuals are +// truncated — the field is informational ("expires in ~N minutes") and +// fractional minutes don't change what the UI displays. +func FormatLeadMinutes(d time.Duration) string { + return strconv.Itoa(int(d / time.Minute)) +} + +// ParseLeadMinutes decodes a MetaSessionLeadMinutes value. Returns 0 +// and the parse error for malformed input; consumers that prefer a +// silent fallback can simply ignore the error. +func ParseLeadMinutes(s string) (int, error) { + return strconv.Atoi(s) +} diff --git a/client/internal/auth/sessionwatch/watcher.go b/client/internal/auth/sessionwatch/watcher.go new file mode 100644 index 000000000..f71689df1 --- /dev/null +++ b/client/internal/auth/sessionwatch/watcher.go @@ -0,0 +1,487 @@ +// Package sessionwatch tracks the SSO session expiry deadline that the +// management server publishes via LoginResponse / SyncResponse and fires +// two warning events at fixed lead times before expiry: an interactive +// T-WarningLead notification and a dismiss-gated T-FinalWarningLead +// fallback dialog. +// +// The deadline is an absolute wall-clock instant, so the watcher compares +// it against the wall clock on a ticker rather than arming a relative +// timer for it. A relative timer runs on the monotonic clock, which does +// not advance while a device is suspended: it fires once that much awake +// time has passed, which can be long after the deadline, and nothing +// re-evaluates when the device wakes up. Polling makes every tick after a +// resume (or after an NTP correction) see the real remaining time. +// +// The watcher is idempotent: Update may be called as often as the network +// map snapshots arrive. Repeating the same deadline is a no-op; a new +// deadline starts a fresh warning cycle. +// +// Warning firing is edge-detected. Each unique deadline value publishes +// each warning at most once. +package sessionwatch + +import ( + "errors" + "fmt" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + cProto "github.com/netbirdio/netbird/client/proto" +) + +const ( + maxPastHorizon = 30 * 24 * time.Hour + + // maxDeadlineHorizon caps how far in the future an accepted deadline + // can sit. A timestamp beyond this is almost certainly a protocol + // glitch, and silently tracking a 100-year deadline would hide the bug. + maxDeadlineHorizon = 10 * 365 * 24 * time.Hour + + // defaultEvalInterval is how often the tracked deadline is compared + // against the wall clock. The leads are minutes, so a coarse tick costs + // nothing in accuracy and keeps the wakeup cheap on battery-powered + // devices. + defaultEvalInterval = 10 * time.Second + + // WarningLead is how far before expiry the first (interactive) + // warning fires. Drives the T-10 OS notification with + // Extend/Dismiss actions. + WarningLead = 10 * time.Minute + + // FinalWarningLead is how far before expiry the fallback final + // warning fires. Drives the auto-opened SessionAboutToExpire dialog, + // but only when the user has not dismissed the T-WarningLead warning + // for the same deadline. Must be strictly less than WarningLead. + FinalWarningLead = 2 * time.Minute +) + +var ( + // ErrDeadlineBeforeEpoch is returned by Update when the supplied + // deadline pre-dates 1970-01-01. + ErrDeadlineBeforeEpoch = errors.New("session deadline before unix epoch") + + // ErrDeadlineTooFarFuture is returned by Update when the supplied + // deadline is more than maxDeadlineHorizon in the future. + ErrDeadlineTooFarFuture = errors.New("session deadline too far in the future") + + // ErrDeadlineInPast is returned by Update when the supplied deadline + // is more than maxPastHorizon in the past. + ErrDeadlineInPast = errors.New("session deadline in the past") +) + +// StatusRecorder is the side-effect surface the watcher drives on every +// state transition. Production wires this to peer.Status (SetSessionExpiresAt +// for deadline change/clear, PublishEvent for the two warnings); tests pass +// a fake recorder so the same surface is observable without an engine. +// +// While the watcher runs, it owns the deadline propagated to the recorder: +// every set, clear and sanity-check rejection routes the value through +// SetSessionExpiresAt, so the SubscribeStatus snapshot the UI reads can +// never drift from the watcher's timer state. (SetSessionExpiresAt fans +// out its own state-change notification, so no separate notify is needed.) +// The recorder is server-scoped and outlives this engine-scoped watcher; +// Close deliberately leaves the recorder value in place so transient engine +// restarts don't blank it — the client run loop clears it on real teardown. +// +// PublishEvent's signature mirrors peer.Status.PublishEvent: the watcher +// composes the metadata internally so the wire format (MetaSession*) is +// owned by sessionwatch, not the caller. +type StatusRecorder interface { + SetSessionExpiresAt(deadline time.Time) + PublishEvent( + severity cProto.SystemEvent_Severity, + category cProto.SystemEvent_Category, + message string, + userMessage string, + metadata map[string]string, + ) +} + +// Watcher observes the latest session deadline and fires two warnings +// before it expires: the interactive T-WarningLead notification, and the +// fallback T-FinalWarningLead dialog (suppressed when the user dismissed +// the first one for the same deadline). Safe for concurrent use. +type Watcher struct { + lead time.Duration + finalLead time.Duration + interval time.Duration + deadlineOnly bool // record the deadline, leave the warnings to the caller + + mu sync.Mutex + current time.Time + firedAt time.Time // deadline value the T-WarningLead warning last published for + finalFiredAt time.Time // deadline value the T-FinalWarningLead warning last published for + dismissedAt time.Time // deadline value the user dismissed via Dismiss(); gates the final warning + announcedAt time.Time // deadline value the recorder has been told about; gates publishing + closed bool + recorder StatusRecorder + nowFn func() time.Time + stop chan struct{} // closed to stop the evaluation loop; nil while it is not running + done chan struct{} // closed by the loop on its way out + wake chan struct{} // buffered nudge asking the loop to evaluate before its next tick +} + +// New returns a watcher with the package defaults WarningLead and +// FinalWarningLead. Pass nil for recorder to silence side effects (handy +// in unit tests that exercise sanity checks without observing the publish +// path). +func New(recorder StatusRecorder) *Watcher { + return NewWithLeads(WarningLead, FinalWarningLead, recorder) +} + +// NewWithLeads returns a watcher with custom lead times. Useful for tests. +// final must be strictly less than lead; otherwise the final warning takes +// over the whole warning window and the interactive notification never +// shows. A zero final lead disables the final warning entirely (see +// evaluate), leaving the interactive one as the only warning. +func NewWithLeads(lead, final time.Duration, recorder StatusRecorder) *Watcher { + return &Watcher{ + lead: lead, + finalLead: final, + interval: defaultEvalInterval, + recorder: recorder, + nowFn: time.Now, + } +} + +// NewDeadlineOnly returns a watcher that validates and records deadlines +// but publishes no warnings about them, and runs no evaluation loop to +// decide. Used where the deadline is handed on to something that schedules +// the warnings itself, such as the Android app. +func NewDeadlineOnly(recorder StatusRecorder) *Watcher { + w := New(recorder) + w.deadlineOnly = true + return w +} + +// Update sets the latest deadline. Pass the zero time to clear (e.g. when +// a Sync push from the server omits the field because login expiration +// was disabled). +// +// Same-value updates are no-ops. A different non-zero value resets the +// "already fired" guards and starts a fresh warning cycle, evaluated +// immediately so a deadline that already sits inside a warning window +// warns without waiting for the next tick. A deadline already in the past +// (within maxPastHorizon) is recorded as-is and warns nothing: the session +// has expired and consumers render it that way. +// +// Returns one of the sentinel Err* values when the deadline fails the +// sanity checks (pre-epoch, far future, or past beyond maxPastHorizon). +// In every error case the watcher first clears its state so it stays +// consistent with what the caller will push into its other sinks (e.g. +// applySessionDeadline forces a zero deadline into the status recorder +// after a non-nil error). +func (w *Watcher) Update(deadline time.Time) error { + w.mu.Lock() + if w.closed { + w.mu.Unlock() + return nil + } + + if deadline.IsZero() { + w.clearLocked() + return nil + } + + now := w.nowFn() + switch { + case deadline.Before(time.Unix(0, 0)): + w.clearLocked() + return fmt.Errorf("%w: %v", ErrDeadlineBeforeEpoch, deadline) + case deadline.After(now.Add(maxDeadlineHorizon)): + w.clearLocked() + return fmt.Errorf("%w: %v", ErrDeadlineTooFarFuture, deadline) + case deadline.Before(now.Add(-maxPastHorizon)): + w.clearLocked() + return fmt.Errorf("%w: %v (now=%v)", ErrDeadlineInPast, deadline, now) + } + + if deadline.Equal(w.current) { + w.mu.Unlock() + return nil + } + + w.current = deadline + // Reset every per-deadline guard so a refreshed deadline starts a fresh + // warning cycle: both edge triggers and the user Dismiss decision + // (the user agreed to the old deadline expiring; a new deadline + // restarts the contract). + w.firedAt = time.Time{} + w.finalFiredAt = time.Time{} + w.dismissedAt = time.Time{} + w.announcedAt = time.Time{} + + // Poll every accepted deadline, including one that reads as already + // expired: the clock may be running ahead of real time and get + // corrected later, and evaluate ignores a deadline that has genuinely + // passed anyway. + if !w.deadlineOnly { + w.startPollLocked() + } + recorder := w.recorder + w.mu.Unlock() + if recorder != nil { + recorder.SetSessionExpiresAt(deadline) + } + log.Infof("auth session deadline set to: %s (in %s)", deadline.Format(time.RFC3339), time.Until(deadline).Round(time.Second)) + + // Open the gate only once the recorder knows the new deadline, so a + // warning that refers to it can never reach consumers before the state + // change itself. A tick landing in between finds the gate shut. + w.mu.Lock() + if w.closed || !w.current.Equal(deadline) { + w.mu.Unlock() + return nil + } + w.announcedAt = deadline + wake := w.wake + w.mu.Unlock() + + // Hand the evaluation to the loop rather than running it here, so a + // deadline that already sits inside a warning window is published at + // once without this goroutine ever touching the recorder. + if wake != nil { + select { + case wake <- struct{}{}: + default: + } + } + return nil +} + +// Deadline returns the most recently observed deadline. Zero when no +// deadline is currently tracked. +func (w *Watcher) Deadline() time.Time { + w.mu.Lock() + defer w.mu.Unlock() + return w.current +} + +// Dismiss records the user's "Dismiss" action against the current deadline +// and suppresses the final warning for that deadline. Idempotent: repeated +// calls are no-ops. A subsequent Update with a fresh deadline resets the +// dismissal so the final-warning cycle starts over. +// +// No-op when the watcher holds no deadline or has been closed. +func (w *Watcher) Dismiss() { + w.mu.Lock() + defer w.mu.Unlock() + if w.closed || w.current.IsZero() { + return + } + if w.dismissedAt.Equal(w.current) { + return + } + w.dismissedAt = w.current + log.Infof("auth session final-warning dismissed for deadline %s", w.current.Format(time.RFC3339)) +} + +// Close stops the evaluation loop and waits for it to exit. Update calls +// after Close are ignored. The recorder keeps its deadline: the watcher is +// engine-scoped and closes on every engine restart (network change, +// sleep/wake, stream errors) while the SSO deadline stays valid across +// those, so clearing here would blank the UI's "expires in" row on every +// transient reconnect. The client run loop clears the server-scoped +// recorder when it exits for real (Down, profile switch, permanent login +// failure). +func (w *Watcher) Close() { + w.mu.Lock() + if w.closed { + // A concurrent Close is already tearing down. w.done outlives it, + // so this caller waits for the same loop rather than returning + // while a warning is still on its way to the recorder. + done := w.done + w.mu.Unlock() + if done != nil { + <-done + } + return + } + w.closed = true + w.current = time.Time{} + w.firedAt = time.Time{} + w.finalFiredAt = time.Time{} + w.dismissedAt = time.Time{} + w.announcedAt = time.Time{} + // Copy the channels out before releasing the lock: the loop takes w.mu + // on every tick, so waiting for it while holding the lock would + // deadlock. w.done stays on the receiver for the branch above. + stop, done := w.stop, w.done + w.stop, w.wake = nil, nil + w.mu.Unlock() + + if stop == nil { + return + } + close(stop) + <-done +} + +// clearLocked drops the tracked deadline and notifies the recorder so +// downstream consumers (SubscribeStatus stream, UI) drop their anchor. +// The caller must hold w.mu; this helper releases it before invoking +// the recorder. +func (w *Watcher) clearLocked() { + if w.current.IsZero() { + w.mu.Unlock() + return + } + w.current = time.Time{} + w.firedAt = time.Time{} + w.finalFiredAt = time.Time{} + w.dismissedAt = time.Time{} + w.announcedAt = time.Time{} + recorder := w.recorder + w.mu.Unlock() + if recorder != nil { + recorder.SetSessionExpiresAt(time.Time{}) + } + log.Infof("auth session deadline cleared") +} + +// startPollLocked starts the evaluation loop unless it is already +// running. The loop starts lazily on the first accepted deadline, so a +// client whose server never publishes a session expiry never pays for a +// ticker, and it runs until Close: the watcher is engine-scoped, and a +// cleared deadline is normally followed by a fresh one on the next sync. +// Caller must hold w.mu. +func (w *Watcher) startPollLocked() { + if w.stop != nil { + return + } + stop := make(chan struct{}) + done := make(chan struct{}) + wake := make(chan struct{}, 1) + w.stop, w.done, w.wake = stop, done, wake + go w.poll(stop, done, wake, w.interval) +} + +// poll re-evaluates the tracked deadline every interval, and as soon as a +// new deadline is announced. It is the only caller of evaluate, so Close +// waiting for it to exit is enough to know no warning is still on its way +// to the recorder. Its channels and interval are passed in rather than +// read off the receiver, so Close can clear them without racing it. +func (w *Watcher) poll(stop <-chan struct{}, done chan<- struct{}, wake <-chan struct{}, interval time.Duration) { + defer close(done) + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-stop: + return + case <-wake: + w.evaluate() + case <-ticker.C: + w.evaluate() + } + } +} + +// evaluate compares the tracked deadline against the wall clock and +// publishes whichever warning the remaining time calls for. Inside the +// final-warning window the interactive warning is stale, so the final one +// is published in its place: a device that resumes there never saw the +// T-WarningLead notification. +func (w *Watcher) evaluate() { + w.mu.Lock() + if w.closed || w.deadlineOnly || w.current.IsZero() { + w.mu.Unlock() + return + } + + deadline := w.current + if !w.announcedAt.Equal(deadline) { + // Update is still on its way to the recorder with this deadline. + w.mu.Unlock() + return + } + // Round(0) strips the monotonic reading so the comparison is wall + // clock on both sides, whether the deadline came off the wire or from + // a caller that derived it from time.Now. + remaining := deadline.Round(0).Sub(w.nowFn().Round(0)) + + switch { + case remaining <= 0: + // Already expired: the post-mortem SessionExpired flow owns it. + w.mu.Unlock() + case w.finalLead > 0 && remaining <= w.finalLead: + w.publishFinalLocked(deadline, remaining) + case remaining <= w.lead: + w.publishWarningLocked(deadline, remaining) + default: + w.mu.Unlock() + } +} + +// publishWarningLocked emits the interactive T-WarningLead warning, at +// most once per deadline value. Caller must hold w.mu; this helper +// releases it. +func (w *Watcher) publishWarningLocked(deadline time.Time, remaining time.Duration) { + if w.firedAt.Equal(deadline) { + w.mu.Unlock() + return + } + w.firedAt = deadline + recorder := w.recorder + w.mu.Unlock() + if recorder == nil { + return + } + log.Infof("auth session expiry soon warning fired for deadline %s (in %s)", + deadline.Format(time.RFC3339), remaining.Round(time.Second)) + publishWarning(recorder, deadline, false) +} + +// publishFinalLocked emits the final warning, at most once per deadline +// value and never once the user dismissed that deadline. It marks the +// interactive warning as handled too: the final window is open, so a +// "expires in WarningLead minutes" notification would be wrong. Caller +// must hold w.mu; this helper releases it. +func (w *Watcher) publishFinalLocked(deadline time.Time, remaining time.Duration) { + if w.finalFiredAt.Equal(deadline) || w.dismissedAt.Equal(deadline) { + w.mu.Unlock() + return + } + w.firedAt = deadline + w.finalFiredAt = deadline + recorder := w.recorder + w.mu.Unlock() + if recorder == nil { + return + } + log.Infof("auth session final-warning fired for deadline %s (in %s)", + deadline.Format(time.RFC3339), remaining.Round(time.Second)) + publishWarning(recorder, deadline, true) +} + +// publishWarning composes the SystemEvent for a watcher-fired warning and +// pushes it through the recorder. Severity is CRITICAL on both — bypassing +// the user's Notifications toggle is deliberate: missing the warning +// window forces the post-mortem SessionExpired flow (tunnel torn down, +// lock icon, manual re-login), which is the UX we are trying to avoid. +func publishWarning(recorder StatusRecorder, deadline time.Time, final bool) { + lead := WarningLead + message := "session expiry warning" + meta := map[string]string{ + MetaSessionWarning: "true", + MetaSessionExpiresAt: FormatExpiresAt(deadline), + } + if final { + lead = FinalWarningLead + message = "session expiry final warning" + meta[MetaSessionFinal] = "true" + } + meta[MetaSessionLeadMinutes] = FormatLeadMinutes(lead) + + recorder.PublishEvent( + cProto.SystemEvent_CRITICAL, + cProto.SystemEvent_AUTHENTICATION, + message, + "", + meta, + ) +} diff --git a/client/internal/auth/sessionwatch/watcher_test.go b/client/internal/auth/sessionwatch/watcher_test.go new file mode 100644 index 000000000..f0cc3ed88 --- /dev/null +++ b/client/internal/auth/sessionwatch/watcher_test.go @@ -0,0 +1,919 @@ +package sessionwatch + +import ( + "errors" + "sync" + "testing" + "time" + + cProto "github.com/netbirdio/netbird/client/proto" +) + +// fakeRecorder satisfies StatusRecorder and records every call so tests +// can observe what the watcher emits. SetSessionExpiresAt and PublishEvent +// land in the same ordered events slice (with the Kind distinguishing +// them) so tests that care about ordering still work. lastDeadline holds +// the most recent value passed to SetSessionExpiresAt so tests can assert +// the recorder ended up cleared/set as expected. +type fakeRecorder struct { + mu sync.Mutex + events []event + lastDeadline time.Time + // setDelay stalls SetSessionExpiresAt, widening the window in which a + // concurrent evaluation could publish a warning out of order. + setDelay time.Duration +} + +type eventKind int + +const ( + stateChange eventKind = iota + publish +) + +type event struct { + kind eventKind + // Set only for publish events. + severity cProto.SystemEvent_Severity + category cProto.SystemEvent_Category + message string + meta map[string]string +} + +// SetSessionExpiresAt mirrors peer.Status: a same-value write is a no-op, +// a real change records the new value and fans out a state-change (the +// production recorder calls notifyStateChange internally). The baseline +// is the zero time, so an initial clear before any deadline is set emits +// nothing — matching the real recorder. +func (r *fakeRecorder) SetSessionExpiresAt(deadline time.Time) { + r.mu.Lock() + delay := r.setDelay + r.mu.Unlock() + // Stall without the lock held, so a concurrent publish can still record. + time.Sleep(delay) + + r.mu.Lock() + defer r.mu.Unlock() + if r.lastDeadline.Equal(deadline) { + return + } + r.lastDeadline = deadline + r.events = append(r.events, event{kind: stateChange}) +} + +func (r *fakeRecorder) deadline() time.Time { + r.mu.Lock() + defer r.mu.Unlock() + return r.lastDeadline +} + +func (r *fakeRecorder) PublishEvent( + severity cProto.SystemEvent_Severity, + category cProto.SystemEvent_Category, + message string, + _ string, + metadata map[string]string, +) { + r.mu.Lock() + defer r.mu.Unlock() + r.events = append(r.events, event{ + kind: publish, + severity: severity, + category: category, + message: message, + meta: metadata, + }) +} + +func (r *fakeRecorder) snapshot() []event { + r.mu.Lock() + defer r.mu.Unlock() + out := make([]event, len(r.events)) + copy(out, r.events) + return out +} + +func (e event) isFinalWarning() bool { + return e.kind == publish && e.meta[MetaSessionFinal] == "true" +} + +func (e event) isWarning() bool { + return e.kind == publish && e.meta[MetaSessionWarning] == "true" && e.meta[MetaSessionFinal] != "true" +} + +func countWhere(events []event, pred func(event) bool) int { + n := 0 + for _, e := range events { + if pred(e) { + n++ + } + } + return n +} + +func waitForEvents(t *testing.T, r *fakeRecorder, want int) []event { + t.Helper() + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + if got := r.snapshot(); len(got) >= want { + return got + } + time.Sleep(5 * time.Millisecond) + } + got := r.snapshot() + t.Fatalf("timed out waiting for %d events, got %d: %+v", want, len(got), got) + return nil +} + +// testInterval keeps the ticker-driven tests fast; the leads they use are +// in the same millisecond scale. +const testInterval = 2 * time.Millisecond + +// newWatcher builds a watcher with the final warning disabled (finalLead=0), +// matching the lead-only behaviour the pre-final-warning tests assume. +func newWatcher(lead time.Duration, r *fakeRecorder) *Watcher { + return newWatcherWithLeads(lead, 0, r) +} + +// newWatcherWithLeads builds a watcher that evaluates on testInterval. The +// interval is set before Update, so the evaluation loop does not exist yet +// and the write cannot race it. +func newWatcherWithLeads(lead, final time.Duration, r StatusRecorder) *Watcher { + w := NewWithLeads(lead, final, r) + w.interval = testInterval + return w +} + +// fakeClock is the watcher's wall clock under test. Reads come from the +// evaluation goroutine while the test writes, so both go through the mutex. +type fakeClock struct { + mu sync.Mutex + t time.Time +} + +func newFakeClock(t time.Time) *fakeClock { + return &fakeClock{t: t} +} + +func (c *fakeClock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +// set jumps the clock, standing in for a resume from suspension or for an +// NTP correction. +func (c *fakeClock) set(t time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = t +} + +// settle waits out a handful of evaluation ticks so a test can assert that +// nothing was published. +func settle() { + time.Sleep(20 * testInterval) +} + +func TestUpdateZeroBeforeAnythingIsNoop(t *testing.T) { + r := &fakeRecorder{} + w := newWatcher(50*time.Millisecond, r) + defer w.Close() + + _ = w.Update(time.Time{}) + + if got := r.snapshot(); len(got) != 0 { + t.Fatalf("expected no events on initial zero, got %+v", got) + } +} + +func TestUpdateNonZeroFiresStateChange(t *testing.T) { + r := &fakeRecorder{} + w := newWatcher(50*time.Millisecond, r) + defer w.Close() + + d := time.Now().Add(time.Hour) + _ = w.Update(d) + + events := waitForEvents(t, r, 1) + if events[0].kind != stateChange { + t.Fatalf("expected stateChange, got %+v", events[0]) + } + if !w.Deadline().Equal(d) { + t.Fatalf("deadline mismatch: %v vs %v", w.Deadline(), d) + } +} + +func TestSameDeadlineIsNoop(t *testing.T) { + r := &fakeRecorder{} + w := newWatcher(50*time.Millisecond, r) + defer w.Close() + + d := time.Now().Add(time.Hour) + _ = w.Update(d) + _ = w.Update(d) + _ = w.Update(d) + + events := waitForEvents(t, r, 1) + if len(events) != 1 { + t.Fatalf("expected exactly 1 event for repeated same deadline, got %d: %+v", len(events), events) + } +} + +func TestWarningFiresOnceWithinLeadWindow(t *testing.T) { + r := &fakeRecorder{} + lead := 50 * time.Millisecond + w := newWatcher(lead, r) + defer w.Close() + + // Deadline 80ms out — warning should fire after ~30ms. + d := time.Now().Add(80 * time.Millisecond) + _ = w.Update(d) + + events := waitForEvents(t, r, 2) + if events[0].kind != stateChange { + t.Fatalf("event[0] should be stateChange, got %+v", events[0]) + } + if !events[1].isWarning() { + t.Fatalf("event[1] should be a warning publish, got %+v", events[1]) + } +} + +func TestWarningFiresImmediatelyWhenAlreadyInsideWindow(t *testing.T) { + r := &fakeRecorder{} + w := newWatcher(time.Hour, r) // lead > delta => fire immediately + defer w.Close() + + d := time.Now().Add(10 * time.Millisecond) + _ = w.Update(d) + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected immediate warning publish, got %+v", events[1]) + } +} + +func TestNewDeadlineCancelsPriorTimer(t *testing.T) { + r := &fakeRecorder{} + lead := 50 * time.Millisecond + w := newWatcher(lead, r) + defer w.Close() + + first := time.Now().Add(80 * time.Millisecond) // would fire warning ~30ms in + _ = w.Update(first) + + // Replace with a far-future deadline before the warning fires. + time.Sleep(5 * time.Millisecond) + second := time.Now().Add(time.Hour) + _ = w.Update(second) + + // Wait past when first's warning would have fired. + time.Sleep(80 * time.Millisecond) + + if n := countWhere(r.snapshot(), event.isWarning); n != 0 { + t.Fatalf("warning fired for cancelled deadline: %+v", r.snapshot()) + } +} + +func TestRefreshAfterFireArmsNewWarning(t *testing.T) { + r := &fakeRecorder{} + lead := 150 * time.Millisecond + w := newWatcher(lead, r) + defer w.Close() + + // Warning fires ~20ms in; the deadline itself stays 150ms away so the + // replacement below lands well before it. + first := time.Now().Add(170 * time.Millisecond) + _ = w.Update(first) + + // Wait for stateChange + warning of the first cycle. + waitForEvents(t, r, 2) + + // Simulate a successful extend: brand new deadline. + second := time.Now().Add(60 * time.Millisecond) + _ = w.Update(second) + + // 4 events total: stateChange, warning (first), stateChange, warning (second). + events := waitForEvents(t, r, 4) + if events[2].kind != stateChange { + t.Fatalf("event[2] should be stateChange for the new deadline, got %+v", events[2]) + } + if !events[3].isWarning() { + t.Fatalf("event[3] should be a warning publish for the new deadline, got %+v", events[3]) + } +} + +func TestUpdateZeroAfterNonZeroClearsState(t *testing.T) { + r := &fakeRecorder{} + w := newWatcher(time.Hour, r) + defer w.Close() + + d := time.Now().Add(2 * time.Hour) + _ = w.Update(d) + waitForEvents(t, r, 1) + + _ = w.Update(time.Time{}) + + events := waitForEvents(t, r, 2) + if events[1].kind != stateChange { + t.Fatalf("expected stateChange on clear, got %+v", events[1]) + } + if !w.Deadline().IsZero() { + t.Fatalf("Deadline should be zero after clear") + } +} + +func TestUpdateRejectsBeforeEpoch(t *testing.T) { + r := &fakeRecorder{} + w := newWatcher(50*time.Millisecond, r) + defer w.Close() + + good := time.Now().Add(time.Hour) + if err := w.Update(good); err != nil { + t.Fatalf("seed Update: %v", err) + } + + err := w.Update(time.Unix(-100, 0)) + if !errors.Is(err, ErrDeadlineBeforeEpoch) { + t.Fatalf("want ErrDeadlineBeforeEpoch, got %v", err) + } + if !w.Deadline().IsZero() { + t.Fatalf("rejected pre-epoch update must clear deadline; got %v", w.Deadline()) + } +} + +func TestUpdateRejectsTooFarFuture(t *testing.T) { + r := &fakeRecorder{} + w := newWatcher(50*time.Millisecond, r) + defer w.Close() + + good := time.Now().Add(time.Hour) + if err := w.Update(good); err != nil { + t.Fatalf("seed Update: %v", err) + } + + err := w.Update(time.Now().Add(50 * 365 * 24 * time.Hour)) + if !errors.Is(err, ErrDeadlineTooFarFuture) { + t.Fatalf("want ErrDeadlineTooFarFuture, got %v", err) + } + if !w.Deadline().IsZero() { + t.Fatalf("rejected far-future update must clear deadline; got %v", w.Deadline()) + } +} + +func TestUpdateRecentPastRecordedAsExpired(t *testing.T) { + r := &fakeRecorder{} + w := newWatcher(50*time.Millisecond, r) + defer w.Close() + + d := time.Now().Add(-1 * time.Hour) + if err := w.Update(d); err != nil { + t.Fatalf("recent-past Update should succeed, got %v", err) + } + if !w.Deadline().Equal(d) { + t.Fatalf("expected deadline to be recorded, got %v want %v", w.Deadline(), d) + } + if got := r.deadline(); !got.Equal(d) { + t.Fatalf("recorder deadline = %v, want %v", got, d) + } + + time.Sleep(80 * time.Millisecond) + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("no warning events may fire for an already-past deadline, got %+v", r.snapshot()) + } +} + +func TestUpdateAncientPastRejected(t *testing.T) { + r := &fakeRecorder{} + w := newWatcher(50*time.Millisecond, r) + defer w.Close() + + good := time.Now().Add(time.Hour) + if err := w.Update(good); err != nil { + t.Fatalf("seed Update: %v", err) + } + // Drain the stateChange from the seed. + waitForEvents(t, r, 1) + + err := w.Update(time.Now().Add(-31 * 24 * time.Hour)) + if !errors.Is(err, ErrDeadlineInPast) { + t.Fatalf("want ErrDeadlineInPast, got %v", err) + } + if !w.Deadline().IsZero() { + t.Fatalf("rejected ancient-past update must clear the deadline, got %v", w.Deadline()) + } + events := waitForEvents(t, r, 2) + if events[1].kind != stateChange { + t.Fatalf("expected stateChange on clear, got %+v", events[1]) + } +} + +func TestCloseSilencesUpdates(t *testing.T) { + r := &fakeRecorder{} + w := newWatcher(50*time.Millisecond, r) + w.Close() + + if err := w.Update(time.Now().Add(time.Hour)); err != nil { + t.Fatalf("Update after Close: want nil, got %v", err) + } + if got := r.snapshot(); len(got) != 0 { + t.Fatalf("expected no events after Close, got %+v", got) + } +} + +// TestCloseKeepsRecorderDeadline pins the reconnect-flap fix: the watcher +// closes on every engine restart (network change, sleep/wake) while the +// SSO deadline stays valid across those, so Close must leave the +// server-scoped recorder's value in place. The client run loop clears the +// recorder when it exits for real. +func TestCloseKeepsRecorderDeadline(t *testing.T) { + r := &fakeRecorder{} + w := newWatcher(time.Hour, r) + + d := time.Now().Add(2 * time.Hour) + if err := w.Update(d); err != nil { + t.Fatalf("seed Update: %v", err) + } + if got := r.deadline(); !got.Equal(d) { + t.Fatalf("recorder deadline after Update = %v, want %v", got, d) + } + + w.Close() + + if got := r.deadline(); !got.Equal(d) { + t.Fatalf("recorder deadline after Close = %v, want %v", got, d) + } +} + +// TestCloseWithoutDeadlineLeavesRecorderUntouched guards the symmetric +// case: closing a watcher that never held a deadline must not emit a +// redundant clear (the recorder may legitimately hold a value written by +// some other path; the watcher only owns what it set). +func TestCloseWithoutDeadlineLeavesRecorderUntouched(t *testing.T) { + r := &fakeRecorder{} + w := newWatcher(time.Hour, r) + + w.Close() + + if got := r.snapshot(); len(got) != 0 { + t.Fatalf("expected no events from Close on an empty watcher, got %+v", got) + } +} + +func TestFinalWarningFiresAfterRegularWarning(t *testing.T) { + r := &fakeRecorder{} + // Warning fires at deadline-80ms, final at deadline-30ms. + w := newWatcherWithLeads(80*time.Millisecond, 30*time.Millisecond, r) + defer w.Close() + + d := time.Now().Add(100 * time.Millisecond) + _ = w.Update(d) + + // Expect stateChange + warning + final-warning. + events := waitForEvents(t, r, 3) + + if countWhere(events, func(e event) bool { return e.kind == stateChange }) != 1 { + t.Fatalf("expected exactly 1 stateChange, got %+v", events) + } + if countWhere(events, event.isWarning) != 1 { + t.Fatalf("expected exactly 1 warning publish, got %+v", events) + } + if countWhere(events, event.isFinalWarning) != 1 { + t.Fatalf("expected exactly 1 final-warning publish, got %+v", events) + } + + // Warning must precede final (same deadline, longer lead fires first). + var wIdx, fIdx int + for i, e := range events { + switch { + case e.isWarning(): + wIdx = i + case e.isFinalWarning(): + fIdx = i + } + } + if wIdx > fIdx { + t.Fatalf("warning must publish before final-warning, got order %+v", events) + } +} + +func TestDismissSuppressesFinalWarning(t *testing.T) { + r := &fakeRecorder{} + w := newWatcherWithLeads(80*time.Millisecond, 30*time.Millisecond, r) + defer w.Close() + + d := time.Now().Add(100 * time.Millisecond) + _ = w.Update(d) + + // Wait for the warning publish so we know we're inside the warning + // window, then dismiss before the final timer would fire. + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + if countWhere(r.snapshot(), event.isWarning) >= 1 { + break + } + time.Sleep(2 * time.Millisecond) + } + if countWhere(r.snapshot(), event.isWarning) < 1 { + t.Fatalf("warning did not publish in time, events=%+v", r.snapshot()) + } + + w.Dismiss() + + // Now wait past when the final would have fired. + time.Sleep(120 * time.Millisecond) + + if n := countWhere(r.snapshot(), event.isFinalWarning); n != 0 { + t.Fatalf("final-warning published after Dismiss(), events=%+v", r.snapshot()) + } +} + +func TestDismissResetByNewDeadline(t *testing.T) { + r := &fakeRecorder{} + w := newWatcherWithLeads(80*time.Millisecond, 30*time.Millisecond, r) + defer w.Close() + + first := time.Now().Add(100 * time.Millisecond) + _ = w.Update(first) + + // Dismiss against the first deadline. + w.Dismiss() + + // Replace with a fresh deadline before the first's timers complete. + time.Sleep(10 * time.Millisecond) + second := time.Now().Add(100 * time.Millisecond) + _ = w.Update(second) + + // The second cycle must publish a final-warning (the dismiss state + // did not carry over). + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + if countWhere(r.snapshot(), event.isFinalWarning) >= 1 { + break + } + time.Sleep(5 * time.Millisecond) + } + if countWhere(r.snapshot(), event.isFinalWarning) < 1 { + t.Fatalf("final-warning did not publish on fresh deadline after Dismiss reset, events=%+v", r.snapshot()) + } +} + +func TestDismissBeforeUpdateIsNoop(t *testing.T) { + r := &fakeRecorder{} + w := newWatcherWithLeads(80*time.Millisecond, 30*time.Millisecond, r) + defer w.Close() + + // No deadline tracked yet; Dismiss must be a no-op (no panic, no state). + w.Dismiss() + + d := time.Now().Add(100 * time.Millisecond) + _ = w.Update(d) + + // Final warning should still publish — Dismiss only acts on the current + // deadline, and there was none at the time of the call. + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + if countWhere(r.snapshot(), event.isFinalWarning) >= 1 { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("final-warning did not publish after no-op pre-Update Dismiss, events=%+v", r.snapshot()) +} + +// The tests below drive the watcher's wall clock directly. The deadline sits +// an hour out in real time and the fake clock jumps, standing in for a device +// that was suspended and resumed somewhere inside — or past — the warning +// windows. The evaluation loop is what reacts to the jump, so these exercise +// the same path production takes on a resume. +func newResumeWatcher(t *testing.T, r *fakeRecorder) (*Watcher, *fakeClock, time.Time) { + t.Helper() + + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + start := time.Now() + clock := newFakeClock(start) + // Set before Update: the evaluation loop does not exist yet. + w.nowFn = clock.now + t.Cleanup(w.Close) + + deadline := start.Add(time.Hour).Round(0) + if err := w.Update(deadline); err != nil { + t.Fatalf("Update: %v", err) + } + return w, clock, deadline +} + +func TestResumeInsideWarningWindowWarns(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(-5 * time.Minute)) + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected the interactive warning after the resume, got %+v", events[1]) + } + if n := countWhere(events, event.isFinalWarning); n != 0 { + t.Fatalf("final-warning must wait for its own window, got %d: %+v", n, events) + } +} + +func TestResumeInsideFinalWindowSendsFinalWarningOnly(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(-time.Minute)) + + events := waitForEvents(t, r, 2) + if !events[1].isFinalWarning() { + t.Fatalf("expected the final warning after the resume, got %+v", events[1]) + } + settle() + if n := countWhere(r.snapshot(), event.isWarning); n != 0 { + t.Fatalf("the interactive warning is stale inside the final window, got %d: %+v", n, r.snapshot()) + } +} + +func TestResumePastDeadlinePublishesNothing(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(time.Minute)) + + settle() + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("an expired session must not warn, got %d publishes: %+v", n, r.snapshot()) + } +} + +func TestWarningPublishesOncePerDeadline(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(-5 * time.Minute)) + waitForEvents(t, r, 2) + + // Many ticks pass inside the same window. + settle() + if n := countWhere(r.snapshot(), event.isWarning); n != 1 { + t.Fatalf("expected exactly 1 warning publish across ticks, got %d: %+v", n, r.snapshot()) + } +} + +// TestWarningRecoversFromAClockRunningAhead covers the device that boots +// before NTP has corrected it: the deadline looks long gone, nothing is +// published, and the warning still arrives once the clock is fixed. +func TestWarningRecoversFromAClockRunningAhead(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(2 * time.Hour)) + settle() + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("a deadline that looks expired must not warn, got %d publishes: %+v", n, r.snapshot()) + } + + clock.set(deadline.Add(-5 * time.Minute)) + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected the warning once the clock was corrected, got %+v", events[1]) + } +} + +func TestResumeInsideFinalWindowRespectsDismiss(t *testing.T) { + r := &fakeRecorder{} + w, clock, deadline := newResumeWatcher(t, r) + + // The user dismissed the warning for this deadline before the device + // was suspended; resuming inside the final window must not reopen it. + w.Dismiss() + clock.set(deadline.Add(-time.Minute)) + + settle() + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("a dismissed deadline must not warn on resume, got %d: %+v", n, r.snapshot()) + } +} + +func TestCloseStopsTheEvaluationLoop(t *testing.T) { + r := &fakeRecorder{} + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + start := time.Now() + clock := newFakeClock(start) + w.nowFn = clock.now + + deadline := start.Add(time.Hour).Round(0) + if err := w.Update(deadline); err != nil { + t.Fatalf("Update: %v", err) + } + w.Close() + + clock.set(deadline.Add(-5 * time.Minute)) + settle() + + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("a closed watcher must not publish, got %d: %+v", n, r.snapshot()) + } +} + +func TestDeadlineOnlyRecordsDeadlineWithoutWarnings(t *testing.T) { + r := &fakeRecorder{} + w := NewDeadlineOnly(r) + defer w.Close() + + // With the default leads this deadline sits inside the final-warning + // window, so a watcher that warns at all would publish on the spot. + d := time.Now().Add(50 * time.Millisecond).Round(0) + if err := w.Update(d); err != nil { + t.Fatalf("Update: %v", err) + } + if got := r.deadline(); !got.Equal(d) { + t.Fatalf("expected recorder deadline %v, got %v", d, got) + } + + time.Sleep(100 * time.Millisecond) + + events := r.snapshot() + if got := countWhere(events, func(e event) bool { return e.kind == publish }); got != 0 { + t.Fatalf("expected no publish in deadline-only mode, got %d: %+v", got, events) + } + w.mu.Lock() + polling := w.stop != nil + w.mu.Unlock() + if polling { + t.Fatal("expected no evaluation loop in deadline-only mode") + } +} + +func TestDeadlineOnlyStillRejectsOutOfRangeDeadlines(t *testing.T) { + r := &fakeRecorder{} + w := NewDeadlineOnly(r) + defer w.Close() + + if err := w.Update(time.Now().Add(time.Hour)); err != nil { + t.Fatalf("Update: %v", err) + } + + err := w.Update(time.Now().Add(-maxPastHorizon - time.Hour)) + if !errors.Is(err, ErrDeadlineInPast) { + t.Fatalf("expected ErrDeadlineInPast, got %v", err) + } + if got := r.deadline(); !got.IsZero() { + t.Fatalf("expected recorder cleared after rejection, got %v", got) + } +} + +// TestClockAheadAtUpdateStillWarnsOnceCorrected covers a client that connects +// while its clock runs ahead of real time: the deadline reads as already +// expired when it arrives, so nothing publishes, and the warning must still +// come once NTP pulls the clock back. +func TestClockAheadAtUpdateStillWarnsOnceCorrected(t *testing.T) { + r := &fakeRecorder{} + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + deadline := time.Now().Add(time.Hour).Round(0) + // Two hours past the deadline from the device's point of view, well + // inside maxPastHorizon, so Update accepts and records it. + clock := newFakeClock(deadline.Add(2 * time.Hour)) + w.nowFn = clock.now + t.Cleanup(w.Close) + + if err := w.Update(deadline); err != nil { + t.Fatalf("Update: %v", err) + } + + settle() + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("a deadline that reads as expired must not warn, got %d: %+v", n, r.snapshot()) + } + + clock.set(deadline.Add(-5 * time.Minute)) + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected the warning once the clock was corrected, got %+v", events[1]) + } +} + +// TestWarningNeverPrecedesTheDeadlineStateChange pins the ordering Update +// documents: consumers learn the new deadline before they see a warning that +// refers to it. The deadline here already sits inside the warning window, and +// the recorder stalls, so the evaluation loop gets many chances to publish +// while Update is still announcing. +func TestWarningNeverPrecedesTheDeadlineStateChange(t *testing.T) { + r := &fakeRecorder{setDelay: 50 * time.Millisecond} + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + deadline := time.Now().Add(time.Hour).Round(0) + clock := newFakeClock(deadline.Add(-5 * time.Minute)) + w.nowFn = clock.now + t.Cleanup(w.Close) + + if err := w.Update(deadline); err != nil { + t.Fatalf("Update: %v", err) + } + + events := waitForEvents(t, r, 2) + if events[0].kind != stateChange { + t.Fatalf("event[0] should be the deadline state change, got %+v", events) + } + if !events[1].isWarning() { + t.Fatalf("event[1] should be the warning, got %+v", events) + } +} + +// TestUpdateWakesTheLoopWithoutWaitingForATick pins the hand-off: Update +// publishes nothing itself, it nudges the evaluation loop, so a deadline that +// already sits inside a warning window is warned about at once even though the +// ticker here would not fire for an hour. +func TestUpdateWakesTheLoopWithoutWaitingForATick(t *testing.T) { + r := &fakeRecorder{} + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + w.interval = time.Hour + t.Cleanup(w.Close) + + d := time.Now().Add(5 * time.Minute).Round(0) + if err := w.Update(d); err != nil { + t.Fatalf("Update: %v", err) + } + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected the warning on the wake-up, got %+v", events[1]) + } +} + +// blockingRecorder holds a publish open until the test releases it, so the +// evaluation loop can be parked mid-publish while Close is called. +type blockingRecorder struct { + fakeRecorder + entered chan struct{} + release chan struct{} +} + +func (r *blockingRecorder) PublishEvent( + severity cProto.SystemEvent_Severity, + category cProto.SystemEvent_Category, + message string, + userMessage string, + metadata map[string]string, +) { + select { + case r.entered <- struct{}{}: + default: + } + <-r.release + r.fakeRecorder.PublishEvent(severity, category, message, userMessage, metadata) +} + +// TestConcurrentCloseWaitsForTheLoop pins the contract for the caller that +// loses the race: Close returns only once the loop is done, whichever of the +// two calls got there first. +func TestConcurrentCloseWaitsForTheLoop(t *testing.T) { + r := &blockingRecorder{entered: make(chan struct{}, 1), release: make(chan struct{})} + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(r.release) }) } + + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + w.interval = time.Hour + // Order matters: Close waits for the loop, which stays parked in the + // publish until the release, so a t.Fatal below would hang the cleanup + // instead of reporting the failure. + t.Cleanup(func() { + release() + w.Close() + }) + + d := time.Now().Add(5 * time.Minute).Round(0) + if err := w.Update(d); err != nil { + t.Fatalf("Update: %v", err) + } + + // The loop is now parked inside the warning publish. + select { + case <-r.entered: + case <-time.After(2 * time.Second): + t.Fatal("the loop never reached the publish") + } + + first := make(chan struct{}) + second := make(chan struct{}) + go func() { w.Close(); close(first) }() + go func() { w.Close(); close(second) }() + + select { + case <-first: + t.Fatal("Close returned while the loop was still publishing") + case <-second: + t.Fatal("Close returned while the loop was still publishing") + case <-time.After(100 * time.Millisecond): + } + + release() + for _, done := range []chan struct{}{first, second} { + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("Close did not return after the publish completed") + } + } +} diff --git a/client/internal/auth/util.go b/client/internal/auth/util.go index 31c81d701..1800584a2 100644 --- a/client/internal/auth/util.go +++ b/client/internal/auth/util.go @@ -20,14 +20,26 @@ func randomBytesInHex(count int) (string, error) { return hex.EncodeToString(buf), nil } -// isValidAccessToken is a simple validation of the access token -func isValidAccessToken(token string, audience string) error { +// validateTokenAudience checks that the token is a well-formed JWT whose +// audience claim matches the expected audience. +// +// It does NOT verify the token's cryptographic signature and therefore must not +// be treated as an authenticity check. The token is obtained by the client +// directly from the IdP token endpoint over TLS, and its signature is verified +// server-side by the management server against the IdP's JWKS +// (see shared/auth/jwt/validator.go). This function is only a client-side +// sanity check that the returned token targets the expected audience. +func validateTokenAudience(token string, audience string) error { if token == "" { return fmt.Errorf("token received is empty") } - encodedClaims := strings.Split(token, ".")[1] - claimsString, err := base64.RawURLEncoding.DecodeString(encodedClaims) + parts := strings.Split(token, ".") + if len(parts) != 3 { + return fmt.Errorf("token is not a well-formed JWT") + } + + claimsString, err := base64.RawURLEncoding.DecodeString(parts[1]) if err != nil { return err } diff --git a/client/internal/auth/util_test.go b/client/internal/auth/util_test.go new file mode 100644 index 000000000..7f225bb86 --- /dev/null +++ b/client/internal/auth/util_test.go @@ -0,0 +1,108 @@ +package auth + +import ( + "encoding/base64" + "encoding/json" + "testing" +) + +// makeJWT builds an unsigned JWT-shaped string (header.payload.signature) with +// the given claims payload. The signature part is arbitrary because +// validateTokenAudience intentionally does not verify it. +func makeJWT(t *testing.T, claims map[string]interface{}) string { + t.Helper() + header := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"RS256","typ":"JWT"}`)) + payloadBytes, err := json.Marshal(claims) + if err != nil { + t.Fatalf("marshal claims: %v", err) + } + payload := base64.RawURLEncoding.EncodeToString(payloadBytes) + return header + "." + payload + ".unverified-signature" +} + +func TestValidateTokenAudience(t *testing.T) { + tests := []struct { + name string + token string + audience string + wantErr bool + }{ + { + name: "empty token", + token: "", + audience: "netbird", + wantErr: true, + }, + { + name: "not a JWT - no dots", + token: "notajwt", + audience: "netbird", + wantErr: true, + }, + { + name: "not a JWT - two parts only", + token: "header.payload", + audience: "netbird", + wantErr: true, + }, + { + name: "matching string audience", + token: makeJWT(t, map[string]interface{}{"aud": "netbird"}), + audience: "netbird", + wantErr: false, + }, + { + name: "mismatching string audience", + token: makeJWT(t, map[string]interface{}{"aud": "other"}), + audience: "netbird", + wantErr: true, + }, + { + name: "matching audience in array", + token: makeJWT(t, map[string]interface{}{"aud": []interface{}{"other", "netbird"}}), + audience: "netbird", + wantErr: false, + }, + { + name: "mismatching audience array", + token: makeJWT(t, map[string]interface{}{"aud": []interface{}{"a", "b"}}), + audience: "netbird", + wantErr: true, + }, + { + name: "missing audience claim", + token: makeJWT(t, map[string]interface{}{"sub": "user"}), + audience: "netbird", + wantErr: true, + }, + { + name: "invalid base64 payload", + token: "header.!!!not-base64!!!.sig", + audience: "netbird", + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := validateTokenAudience(tc.token, tc.audience) + if tc.wantErr && err == nil { + t.Fatalf("expected error, got nil") + } + if !tc.wantErr && err != nil { + t.Fatalf("expected no error, got %v", err) + } + }) + } +} + +// TestValidateTokenAudienceNoPanic guards the regression where a non-empty +// token without the JWT dot structure caused an index-out-of-range panic. +func TestValidateTokenAudienceNoPanic(t *testing.T) { + inputs := []string{"a", ".", "a.", "aaaa", "no-dots-here"} + for _, in := range inputs { + if err := validateTokenAudience(in, "netbird"); err == nil { + t.Fatalf("expected error for malformed token %q, got nil", in) + } + } +} diff --git a/client/internal/certproof/collect.go b/client/internal/certproof/collect.go new file mode 100644 index 000000000..310fd2b19 --- /dev/null +++ b/client/internal/certproof/collect.go @@ -0,0 +1,168 @@ +package certproof + +import ( + "context" + "crypto" + "crypto/sha256" + "crypto/x509" + "time" + + log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// Collect answers the certificate challenges in checks: for each challenge it picks a +// stored certificate that chains to the challenge's CAs and signs the nonce with its +// key. The same certificate is proven once even if several checks accept it. +func Collect(ctx context.Context, store Store, checks []*proto.Checks, peerKey []byte) []certposture.Proof { + challenges := certificateChallenges(checks) + if len(challenges) == 0 { + logNoChallenges(checks) + return nil + } + return CollectChallenges(ctx, store, challenges, peerKey) +} + +func logNoChallenges(checks []*proto.Checks) { + if len(checks) > 0 { + log.Debugf("certificate posture: %d posture checks received, none carries a certificate challenge", len(checks)) + } +} + +// CollectChallenges answers challenges already extracted from the posture checks, so a +// caller that ships them across a process boundary reuses the same matching and signing. +// Only nonces of the size management issues are signed, for a peer key of the size of +// ours, so the keys behind the store never sign arbitrary caller-chosen data. +func CollectChallenges(ctx context.Context, store Store, challenges []*proto.CertificateChallenge, peerKey []byte) []certposture.Proof { + if len(peerKey) != wgtypes.KeyLen { + log.Warnf("certificate posture: refusing to sign for a %d byte peer key", len(peerKey)) + return nil + } + challenges = wellFormed(challenges) + if len(challenges) == 0 { + return nil + } + log.Debugf("certificate posture: answering %d certificate challenges from store %T", len(challenges), store) + + candidates, err := store.Candidates(ctx) + if err != nil { + log.Warnf("failed loading certificates for posture checks: %v", err) + return nil + } + if len(candidates) == 0 { + log.Debug("certificate posture: certificate store holds no candidates, no proof will be sent") + return nil + } + log.Debugf("certificate posture: store holds %d candidate certificates", len(candidates)) + + now := time.Now() + proven := make(map[[sha256.Size]byte]struct{}) + var proofs []certposture.Proof + for i, challenge := range challenges { + roots, err := certposture.ParseCAs(challenge.GetCaCertificates()) + if err != nil { + log.Warnf("skipping certificate challenge with invalid CA certificates: %v", err) + continue + } + log.Debugf("certificate posture: challenge %d accepts %d CA certificates, nonce is %d bytes", i, len(challenge.GetCaCertificates()), len(challenge.GetNonce())) + + matched := false + for _, candidate := range candidates { + if len(candidate.Chain) == 0 { + continue + } + leaf := candidate.Chain[0] + chain, err := certposture.VerifiedChain(leaf, candidate.issuers(), roots, now) + if err != nil { + log.Debugf("certificate posture: challenge %d rejected %q issued by %q: %v", i, leaf.Subject, leaf.Issuer, err) + continue + } + matched = true + + // The same leaf can chain to different CAs for different challenges, and + // management checks each chain against each check's CAs, so a proof is + // deduplicated by its whole chain rather than by its leaf. + fingerprint := chainFingerprint(chain) + if _, done := proven[fingerprint]; done { + log.Debugf("certificate posture: challenge %d matched %q, already proven for an earlier challenge", i, leaf.Subject) + break + } + proof, err := prove(candidate.Signer, chain, challenge.GetNonce(), peerKey) + if err != nil { + log.Warnf("failed signing certificate proof for %s: %v", leaf.Subject, err) + continue + } + log.Debugf("certificate posture: challenge %d proven by %q with %s, signature %d bytes, chain of %d", i, leaf.Subject, proof.SigAlg, len(proof.Signature), len(proof.Chain)) + proven[fingerprint] = struct{}{} + proofs = append(proofs, proof) + break + } + if !matched { + log.Debugf("certificate posture: challenge %d matched none of the %d candidates", i, len(candidates)) + } + } + log.Debugf("certificate posture: %d challenges produced %d proofs", len(challenges), len(proofs)) + return proofs +} + +// HasChallenges reports whether any of checks asks for a certificate proof. +func HasChallenges(checks []*proto.Checks) bool { + return len(certificateChallenges(checks)) > 0 +} + +func certificateChallenges(checks []*proto.Checks) []*proto.CertificateChallenge { + var challenges []*proto.CertificateChallenge + for _, check := range checks { + if challenge := check.GetCertificateChallenge(); challenge != nil { + challenges = append(challenges, challenge) + } + } + return wellFormed(challenges) +} + +// wellFormed drops challenges whose nonce is not one management could have issued. +func wellFormed(challenges []*proto.CertificateChallenge) []*proto.CertificateChallenge { + var kept []*proto.CertificateChallenge + for _, challenge := range challenges { + if len(challenge.GetNonce()) != certposture.NonceSize { + log.Debugf("certificate posture: skipping challenge with a %d byte nonce", len(challenge.GetNonce())) + continue + } + kept = append(kept, challenge) + } + return kept +} + +func prove(signer crypto.Signer, chain []*x509.Certificate, nonce, peerKey []byte) (certposture.Proof, error) { + sigAlg, sig, err := certposture.Sign(signer, nonce, peerKey) + if err != nil { + return certposture.Proof{}, err + } + der := make([][]byte, 0, len(chain)) + for _, cert := range chain { + der = append(der, cert.Raw) + } + return certposture.Proof{Nonce: nonce, Chain: der, SigAlg: sigAlg, Signature: sig}, nil +} + +func chainFingerprint(chain []*x509.Certificate) [sha256.Size]byte { + ders := make([][]byte, 0, len(chain)) + for _, cert := range chain { + ders = append(ders, cert.Raw) + } + return chainDERFingerprint(ders) +} + +// chainDERFingerprint identifies a chain of DER certificates by the hashes of all of them, +// in order. +func chainDERFingerprint(chain [][]byte) [sha256.Size]byte { + buf := make([]byte, 0, len(chain)*sha256.Size) + for _, der := range chain { + certHash := sha256.Sum256(der) + buf = append(buf, certHash[:]...) + } + return sha256.Sum256(buf) +} diff --git a/client/internal/certproof/collect_darwin.go b/client/internal/certproof/collect_darwin.go new file mode 100644 index 000000000..cc21563c1 --- /dev/null +++ b/client/internal/certproof/collect_darwin.go @@ -0,0 +1,137 @@ +//go:build !ios + +package certproof + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +const helperTimeout = 30 * time.Second + +// userHelperBackoff holds off asking a user's keychain again after it proved nothing. +var userHelperBackoff = newHelperBackoff() + +// CollectProofs answers the certificate challenges in checks from every store this Mac +// can reach. The root daemon reads the System keychain itself, which is where MDM +// installs device identities, and reaches the console user's login keychain only by +// launching a helper into that user's session. A Mac sitting at the login window +// therefore yields device proofs alone. +func CollectProofs(ctx context.Context, checks []*proto.Checks, peerKey []byte, cfg Config) []certposture.Proof { + challenges := certificateChallenges(checks) + if len(challenges) == 0 { + logNoChallenges(checks) + return nil + } + + // A helper already runs inside the user's session, so it reads its own keychain + // directly and must never launch another one. + if os.Geteuid() != 0 { + return CollectChallenges(ctx, DefaultStore(), challenges, peerKey) + } + + proofs := CollectChallenges(ctx, DefaultStore(), challenges, peerKey) + if cfg.OwnerUnknown { + return proofs + } + + userProofs, err := collectAsConsoleUser(ctx, cfg.ProfileOwner, challenges, peerKey) + if err != nil { + log.Debugf("certificate posture: console user keychain unavailable: %v", err) + } + return mergeProofs(proofs, userProofs) +} + +// UserContext identifies the user whose keychain a collection would include: the console +// user when it owns the active profile, or empty when no user keychain would be asked. A +// change means a collection made earlier no longer reflects what this Mac can prove. +func UserContext(cfg Config) string { + if os.Geteuid() != 0 || cfg.OwnerUnknown { + return "" + } + user, ok := currentConsoleUser() + if !ok || !user.isOwner(cfg.ProfileOwner) { + return "" + } + return strconv.FormatUint(uint64(user.UID), 10) + ":" + user.Name +} + +// collectAsConsoleUser runs the helper inside the desktop session of the logged-in +// user. Dropping to their uid is not enough: keychain access is an XPC call to a +// per-session securityd, so the helper has to enter their Mach bootstrap namespace, +// which is what launchctl asuser does. +func collectAsConsoleUser(ctx context.Context, owner string, challenges []*proto.CertificateChallenge, peerKey []byte) ([]certposture.Proof, error) { + user, ok := currentConsoleUser() + if !ok { + return nil, nil + } + if !user.isOwner(owner) { + log.Debugf("certificate posture: console user %s does not own the active profile, no user keychain is asked", user.Name) + return nil, nil + } + + uid := strconv.FormatUint(uint64(user.UID), 10) + backoffKey := helperBackoffKey(uid, challenges) + if !userHelperBackoff.allow(backoffKey, time.Now()) { + log.Debugf("certificate posture: the keychain of uid %s proved nothing recently, not asking again yet", uid) + return nil, nil + } + + binary, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("resolve own binary: %w", err) + } + + parent := ctx + ctx, cancel := context.WithTimeout(ctx, helperTimeout) + defer cancel() + + // Absolute paths, because the daemon's PATH is configurable through the service + // environment, and sudo selects the user by uid so the name never has to round-trip. + cmd := exec.CommandContext(ctx, "/bin/launchctl", "asuser", uid, "/usr/bin/sudo", "-u", "#"+uid, "-H", "--", binary, "posture", "cert-proof") + killHelperGroupOnCancel(cmd) + + log.Debugf("certificate posture: asking the desktop session of uid %s to answer %d challenges", uid, len(challenges)) + proofs, err := runHelperCmd(cmd, helperRequest(challenges, peerKey)) + // A run that completed, or ran into the timeout waiting on a prompt nobody answered, + // tells whether the keychain proves anything. A launch or output failure, or a run the + // caller cut short, says nothing about it and must not hold off the next one. + timedOut := errors.Is(ctx.Err(), context.DeadlineExceeded) && parent.Err() == nil + if err == nil || timedOut { + userHelperBackoff.record(backoffKey, err == nil && len(proofs) > 0, time.Now()) + } + if err != nil { + return nil, fmt.Errorf("run helper as uid %s: %w", uid, err) + } + log.Debugf("certificate posture: desktop session of uid %s returned %d proofs", uid, len(proofs)) + return proofs, nil +} + +// helperStore is the store the helper reads: the user's login keychain alone. The +// session's search list also holds the System keychain, which the daemon reads itself, +// and using a System keychain key from the user's session would ask for an +// administrator's approval. +func helperStore() Store { + home, err := os.UserHomeDir() + if err != nil { + log.Debugf("certificate posture: no home directory, searching the default keychain list: %v", err) + return newKeychainStore() + } + login := filepath.Join(home, "Library", "Keychains", "login.keychain-db") + if _, err := os.Stat(login); err != nil { + // Keychains created before macOS 10.12 keep the old file name. + login = filepath.Join(home, "Library", "Keychains", "login.keychain") + } + return newKeychainStore(login) +} diff --git a/client/internal/certproof/collect_js.go b/client/internal/certproof/collect_js.go new file mode 100644 index 000000000..fd7c4c690 --- /dev/null +++ b/client/internal/certproof/collect_js.go @@ -0,0 +1,31 @@ +//go:build js + +package certproof + +import ( + "context" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// CollectProofs proves nothing in a browser: it has no TPM, certificate store or file +// store to sign with, so the stores stay out of the WebAssembly build. +func CollectProofs(context.Context, []*proto.Checks, []byte, Config) []certposture.Proof { + return nil +} + +// UserContext identifies the user whose certificates a collection would include. A +// browser has no per-user store, so it never changes. +func UserContext(Config) string { + return "" +} + +// DefaultStore is empty in a browser. +func DefaultStore() Store { + return Stores{} +} + +func helperStore() Store { + return DefaultStore() +} diff --git a/client/internal/certproof/collect_other.go b/client/internal/certproof/collect_other.go new file mode 100644 index 000000000..785d3db65 --- /dev/null +++ b/client/internal/certproof/collect_other.go @@ -0,0 +1,31 @@ +//go:build ((!darwin && !windows) || ios) && !js + +package certproof + +import ( + "context" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// CollectProofs answers the certificate challenges in checks from the PEM directory cfg +// names, joined by its PKCS#11 token when it names one. Only macOS and Windows keep +// per-user certificates out of reach of a privileged daemon, so every other platform +// reads its store in the daemon itself. +func CollectProofs(ctx context.Context, checks []*proto.Checks, peerKey []byte, cfg Config) []certposture.Proof { + return Collect(ctx, storeWithToken(cfg), checks, peerKey) +} + +// UserContext identifies the user whose certificates a collection would include. These +// platforms have no per-user store, so it never changes. +func UserContext(Config) string { + return "" +} + +// helperStore is the store the helper reads. Nothing launches a helper on these +// platforms, so it is the store the daemon reads, configured from the same environment, +// which lets an administrator check a setup by running the helper by hand. +func helperStore() Store { + return storeWithToken(Config{PKCS11: PKCS11FromEnv()}) +} diff --git a/client/internal/certproof/collect_test.go b/client/internal/certproof/collect_test.go new file mode 100644 index 000000000..aa033dea5 --- /dev/null +++ b/client/internal/certproof/collect_test.go @@ -0,0 +1,258 @@ +package certproof + +import ( + "context" + "crypto/rand" + "crypto/x509" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/certposture/certtest" + "github.com/netbirdio/netbird/shared/management/proto" +) + +var peerKey = []byte("peer-public-key-aaaaaaaaaaaaaaaa") + +func TestCollect_ProvesOneMatchingCertificatePerChallenge(t *testing.T) { + corpCA := certtest.NewCA(t, "corp-root") + otherCA := certtest.NewCA(t, "other-root") + unrelatedCA := certtest.NewCA(t, "unrelated-root") + + dir := storeDir(t) + deviceKey := certtest.ECDSAKey(t) + device := corpCA.Issue(t, deviceKey, "device") + writeFile(t, dir, "device.pem", certtest.CertPEM(device)+certtest.KeyPEM(t, deviceKey)) + + otherKey := certtest.RSAKey(t) + writeFile(t, dir, "other.crt", certtest.CertPEM(otherCA.Issue(t, otherKey, "other"))) + writeFile(t, dir, "other.key", certtest.KeyPEM(t, otherKey)) + + writeFile(t, dir, "keyless.crt", certtest.CertPEM(corpCA.Issue(t, certtest.ECDSAKey(t), "keyless"))) + writeFile(t, dir, "notes.txt", "ignored") + + challenger := certposture.NewChallenger([]byte("secret")) + nonce := challenger.Nonce(peerKey, time.Now()) + challenge := func(cas ...string) *proto.Checks { + return &proto.Checks{CertificateChallenge: &proto.CertificateChallenge{Nonce: nonce, CaCertificates: cas}} + } + checks := []*proto.Checks{ + {Files: []string{"/usr/bin/agent"}}, + challenge(corpCA.PEM), + challenge(corpCA.PEM), + challenge(otherCA.PEM), + challenge(unrelatedCA.PEM), + challenge("not a pem"), + } + + proofs := Collect(context.Background(), newFileStore(dir), checks, peerKey) + + require.Len(t, proofs, 2) + var subjects []string + for _, p := range proofs { + chain, err := challenger.Verify(p, peerKey, time.Now()) + require.NoError(t, err) + subjects = append(subjects, chain[0].Subject.CommonName) + } + assert.ElementsMatch(t, []string{"device", "other"}, subjects) +} + +func TestCollect_NothingToProve(t *testing.T) { + dir := storeDir(t) + key := certtest.ECDSAKey(t) + ca := certtest.NewCA(t, "root") + writeFile(t, dir, "device.pem", certtest.CertPEM(ca.Issue(t, key, "device"))+certtest.KeyPEM(t, key)) + + tests := []struct { + name string + store Store + checks []*proto.Checks + }{ + {"no checks", newFileStore(dir), nil}, + {"files only", newFileStore(dir), []*proto.Checks{{Files: []string{"/bin/x"}}}}, + {"challenge without nonce", newFileStore(dir), []*proto.Checks{{CertificateChallenge: &proto.CertificateChallenge{CaCertificates: []string{ca.PEM}}}}}, + {"missing store dir", newFileStore(filepath.Join(dir, "missing")), []*proto.Checks{{CertificateChallenge: &proto.CertificateChallenge{Nonce: make([]byte, certposture.NonceSize), CaCertificates: []string{ca.PEM}}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Nil(t, Collect(context.Background(), tt.store, tt.checks, peerKey)) + }) + } +} + +func TestFileStore_ChainWithIntermediate(t *testing.T) { + root := certtest.NewCA(t, "root") + intermediate := certtest.NewIntermediate(t, root, "intermediate") + key := certtest.ECDSAKey(t) + leaf := intermediate.Issue(t, key, "device") + + dir := storeDir(t) + writeFile(t, dir, "device.pem", certtest.CertPEM(leaf)+certtest.CertPEM(intermediate.Cert)+certtest.KeyPEM(t, key)) + + candidates, err := newFileStore(dir).Candidates(context.Background()) + require.NoError(t, err) + require.Len(t, candidates, 1) + require.Len(t, candidates[0].Chain, 2) + + roots, err := certposture.ParseCAs([]string{root.PEM}) + require.NoError(t, err) + assert.NoError(t, certposture.VerifyChain(candidates[0].Chain, roots, time.Now())) +} + +// storeDir is a PEM directory the store accepts: t.TempDir follows the umask, which +// leaves the directory group-writable on systems with a user-private group umask. +func storeDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.Chmod(dir, 0o700)) + return dir +} + +func writeFile(t *testing.T, dir, name, content string) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600)) +} + +func TestCollectChallenges_RefusesMalformedInput(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + dir := storeDir(t) + key := certtest.ECDSAKey(t) + writeFile(t, dir, "device.pem", certtest.CertPEM(ca.Issue(t, key, "device"))+certtest.KeyPEM(t, key)) + store := newFileStore(dir) + nonce := certposture.NewChallenger([]byte("secret")).Nonce(peerKey, time.Now()) + + tests := []struct { + name string + nonce []byte + peerKey []byte + want int + }{ + {"issued nonce and peer key are signed", nonce, peerKey, 1}, + {"short nonce is not signed", nonce[:8], peerKey, 0}, + {"oversized nonce is not signed", append(append([]byte{}, nonce...), 0), peerKey, 0}, + {"short peer key is not signed", nonce, peerKey[:16], 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + challenges := []*proto.CertificateChallenge{{Nonce: tt.nonce, CaCertificates: []string{ca.PEM}}} + assert.Len(t, CollectChallenges(context.Background(), store, challenges, tt.peerKey), tt.want, + "the device key signs only what management could have issued") + }) + } +} + +func TestFileStore_SkipsKeyOfAnotherCertificate(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + dir := storeDir(t) + + // A stale key next to a renewed certificate, sorted before the good pair, must not + // produce a proof that management rejects and stop the search there. + writeFile(t, dir, "a-renewed.crt", certtest.CertPEM(ca.Issue(t, certtest.ECDSAKey(t), "renewed"))) + writeFile(t, dir, "a-renewed.key", certtest.KeyPEM(t, certtest.ECDSAKey(t))) + goodKey := certtest.ECDSAKey(t) + good := ca.Issue(t, goodKey, "good") + writeFile(t, dir, "b-good.pem", certtest.CertPEM(good)+certtest.KeyPEM(t, goodKey)) + + candidates, err := newFileStore(dir).Candidates(context.Background()) + require.NoError(t, err) + require.Len(t, candidates, 1, "only the certificate whose key matches is a candidate") + assert.True(t, good.Equal(candidates[0].Chain[0]), "the matching pair is kept") + + challenger := certposture.NewChallenger([]byte("secret")) + now := time.Now() + nonce := challenger.Nonce(peerKey, now) + proofs := CollectChallenges(context.Background(), newFileStore(dir), []*proto.CertificateChallenge{{Nonce: nonce, CaCertificates: []string{ca.PEM}}}, peerKey) + require.Len(t, proofs, 1) + _, err = challenger.Verify(proofs[0], peerKey, now) + assert.NoError(t, err, "the proof sent is one management accepts") +} + +// staticStore hands out fixed candidates, for scenarios no on-disk layout can express. +type staticStore []Candidate + +func (s staticStore) Candidates(context.Context) ([]Candidate, error) { return s, nil } + +func TestCollectChallenges_RoutesAroundExpiredCopyOfRenewedIntermediate(t *testing.T) { + root := certtest.NewCA(t, "root") + intermediate := certtest.NewIntermediate(t, root, "issuing-ca") + + // Renewing a CA with the same key pair leaves two certificates with the same + // subject and key in the store. The expired one sorts first here. + expiredTmpl := *intermediate.Cert + expiredTmpl.SerialNumber = big.NewInt(1) + expiredTmpl.NotBefore = time.Now().Add(-72 * time.Hour) + expiredTmpl.NotAfter = time.Now().Add(-48 * time.Hour) + der, err := x509.CreateCertificate(rand.Reader, &expiredTmpl, root.Cert, intermediate.Key.Public(), root.Key) + require.NoError(t, err) + expired, err := x509.ParseCertificate(der) + require.NoError(t, err) + + key := certtest.ECDSAKey(t) + leaf := intermediate.Issue(t, key, "device") + pool := []*x509.Certificate{expired, intermediate.Cert} + + chain := buildChain(leaf, pool) + require.Len(t, chain, 2) + require.True(t, expired.Equal(chain[1]), "precondition: the first-match chain runs through the expired copy") + + challenger := certposture.NewChallenger([]byte("secret")) + now := time.Now() + nonce := challenger.Nonce(peerKey, now) + store := staticStore{{Chain: chain, Signer: key, Intermediates: pool}} + + proofs := CollectChallenges(context.Background(), store, []*proto.CertificateChallenge{{Nonce: nonce, CaCertificates: []string{root.PEM}}}, peerKey) + + require.Len(t, proofs, 1, "a valid path through the renewed intermediate exists, so the challenge is answered") + verified, err := challenger.Verify(proofs[0], peerKey, now) + require.NoError(t, err) + assert.True(t, intermediate.Cert.Equal(verified[1]), "the proof carries the valid intermediate, not the expired copy") + assert.True(t, certposture.ChainMatchesCAs(certposture.EncodeChainPEM(verified), []string{root.PEM}, now), + "management's own check accepts the chain the proof carries") +} + +func TestCollectChallenges_ProvesALeafOncePerDistinctChain(t *testing.T) { + rootA := certtest.NewCA(t, "root-a") + rootB := certtest.NewCA(t, "root-b") + issuer := certtest.NewIntermediate(t, rootA, "issuing-ca") + + // The same issuing CA cross-signed by a second root: one leaf, two valid paths. + crossTmpl := *issuer.Cert + crossTmpl.SerialNumber = big.NewInt(2) + der, err := x509.CreateCertificate(rand.Reader, &crossTmpl, rootB.Cert, issuer.Key.Public(), rootB.Key) + require.NoError(t, err) + cross, err := x509.ParseCertificate(der) + require.NoError(t, err) + + key := certtest.ECDSAKey(t) + leaf := issuer.Issue(t, key, "device") + pool := []*x509.Certificate{issuer.Cert, cross} + store := staticStore{{Chain: buildChain(leaf, pool), Signer: key, Intermediates: pool}} + + challenger := certposture.NewChallenger([]byte("secret")) + now := time.Now() + nonce := challenger.Nonce(peerKey, now) + challenges := []*proto.CertificateChallenge{ + {Nonce: nonce, CaCertificates: []string{rootA.PEM}}, + {Nonce: nonce, CaCertificates: []string{rootB.PEM}}, + {Nonce: nonce, CaCertificates: []string{rootA.PEM}}, + } + + proofs := CollectChallenges(context.Background(), store, challenges, peerKey) + + require.Len(t, proofs, 2, "one proof per distinct chain, the repeated root-a challenge reuses the first") + for _, root := range []*certtest.CA{rootA, rootB} { + matched := false + for _, p := range proofs { + chain, err := challenger.Verify(p, peerKey, now) + require.NoError(t, err) + matched = matched || certposture.ChainMatchesCAs(certposture.EncodeChainPEM(chain), []string{root.PEM}, now) + } + assert.True(t, matched, "management can match a chain for the check that trusts %s", root.Cert.Subject.CommonName) + } +} diff --git a/client/internal/certproof/collect_windows.go b/client/internal/certproof/collect_windows.go new file mode 100644 index 000000000..3b21ec147 --- /dev/null +++ b/client/internal/certproof/collect_windows.go @@ -0,0 +1,108 @@ +package certproof + +import ( + "context" + "fmt" + "os" + "os/exec" + "syscall" + "time" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +const helperTimeout = 30 * time.Second + +// CollectProofs answers the certificate challenges in checks from every store this +// machine can reach. The service reads the local machine store itself, where AD and +// Intune enrol device certificates, and reaches the signed-in user's store by launching +// a helper with that session's token. A machine at the sign-in screen therefore proves +// device certificates alone. +func CollectProofs(ctx context.Context, checks []*proto.Checks, peerKey []byte, cfg Config) []certposture.Proof { + challenges := certificateChallenges(checks) + if len(challenges) == 0 { + logNoChallenges(checks) + return nil + } + + proofs := CollectChallenges(ctx, DefaultStore(), challenges, peerKey) + + // The helper already runs as the signed-in user, and an ordinary process has no + // right to a session token, so only the service goes looking for one. + if !runningAsLocalSystem() || cfg.OwnerUnknown { + return proofs + } + + userProofs, err := collectAsDesktopUser(ctx, cfg.ProfileOwner, challenges, peerKey) + if err != nil { + log.Debugf("certificate posture: user certificate store unavailable: %v", err) + } + return mergeProofs(proofs, userProofs) +} + +// UserContext identifies the session whose store a collection would include: a session +// of the profile owner, or empty when no user store would be asked. A change means a +// collection made earlier no longer reflects what this machine can prove. +func UserContext(cfg Config) string { + if !runningAsLocalSystem() || cfg.OwnerUnknown { + return "" + } + user, ok := currentDesktopUser(cfg.ProfileOwner) + if !ok { + return "" + } + defer user.Close() + return fmt.Sprintf("%d:%s", user.Session, user.Name) +} + +// helperStore is the store the helper reads. It runs as the signed-in user, so it wants +// that user's store rather than the machine store the service already read. +func helperStore() Store { + return newUserStore() +} + +// collectAsDesktopUser runs the helper inside the interactive session of the signed-in +// user. Unlike a keychain on macOS, a Windows service can assume a user identity +// directly, so the session token goes straight into the child process. +func collectAsDesktopUser(ctx context.Context, owner string, challenges []*proto.CertificateChallenge, peerKey []byte) ([]certposture.Proof, error) { + user, ok := currentDesktopUser(owner) + if !ok { + return nil, nil + } + defer user.Close() + + binary, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("resolve own binary: %w", err) + } + + // The user's own environment, not the service's: the service environment may carry + // secrets such as a setup key that the signed-in user must not be able to read. + env, err := user.Token.Environ(false) + if err != nil { + return nil, fmt.Errorf("build environment of %s: %w", user.Name, err) + } + + ctx, cancel := context.WithTimeout(ctx, helperTimeout) + defer cancel() + + cmd := exec.CommandContext(ctx, binary, "posture", "cert-proof") + cmd.Env = env + cmd.SysProcAttr = &syscall.SysProcAttr{ + Token: syscall.Token(user.Token), + HideWindow: true, + CreationFlags: windows.CREATE_NO_WINDOW, + } + + log.Debugf("certificate posture: asking session %d to answer %d challenges", user.Session, len(challenges)) + proofs, err := runHelperCmd(cmd, helperRequest(challenges, peerKey)) + if err != nil { + return nil, fmt.Errorf("run helper in session %d: %w", user.Session, err) + } + log.Debugf("certificate posture: session %d returned %d proofs", user.Session, len(proofs)) + return proofs, nil +} diff --git a/client/internal/certproof/collector.go b/client/internal/certproof/collector.go new file mode 100644 index 000000000..2931f4b00 --- /dev/null +++ b/client/internal/certproof/collector.go @@ -0,0 +1,189 @@ +package certproof + +import ( + "context" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +const ( + collectTimeout = 45 * time.Second + + // lostAfter is how many deadlines an abandoned collection is waited for before a new + // one may start beside it. A store that is merely slow finishes well within that and + // releases its slot; only one wedged for good is worked around. + lostAfter = 10 + + // maxInFlight bounds how many abandoned collections may pile up. A store that never + // answers would otherwise leak a goroutine on every collection for the life of the + // daemon. + maxInFlight = 2 +) + +// collectSlots tracks the collections running in the process. Collectors share one by +// default: a collection abandoned in a token, TPM or keychain call keeps running after +// the engine that started it has stopped, and the next engine must account for it. +type collectSlots struct { + mu sync.Mutex + next uint64 + // running holds the start time of each collection still running, by slot id. + running map[uint64]time.Time +} + +var processSlots collectSlots + +// Collector runs CollectProofs with a deadline and, in the normal case, one collection +// at a time in the process. Token, TPM and keychain calls cannot be interrupted, and on +// Linux there is not even a process to kill, so a collection that overruns is abandoned +// rather than awaited. A single wedged call would then refuse every later collection for +// the life of the daemon, so once an abandoned collection has been running for lostAfter +// deadlines another may start beside it, up to maxInFlight. The zero value is ready to +// use. +type Collector struct { + // slots overrides the process-wide slots when set. + slots *collectSlots + // timeout overrides collectTimeout when set. + timeout time.Duration + // now overrides the clock when set. + now func() time.Time +} + +// Collect answers the certificate challenges in checks, returning no proofs when there +// are no challenges, when previous collections still hold the slots, or when this one +// does not finish in time. Missing proofs fail the certificate check on management. +func (c *Collector) Collect(ctx context.Context, checks []*proto.Checks, peerKey []byte, cfg Config) []certposture.Proof { + return c.collect(ctx, checks, func(ctx context.Context) []certposture.Proof { + return CollectProofs(ctx, checks, peerKey, cfg) + }) +} + +func (c *Collector) collect(ctx context.Context, checks []*proto.Checks, run func(context.Context) []certposture.Proof) []certposture.Proof { + if len(certificateChallenges(checks)) == 0 { + return nil + } + slots := c.slotsInUse() + slot, ok := slots.start(c.clock(), time.Duration(lostAfter)*c.deadline()) + if !ok { + return nil + } + + ctx, cancel := context.WithTimeout(ctx, c.deadline()) + defer cancel() + + done := make(chan []certposture.Proof, 1) + go func() { + // The slot is freed before the result is delivered, so a caller that starts the + // next collection right after this one returned is not turned away. + proofs := run(ctx) + slots.finish(slot) + done <- proofs + }() + + select { + case proofs := <-done: + return proofs + case <-ctx.Done(): + log.Warnf("certificate posture: proof collection did not finish within %s, sending no proofs", c.deadline()) + return nil + } +} + +// Stuck reports whether lost collections hold every slot. No collection starts again +// until one of them returns, which for a store that never answers means a restart. +func (c *Collector) Stuck() bool { + return c.slotsInUse().allLost(c.clock(), time.Duration(lostAfter)*c.deadline()) +} + +func (c *Collector) slotsInUse() *collectSlots { + if c.slots != nil { + return c.slots + } + return &processSlots +} + +func (c *Collector) clock() time.Time { + if c.now != nil { + return c.now() + } + return time.Now() +} + +func (c *Collector) deadline() time.Duration { + if c.timeout > 0 { + return c.timeout + } + return collectTimeout +} + +// start claims a slot at now, reporting its id and whether the caller may collect. A +// collection that has held a slot for lost is taken to be wedged. +func (s *collectSlots) start(now time.Time, lost time.Duration) (uint64, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.running) > 0 { + oldest := s.oldest() + switch { + case len(s.running) >= maxInFlight: + log.Warnf("certificate posture: %d proof collections are wedged, sending no proofs", len(s.running)) + return 0, false + case now.Sub(oldest) < lost: + log.Warnf("certificate posture: previous proof collection is still running, sending no proofs") + return 0, false + default: + log.Warnf("certificate posture: a proof collection has been stuck since %s, starting another", oldest.Format(time.RFC3339)) + } + } + if s.running == nil { + s.running = make(map[uint64]time.Time, maxInFlight) + } + s.next++ + s.running[s.next] = now + return s.next, true +} + +// finish releases the slot with the given id. +func (s *collectSlots) finish(slot uint64) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.running, slot) +} + +// allLost reports whether every slot is held by a collection that has run for lost. +func (s *collectSlots) allLost(now time.Time, lost time.Duration) bool { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.running) < maxInFlight { + return false + } + for _, started := range s.running { + if now.Sub(started) < lost { + return false + } + } + return true +} + +// idle reports whether no collection is running. +func (s *collectSlots) idle() bool { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.running) == 0 +} + +// oldest returns the start time of the longest-running collection. The caller holds mu +// and has checked that one is running. +func (s *collectSlots) oldest() time.Time { + var oldest time.Time + for _, started := range s.running { + if oldest.IsZero() || started.Before(oldest) { + oldest = started + } + } + return oldest +} diff --git a/client/internal/certproof/collector_test.go b/client/internal/certproof/collector_test.go new file mode 100644 index 000000000..a28c47da2 --- /dev/null +++ b/client/internal/certproof/collector_test.go @@ -0,0 +1,207 @@ +package certproof + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +var challengeChecks = []*proto.Checks{{CertificateChallenge: &proto.CertificateChallenge{ + Nonce: certposture.NewChallenger([]byte("secret")).Nonce(peerKey, time.Now()), +}}} + +func TestCollector_SkipsChecksWithoutChallenges(t *testing.T) { + var c Collector + called := false + + proofs := c.collect(context.Background(), []*proto.Checks{{Files: []string{"/bin/agent"}}}, func(context.Context) []certposture.Proof { + called = true + return nil + }) + + assert.Nil(t, proofs) + assert.False(t, called, "no store is touched when no check carries a challenge") +} + +func TestCollector_ReturnsProofs(t *testing.T) { + var c Collector + want := []certposture.Proof{{Nonce: []byte("nonce")}} + + proofs := c.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { return want }) + + assert.Equal(t, want, proofs, "a collection that finishes in time is returned as is") +} + +func TestCollector_AbandonsStuckCollection(t *testing.T) { + c := Collector{timeout: 50 * time.Millisecond, slots: &collectSlots{}} + release := make(chan struct{}) + finished := make(chan struct{}) + + // A token or keychain call that ignores its context and blocks well past the deadline. + stuck := func(context.Context) []certposture.Proof { + defer close(finished) + <-release + return []certposture.Proof{{Nonce: []byte("late")}} + } + + start := time.Now() + proofs := c.collect(context.Background(), challengeChecks, stuck) + assert.Nil(t, proofs, "an overrunning collection yields no proofs") + assert.Less(t, time.Since(start), time.Second, "the caller is released at the deadline, not when the call returns") + + called := false + proofs = c.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { + called = true + return nil + }) + assert.Nil(t, proofs) + assert.False(t, called, "no second collection starts while the first is still running") + + close(release) + <-finished + require.Eventually(t, func() bool { return c.slots.idle() }, time.Second, 5*time.Millisecond, "the collector frees up once the stuck call returns") + + want := []certposture.Proof{{Nonce: []byte("nonce")}} + assert.Equal(t, want, c.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { return want }), + "collection works again after the stuck call returned") +} + +func TestCollector_CancelsContextAtDeadline(t *testing.T) { + c := Collector{timeout: 20 * time.Millisecond, slots: &collectSlots{}} + cancelled := make(chan struct{}) + + c.collect(context.Background(), challengeChecks, func(ctx context.Context) []certposture.Proof { + <-ctx.Done() + close(cancelled) + return nil + }) + + select { + case <-cancelled: + case <-time.After(time.Second): + t.Fatal("a collection that honours its context, like the helper process, must see it cancelled") + } +} + +// TestCollector_SingleFlightAcrossCollectors: each engine has its own Collector, and a +// collection stuck in a token call outlives the engine that started it, so the next +// engine's Collector must not start another one until it has finished. +func TestCollector_SingleFlightAcrossCollectors(t *testing.T) { + shared := &collectSlots{} + first := Collector{timeout: 20 * time.Millisecond, slots: shared} + second := Collector{timeout: time.Second, slots: shared} + + release := make(chan struct{}) + stuck := func(context.Context) []certposture.Proof { + <-release + return nil + } + assert.Nil(t, first.collect(context.Background(), challengeChecks, stuck), "the first collection is abandoned at its deadline") + + called := false + proofs := second.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { + called = true + return []certposture.Proof{{}} + }) + assert.Nil(t, proofs, "no proofs while the abandoned collection still runs") + assert.False(t, called, "a second collection does not start on top of the abandoned one") + + close(release) + require.Eventually(t, func() bool { return shared.idle() }, time.Second, 5*time.Millisecond) + assert.Len(t, second.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { return []certposture.Proof{{}} }), 1, + "collections resume once the abandoned one finished") +} + +// TestCollector_ZeroValuesShareTheProcessFlag: the zero value uses the process-wide flag. +func TestCollector_ZeroValuesShareTheProcessFlag(t *testing.T) { + var a, b Collector + assert.Same(t, a.slotsInUse(), b.slotsInUse(), "separate Collectors share the process slots") +} + +// TestCollector_LostCollectionDoesNotBlockForever: a store call that never answers, such +// as a wedged token on Linux where there is no process to kill, must not refuse every +// later collection. Once it has run for lostAfter deadlines another starts beside it, +// and no more than maxInFlight ever run. +func TestCollector_LostCollectionDoesNotBlockForever(t *testing.T) { + now := time.Now() + slots := &collectSlots{} + c := Collector{timeout: 10 * time.Millisecond, slots: slots, now: func() time.Time { return now }} + + release := make(chan struct{}) + defer close(release) + var started atomic.Int32 + wedged := func(context.Context) []certposture.Proof { + started.Add(1) + <-release + return nil + } + + c.collect(context.Background(), challengeChecks, wedged) + waitStarted(t, &started, 1) + c.collect(context.Background(), challengeChecks, wedged) + assert.Equal(t, int32(1), started.Load(), "a second collection is refused while the first may still finish") + assert.False(t, c.Stuck(), "one slow collection is not stuck yet") + + now = now.Add(time.Duration(lostAfter) * c.deadline()) + c.collect(context.Background(), challengeChecks, wedged) + waitStarted(t, &started, 2) + assert.False(t, c.Stuck(), "the second collection may still return, so the collector is not stuck yet") + + now = now.Add(time.Duration(lostAfter) * c.deadline()) + c.collect(context.Background(), challengeChecks, wedged) + assert.Equal(t, int32(2), started.Load(), "no more than maxInFlight collections run") + assert.True(t, c.Stuck(), "with every slot held by a lost collection the collector reports itself stuck") +} + +// TestCollector_LostWindowFollowsTheOldestRunningCollection: A wedges, B starts beside it +// once A counts as lost, then A returns. B is recent, so a third collection must wait for +// B to be lost in its own right rather than inherit A's start time. +func TestCollector_LostWindowFollowsTheOldestRunningCollection(t *testing.T) { + now := time.Now() + slots := &collectSlots{} + c := Collector{timeout: 10 * time.Millisecond, slots: slots, now: func() time.Time { return now }} + lost := time.Duration(lostAfter) * c.deadline() + + releaseA := make(chan struct{}) + releaseB := make(chan struct{}) + defer close(releaseB) + var started atomic.Int32 + blockOn := func(release chan struct{}) func(context.Context) []certposture.Proof { + return func(context.Context) []certposture.Proof { + started.Add(1) + <-release + return nil + } + } + + c.collect(context.Background(), challengeChecks, blockOn(releaseA)) + waitStarted(t, &started, 1) + now = now.Add(lost) + c.collect(context.Background(), challengeChecks, blockOn(releaseB)) + waitStarted(t, &started, 2) + + close(releaseA) + require.Eventually(t, func() bool { + slots.mu.Lock() + defer slots.mu.Unlock() + return len(slots.running) == 1 + }, time.Second, 5*time.Millisecond, "A frees its slot once it returns") + + c.collect(context.Background(), challengeChecks, blockOn(make(chan struct{}))) + assert.Equal(t, int32(2), started.Load(), "a third collection waits while B is recent") +} + +// waitStarted waits for the collections the test launched to have started. collect may +// return at its deadline before the goroutine running the store call was scheduled. +func waitStarted(t *testing.T, started *atomic.Int32, n int32) { + t.Helper() + require.Eventually(t, func() bool { return started.Load() == n }, time.Second, time.Millisecond, + "%d collections should have started", n) +} diff --git a/client/internal/certproof/consoleuser_darwin.go b/client/internal/certproof/consoleuser_darwin.go new file mode 100644 index 000000000..1045bc014 --- /dev/null +++ b/client/internal/certproof/consoleuser_darwin.go @@ -0,0 +1,123 @@ +//go:build !ios + +package certproof + +import ( + "bytes" + "fmt" + "strconv" + "sync" + + "github.com/ebitengine/purego" + log "github.com/sirupsen/logrus" +) + +const ( + systemConfigurationFramework = "/System/Library/Frameworks/SystemConfiguration.framework/SystemConfiguration" + + encodingUTF8 = 0x08000100 + consoleNameBufSize = 256 +) + +var ( + consoleOnce sync.Once + consoleErr error + + scDynamicStoreCopyConsoleUser func(store uintptr, uid, gid *uint32) uintptr + cfStringGetCString func(str uintptr, buffer *byte, size int, encoding uint32) bool +) + +// ConsoleUser is the account whose desktop session owns the display. Its login keychain +// is the only user keychain a NetBird daemon can reach, and only while it is logged in. +type ConsoleUser struct { + Name string + UID uint32 + GID uint32 +} + +// currentConsoleUser reports the user sitting at the desktop. The second return value is +// false when nobody is: at the login window macOS either reports no console user at all +// or attributes the session to root, and neither has a login keychain to offer. +func currentConsoleUser() (ConsoleUser, bool) { + if err := loadConsoleUser(); err != nil { + log.Debugf("console user lookup unavailable: %v", err) + return ConsoleUser{}, false + } + + var uid, gid uint32 + name := scDynamicStoreCopyConsoleUser(0, &uid, &gid) + if name == 0 { + log.Debug("no console user is logged in, no login keychain is reachable") + return ConsoleUser{}, false + } + defer release(name) + + user := ConsoleUser{Name: cfString(name), UID: uid, GID: gid} + if !user.hasDesktop() { + log.Debugf("console session belongs to %q uid=%d, which is not a desktop login, no login keychain is reachable", user.Name, user.UID) + return ConsoleUser{}, false + } + return user, true +} + +// hasDesktop reports whether the console session is a real user desktop. The login +// window runs as root and some macOS releases name it "loginwindow" instead. +func (u ConsoleUser) hasDesktop() bool { + switch u.Name { + case "", "root", "loginwindow": + return false + } + return u.UID != 0 +} + +// isOwner reports whether the console user is owner, the account of the active profile, +// which is recorded as a short user name or, for an account without one, a numeric uid. +// With no owner the console user counts, as macOS has a single console user. +func (u ConsoleUser) isOwner(owner string) bool { + return owner == "" || owner == u.Name || owner == strconv.FormatUint(uint64(u.UID), 10) +} + +func cfString(str uintptr) string { + buf := make([]byte, consoleNameBufSize) + if !cfStringGetCString(str, &buf[0], len(buf), encodingUTF8) { + return "" + } + if end := bytes.IndexByte(buf, 0); end >= 0 { + return string(buf[:end]) + } + return string(buf) +} + +// loadConsoleUser resolves the console user symbols. It loads the keychain bindings +// first because CFRelease is resolved there and released strings depend on it. +func loadConsoleUser() error { + if err := loadKeychain(); err != nil { + return err + } + consoleOnce.Do(func() { consoleErr = resolveConsoleUser() }) + return consoleErr +} + +func resolveConsoleUser() error { + systemConfiguration, err := purego.Dlopen(systemConfigurationFramework, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return fmt.Errorf("open %s: %w", systemConfigurationFramework, err) + } + coreFoundation, err := purego.Dlopen(coreFoundationFramework, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return fmt.Errorf("open %s: %w", coreFoundationFramework, err) + } + + symbol, err := purego.Dlsym(systemConfiguration, "SCDynamicStoreCopyConsoleUser") + if err != nil { + return fmt.Errorf("resolve SCDynamicStoreCopyConsoleUser: %w", err) + } + purego.RegisterFunc(&scDynamicStoreCopyConsoleUser, symbol) + + symbol, err = purego.Dlsym(coreFoundation, "CFStringGetCString") + if err != nil { + return fmt.Errorf("resolve CFStringGetCString: %w", err) + } + purego.RegisterFunc(&cfStringGetCString, symbol) + return nil +} diff --git a/client/internal/certproof/consoleuser_darwin_test.go b/client/internal/certproof/consoleuser_darwin_test.go new file mode 100644 index 000000000..051cc4ab0 --- /dev/null +++ b/client/internal/certproof/consoleuser_darwin_test.go @@ -0,0 +1,51 @@ +//go:build !ios + +package certproof + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConsoleUser_OnlyADesktopSessionCanBeValidated(t *testing.T) { + tests := []struct { + name string + user ConsoleUser + desktop bool + }{ + {"logged in user", ConsoleUser{Name: "maycon", UID: 501, GID: 20}, true}, + {"login window as root", ConsoleUser{Name: "root", UID: 0}, false}, + {"login window by name", ConsoleUser{Name: "loginwindow", UID: 0}, false}, + {"named user still at uid 0", ConsoleUser{Name: "admin", UID: 0}, false}, + {"no console user", ConsoleUser{}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.desktop, tt.user.hasDesktop(), "only a real desktop session offers a login keychain") + }) + } +} + +// currentConsoleUser runs against the real SystemConfiguration framework. A machine with +// a desktop open must report a non-root user; a headless runner must report none. +func TestCurrentConsoleUser_AgreesWithItself(t *testing.T) { + user, ok := currentConsoleUser() + if !ok { + t.Log("no console user, running headless") + return + } + assert.NotEmpty(t, user.Name, "a console user must have a name") + assert.NotZero(t, user.UID, "a desktop session never belongs to uid 0") + assert.True(t, user.hasDesktop(), "a reported console user must be a desktop session") +} + +func TestConsoleUser_IsOwner(t *testing.T) { + user := ConsoleUser{Name: "maycon", UID: 501, GID: 20} + + assert.True(t, user.isOwner(""), "a profile without owner accepts the single console user") + assert.True(t, user.isOwner("maycon"), "the owner by short name") + assert.True(t, user.isOwner("501"), "the owner recorded as a numeric uid") + assert.False(t, user.isOwner("viktor"), "another account's profile must not read this user's keychain") + assert.False(t, user.isOwner("502"), "another uid is another account") +} diff --git a/client/internal/certproof/desktopuser_windows.go b/client/internal/certproof/desktopuser_windows.go new file mode 100644 index 000000000..0a9d63b36 --- /dev/null +++ b/client/internal/certproof/desktopuser_windows.go @@ -0,0 +1,182 @@ +package certproof + +import ( + "fmt" + "strings" + "unsafe" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" +) + +const ( + noActiveSession = 0xFFFFFFFF + servicesSession = 0 + + // wtsCurrentServer is WTS_CURRENT_SERVER_HANDLE, which x/sys/windows does not export. + wtsCurrentServer = windows.Handle(0) +) + +// DesktopUser is an interactive session and the account signed into it. The user's +// certificate store is readable only from a process running as that account, because +// its private keys are protected against the user profile rather than the machine. +type DesktopUser struct { + Session uint32 + Name string + Token windows.Token +} + +// Close releases the session token. +func (u DesktopUser) Close() { + if err := u.Token.Close(); err != nil { + log.Debugf("failed closing desktop session token: %v", err) + } +} + +// currentDesktopUser returns a token for the session whose certificate store should be +// asked: a session of owner, the account the active profile belongs to, with the +// physical console preferred over remote sessions. With no owner only the console user +// counts. Picking any signed-in user instead would let whoever else is logged in to a +// terminal server or VDI host decide the result. The second return is false when no +// such session exists, and only machine certificates can then be proven. +// +// Obtaining the token needs SE_TCB_NAME, which the LocalSystem service has and an +// ordinary process does not. +func currentDesktopUser(owner string) (DesktopUser, bool) { + console := windows.WTSGetActiveConsoleSessionId() + if owner == "" { + return consoleUser(console) + } + + sessions, err := userSessions(console) + if err != nil { + log.Debugf("cannot enumerate terminal sessions: %v", err) + return DesktopUser{}, false + } + var found DesktopUser + var ok bool + for _, session := range sessions { + user, signedIn := desktopUser(session) + if !signedIn { + continue + } + switch { + case !sameAccountName(user.Name, owner): + user.Close() + case !ok: + found, ok = user, true + case strings.EqualFold(user.Name, found.Name): + // Another session of the same account; the first one in preference order wins. + user.Close() + default: + // An owner recorded without a domain matches accounts of several domains here. + // Picking one would let another domain's user answer for the owner. + log.Debugf("certificate posture: profile owner %s matches both %s and %s, no user certificate store is used", owner, found.Name, user.Name) + user.Close() + found.Close() + return DesktopUser{}, false + } + } + if !ok { + log.Debugf("certificate posture: profile owner %s has no signed-in session, no user certificate store is reachable", owner) + } + return found, ok +} + +func consoleUser(console uint32) (DesktopUser, bool) { + if console == noActiveSession || console == servicesSession { + log.Debug("no console session, no user certificate store is reachable") + return DesktopUser{}, false + } + user, ok := desktopUser(console) + if !ok { + log.Debugf("console session %d has nobody signed in, no user certificate store is reachable", console) + } + return user, ok +} + +// sameAccountName compares DOMAIN\account names case-insensitively, as Windows does, and an +// owner given without a domain against the account part alone. The session side comes from +// the session token's own SID, which Windows resolves from its cache of signed-in users. +// Resolving the owner name to a SID instead would ask the domain controller, which on a +// laptop that cannot reach it yet blocks for tens of seconds, past the collection deadline. +func sameAccountName(sessionName, owner string) bool { + if strings.EqualFold(sessionName, owner) { + return true + } + if strings.Contains(owner, `\`) { + return false + } + _, account, found := strings.Cut(sessionName, `\`) + return found && strings.EqualFold(account, owner) +} + +func desktopUser(session uint32) (DesktopUser, bool) { + var token windows.Token + if err := windows.WTSQueryUserToken(session, &token); err != nil { + log.Debugf("no user token for session %d: %v", session, err) + return DesktopUser{}, false + } + + name, err := tokenAccount(token) + if err != nil { + log.Debugf("session %d token has no readable account: %v", session, err) + if closeErr := token.Close(); closeErr != nil { + log.Debugf("failed closing session token: %v", closeErr) + } + return DesktopUser{}, false + } + return DesktopUser{Session: session, Name: name, Token: token}, true +} + +func tokenAccount(token windows.Token) (string, error) { + user, err := token.GetTokenUser() + if err != nil { + return "", fmt.Errorf("read token user: %w", err) + } + account, domain, _, err := user.User.Sid.LookupAccount("") + if err != nil { + return "", fmt.Errorf("look up account: %w", err) + } + if domain == "" { + return account, nil + } + return domain + `\` + account, nil +} + +// userSessions lists the sessions a user can be signed in to, console first, then +// active remote sessions, then disconnected ones, whose user is still signed in. Session +// 0 is skipped: it hosts services and never belongs to an interactive user. +func userSessions(console uint32) ([]uint32, error) { + var info *windows.WTS_SESSION_INFO + var count uint32 + if err := windows.WTSEnumerateSessions(wtsCurrentServer, 0, 1, &info, &count); err != nil { + return nil, fmt.Errorf("enumerate sessions: %w", err) + } + defer windows.WTSFreeMemory(uintptr(unsafe.Pointer(info))) + + var sessions []uint32 + if console != noActiveSession && console != servicesSession { + sessions = append(sessions, console) + } + for _, state := range []uint32{windows.WTSActive, windows.WTSDisconnected} { + for _, session := range unsafe.Slice(info, count) { + if session.SessionID == servicesSession || session.SessionID == console || session.State != state { + continue + } + sessions = append(sessions, session.SessionID) + } + } + return sessions, nil +} + +// runningAsLocalSystem reports whether this process is the service. The helper runs as +// the signed-in user and must read its own store rather than launching another helper. +func runningAsLocalSystem() bool { + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + log.Debugf("failed reading own token user: %v", err) + return false + } + return user.User.Sid.IsWellKnown(windows.WinLocalSystemSid) +} diff --git a/client/internal/certproof/desktopuser_windows_test.go b/client/internal/certproof/desktopuser_windows_test.go new file mode 100644 index 000000000..c3c4022e5 --- /dev/null +++ b/client/internal/certproof/desktopuser_windows_test.go @@ -0,0 +1,61 @@ +package certproof + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSameAccountName(t *testing.T) { + tests := []struct { + session, owner string + want bool + }{ + {`CORP\alice`, `CORP\alice`, true}, + {`CORP\alice`, `corp\ALICE`, true}, + {`CORP\alice`, `alice`, true}, + {`CORP\alice`, `OTHER\alice`, false}, + {`CORP\alice`, `bob`, false}, + {`alice`, `alice`, true}, + {`CORP\alice`, `CORP\alic`, false}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, sameAccountName(tt.session, tt.owner), "session %q, owner %q", tt.session, tt.owner) + } +} + +// currentDesktopUser against the real session manager: an owner no session belongs to +// must never yield a session, whoever else is signed in. +func TestCurrentDesktopUser_UnknownOwnerHasNoSession(t *testing.T) { + user, ok := currentDesktopUser(`NO-SUCH-DOMAIN\no-such-user-netbird`) + if ok { + user.Close() + } + assert.False(t, ok, "a profile owner without a session must not borrow another user's store") +} + +// TestCurrentDesktopUser_FindsTheOwnersSession runs against a real machine as LocalSystem, +// with NB_TEST_DESKTOP_OWNER naming an account that is signed in. That account's session +// must be found, and only that account's. +func TestCurrentDesktopUser_FindsTheOwnersSession(t *testing.T) { + owner := os.Getenv("NB_TEST_DESKTOP_OWNER") + if owner == "" { + t.Skip("set NB_TEST_DESKTOP_OWNER to a signed-in account and run as LocalSystem") + } + user, ok := currentDesktopUser(owner) + require.True(t, ok, "the signed-in owner %s must have a session", owner) + defer user.Close() + t.Logf("owner %s resolved to session %d as %s", owner, user.Session, user.Name) + assert.True(t, sameAccountName(user.Name, owner), "the session found belongs to %s, not %s", owner, user.Name) + assert.NotZero(t, user.Session, "session 0 hosts services and never belongs to the owner") + + console, consoleOK := currentDesktopUser("") + if consoleOK { + defer console.Close() + t.Logf("without an owner the console user counts: session %d as %s", console.Session, console.Name) + } else { + t.Log("without an owner nobody counts: no user at the console") + } +} diff --git a/client/internal/certproof/doc.go b/client/internal/certproof/doc.go new file mode 100644 index 000000000..c8dc62f0c --- /dev/null +++ b/client/internal/certproof/doc.go @@ -0,0 +1,26 @@ +// Package certproof answers certificate posture challenges: it finds the certificates a +// peer holds a private key for and signs the challenge nonce, bound to the peer's +// WireGuard key, with that key. Management verifies the signature and the chain against +// the CAs of the check, so only a holder of the key passes and a proof made for one peer +// cannot be replayed by another. Keys in the OS stores, the TPM and a PKCS#11 token are +// used through the platform, which signs; a plain PEM key file is the exception, parsed +// and used in the daemon's memory. +// +// Machine stores are read by the daemon itself: the Windows LocalMachine store, the macOS +// System keychain, and on Linux a directory of PEM files, TSS2 key files the TPM signs +// with, and a PKCS#11 token. On Linux the directory and the token URI come from the +// daemon's environment (NB_CERT_STORE_DIR, NB_CERT_PKCS11_URI), as does the token PIN +// (NB_TPM_PIN); none of them is read from the profile config. +// +// User stores cannot be read by a privileged daemon, so it starts this binary as +// "netbird posture cert-proof" inside the user's session and receives only signatures +// and chains on its stdout. On macOS uid is not what unlocks a login keychain, the +// session's bootstrap namespace is, which is why the helper enters it through launchctl +// asuser. On Windows a service opening CURRENT_USER silently reads its own empty hive, so +// the helper runs with the session's token instead. Its output is untrusted: sizes are +// capped and only proofs for requested nonces are kept. +// +// A user proof shows that some process in that user's session could use a key whose +// certificate chains to the CA. It does not show which binary signed, or that the key is +// hardware-bound; device stores are the stronger signal. +package certproof diff --git a/client/internal/certproof/helper.go b/client/internal/certproof/helper.go new file mode 100644 index 000000000..81a72ac70 --- /dev/null +++ b/client/internal/certproof/helper.go @@ -0,0 +1,66 @@ +package certproof + +import ( + "context" + "encoding/json" + "fmt" + "io" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// HelperRequest is the work the daemon hands to a helper running in a user session. The +// peer key binds every signature to this machine, so a proof cannot be replayed onto +// another peer. +type HelperRequest struct { + PeerKey []byte `json:"peerKey"` + Challenges []HelperChallenge `json:"challenges"` +} + +// HelperChallenge is one certificate challenge in the form the helper needs. +type HelperChallenge struct { + Nonce []byte `json:"nonce"` + CACertificates []string `json:"caCertificates"` +} + +// HelperResponse carries the proofs the helper produced from its own keychain. +type HelperResponse struct { + Proofs []certposture.Proof `json:"proofs"` +} + +// RunHelper answers the challenges on in from the store of the user running this +// process and writes the proofs to out. It is the child half of the console user +// lookup: the daemon cannot read a login keychain, so it launches this in the user's +// session instead. Only the signature crosses back, never the private key. +func RunHelper(ctx context.Context, in io.Reader, out io.Writer) error { + return runHelper(ctx, helperStore(), in, out) +} + +func runHelper(ctx context.Context, store Store, in io.Reader, out io.Writer) error { + var req HelperRequest + if err := json.NewDecoder(in).Decode(&req); err != nil { + return fmt.Errorf("decode helper request: %w", err) + } + + challenges := make([]*proto.CertificateChallenge, 0, len(req.Challenges)) + for _, challenge := range req.Challenges { + challenges = append(challenges, &proto.CertificateChallenge{ + Nonce: challenge.Nonce, + CaCertificates: challenge.CACertificates, + }) + } + + var proofs []certposture.Proof + if len(challenges) > 0 { + proofs = CollectChallenges(ctx, store, challenges, req.PeerKey) + } + log.Debugf("certificate posture helper: answering %d challenges with %d proofs", len(challenges), len(proofs)) + + if err := json.NewEncoder(out).Encode(HelperResponse{Proofs: proofs}); err != nil { + return fmt.Errorf("encode helper response: %w", err) + } + return nil +} diff --git a/client/internal/certproof/helper_backoff.go b/client/internal/certproof/helper_backoff.go new file mode 100644 index 000000000..3372b026b --- /dev/null +++ b/client/internal/certproof/helper_backoff.go @@ -0,0 +1,78 @@ +package certproof + +import ( + "crypto/sha256" + "encoding/hex" + "slices" + "strings" + "sync" + "time" + + "github.com/netbirdio/netbird/shared/management/proto" +) + +// helperQuietPeriod is how long a user whose helper proved nothing is not asked again +// for the same CAs. On macOS each helper run may show a keychain prompt, so retrying on +// every collection would put up a new one each time the user ignored or denied the last. +const helperQuietPeriod = time.Hour + +// helperBackoff remembers, per user and set of CAs asked about, until when the helper +// is not launched again because its last run proved nothing. +type helperBackoff struct { + mu sync.Mutex + until map[string]time.Time +} + +func newHelperBackoff() *helperBackoff { + return &helperBackoff{until: map[string]time.Time{}} +} + +// allow reports whether the helper may be launched for key now. +func (b *helperBackoff) allow(key string, now time.Time) bool { + b.mu.Lock() + defer b.mu.Unlock() + return !now.Before(b.until[key]) +} + +// record stores the outcome of a helper run for key: one that proved something clears +// the back-off, one that proved nothing starts it. Expired entries are dropped, so the +// map holds only users currently held off. +func (b *helperBackoff) record(key string, proven bool, now time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + for other, until := range b.until { + if !now.Before(until) { + delete(b.until, other) + } + } + if proven { + delete(b.until, key) + return + } + b.until[key] = now.Add(helperQuietPeriod) +} + +// helperBackoffKey identifies a user and the set of CAs the challenges accept, in any +// order, leaving out the nonces, which rotate without changing what the user is asked to +// prove. +func helperBackoffKey(user string, challenges []*proto.CertificateChallenge) string { + sets := make([]string, 0, len(challenges)) + for _, challenge := range challenges { + cas := make([]string, 0, len(challenge.GetCaCertificates())) + for _, ca := range challenge.GetCaCertificates() { + sum := sha256.Sum256([]byte(ca)) + cas = append(cas, hex.EncodeToString(sum[:])) + } + slices.Sort(cas) + sets = append(sets, strings.Join(cas, ",")) + } + slices.Sort(sets) + + h := sha256.New() + h.Write([]byte(user)) + for _, set := range sets { + h.Write([]byte{0}) + h.Write([]byte(set)) + } + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/client/internal/certproof/helper_backoff_test.go b/client/internal/certproof/helper_backoff_test.go new file mode 100644 index 000000000..0d3cafea1 --- /dev/null +++ b/client/internal/certproof/helper_backoff_test.go @@ -0,0 +1,69 @@ +package certproof + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/netbirdio/netbird/shared/management/proto" +) + +// TestHelperBackoff: a user whose helper proved nothing, for one ignoring or denying a +// keychain prompt, is not asked again for the same CAs within the quiet period, while a +// success or other CAs are asked at once. +func TestHelperBackoff(t *testing.T) { + now := time.Now() + b := newHelperBackoff() + key := helperBackoffKey("501", []*proto.CertificateChallenge{{CaCertificates: []string{"ca-a"}}}) + + assert.True(t, b.allow(key, now), "a user never asked is asked") + + b.record(key, false, now) + assert.False(t, b.allow(key, now.Add(helperQuietPeriod-time.Second)), "no new prompt within the quiet period") + assert.True(t, b.allow(key, now.Add(helperQuietPeriod)), "asked again once the quiet period passed") + + other := helperBackoffKey("501", []*proto.CertificateChallenge{{CaCertificates: []string{"ca-b"}}}) + assert.True(t, b.allow(other, now), "a check for other CAs is asked at once") + assert.True(t, b.allow(helperBackoffKey("502", []*proto.CertificateChallenge{{CaCertificates: []string{"ca-a"}}}), now), "another user is asked at once") + + b.record(key, true, now) + assert.True(t, b.allow(key, now), "a run that proved something clears the quiet period") +} + +// TestHelperBackoffKey_IgnoresNonces: nonces rotate every window without changing what +// the user is asked to prove, so they do not reset the quiet period. +func TestHelperBackoffKey_IgnoresNonces(t *testing.T) { + a := helperBackoffKey("501", []*proto.CertificateChallenge{{Nonce: []byte("one"), CaCertificates: []string{"ca"}}}) + b := helperBackoffKey("501", []*proto.CertificateChallenge{{Nonce: []byte("two"), CaCertificates: []string{"ca"}}}) + assert.Equal(t, a, b, "the key does not depend on the nonce") +} + +// TestHelperBackoffKey_IgnoresOrder: the same CAs, listed in another order or with the +// challenges reordered, are the same question to the user. +func TestHelperBackoffKey_IgnoresOrder(t *testing.T) { + a := helperBackoffKey("501", []*proto.CertificateChallenge{ + {CaCertificates: []string{"ca-1", "ca-2"}}, + {CaCertificates: []string{"ca-3"}}, + }) + b := helperBackoffKey("501", []*proto.CertificateChallenge{ + {CaCertificates: []string{"ca-3"}}, + {CaCertificates: []string{"ca-2", "ca-1"}}, + }) + assert.Equal(t, a, b, "the key does not depend on the order of CAs or challenges") + + c := helperBackoffKey("501", []*proto.CertificateChallenge{{CaCertificates: []string{"ca-1", "ca-2", "ca-3"}}}) + assert.NotEqual(t, a, c, "the same CAs grouped into other challenges are another question") +} + +// TestHelperBackoff_PrunesExpired: entries whose quiet period passed are dropped when the +// next outcome is recorded, so the map does not grow with every user and CA set seen. +func TestHelperBackoff_PrunesExpired(t *testing.T) { + now := time.Now() + b := newHelperBackoff() + b.record("old", false, now) + b.record("new", false, now.Add(helperQuietPeriod)) + + assert.NotContains(t, b.until, "old", "an expired entry is pruned") + assert.Contains(t, b.until, "new", "a current entry is kept") +} diff --git a/client/internal/certproof/helper_kill_other.go b/client/internal/certproof/helper_kill_other.go new file mode 100644 index 000000000..2ececb27e --- /dev/null +++ b/client/internal/certproof/helper_kill_other.go @@ -0,0 +1,10 @@ +//go:build !unix && !windows + +package certproof + +import "os/exec" + +// startHelper starts cmd. +func startHelper(cmd *exec.Cmd) (func(), error) { + return func() {}, cmd.Start() +} diff --git a/client/internal/certproof/helper_kill_unix.go b/client/internal/certproof/helper_kill_unix.go new file mode 100644 index 000000000..e434d4962 --- /dev/null +++ b/client/internal/certproof/helper_kill_unix.go @@ -0,0 +1,35 @@ +//go:build unix + +package certproof + +import ( + "errors" + "os/exec" + "syscall" +) + +// killHelperGroupOnCancel puts cmd in a process group of its own and kills the whole +// group when cmd's context ends. The helper may run below launchers such as launchctl +// and sudo, which do not pass a kill on, and it keeps the output pipes open, so killing +// only the direct child would leave the helper running and Wait blocked until the +// helper exits on its own, for one waiting on a keychain prompt nobody answers. +func killHelperGroupOnCancel(cmd *exec.Cmd) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.Setpgid = true + cmd.Cancel = func() error { + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if errors.Is(err, syscall.ESRCH) { + return nil + } + return err + } + cmd.WaitDelay = helperWaitDelay +} + +// startHelper starts cmd. Killing the helper with everything below it is arranged by +// killHelperGroupOnCancel where the helper runs under a launcher. +func startHelper(cmd *exec.Cmd) (func(), error) { + return func() {}, cmd.Start() +} diff --git a/client/internal/certproof/helper_kill_windows.go b/client/internal/certproof/helper_kill_windows.go new file mode 100644 index 000000000..1e143c132 --- /dev/null +++ b/client/internal/certproof/helper_kill_windows.go @@ -0,0 +1,77 @@ +//go:build windows + +package certproof + +import ( + "fmt" + "os/exec" + "unsafe" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" +) + +// startHelper starts cmd inside a job object that is terminated when cmd's context ends +// and closed, killing whatever is left in it, once the helper has been awaited. Killing +// only the helper on Windows leaves its descendants running, and one that holds the +// output pipe would block Wait until it exits, so WaitDelay bounds that wait as well. +func startHelper(cmd *exec.Cmd) (func(), error) { + job, err := newKillOnCloseJob() + if err != nil { + return nil, err + } + closeJob := func() { + if err := windows.CloseHandle(job); err != nil { + log.Debugf("failed to close certificate proof helper job: %v", err) + } + } + + cmd.Cancel = func() error { + return windows.TerminateJobObject(job, 1) + } + cmd.WaitDelay = helperWaitDelay + if err := cmd.Start(); err != nil { + closeJob() + return nil, err + } + + // A helper outside the job is still killed directly and bounded by WaitDelay; only + // its descendants would outlive it. + if err := assignToJob(job, cmd.Process.Pid); err != nil { + log.Debugf("failed to put certificate proof helper %d in its job: %v", cmd.Process.Pid, err) + cmd.Cancel = func() error { return cmd.Process.Kill() } + } + return closeJob, nil +} + +// newKillOnCloseJob creates a job object whose processes are killed when its last handle +// is closed. +func newKillOnCloseJob() (windows.Handle, error) { + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return 0, fmt.Errorf("create job object: %w", err) + } + info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{ + BasicLimitInformation: windows.JOBOBJECT_BASIC_LIMIT_INFORMATION{ + LimitFlags: windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + }, + } + if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(info))); err != nil { + _ = windows.CloseHandle(job) + return 0, fmt.Errorf("set job object limits: %w", err) + } + return job, nil +} + +// assignToJob puts the process pid into job. +func assignToJob(job windows.Handle, pid int) error { + process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(pid)) + if err != nil { + return fmt.Errorf("open process: %w", err) + } + defer func() { + _ = windows.CloseHandle(process) + }() + return windows.AssignProcessToJobObject(job, process) +} diff --git a/client/internal/certproof/helper_kill_windows_test.go b/client/internal/certproof/helper_kill_windows_test.go new file mode 100644 index 000000000..4778d253b --- /dev/null +++ b/client/internal/certproof/helper_kill_windows_test.go @@ -0,0 +1,29 @@ +//go:build windows + +package certproof + +import ( + "context" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRunHelperCmd_KillsHelperTree: the helper's child holds stdout and runs for a minute, +// like a helper stuck below a launcher. Killing only the helper would leave Wait blocked +// on the child's pipe until WaitDelay gives up; terminating the job ends both at once. +func TestRunHelperCmd_KillsHelperTree(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + + cmd := exec.CommandContext(ctx, "cmd.exe", "/c", "ping -n 60 127.0.0.1") + + start := time.Now() + _, err := runHelperCmd(cmd, HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}}) + + require.Error(t, err) + assert.Less(t, time.Since(start), helperWaitDelay, "the whole tree is killed at the timeout, not left for WaitDelay") +} diff --git a/client/internal/certproof/helper_run.go b/client/internal/certproof/helper_run.go new file mode 100644 index 000000000..52c9db775 --- /dev/null +++ b/client/internal/certproof/helper_run.go @@ -0,0 +1,126 @@ +package certproof + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os/exec" + "strings" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" +) + +const ( + maxHelperStdout = 1 << 20 + maxHelperStderr = 4 << 10 + + // helperWaitDelay bounds how long a helper's output is awaited after it was killed. + helperWaitDelay = 2 * time.Second +) + +var errHelperOutputTooLarge = errors.New("helper output exceeds the size limit") + +// runHelperCmd feeds req to the helper process cmd and returns the proofs it answered. +// The helper runs as an unprivileged user who can control its output, so both streams +// are capped and only proofs for a nonce req asked about are kept, one per challenge. +func runHelperCmd(cmd *exec.Cmd, req HelperRequest) ([]certposture.Proof, error) { + payload, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("encode helper request: %w", err) + } + + stdout := &cappedBuffer{limit: maxHelperStdout} + stderr := &cappedBuffer{limit: maxHelperStderr} + cmd.Stdin = bytes.NewReader(payload) + cmd.Stdout = stdout + cmd.Stderr = stderr + + release, err := startHelper(cmd) + if err != nil { + return nil, fmt.Errorf("start helper: %w", err) + } + defer release() + if err := cmd.Wait(); err != nil { + return nil, fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String())) + } + logHelperStderr(stderr.String()) + if stdout.truncated { + return nil, errHelperOutputTooLarge + } + + var resp HelperResponse + if err := json.Unmarshal(stdout.Bytes(), &resp); err != nil { + return nil, fmt.Errorf("decode helper response: %w", err) + } + return requestedProofs(req, resp.Proofs), nil +} + +// logHelperStderr records what a helper that succeeded wrote to stderr, such as a user +// certificate it could not sign with, which is otherwise only visible in the user's +// session. Each line is quoted because the helper's user controls its content. +func logHelperStderr(stderr string) { + for _, line := range strings.Split(strings.TrimSpace(stderr), "\n") { + if line = strings.TrimSpace(line); line != "" { + log.Debugf("certificate posture helper: %q", line) + } + } +} + +// requestedProofs keeps the proofs whose nonce belongs to one of req's challenges, at +// most as many as req has challenges. +func requestedProofs(req HelperRequest, proofs []certposture.Proof) []certposture.Proof { + var kept []certposture.Proof + for _, proof := range proofs { + if len(kept) == len(req.Challenges) { + break + } + if !req.asked(proof.Nonce) { + log.Debugf("certificate posture: dropping helper proof for a nonce that was not requested") + continue + } + kept = append(kept, proof) + } + return kept +} + +func (r HelperRequest) asked(nonce []byte) bool { + for _, challenge := range r.Challenges { + if len(challenge.Nonce) > 0 && bytes.Equal(challenge.Nonce, nonce) { + return true + } + } + return false +} + +// cappedBuffer keeps the first limit bytes written to it and discards the rest, so a +// misbehaving child cannot grow the parent's memory without bound. The buffer is a +// named field rather than embedded: an embedded bytes.Buffer would promote ReadFrom, +// which io.Copy prefers over Write, bypassing the cap. +type cappedBuffer struct { + buf bytes.Buffer + limit int + truncated bool +} + +func (b *cappedBuffer) Write(p []byte) (int, error) { + if room := b.limit - b.buf.Len(); room < len(p) { + b.truncated = true + if room > 0 { + b.buf.Write(p[:room]) + } + return len(p), nil + } + return b.buf.Write(p) +} + +func (b *cappedBuffer) Bytes() []byte { + return b.buf.Bytes() +} + +func (b *cappedBuffer) String() string { + return b.buf.String() +} diff --git a/client/internal/certproof/helper_run_test.go b/client/internal/certproof/helper_run_test.go new file mode 100644 index 000000000..ea1246e93 --- /dev/null +++ b/client/internal/certproof/helper_run_test.go @@ -0,0 +1,119 @@ +//go:build !windows && !js + +package certproof + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" +) + +// fakeHelper is a child process that drains its stdin and then prints script's output, +// standing in for a helper running in a user session the daemon cannot trust. +func fakeHelper(t *testing.T, script string) *exec.Cmd { + t.Helper() + return exec.Command("/bin/sh", "-c", "cat >/dev/null; "+script) +} + +func printJSON(t *testing.T, v any) string { + t.Helper() + out, err := json.Marshal(v) + require.NoError(t, err) + return fmt.Sprintf("printf '%%s' '%s'", out) +} + +func TestRunHelperCmd_KeepsOnlyRequestedProofs(t *testing.T) { + req := HelperRequest{PeerKey: peerKey, Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + resp := HelperResponse{Proofs: []certposture.Proof{ + {Nonce: []byte("injected"), Signature: []byte("x")}, + {Nonce: []byte("asked"), Signature: []byte("first")}, + {Nonce: []byte("asked"), Signature: []byte("second")}, + }} + + proofs, err := runHelperCmd(fakeHelper(t, printJSON(t, resp)), req) + + require.NoError(t, err) + require.Len(t, proofs, 1, "a helper may not return more proofs than challenges or proofs for nonces it was not asked about") + assert.Equal(t, []byte("first"), proofs[0].Signature, "the first proof for the requested nonce is kept") +} + +func TestRunHelperCmd_RejectsOversizedOutput(t *testing.T) { + req := HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + script := fmt.Sprintf("head -c %d /dev/zero", maxHelperStdout+1) + + _, err := runHelperCmd(fakeHelper(t, script), req) + + assert.ErrorIs(t, err, errHelperOutputTooLarge) +} + +func TestRunHelperCmd_CapsStderrInError(t *testing.T) { + req := HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + script := fmt.Sprintf("head -c %d /dev/zero | tr '\\0' 'a' >&2; exit 3", 10*maxHelperStderr) + + _, err := runHelperCmd(fakeHelper(t, script), req) + + require.Error(t, err) + assert.LessOrEqual(t, len(err.Error()), maxHelperStderr+100, "a chatty helper must not blow up the daemon's error or log line") + assert.True(t, strings.Contains(err.Error(), "exit status 3"), "the exit status is kept: %v", err) +} + +// TestRunHelperCmd_LogsStderrOfSuccessfulHelper covers a helper that answers but warns, +// for one about a certificate it could not sign with: the warning reaches the daemon's +// log, quoted so a helper cannot forge log lines with embedded newlines. +func TestRunHelperCmd_LogsStderrOfSuccessfulHelper(t *testing.T) { + var logged bytes.Buffer + log.SetOutput(&logged) + level := log.GetLevel() + log.SetLevel(log.DebugLevel) + t.Cleanup(func() { + log.SetOutput(os.Stderr) + log.SetLevel(level) + }) + + req := HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + script := "printf 'failed signing certificate proof for CN=user\\nforged line\\n' >&2; " + printJSON(t, HelperResponse{}) + + _, err := runHelperCmd(fakeHelper(t, script), req) + + require.NoError(t, err) + assert.Contains(t, logged.String(), "failed signing certificate proof for CN=user", "the helper's warning reaches the daemon log") + assert.NotContains(t, logged.String(), "\nforged line", "helper output cannot start a log line of its own") +} + +// TestRunHelperCmd_KillsHelperBelowLauncher stands in for launchctl and sudo starting the +// helper: the direct child spawns a grandchild that holds stdout and never exits, like a +// helper waiting on a keychain prompt. The timeout must end both, not wait for the +// grandchild. +func TestRunHelperCmd_KillsHelperBelowLauncher(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", "sleep 30 & cat >/dev/null; wait") + killHelperGroupOnCancel(cmd) + + start := time.Now() + _, err := runHelperCmd(cmd, HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}}) + + require.Error(t, err) + assert.Less(t, time.Since(start), 5*time.Second, "the helper is killed at the timeout, not awaited until it exits") +} + +func TestRunHelperCmd_RejectsGarbage(t *testing.T) { + req := HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + + _, err := runHelperCmd(fakeHelper(t, "printf 'not json'"), req) + + assert.ErrorContains(t, err, "decode helper response") +} diff --git a/client/internal/certproof/helper_spawn.go b/client/internal/certproof/helper_spawn.go new file mode 100644 index 000000000..54ed265ec --- /dev/null +++ b/client/internal/certproof/helper_spawn.go @@ -0,0 +1,18 @@ +//go:build (darwin && !ios) || windows + +package certproof + +import ( + "github.com/netbirdio/netbird/shared/management/proto" +) + +func helperRequest(challenges []*proto.CertificateChallenge, peerKey []byte) HelperRequest { + req := HelperRequest{PeerKey: peerKey, Challenges: make([]HelperChallenge, 0, len(challenges))} + for _, challenge := range challenges { + req.Challenges = append(req.Challenges, HelperChallenge{ + Nonce: challenge.GetNonce(), + CACertificates: challenge.GetCaCertificates(), + }) + } + return req +} diff --git a/client/internal/certproof/helper_spawn_test.go b/client/internal/certproof/helper_spawn_test.go new file mode 100644 index 000000000..74a792a49 --- /dev/null +++ b/client/internal/certproof/helper_spawn_test.go @@ -0,0 +1,29 @@ +//go:build (darwin && !ios) || windows + +package certproof + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture/certtest" + "github.com/netbirdio/netbird/shared/management/proto" +) + +func TestHelperRequest_CarriesEveryChallenge(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + challenges := []*proto.CertificateChallenge{ + {Nonce: []byte("first"), CaCertificates: []string{ca.PEM}}, + {Nonce: []byte("second")}, + } + + req := helperRequest(challenges, peerKey) + + require.Len(t, req.Challenges, 2, "every challenge must reach the helper") + assert.Equal(t, peerKey, req.PeerKey, "the peer key binds the signature to this machine") + assert.Equal(t, []byte("first"), req.Challenges[0].Nonce, "the nonce must survive unchanged") + assert.Equal(t, []string{ca.PEM}, req.Challenges[0].CACertificates, "the accepted CAs must survive unchanged") + assert.Empty(t, req.Challenges[1].CACertificates, "a challenge without CAs stays without CAs") +} diff --git a/client/internal/certproof/helper_test.go b/client/internal/certproof/helper_test.go new file mode 100644 index 000000000..61fcc8532 --- /dev/null +++ b/client/internal/certproof/helper_test.go @@ -0,0 +1,66 @@ +package certproof + +import ( + "bytes" + "context" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/certposture/certtest" +) + +// TestRunHelper_AnswersAChallengeFromItsStore runs the helper's request and response +// handling in-process: the proof it writes verifies as the daemon will verify it. The +// process boundary itself is covered by the runHelperCmd tests. +func TestRunHelper_AnswersAChallengeFromItsStore(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + dir := storeDir(t) + key := certtest.ECDSAKey(t) + writeFile(t, dir, "device.pem", certtest.CertPEM(ca.Issue(t, key, "device"))+certtest.KeyPEM(t, key)) + + challenger := certposture.NewChallenger([]byte("secret")) + nonce := challenger.Nonce(peerKey, time.Now()) + request, err := json.Marshal(HelperRequest{ + PeerKey: peerKey, + Challenges: []HelperChallenge{{Nonce: nonce, CACertificates: []string{ca.PEM}}}, + }) + require.NoError(t, err, "request must encode") + + var stdout bytes.Buffer + require.NoError(t, runHelper(context.Background(), newFileStore(dir), bytes.NewReader(request), &stdout)) + + var resp HelperResponse + require.NoError(t, json.Unmarshal(stdout.Bytes(), &resp), "helper must emit decodable JSON") + require.Len(t, resp.Proofs, 1, "the matching certificate should produce one proof") + + // Verify exactly as management does, so the proof is proven to survive the encode, + // the process boundary and the decode intact. + chain, err := challenger.Verify(resp.Proofs[0], peerKey, time.Now()) + require.NoError(t, err, "the decoded proof must verify against the issued nonce") + assert.Equal(t, "device", chain[0].Subject.CommonName, "the proven leaf should be the device certificate") +} + +func TestRunHelper_NoChallengesYieldsEmptyResponse(t *testing.T) { + request, err := json.Marshal(HelperRequest{PeerKey: peerKey}) + require.NoError(t, err) + + var stdout bytes.Buffer + require.NoError(t, runHelper(context.Background(), newFileStore(storeDir(t)), bytes.NewReader(request), &stdout)) + + var resp HelperResponse + require.NoError(t, json.Unmarshal(stdout.Bytes(), &resp), "an empty request must still emit valid JSON") + assert.Empty(t, resp.Proofs, "no challenges should produce no proofs") +} + +func TestRunHelper_RejectsMalformedRequest(t *testing.T) { + var stdout bytes.Buffer + err := runHelper(context.Background(), newFileStore(storeDir(t)), bytes.NewReader([]byte("not json")), &stdout) + + require.Error(t, err, "a malformed request must fail rather than emit an empty proof set") + assert.Empty(t, stdout.String(), "nothing should be written to stdout on a decode failure") +} diff --git a/client/internal/certproof/keychain_darwin.go b/client/internal/certproof/keychain_darwin.go new file mode 100644 index 000000000..5b1426997 --- /dev/null +++ b/client/internal/certproof/keychain_darwin.go @@ -0,0 +1,490 @@ +//go:build !ios + +package certproof + +import ( + "bytes" + "context" + "crypto" + "crypto/x509" + "errors" + "fmt" + "io" + "os" + "sync" + "unsafe" + + "github.com/ebitengine/purego" + log "github.com/sirupsen/logrus" +) + +const ( + securityFramework = "/System/Library/Frameworks/Security.framework/Security" + coreFoundationFramework = "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation" + + errSecItemNotFound = -25300 + errSecInteractionNotAllowed = -25308 +) + +// errKeyNeedsApproval reports a key whose access list does not include netbird, so using +// it needs the user's approval, which a daemon has no UI to ask for. +var errKeyNeedsApproval = errors.New("the key's access control requires user approval for netbird; " + + "import the identity with netbird allowed (security import -k " + + "-T /Applications/NetBird.app/Contents/MacOS/netbird), " + + "or set AllowAllAppsAccess in the MDM certificate payload, which allows every application") + +var ( + keychainOnce sync.Once + keychainErr error + + secItemCopyMatching func(query uintptr, result *uintptr) int32 + secIdentityCopyCertificate func(identity uintptr, cert *uintptr) int32 + secIdentityCopyPrivateKey func(identity uintptr, key *uintptr) int32 + secCertificateCopyData func(cert uintptr) uintptr + secKeyCreateSignature func(key, algorithm, data uintptr, err *uintptr) uintptr + + secKeychainOpen func(path *byte, keychain *uintptr) int32 + secKeychainCopySearchList func(searchList *uintptr) int32 + secKeychainGetPath func(keychain uintptr, pathLength *uint32, path *byte) int32 + cfArrayCreate func(alloc uintptr, values *uintptr, count int, callBacks uintptr) uintptr + + cfDictionaryCreate func(alloc uintptr, keys, values *uintptr, count int, keyCallBacks, valueCallBacks uintptr) uintptr + cfArrayGetCount func(array uintptr) int + cfArrayGetValueAtIndex func(array uintptr, index int) uintptr + cfDataCreate func(alloc uintptr, data *byte, length int) uintptr + cfDataGetLength func(data uintptr) int + cfDataGetBytePtr func(data uintptr) unsafe.Pointer + cfErrorGetCode func(err uintptr) int + cfRelease func(ref uintptr) + + kSecClass, kSecClassIdentity, kSecClassCertificate, kSecMatchLimit, kSecMatchLimitAll, kSecReturnRef uintptr + kSecMatchSearchList uintptr + kSecKeyAlgorithmECDSASHA256, kSecKeyAlgorithmECDSASHA384, kSecKeyAlgorithmRSAPSSSHA256 uintptr + kCFBooleanTrue, kCFTypeDictionaryKeyCallBacks, kCFTypeDictionaryValueCallBacks, kCFTypeArrayCallBacks uintptr +) + +// systemKeychain is where MDM installs device identities. +const systemKeychain = "/Library/Keychains/System.keychain" + +// DefaultStore is the System keychain for the root daemon, so it never touches a user's +// keychain, whose keys would ask for approval in a session the daemon has no UI in. Run +// by an ordinary user, it is that user's keychain search list. +func DefaultStore() Store { + if os.Geteuid() == 0 { + return newKeychainStore(systemKeychain) + } + return newKeychainStore() +} + +// keychainStore yields the identities of a set of keychains, or of the process's keychain +// search list when none is named, reached through purego so the client keeps building +// with CGO_ENABLED=0. +type keychainStore struct { + keychains []string +} + +// newKeychainStore returns a store that searches the keychain files at paths, or the +// process's keychain search list when no path is given. +func newKeychainStore(paths ...string) *keychainStore { + return &keychainStore{keychains: paths} +} + +func (s *keychainStore) Candidates(_ context.Context) ([]Candidate, error) { + if err := loadKeychain(); err != nil { + return nil, err + } + searchList, done, err := searchListOf(s.keychains) + if err != nil { + return nil, err + } + defer done() + + var leaves []*x509.Certificate + err = eachIdentity(searchList, func(_ uintptr, der []byte) (bool, error) { + cert, err := x509.ParseCertificate(der) + if err != nil { + log.Warnf("skipping keychain identity: %v", err) + return false, nil + } + log.Debugf("keychain identity: subject=%q issuer=%q serial=%s expires=%s", cert.Subject, cert.Issuer, cert.SerialNumber, cert.NotAfter) + leaves = append(leaves, cert) + return false, nil + }) + if err != nil { + return nil, err + } + // The certificate query runs even without identities: it separates a keychain that is + // readable but holds no identity from one the process cannot read at all. + pool, err := keychainCertificates(searchList) + if err != nil { + return nil, err + } + if len(leaves) == 0 { + log.Debugf("keychain search list holds no identities usable for certificate posture, but %d readable certificates: an identity needs its private key in the same keychain", len(pool)) + return nil, nil + } + log.Debugf("keychain search list holds %d identities and %d certificates for chain building", len(leaves), len(pool)) + + candidates := make([]Candidate, 0, len(leaves)) + for _, leaf := range leaves { + chain := buildChain(leaf, pool) + log.Debugf("keychain candidate %q issued by %q built a chain of %d certificates", leaf.Subject, leaf.Issuer, len(chain)) + if len(chain) == 1 && leaf.CheckSignatureFrom(leaf) != nil { + log.Debugf("keychain candidate %q has no issuer in the keychain, its proof carries the leaf alone and only verifies if the challenge supplies %q", leaf.Subject, leaf.Issuer) + } + candidates = append(candidates, Candidate{Chain: chain, Signer: &keychainSigner{leaf: leaf, keychains: s.keychains}, Intermediates: pool}) + } + return candidates, nil +} + +// keychainSigner holds only the certificate; the identity is looked up again at signing +// time so no keychain references outlive a call. +type keychainSigner struct { + leaf *x509.Certificate + keychains []string +} + +func (s *keychainSigner) Public() crypto.PublicKey { + return s.leaf.PublicKey +} + +func (s *keychainSigner) Sign(_ io.Reader, digest []byte, opts crypto.SignerOpts) ([]byte, error) { + scheme, err := schemeFor(s.leaf.PublicKey, opts) + if err != nil { + return nil, err + } + log.Debugf("signing certificate posture challenge with keychain key of %q", s.leaf.Subject) + + searchList, done, err := searchListOf(s.keychains) + if err != nil { + return nil, err + } + defer done() + + algorithm := keychainAlgorithm(scheme) + var signature []byte + var attempts signAttempts + err = eachIdentity(searchList, func(identity uintptr, der []byte) (bool, error) { + if !bytes.Equal(der, s.leaf.Raw) { + return false, nil + } + // The same certificate can sit in the keychain with more than one key, of which + // only one may be usable, so a failure moves on to the next match. + var ok bool + signature, ok = attempts.try(func() ([]byte, error) { return signWithIdentity(identity, algorithm, digest) }) + return ok, nil + }) + if err != nil { + return nil, err + } + if signature == nil { + if err := attempts.err(); err != nil { + return nil, err + } + return nil, errors.New("certificate is no longer in the keychain") + } + log.Debugf("keychain signed certificate posture challenge for %q, %d bytes", s.leaf.Subject, len(signature)) + return signature, nil +} + +func keychainAlgorithm(scheme sigScheme) uintptr { + switch scheme { + case schemeECDSASHA384: + return kSecKeyAlgorithmECDSASHA384 + case schemeRSAPSSSHA256: + return kSecKeyAlgorithmRSAPSSSHA256 + default: + return kSecKeyAlgorithmECDSASHA256 + } +} + +func signWithIdentity(identity, algorithm uintptr, digest []byte) ([]byte, error) { + var key uintptr + if status := secIdentityCopyPrivateKey(identity, &key); status != 0 { + return nil, fmt.Errorf("SecIdentityCopyPrivateKey: %d", status) + } + defer release(key) + + data := cfDataCreate(0, &digest[0], len(digest)) + if data == 0 { + return nil, errors.New("CFDataCreate returned NULL") + } + defer release(data) + + var cfErr uintptr + signature := secKeyCreateSignature(key, algorithm, data, &cfErr) + if signature == 0 { + if cfErr == 0 { + return nil, errors.New("SecKeyCreateSignature failed without a CFError") + } + defer release(cfErr) + code := cfErrorGetCode(cfErr) + if code == errSecInteractionNotAllowed { + return nil, fmt.Errorf("SecKeyCreateSignature: CFError %d: %w", code, errKeyNeedsApproval) + } + return nil, fmt.Errorf("SecKeyCreateSignature: CFError %d", code) + } + defer release(signature) + return dataBytes(signature), nil +} + +// searchListOf opens the keychain files at paths as a CFArray for kSecMatchSearchList, +// or yields 0, the process's own search list, when paths is empty. The caller calls done +// once it no longer uses the list. +func searchListOf(paths []string) (searchList uintptr, done func(), err error) { + if len(paths) == 0 { + return 0, func() {}, nil + } + list, err := openSearchList(paths) + if err != nil { + return 0, nil, err + } + return list, func() { release(list) }, nil +} + +// openSearchList opens the keychain files at paths, at least one, as a CFArray. The +// caller releases the array. +func openSearchList(paths []string) (uintptr, error) { + refs := make([]uintptr, 0, len(paths)) + defer func() { + for _, ref := range refs { + release(ref) + } + }() + for _, path := range paths { + cpath := append([]byte(path), 0) + var keychain uintptr + if status := secKeychainOpen(&cpath[0], &keychain); status != 0 { + return 0, fmt.Errorf("SecKeychainOpen %s: %d", path, status) + } + refs = append(refs, keychain) + } + // The array retains the keychains, so the references opened here are released. + list := cfArrayCreate(0, &refs[0], len(refs), kCFTypeArrayCallBacks) + if list == 0 { + return 0, errors.New("CFArrayCreate returned NULL") + } + return list, nil +} + +// eachIdentity calls fn with every identity in searchList, or in the process's search +// list when it is 0, and its certificate. An identity whose certificate cannot be read is +// skipped rather than ending the walk. +func eachIdentity(searchList uintptr, fn func(identity uintptr, der []byte) (bool, error)) error { + return eachMatching(searchList, kSecClassIdentity, "identity", func(identity uintptr) (bool, error) { + var cert uintptr + if status := secIdentityCopyCertificate(identity, &cert); status != 0 { + log.Debugf("skipping keychain identity: SecIdentityCopyCertificate: %d", status) + return false, nil + } + der := certificateDER(cert) + release(cert) + if der == nil { + log.Debug("skipping keychain identity whose certificate has no DER data") + return false, nil + } + return fn(identity, der) + }) +} + +func keychainCertificates(searchList uintptr) ([]*x509.Certificate, error) { + var certs []*x509.Certificate + var unparsable int + err := eachMatching(searchList, kSecClassCertificate, "certificate", func(item uintptr) (bool, error) { + if cert, err := x509.ParseCertificate(certificateDER(item)); err == nil { + certs = append(certs, cert) + return false, nil + } + unparsable++ + return false, nil + }) + log.Debugf("keychain holds %d parsable certificates, %d unparsable", len(certs), unparsable) + return certs, err +} + +func eachMatching(searchList, class uintptr, name string, fn func(item uintptr) (bool, error)) error { + keys := []uintptr{kSecClass, kSecMatchLimit, kSecReturnRef} + values := []uintptr{class, kSecMatchLimitAll, kCFBooleanTrue} + if searchList != 0 { + keys = append(keys, kSecMatchSearchList) + values = append(values, searchList) + } + query := cfDictionaryCreate(0, &keys[0], &values[0], len(keys), kCFTypeDictionaryKeyCallBacks, kCFTypeDictionaryValueCallBacks) + defer release(query) + + var items uintptr + switch status := secItemCopyMatching(query, &items); status { + case 0: + case errSecItemNotFound: + log.Debugf("keychain %s query returned errSecItemNotFound (%d): the search list holds no item of this class", name, errSecItemNotFound) + return nil + default: + log.Debugf("keychain %s query returned OSStatus %d", name, status) + return fmt.Errorf("SecItemCopyMatching: %d", status) + } + defer release(items) + + n := cfArrayGetCount(items) + log.Debugf("keychain %s query returned %d items", name, n) + for i := 0; i < n; i++ { + if stop, err := fn(cfArrayGetValueAtIndex(items, i)); stop || err != nil { + return err + } + } + return nil +} + +// certificateDER returns the DER form of cert, or nil when SecCertificateCopyData +// returns NULL, which it does for an object that is not a valid certificate. +func certificateDER(cert uintptr) []byte { + data := secCertificateCopyData(cert) + if data == 0 { + return nil + } + defer release(data) + return dataBytes(data) +} + +func dataBytes(data uintptr) []byte { + n := cfDataGetLength(data) + if n <= 0 { + return nil + } + return bytes.Clone(unsafe.Slice((*byte)(cfDataGetBytePtr(data)), n)) +} + +// release drops a CoreFoundation reference. CFRelease crashes the process on NULL, and +// several Security calls return NULL on failure, so every release goes through here. +func release(ref uintptr) { + if ref != 0 { + cfRelease(ref) + } +} + +func loadKeychain() error { + keychainOnce.Do(func() { + if keychainErr = resolveKeychain(); keychainErr != nil { + log.Debugf("macOS keychain unavailable for certificate posture: %v", keychainErr) + return + } + log.Debugf("macOS Security framework loaded for certificate posture, running as uid=%d euid=%d", os.Getuid(), os.Geteuid()) + logSearchList() + }) + return keychainErr +} + +// logSearchList reports the keychains the process searches. The root daemon sees the +// System keychain and System Roots, never a user's login keychain. +func logSearchList() { + if secKeychainCopySearchList == nil || secKeychainGetPath == nil { + log.Debug("keychain search list diagnostics unavailable on this macOS version") + return + } + + var list uintptr + if status := secKeychainCopySearchList(&list); status != 0 { + log.Debugf("SecKeychainCopySearchList returned OSStatus %d", status) + return + } + defer release(list) + + n := cfArrayGetCount(list) + log.Debugf("keychain search list contains %d keychains", n) + for i := 0; i < n; i++ { + log.Debugf("keychain search list[%d]: %s", i, keychainPath(cfArrayGetValueAtIndex(list, i))) + } +} + +func keychainPath(keychain uintptr) string { + path := make([]byte, 1024) + length := uint32(len(path)) + if status := secKeychainGetPath(keychain, &length, &path[0]); status != 0 { + return fmt.Sprintf("", status) + } + return string(path[:length]) +} + +func resolveKeychain() error { + security, err := purego.Dlopen(securityFramework, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return fmt.Errorf("open %s: %w", securityFramework, err) + } + coreFoundation, err := purego.Dlopen(coreFoundationFramework, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return fmt.Errorf("open %s: %w", coreFoundationFramework, err) + } + + for _, fn := range []struct { + ptr any + lib uintptr + name string + }{ + {&secItemCopyMatching, security, "SecItemCopyMatching"}, + {&secIdentityCopyCertificate, security, "SecIdentityCopyCertificate"}, + {&secIdentityCopyPrivateKey, security, "SecIdentityCopyPrivateKey"}, + {&secCertificateCopyData, security, "SecCertificateCopyData"}, + {&secKeyCreateSignature, security, "SecKeyCreateSignature"}, + {&secKeychainOpen, security, "SecKeychainOpen"}, + {&cfArrayCreate, coreFoundation, "CFArrayCreate"}, + {&cfDictionaryCreate, coreFoundation, "CFDictionaryCreate"}, + {&cfArrayGetCount, coreFoundation, "CFArrayGetCount"}, + {&cfArrayGetValueAtIndex, coreFoundation, "CFArrayGetValueAtIndex"}, + {&cfDataCreate, coreFoundation, "CFDataCreate"}, + {&cfDataGetLength, coreFoundation, "CFDataGetLength"}, + {&cfDataGetBytePtr, coreFoundation, "CFDataGetBytePtr"}, + {&cfErrorGetCode, coreFoundation, "CFErrorGetCode"}, + {&cfRelease, coreFoundation, "CFRelease"}, + } { + symbol, err := purego.Dlsym(fn.lib, fn.name) + if err != nil { + return fmt.Errorf("resolve %s: %w", fn.name, err) + } + purego.RegisterFunc(fn.ptr, symbol) + } + + for _, global := range []struct { + ptr *uintptr + lib uintptr + name string + deref bool + }{ + {&kSecClass, security, "kSecClass", true}, + {&kSecClassIdentity, security, "kSecClassIdentity", true}, + {&kSecClassCertificate, security, "kSecClassCertificate", true}, + {&kSecMatchLimit, security, "kSecMatchLimit", true}, + {&kSecMatchLimitAll, security, "kSecMatchLimitAll", true}, + {&kSecReturnRef, security, "kSecReturnRef", true}, + {&kSecMatchSearchList, security, "kSecMatchSearchList", true}, + {&kSecKeyAlgorithmECDSASHA256, security, "kSecKeyAlgorithmECDSASignatureDigestX962SHA256", true}, + {&kSecKeyAlgorithmECDSASHA384, security, "kSecKeyAlgorithmECDSASignatureDigestX962SHA384", true}, + {&kSecKeyAlgorithmRSAPSSSHA256, security, "kSecKeyAlgorithmRSASignatureDigestPSSSHA256", true}, + {&kCFBooleanTrue, coreFoundation, "kCFBooleanTrue", true}, + {&kCFTypeDictionaryKeyCallBacks, coreFoundation, "kCFTypeDictionaryKeyCallBacks", false}, + {&kCFTypeDictionaryValueCallBacks, coreFoundation, "kCFTypeDictionaryValueCallBacks", false}, + {&kCFTypeArrayCallBacks, coreFoundation, "kCFTypeArrayCallBacks", false}, + } { + addr, err := purego.Dlsym(global.lib, global.name) + if err != nil { + return fmt.Errorf("resolve %s: %w", global.name, err) + } + if global.deref { + addr = **(**uintptr)(unsafe.Pointer(&addr)) + } + *global.ptr = addr + } + + resolveOptional(security, "SecKeychainCopySearchList", &secKeychainCopySearchList) + resolveOptional(security, "SecKeychainGetPath", &secKeychainGetPath) + return nil +} + +// resolveOptional binds a diagnostic-only symbol, leaving it nil when the framework no +// longer exports it so keychain lookups keep working without it. +func resolveOptional(lib uintptr, name string, ptr any) { + symbol, err := purego.Dlsym(lib, name) + if err != nil { + log.Debugf("keychain diagnostics: %s unavailable: %v", name, err) + return + } + purego.RegisterFunc(ptr, symbol) +} diff --git a/client/internal/certproof/keystore.go b/client/internal/certproof/keystore.go new file mode 100644 index 000000000..7ce676b4c --- /dev/null +++ b/client/internal/certproof/keystore.go @@ -0,0 +1,77 @@ +package certproof + +import ( + "crypto" + "crypto/ecdsa" + "crypto/rsa" + "crypto/x509" + "encoding/asn1" + "errors" + "fmt" + "math/big" + "slices" +) + +var errUnsupportedScheme = errors.New("unsupported signature scheme for OS keystore") + +type sigScheme int + +const ( + schemeECDSASHA256 sigScheme = iota + 1 + schemeECDSASHA384 + schemeRSAPSSSHA256 +) + +// schemeFor maps a crypto.Signer request onto the schemes the OS keystores perform. +func schemeFor(pub crypto.PublicKey, opts crypto.SignerOpts) (sigScheme, error) { + switch pub.(type) { + case *ecdsa.PublicKey: + switch opts.HashFunc() { + case crypto.SHA256: + return schemeECDSASHA256, nil + case crypto.SHA384: + return schemeECDSASHA384, nil + } + case *rsa.PublicKey: + if pss, ok := opts.(*rsa.PSSOptions); ok && pss.Hash == crypto.SHA256 { + return schemeRSAPSSSHA256, nil + } + } + return 0, fmt.Errorf("%w: %T with %v", errUnsupportedScheme, pub, opts.HashFunc()) +} + +// buildChain extends leaf with the issuers found in pool up to a self-signed certificate. +func buildChain(leaf *x509.Certificate, pool []*x509.Certificate) []*x509.Certificate { + chain := []*x509.Certificate{leaf} + current := leaf + for current.CheckSignatureFrom(current) != nil { + issuer := issuerIn(current, pool, chain) + if issuer == nil { + break + } + chain = append(chain, issuer) + current = issuer + } + return chain +} + +func issuerIn(cert *x509.Certificate, pool, seen []*x509.Certificate) *x509.Certificate { + for _, candidate := range pool { + if slices.ContainsFunc(seen, candidate.Equal) { + continue + } + if cert.CheckSignatureFrom(candidate) == nil { + return candidate + } + } + return nil +} + +// ecdsaSignatureASN1 converts the fixed-width r||s form emitted by CNG into the DER form Go verifies. +func ecdsaSignatureASN1(raw []byte) ([]byte, error) { + if len(raw) == 0 || len(raw)%2 != 0 { + return nil, errors.New("malformed raw ECDSA signature") + } + half := len(raw) / 2 + return asn1.Marshal(struct{ R, S *big.Int }{new(big.Int).SetBytes(raw[:half]), new(big.Int).SetBytes(raw[half:])}) +} diff --git a/client/internal/certproof/keystore_test.go b/client/internal/certproof/keystore_test.go new file mode 100644 index 000000000..7d028c4fb --- /dev/null +++ b/client/internal/certproof/keystore_test.go @@ -0,0 +1,91 @@ +package certproof + +import ( + "crypto" + "crypto/ecdsa" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/certposture/certtest" +) + +func TestBuildChain_FollowsIssuersThroughThePool(t *testing.T) { + root := certtest.NewCA(t, "root") + intermediate := certtest.NewIntermediate(t, root, "intermediate") + unrelated := certtest.NewCA(t, "unrelated") + leaf := intermediate.Issue(t, certtest.ECDSAKey(t), "device") + pool := []*x509.Certificate{unrelated.Cert, root.Cert, leaf, intermediate.Cert} + + chain := buildChain(leaf, pool) + + require.Equal(t, []*x509.Certificate{leaf, intermediate.Cert, root.Cert}, chain) + roots, err := certposture.ParseCAs([]string{root.PEM}) + require.NoError(t, err) + assert.NoError(t, certposture.VerifyChain(chain, roots, time.Now())) +} + +func TestBuildChain_StopsWhereThePoolEnds(t *testing.T) { + root := certtest.NewCA(t, "root") + intermediate := certtest.NewIntermediate(t, root, "intermediate") + leaf := intermediate.Issue(t, certtest.ECDSAKey(t), "device") + + assert.Equal(t, []*x509.Certificate{leaf}, buildChain(leaf, nil)) + assert.Equal(t, []*x509.Certificate{leaf, intermediate.Cert}, buildChain(leaf, []*x509.Certificate{intermediate.Cert})) +} + +func TestSchemeFor(t *testing.T) { + ecKey := certtest.ECDSAKey(t) + rsaKey := certtest.RSAKey(t) + pss := &rsa.PSSOptions{SaltLength: rsa.PSSSaltLengthEqualsHash, Hash: crypto.SHA256} + + tests := []struct { + name string + pub crypto.PublicKey + opts crypto.SignerOpts + want sigScheme + }{ + {"ecdsa sha256", ecKey.Public(), crypto.SHA256, schemeECDSASHA256}, + {"ecdsa sha384", ecKey.Public(), crypto.SHA384, schemeECDSASHA384}, + {"rsa pss sha256", rsaKey.Public(), pss, schemeRSAPSSSHA256}, + {"rsa pkcs1v15", rsaKey.Public(), crypto.SHA256, 0}, + {"ecdsa sha512", ecKey.Public(), crypto.SHA512, 0}, + {"rsa pss sha384", rsaKey.Public(), &rsa.PSSOptions{SaltLength: rsa.PSSSaltLengthEqualsHash, Hash: crypto.SHA384}, 0}, + {"ed25519", certtest.Ed25519Key(t).Public(), crypto.Hash(0), 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := schemeFor(tc.pub, tc.opts) + if tc.want == 0 { + assert.ErrorIs(t, err, errUnsupportedScheme) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestECDSASignatureASN1(t *testing.T) { + key := certtest.ECDSAKey(t).(*ecdsa.PrivateKey) + digest := sha256.Sum256([]byte("nonce")) + r, s, err := ecdsa.Sign(rand.Reader, key, digest[:]) + require.NoError(t, err) + raw := append(r.FillBytes(make([]byte, 32)), s.FillBytes(make([]byte, 32))...) + + der, err := ecdsaSignatureASN1(raw) + require.NoError(t, err) + assert.True(t, ecdsa.VerifyASN1(&key.PublicKey, digest[:], der)) + + _, err = ecdsaSignatureASN1(raw[:63]) + assert.Error(t, err, "an odd length cannot split into r and s") + _, err = ecdsaSignatureASN1(nil) + assert.Error(t, err, "an empty signature is malformed") +} diff --git a/client/internal/certproof/merge.go b/client/internal/certproof/merge.go new file mode 100644 index 000000000..d539ce927 --- /dev/null +++ b/client/internal/certproof/merge.go @@ -0,0 +1,50 @@ +package certproof + +import ( + "crypto/sha256" + "crypto/x509" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" +) + +// mergeProofs appends the user session proofs to the device proofs, dropping one whose +// chain the device already proved. Management checks each chain against each check's CAs, +// so the same leaf proven through another chain is kept. +func mergeProofs(device, user []certposture.Proof) []certposture.Proof { + if len(user) == 0 { + return device + } + + seen := make(map[[sha256.Size]byte]struct{}, len(device)+len(user)) + for _, proof := range device { + if len(proof.Chain) > 0 { + seen[chainDERFingerprint(proof.Chain)] = struct{}{} + } + } + + merged := device + for _, proof := range user { + if len(proof.Chain) == 0 { + continue + } + fingerprint := chainDERFingerprint(proof.Chain) + if _, done := seen[fingerprint]; done { + continue + } + seen[fingerprint] = struct{}{} + merged = append(merged, proof) + logUserProof(proof) + } + return merged +} + +func logUserProof(proof certposture.Proof) { + leaf, err := x509.ParseCertificate(proof.Chain[0]) + if err != nil { + log.Debugf("certificate posture: user proof carries an unparsable leaf: %v", err) + return + } + log.Debugf("certificate posture: signed-in user proved %q issued by %q", leaf.Subject, leaf.Issuer) +} diff --git a/client/internal/certproof/merge_test.go b/client/internal/certproof/merge_test.go new file mode 100644 index 000000000..7db2dd421 --- /dev/null +++ b/client/internal/certproof/merge_test.go @@ -0,0 +1,54 @@ +package certproof + +import ( + "crypto/sha256" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/certposture/certtest" +) + +func TestMergeProofs_ProvesACertificateHeldByBothStoresOnce(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + shared := ca.Issue(t, certtest.ECDSAKey(t), "shared") + userOnly := ca.Issue(t, certtest.ECDSAKey(t), "user-only") + + device := []certposture.Proof{{Chain: [][]byte{shared.Raw}}} + user := []certposture.Proof{{Chain: [][]byte{shared.Raw}}, {Chain: [][]byte{userOnly.Raw}}, {}} + + merged := mergeProofs(device, user) + + require.Len(t, merged, 2, "the shared leaf is proven once and the empty chain is dropped") + assert.Equal(t, shared.Raw, merged[0].Chain[0], "the device proof keeps its place") + assert.Equal(t, userOnly.Raw, merged[1].Chain[0], "the user-only certificate is appended") +} + +func TestMergeProofs_KeepsDeviceProofsWhenNoUserSession(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + device := []certposture.Proof{{Chain: [][]byte{ca.Issue(t, certtest.ECDSAKey(t), "device").Raw}}} + + merged := mergeProofs(device, nil) + + require.Len(t, merged, 1, "a machine with nobody signed in still sends its device proof") + assert.Equal(t, sha256.Sum256(device[0].Chain[0]), sha256.Sum256(merged[0].Chain[0]), "the device proof is unchanged") +} + +func TestMergeProofs_KeepsAnotherChainOfTheSameLeaf(t *testing.T) { + root := certtest.NewCA(t, "corp-root") + interA := root.Issue(t, certtest.ECDSAKey(t), "inter-a") + interB := root.Issue(t, certtest.ECDSAKey(t), "inter-b") + leaf := root.Issue(t, certtest.ECDSAKey(t), "leaf") + + // Management checks each chain against each check's CAs, so the same leaf proven + // through another chain is a different proof, not a duplicate. + device := []certposture.Proof{{Chain: [][]byte{leaf.Raw, interA.Raw}}} + user := []certposture.Proof{{Chain: [][]byte{leaf.Raw, interB.Raw}}, {Chain: [][]byte{leaf.Raw, interA.Raw}}} + + merged := mergeProofs(device, user) + + require.Len(t, merged, 2, "the chain only the user store holds is kept, the one both hold is not repeated") + assert.Equal(t, interB.Raw, merged[1].Chain[1], "the user's chain through the other intermediate is appended") +} diff --git a/client/internal/certproof/pinlatch.go b/client/internal/certproof/pinlatch.go new file mode 100644 index 000000000..79e396b11 --- /dev/null +++ b/client/internal/certproof/pinlatch.go @@ -0,0 +1,54 @@ +package certproof + +import ( + "crypto/sha256" + "encoding/binary" + "errors" + "sync" +) + +// errPINRejectedBefore is returned instead of logging in with a PIN the token already +// refused: every failed login counts towards the token's lockout, which for a TPM is +// shared with everything else on the machine, and proofs are collected on every sync. +var errPINRejectedBefore = errors.New("PKCS#11 token rejected this PIN before, not trying it again") + +// errPINNeedsToken refuses a PIN that names no token to log in to. +var errPINNeedsToken = errors.New("a PKCS#11 PIN needs the token named in NB_CERT_PKCS11_URI, as token=