diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 647e04936..ded77ec58 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -46,3 +46,25 @@ updates: wireguard: patterns: - "golang.zx2c4.com/wireguard*" + + # Base images of the source-build Dockerfiles, pinned by digest (Chainguard + # publishes only :latest for free). Dockerfile.release files feed goreleaser + # and keep the published images as they are, so their bases are left alone. + - package-ecosystem: "docker" + directories: + - "/upload-server" + schedule: + interval: "weekly" + open-pull-requests-limit: 3 + groups: + base-images: + patterns: + - "*" + ignore: + - dependency-name: "gcr.io/distroless/base" + # Go minor and major versions move with the rest of the repository; + # patch releases and new digests of the pinned tag still come through. + - dependency-name: "golang" + update-types: + - "version-update:semver-minor" + - "version-update:semver-major" diff --git a/.github/scripts/test-homebrew-cask.sh b/.github/scripts/test-homebrew-cask.sh new file mode 100755 index 000000000..9c1041bdd --- /dev/null +++ b/.github/scripts/test-homebrew-cask.sh @@ -0,0 +1,338 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { + echo "::error::$*" >&2 + exit 1 +} + +if [[ ${RUNNER_ENVIRONMENT:-} != github-hosted || ${RUNNER_OS:-} != macOS || $(uname -s) != Darwin ]]; then + fail "This test installs a system daemon and must run on a disposable GitHub macOS runner." +fi +if [[ $EUID == 0 ]]; then + fail "Run this script as the Homebrew user, not root." +fi + +readonly test_dir="${RUNNER_TEMP:?}/homebrew-cask" +readonly results_dir="$test_dir/results" +readonly app='/Applications/Netbird UI.app' +readonly plist='/Library/LaunchDaemons/netbird.plist' +readonly cask='netbirdio/tap/netbird-ui' +readonly formula='netbirdio/tap/netbird' +readonly published_cask="$test_dir/published-netbird-ui.rb" +readonly legacy_cask="$test_dir/legacy-netbird-ui.rb" +readonly rendered_cask="$test_dir/rendered-netbird-ui.rb" +readonly fixture_dir="$test_dir/fixture" +readonly serve_dir="$test_dir/serve" +readonly fixture_zip="$serve_dir/netbird-ui.zip" +readonly fixture_port=18080 +readonly fixture_url="http://127.0.0.1:$fixture_port/netbird-ui.zip" +readonly marker="$test_dir/installer.marker" + +mkdir -p "$results_dir" "$fixture_dir/netbird_ui_darwin" "$serve_dir" "$test_dir/downloads" +exec > >(tee "$results_dir/test.log") 2>&1 + +sudo -n true +if command -v netbird || [[ -e "$app" || -e "$plist" ]] || pgrep -x netbird-ui; then + fail "The runner already has NetBird installed or running." +fi +if sudo launchctl print system/netbird > "$results_dir/initial-service.log" 2>&1; then + fail "The runner already has a NetBird service loaded." +fi + +install_attempted=false +server_pid='' +daemon_pid='' +version='' + +stop_ui() { + local status=0 + sudo pkill -x netbird-ui || status=$? + # pkill returns 1 when the UI is already closed. + [[ $status == 0 || $status == 1 ]] +} + +cleanup() { + local status=$? + trap - EXIT + set +e + + if [[ $install_attempted == true ]]; then + stop_ui || status=1 + if [[ -S /var/run/netbird.sock ]]; then + sudo netbird down || status=1 + fi + if brew list --cask "$cask" >/dev/null 2>&1 || [[ -e "$app" ]]; then + brew uninstall --cask --force "$cask" || status=1 + fi + # A failed cask install can leave a daemon even after Homebrew rolls back the app. + if sudo launchctl print system/netbird > "$results_dir/cleanup-service.log" 2>&1; then + sudo netbird service stop || status=1 + fi + if [[ -e "$plist" ]]; then + sudo netbird service uninstall || status=1 + fi + fi + if [[ -f /var/log/netbird/client.log ]]; then + sudo cat /var/log/netbird/client.log > "$results_dir/client.log" || status=1 + fi + if command -v netbird >/dev/null; then + brew uninstall --formula "$formula" || status=1 + fi + if [[ -n $server_pid ]]; then + kill "$server_pid" 2>/dev/null || true + fi + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +run_logged() { + local name=$1 + shift + "$@" 2>&1 | tee "$results_dir/$name.log" +} + +cask_field() { + local stanza=$1 file=$2 + sed -nE "s/^[[:space:]]*$stanza \"([^\"]+)\".*/\\1/p" "$file" +} + +release_fields() { + local file=$1 + grep -E '^[[:space:]]*(version|url|sha256|app) ' "$file" +} + +use_cask() { + local file=$1 + cp "$file" "$tap_dir/Casks/netbird-ui.rb" +} + +# The released installer opens the UI as root, which never returns on a headless +# runner. The cask only needs two script paths and a version argument, so the test +# ships a stub bundle that records what it received and starts the daemon. +build_fixture() { + local bundle="$fixture_dir/netbird_ui_darwin" + printf '#!/bin/sh\nexit 0\n' > "$bundle/netbird-ui" + chmod 755 "$bundle/netbird-ui" + # After a bootout launchd keeps tearing the previous daemon down for a couple of + # seconds, and loading the same label again fails until that finishes. + cat > "$bundle/installer.sh" < '$marker' +netbird service install +attempt=0 +until netbird service start; do + attempt=\$((attempt + 1)) + [ "\$attempt" -lt 15 ] || exit 1 + sleep 1 +done +EOF + printf '#!/bin/sh\nexit 0\n' > "$bundle/uninstaller.sh" + # Shipped without the executable bit so the 0755 seen after install can only come from the cask. + chmod 644 "$bundle/installer.sh" "$bundle/uninstaller.sh" + rm -f "$fixture_zip" + (cd "$fixture_dir" && zip -qr "$fixture_zip" netbird_ui_darwin) +} + +start_fixture_server() { + python3 -m http.server "$fixture_port" --bind 127.0.0.1 --directory "$serve_dir" \ + > "$results_dir/fixture-server.log" 2>&1 & + server_pid=$! + local attempt + for attempt in {1..20}; do + if curl --silent --fail --output /dev/null "$fixture_url"; then + return + fi + sleep 0.5 + done + fail "The fixture HTTP server did not come up on port $fixture_port." +} + +assert_published_layout() { + local url archive script + while read -r url; do + archive="$test_dir/downloads/${url##*/}" + curl --fail --location --silent --retry 3 --output "$archive" "$url" + for script in installer.sh uninstaller.sh; do + unzip -l "$archive" | grep -q " netbird_ui_darwin/$script\$" || + fail "The published archive ${url##*/} has no netbird_ui_darwin/$script." + done + done < <(cask_field url "$published_cask") +} + +assert_no_deprecations() { + if grep -Ei '(postflight|uninstall_preflight).*deprecated|deprecated.*(postflight|uninstall_preflight)' "$@"; then + fail "Homebrew reported a deprecated cask lifecycle hook." + fi +} + +wait_for_daemon() { + local attempt + for attempt in {1..30}; do + if sudo launchctl print system/netbird > "$results_dir/service.log" 2>&1 && + grep -Eq '^[[:space:]]*state = running$' "$results_dir/service.log"; then + return + fi + sleep 1 + done + cat "$results_dir/service.log" + fail "The installed daemon did not reach the running state." +} + +wait_for_exit() { + local pid=$1 attempt + for attempt in {1..30}; do + if ! sudo kill -0 "$pid" 2>/dev/null; then + return + fi + sleep 1 + done + fail "Daemon process $pid is still running after removal." +} + +assert_service_absent() { + if sudo launchctl print system/netbird > "$results_dir/removed-service.log" 2>&1; then + fail "The NetBird service is still loaded after removal." + fi +} + +assert_installed() { + local script + [[ -f $marker ]] || fail "The cask did not run installer.sh." + grep -qx "version=$version" "$marker" || fail "installer.sh did not receive the cask version: $(cat "$marker")" + grep -qx 'uid=0' "$marker" || fail "installer.sh did not run as root: $(cat "$marker")" + [[ -d "$app" && -x "$app/netbird-ui" ]] || fail "The UI was not installed." + for script in installer.sh uninstaller.sh; do + [[ $(stat -f '%Lp' "$app/$script") == 755 ]] || fail "Incorrect permissions on $script." + done + [[ -f "$plist" ]] || fail "The installer did not create the daemon plist." + wait_for_daemon + daemon_pid=$(awk '/^[[:space:]]*pid = / { print $3; exit }' "$results_dir/service.log") + [[ $daemon_pid =~ ^[0-9]+$ ]] || fail "The running daemon has no PID." + sudo kill -0 "$daemon_pid" +} + +assert_uninstalled() { + local log=$1 + assert_no_deprecations "$log" + [[ ! -e "$app" ]] || fail "The UI app remains after uninstall." + [[ ! -e "$plist" ]] || fail "The daemon plist remains after uninstall." + assert_service_absent + wait_for_exit "$daemon_pid" + [[ $(netbird version) == "$version" ]] || fail "Cask uninstall removed the CLI dependency." +} + +installed_caskfiles() { + local extension=$1 + find "$(brew --caskroom)/netbird-ui/.metadata" -name "netbird-ui.$extension" 2>/dev/null +} + +assert_legacy_metadata() { + installed_caskfiles rb | grep -q . || fail "The legacy cask did not leave a Ruby caskfile behind." +} + +assert_steps_metadata() { + if installed_caskfiles rb | grep -q .; then + fail "Homebrew still keeps the legacy Ruby caskfile after reinstall." + fi + installed_caskfiles json | grep -q . || fail "Homebrew did not save the reinstalled cask as JSON." +} + +brew --version +sw_vers +brew tap netbirdio/tap "${GITHUB_WORKSPACE:?}/.homebrew-cask-tap" +tap_dir=$(brew --repository netbirdio/tap) +readonly tap_dir + +[[ -f "$tap_dir/Casks/netbird-ui.rb" ]] || fail "The tap has no Casks/netbird-ui.rb." +cp "$tap_dir/Casks/netbird-ui.rb" "$published_cask" +cp "$published_cask" "$results_dir/published-netbird-ui.rb" + +version=$(brew info --json=v2 --formula "$formula" | jq -r '.formulae[0].versions.stable') +readonly version +[[ -n $version && $version != null ]] || fail "Could not read the formula version from the tap." + +assert_published_layout + +build_fixture +fixture_sha=$(shasum -a 256 "$fixture_zip" | cut -d' ' -f1) +readonly fixture_sha +start_fixture_server + +export PROJECT=netbird-ui VERSION="$version" +export AMD="$fixture_zip" ARM="$fixture_zip" AMD_URL="$fixture_url" ARM_URL="$fixture_url" +gomplate -f "$GITHUB_WORKSPACE/client/ui/netbird-ui.rb.tmpl" -o "$rendered_cask" +cp "$rendered_cask" "$results_dir/rendered-netbird-ui.rb" + +sed -E "s|^([[:space:]]*version) \"[^\"]+\"|\\1 \"$version\"|; s|^([[:space:]]*url) \"[^\"]+\"|\\1 \"$fixture_url\"|; s|^([[:space:]]*sha256) \"[^\"]+\"|\\1 \"$fixture_sha\"|" \ + "$published_cask" > "$legacy_cask" +cp "$legacy_cask" "$results_dir/legacy-netbird-ui.rb" +if ! diff <(release_fields "$legacy_cask") <(release_fields "$rendered_cask"); then + fail "The rendered cask changes release data, not only lifecycle stanzas." +fi + +use_cask "$rendered_cask" +brew info --json=v2 --cask "$cask" > "$results_dir/cask.json" 2> "$results_dir/load.log" +cat "$results_dir/load.log" +assert_no_deprecations "$results_dir/load.log" +run_logged style brew style --cask --only-cops=Cask/InstallSteps "$cask" + +run_logged install-cli brew install --formula "$formula" +[[ $(netbird version) == "$version" ]] || fail "The installed CLI does not report the formula version." + +for scenario in running stopped missing; do + echo "::group::Uninstall with $scenario service" + install_attempted=true + sudo rm -f "$marker" + run_logged "install-$scenario" brew install --cask "$cask" + assert_no_deprecations "$results_dir/install-$scenario.log" + assert_installed + stop_ui + + case "$scenario" in + running) ;; + stopped) + run_logged stop-daemon sudo netbird service stop + wait_for_exit "$daemon_pid" + [[ -f "$plist" ]] || fail "Stopping the daemon unexpectedly removed its plist." + ;; + missing) + run_logged stop-missing-daemon sudo netbird service stop + run_logged remove-daemon sudo netbird service uninstall + wait_for_exit "$daemon_pid" + [[ ! -e "$plist" ]] || fail "The missing-service scenario still has a plist." + assert_service_absent + ;; + *) fail "Unknown uninstall scenario: $scenario" ;; + esac + + run_logged "uninstall-$scenario" brew uninstall --cask "$cask" + assert_uninstalled "$results_dir/uninstall-$scenario.log" + echo "::endgroup::" +done + +# Every existing user first meets the new cask through an upgrade of the published +# one, whose legacy flight blocks Homebrew replays from the saved Ruby caskfile. +echo "::group::Reinstall over the published legacy cask" +install_attempted=true +use_cask "$legacy_cask" +sudo rm -f "$marker" +run_logged install-legacy brew install --cask "$cask" +assert_installed +assert_legacy_metadata +stop_ui + +use_cask "$rendered_cask" +sudo rm -f "$marker" +run_logged reinstall-legacy brew reinstall --cask "$cask" +assert_installed +assert_steps_metadata +stop_ui + +run_logged uninstall-legacy brew uninstall --cask "$cask" +assert_uninstalled "$results_dir/uninstall-legacy.log" +echo "::endgroup::" diff --git a/.github/workflows/agent-network-e2e.yml b/.github/workflows/agent-network-e2e.yml index 9501c5fba..41bfe153d 100644 --- a/.github/workflows/agent-network-e2e.yml +++ b/.github/workflows/agent-network-e2e.yml @@ -52,7 +52,7 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-anet-e2e-buildx-${{ hashFiles('go.sum', 'combined/Dockerfile.multistage', 'proxy/Dockerfile.multistage', 'e2e/harness/Dockerfile.client') }} + key: ${{ runner.os }}-anet-e2e-buildx-${{ hashFiles('go.sum', 'combined/Dockerfile.multistage', 'proxy/Dockerfile.multistage', 'e2e/harness/Dockerfile.client', 'e2e/harness/Dockerfile.nblink') }} restore-keys: | ${{ runner.os }}-anet-e2e-buildx- diff --git a/.github/workflows/check-license-dependencies.yml b/.github/workflows/check-license-dependencies.yml index 81d293e4f..27c59f8d6 100644 --- a/.github/workflows/check-license-dependencies.yml +++ b/.github/workflows/check-license-dependencies.yml @@ -34,7 +34,7 @@ jobs: while IFS= read -r dir; do echo "=== Checking $dir ===" # Search for problematic imports, excluding test files - RESULTS=$(grep -r "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\)" "$dir" --include="*.go" 2>/dev/null | grep -v "_test.go" | grep -v "test_" | grep -v "/test/" | grep -v "tools/idp-migrate/" || true) + RESULTS=$(grep -r "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\)" "$dir" --include="*.go" 2>/dev/null | grep -v "_test.go" | grep -v "test_" | grep -v "/test/" | grep -v "tools/idp-migrate/" | grep -v "tools/mysql-migrate/" || true) if [ -n "$RESULTS" ]; then echo "❌ Found problematic dependencies:" echo "$RESULTS" @@ -93,7 +93,7 @@ jobs: IMPORTERS=$(go list -json -deps ./... 2>/dev/null | jq -r "select(.Imports[]? == \"$package\") | .ImportPath") # Check if any importer is NOT in management/signal/relay - BSD_IMPORTER=$(echo "$IMPORTERS" | grep -v "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\|combined\|tools/idp-migrate\)" | head -1) + BSD_IMPORTER=$(echo "$IMPORTERS" | grep -v "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\|combined\|tools/idp-migrate\|tools/mysql-migrate\)" | head -1) if [ -n "$BSD_IMPORTER" ]; then echo "❌ $package ($license) is imported by BSD-licensed code: $BSD_IMPORTER" diff --git a/.github/workflows/docs-ack.yml b/.github/workflows/docs-ack.yml index 7e34e2f8a..edd1eebff 100644 --- a/.github/workflows/docs-ack.yml +++ b/.github/workflows/docs-ack.yml @@ -12,6 +12,8 @@ jobs: docs-ack: name: Require docs PR URL or explicit "not needed" runs-on: ubuntu-latest + # Crowdin's translation-sync service PRs are auto-generated without the PR template. + if: github.event.pull_request.user.login != 'netbirddev' steps: - name: Read PR body diff --git a/.github/workflows/frontend-ui.yml b/.github/workflows/frontend-ui.yml index 014c5c2ae..2ad43c581 100644 --- a/.github/workflows/frontend-ui.yml +++ b/.github/workflows/frontend-ui.yml @@ -38,12 +38,12 @@ jobs: persist-credentials: false - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: "22" - name: Set up pnpm - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3.0.0 + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: version: 11 @@ -79,7 +79,7 @@ jobs: run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" - name: Cache pnpm store - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ${{ steps.pnpm-store.outputs.path }} key: ${{ runner.os }}-pnpm-${{ hashFiles('client/ui/frontend/pnpm-lock.yaml') }} diff --git a/.github/workflows/golang-test-freebsd.yml b/.github/workflows/golang-test-freebsd.yml index 65c39147a..7bd48e3d0 100644 --- a/.github/workflows/golang-test-freebsd.yml +++ b/.github/workflows/golang-test-freebsd.yml @@ -33,7 +33,7 @@ jobs: with: usesh: true copyback: false - release: "15.0" + release: "15.1" envs: "GO_VERSION" prepare: | pkg install -y curl pkgconf xorg diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml index 449eb14fa..880370ede 100644 --- a/.github/workflows/golang-test-linux.yml +++ b/.github/workflows/golang-test-linux.yml @@ -207,6 +207,43 @@ jobs: # regression test, and need no frontend bundle. run: CGO_ENABLED=1 go test -timeout 5m ./client/ui/authsession/... ./client/ui/i18n/... ./client/ui/preferences/... ./client/ui/services/... + test_client_pkcs11: + name: "Client PKCS#11 / Unit" + # The deb and rpm packages ship the client built with the pkcs11 tag, which loads + # PKCS#11 modules through purego. The other jobs build without the tag, so this one + # compiles that driver and signs with a real SoftHSM token, the build release ships. + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Install Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: "go.mod" + cache: false + + - name: Install SoftHSM + run: sudo apt update && sudo apt install -y -q softhsm2 + + - name: Initialise token + run: | + mkdir -p "$RUNNER_TEMP/softhsm/tokens" + printf 'directories.tokendir = %s\nobjectstore.backend = file\n' "$RUNNER_TEMP/softhsm/tokens" > "$RUNNER_TEMP/softhsm/softhsm2.conf" + echo "SOFTHSM2_CONF=$RUNNER_TEMP/softhsm/softhsm2.conf" >> "$GITHUB_ENV" + SOFTHSM2_CONF="$RUNNER_TEMP/softhsm/softhsm2.conf" softhsm2-util --init-token --free --label netbird --pin 1234 --so-pin 1234 + + - name: Vet + run: CGO_ENABLED=0 go vet -tags pkcs11 ./client/internal/pkcs11/... ./client/internal/certproof/... + + - name: Test + env: + NB_TEST_PKCS11_URI: "pkcs11:token=netbird?module-path=/usr/lib/softhsm/libsofthsm2.so&pin-value=1234" + NB_TEST_PKCS11_DISPOSABLE: "1" + run: CGO_ENABLED=0 go test -tags pkcs11 -timeout 5m ./client/internal/pkcs11/... ./client/internal/certproof/... + test_client_on_docker: name: "Client (Docker) / Unit" needs: [build-cache] @@ -270,7 +307,7 @@ jobs: sh -c ' \ apk update; apk add --no-cache \ ca-certificates iptables ip6tables dbus dbus-dev libpcap-dev build-base; \ - go test -buildvcs=false -tags "devcert privileged" -v -timeout 10m -p 1 $(go list -e -buildvcs=false ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui -e /upload-server -e /client/testutil/privileged) + go test -buildvcs=false -tags "devcert privileged" -v -timeout 10m -p 1 $(go list -e -buildvcs=false ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui -e /upload-server -e /client/testutil/privileged -e tools/mysql-migrate) ' test_relay: diff --git a/.github/workflows/golang-test-windows.yml b/.github/workflows/golang-test-windows.yml index ca300f7df..49be3af58 100644 --- a/.github/workflows/golang-test-windows.yml +++ b/.github/workflows/golang-test-windows.yml @@ -76,7 +76,7 @@ jobs: # resolve; the Where-Object pipeline then drops the broken package by # path. Without -e, go list aborts with empty stdout. run: | - $packages = go list -e ./... | Where-Object { $_ -notmatch '/management' } | Where-Object { $_ -notmatch '/relay' } | Where-Object { $_ -notmatch '/signal' } | Where-Object { $_ -notmatch '/proxy' } | Where-Object { $_ -notmatch '/combined' } | Where-Object { $_ -notmatch '/client/ui$' } + $packages = go list -e ./... | Where-Object { $_ -notmatch '/management' } | Where-Object { $_ -notmatch '/relay' } | Where-Object { $_ -notmatch '/signal' } | Where-Object { $_ -notmatch '/proxy' } | Where-Object { $_ -notmatch '/combined' } | Where-Object { $_ -notmatch '/client/ui$' } | Where-Object { $_ -notmatch '/tools/mysql-migrate' } $goExe = "C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe" $cmd = "$goExe test -tags `"devcert privileged`" -timeout 10m -p 1 $($packages -join ' ') > test-out.txt 2>&1" Set-Content -Path "${{ github.workspace }}\run-tests.cmd" -Value $cmd diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 586e1235b..843177b88 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -30,7 +30,7 @@ jobs: # segment by codespell and behave the same across versions; the # recursive "**" form did not take effect with the codespell shipped # by this action. - skip: go.mod,go.sum,*/proxy/web/*,*pnpm-lock.yaml,*package-lock.json,*/locales/de/*,*/locales/es/*,*/locales/fr/*,*/locales/hu/*,*/locales/it/*,*/locales/pt/*,*/locales/ru/*,*/locales/zh-CN/*,*/i18n/TRANSLATING.md + skip: go.mod,go.sum,*/proxy/web/*,*pnpm-lock.yaml,*package-lock.json,*/locales/de/*,*/locales/es/*,*/locales/fr/*,*/locales/gl/*,*/locales/hu/*,*/locales/it/*,*/locales/pt/*,*/locales/ru/*,*/locales/zh-CN/*,*/i18n/TRANSLATING.md golangci: strategy: fail-fast: false @@ -80,3 +80,49 @@ jobs: skip-save-cache: true cache-invalidation-interval: 0 args: --timeout=20m + + # Separate job rather than extra rows in the matrix above: those rows pick a + # GOOS by picking a runner OS, while android/ios are cross-compiled from + # ubuntu — an `include` entry with os: ubuntu-latest would merge into the + # Linux row instead of adding one. The package path is restricted because a + # whole-repo run under GOOS=android pulls *_linux.go files into packages that + # have no android counterpart. + golangci-mobile: + strategy: + fail-fast: false + matrix: + include: + - goos: android + goarch: arm64 + packages: ./client/android/... + display_name: Android + - goos: ios + goarch: arm64 + packages: ./client/ios/... + display_name: iOS + name: ${{ matrix.display_name }} + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + CGO_ENABLED: 0 + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Install Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: "go.mod" + cache: false + - name: golangci-lint + uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee #v9.2.1 + with: + version: latest + install-mode: binary + skip-cache: true + skip-save-cache: true + cache-invalidation-interval: 0 + args: --timeout=20m ${{ matrix.packages }} diff --git a/.github/workflows/mobile-build-validation.yml b/.github/workflows/mobile-build-validation.yml new file mode 100644 index 000000000..613a39b3d --- /dev/null +++ b/.github/workflows/mobile-build-validation.yml @@ -0,0 +1,64 @@ +name: Mobile + +on: + push: + branches: + - main + - "release-*" + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }} + cancel-in-progress: true + +jobs: + android_build: + name: "Android / Build" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + goarch: [arm64, arm, amd64, "386"] + env: + CGO_ENABLED: 0 + GOOS: android + GOARCH: ${{ matrix.goarch }} + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Install Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: "go.mod" + - name: Build Android bridge + run: go build ./client/android/... + - name: Vet Android bridge + if: matrix.goarch == 'arm64' + run: go vet ./client/android/... + + ios_build: + name: "iOS / Build" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + goarch: [arm64, amd64] + env: + CGO_ENABLED: 0 + GOOS: ios + GOARCH: ${{ matrix.goarch }} + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Install Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: "go.mod" + # No `go vet` counterpart: every ios target requires external (cgo) + # linking, which needs an Xcode toolchain the runner does not have. + - name: Build iOS SDK + run: go build ./client/ios/... diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml index 24d81b50f..5b769e2f7 100644 --- a/.github/workflows/pr-title-check.yml +++ b/.github/workflows/pr-title-check.yml @@ -7,6 +7,8 @@ on: jobs: check-title: runs-on: ubuntu-latest + # Crowdin's translation-sync service PRs are auto-generated with a fixed title. + if: github.event.pull_request.user.login != 'netbirddev' steps: - name: Validate PR title prefix uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 diff --git a/.github/workflows/redhat-certify.yml b/.github/workflows/redhat-certify.yml new file mode 100644 index 000000000..96d50de9f --- /dev/null +++ b/.github/workflows/redhat-certify.yml @@ -0,0 +1,207 @@ +name: Red Hat Certification + +# Certify published UBI images in the Red Hat Ecosystem Catalog. Called by +# release.yml on stable tags, or run by hand to (re)certify any released +# version. preflight submits every architecture of an image's manifest list +# to Pyxis; auto-publish on the component makes it public once certified. +# +# Each component's Partner Connect ID comes from the REDHAT_CERT_ID_ +# repository variable, e.g. REDHAT_CERT_ID_CLIENT_ROOTLESS. The run fails +# before certifying anything if a selected component's variable is not set. + +on: + workflow_call: + inputs: + component: + type: string + required: true + version: + type: string + required: true + secrets: + PYXIS_API_TOKEN: + required: true + workflow_dispatch: + inputs: + component: + description: "Component to certify" + type: choice + required: true + default: all + options: + - all + - client-rootless + - reverse-proxy + - netbird-server + - management + - signal + - relay + version: + description: "Released version, e.g. v0.80.0" + type: string + required: true + +permissions: + contents: read + +jobs: + resolve: + name: Resolve components + runs-on: ubuntu-24.04 + outputs: + version: ${{ steps.resolve.outputs.version }} + matrix: ${{ steps.resolve.outputs.matrix }} + steps: + - name: Resolve components and images + id: resolve + env: + COMPONENT: ${{ inputs.component }} + INPUT_VERSION: ${{ inputs.version }} + REPO_VARS: ${{ toJSON(vars) }} + run: | + set -euo pipefail + version="${INPUT_VERSION#v}" + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Only stable x.y.z versions are certified, got '${INPUT_VERSION}'" + exit 1 + fi + # name, image repository, tag suffix (must match .goreleaser.yaml). + # Keep the names in sync with the workflow_dispatch options above. + components=( + "client-rootless ghcr.io/netbirdio/netbird -rootless-ubi" + "reverse-proxy ghcr.io/netbirdio/reverse-proxy -ubi" + "netbird-server ghcr.io/netbirdio/netbird-server -ubi" + "management ghcr.io/netbirdio/management -ubi" + "signal ghcr.io/netbirdio/signal -ubi" + "relay ghcr.io/netbirdio/relay -ubi" + ) + matrix="[]" + missing=() + for c in "${components[@]}"; do + read -r name repo suffix <<< "$c" + [[ "$COMPONENT" == all || "$COMPONENT" == "$name" ]] || continue + var="REDHAT_CERT_ID_${name^^}"; var="${var//-/_}" + id="$(jq -r --arg v "$var" '.[$v] // empty' <<< "$REPO_VARS")" + if [[ -z "$id" ]]; then + missing+=("$var") + continue + fi + matrix="$(jq -c --arg n "$name" --arg t "${version}${suffix}" --arg r "${repo}:${version}${suffix}" --arg i "$id" \ + '. + [{component: $n, tag: $t, ref: $r, component_id: $i}]' <<< "$matrix")" + done + if (( ${#missing[@]} )); then + echo "::error::Set these repository variables to the Partner Connect component IDs: ${missing[*]}" + exit 1 + fi + if [[ "$matrix" == "[]" ]]; then + echo "::error::No component to certify for '${COMPONENT}'" + exit 1 + fi + echo "Components to certify: ${matrix}" + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" + + certify: + name: "Certify ${{ matrix.component }} UBI image" + needs: resolve + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.resolve.outputs.matrix) }} + env: + PREFLIGHT_VERSION: "1.21.0" + # sha256 of preflight-linux-amd64 from the 1.21.0 GitHub release. + # Red Hat publishes no checksum file, so the value is pinned here. + PREFLIGHT_SHA256: "5e653135503c72f8702bbe31d7643197d12937c68086879133dd6b9650a9a449" + steps: + - name: Verify the multi-arch image is on ghcr.io + env: + IMAGE_REF: ${{ matrix.ref }} + run: | + set -euo pipefail + docker buildx imagetools inspect "$IMAGE_REF" --raw > manifest.json + for arch in amd64 arm64; do + if ! jq -e --arg a "$arch" '.manifests[] | select(.platform.architecture == $a)' manifest.json > /dev/null; then + echo "::error::${IMAGE_REF} has no ${arch} manifest" + exit 1 + fi + done + echo "Manifest list for ${IMAGE_REF}:" + jq -r '.manifests[] | "\(.platform.os)/\(.platform.architecture) \(.digest)"' manifest.json + + - name: Install preflight + run: | + set -euo pipefail + curl -fsSL --proto '=https' --proto-redir '=https' -o preflight \ + "https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/download/${PREFLIGHT_VERSION}/preflight-linux-amd64" + echo "${PREFLIGHT_SHA256} preflight" | sha256sum -c - + chmod +x preflight + ./preflight --version + + - name: Run preflight checks and submit to Red Hat + env: + IMAGE_REF: ${{ matrix.ref }} + PFLT_PYXIS_API_TOKEN: ${{ secrets.PYXIS_API_TOKEN }} + PFLT_CERTIFICATION_COMPONENT_ID: ${{ matrix.component_id }} + PFLT_ARTIFACTS: artifacts + PFLT_LOGFILE: artifacts/preflight.log + PFLT_LOGLEVEL: info + PFLT_JUNIT: "true" + run: | + set -euo pipefail + # No --platform: preflight walks the manifest list and submits every + # architecture in one run, grouped under one manifest-list digest. + # preflight does not create the PFLT_LOGFILE directory, and --submit + # fails if the log file is missing. + mkdir -p artifacts + ./preflight check container "$IMAGE_REF" --submit + + - name: Fail if any check did not pass + run: | + set -euo pipefail + shopt -s nullglob globstar + results=(artifacts/**/results.json) + if [[ ${#results[@]} -eq 0 ]]; then + echo "::error::preflight produced no results.json" + exit 1 + fi + status=0 + for f in "${results[@]}"; do + arch="$(basename "$(dirname "$f")")" + passed="$(jq -r '.passed' "$f")" + failed="$(jq -r '[.results.failed[]?.name] | join(", ")' "$f")" + echo "${arch}: passed=${passed} ${failed:+failed checks: ${failed}}" + [[ "$passed" == "true" ]] || status=1 + done + exit $status + + - name: Upload preflight artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: redhat-preflight-${{ matrix.component }}-${{ needs.resolve.outputs.version }} + path: artifacts/ + retention-days: 30 + + - name: Wait for Pyxis to mark both architectures certified + env: + TAG: ${{ matrix.tag }} + COMPONENT_ID: ${{ matrix.component_id }} + PFLT_PYXIS_API_TOKEN: ${{ secrets.PYXIS_API_TOKEN }} + run: | + set -euo pipefail + # Filter on the tag server-side so older versions are found past the first page. + url="https://catalog.redhat.com/api/containers/v1/projects/certification/id/${COMPONENT_ID}/images?filter=repositories.tags.name==${TAG}&page_size=100" + for attempt in $(seq 1 20); do + certified="$(curl -fsS --proto '=https' --proto-redir '=https' -H "X-API-KEY: ${PFLT_PYXIS_API_TOKEN}" "$url" \ + | jq -r --arg t "$TAG" '[.data[] | select(.repositories[]?.tags[]?.name == $t) | select(.certified == true) | .architecture] | unique | join(",")')" + echo "attempt ${attempt}: certified architectures for ${TAG}: ${certified:-none}" + if [[ "$certified" == "amd64,arm64" ]]; then + echo "Both architectures certified. Auto-publish is enabled on the component, so the catalog updates on its own." + exit 0 + fi + sleep 30 + done + echo "::error::Pyxis has not marked both architectures certified after 10 minutes. Check https://connect.redhat.com/component/view/${COMPONENT_ID}/images" + exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 51426a7ce..22ab7f566 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,8 +23,50 @@ concurrency: cancel-in-progress: true jobs: + freebsd_port_trigger: + name: "FreeBSD Port / Trigger" + runs-on: ubuntu-22.04 + outputs: + run: ${{ steps.decide.outputs.run }} + version: ${{ steps.decide.outputs.version }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Decide whether the port job runs + id: decide + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + run=false + case "$GITHUB_EVENT_NAME" in + push) + if [[ "$GITHUB_REF" == "refs/heads/main" ]]; then + run=true + elif [[ "$GITHUB_REF" == refs/tags/* && "$GITHUB_REF_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + # The port tracks stable releases only, so a canary or rc tag has nothing to submit. + run=true + echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + fi + ;; + pull_request) + git fetch --depth=1 origin "$BASE_SHA" + if git diff --name-only "$BASE_SHA" HEAD | \ + grep -qE '^(release_files/freebsd-port-.*\.sh|\.github/workflows/release\.yml)$'; then + run=true + fi + ;; + esac + echo "run=$run" >> "$GITHUB_OUTPUT" + echo "FreeBSD port job runs: $run" + release_freebsd_port: name: "FreeBSD Port / Build & Test" + needs: freebsd_port_trigger + if: needs.freebsd_port_trigger.outputs.run == 'true' runs-on: ubuntu-22.04 steps: - name: Checkout @@ -33,18 +75,19 @@ jobs: persist-credentials: false - name: Generate FreeBSD port diff - run: bash -x release_files/freebsd-port-diff.sh - - - name: Generate FreeBSD port issue body - run: bash -x release_files/freebsd-port-issue-body.sh + env: + TARGET_VERSION: ${{ needs.freebsd_port_trigger.outputs.version }} + run: bash -x release_files/freebsd-port-diff.sh "$TARGET_VERSION" - name: Check if diff was generated id: check_diff run: | - if ls netbird-*.diff 1> /dev/null 2>&1; then - echo "diff_exists=true" >> $GITHUB_OUTPUT + shopt -s nullglob + diffs=(netbird-*.diff) + if [[ ${#diffs[@]} -gt 0 ]]; then + echo "diff_exists=true" >> "$GITHUB_OUTPUT" else - echo "diff_exists=false" >> $GITHUB_OUTPUT + echo "diff_exists=false" >> "$GITHUB_OUTPUT" echo "No diff file generated (port may already be up to date)" fi @@ -52,10 +95,18 @@ jobs: if: steps.check_diff.outputs.diff_exists == 'true' id: version run: | - VERSION=$(ls netbird-*.diff | sed 's/netbird-\(.*\)\.diff/\1/') - echo "version=$VERSION" >> $GITHUB_OUTPUT + diffs=(netbird-*.diff) + VERSION="${diffs[0]#netbird-}" + VERSION="${VERSION%.diff}" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "Generated files for version: $VERSION" - cat netbird-*.diff + cat "${diffs[0]}" + + - name: Generate FreeBSD port issue body + if: steps.check_diff.outputs.diff_exists == 'true' + env: + TARGET_VERSION: ${{ steps.version.outputs.version }} + run: bash -x release_files/freebsd-port-issue-body.sh "" "$TARGET_VERSION" - name: Read Go version from go.mod id: goversion @@ -69,7 +120,7 @@ jobs: with: usesh: true copyback: false - release: "15.0" + release: "15.1" envs: "GO_VERSION" prepare: | # Install required packages @@ -191,6 +242,17 @@ jobs: # requires a changelog. Generated, not committed (see .gitignore). # chglog is a go.mod tool directive, so go.sum pins it and its deps. run: bash release_files/rpm-changelog.sh + - name: Fill the RPM ISA provide version + # nfpm cannot emit rpmbuild's ISA provide and GoReleaser cannot template it. + run: bash release_files/rpm-provides.sh + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + - name: Install proxy web dependencies for license collection + # release_files/collect-licenses.sh -w reads the proxy UI's license terms from node_modules. + working-directory: proxy/web + run: npm ci --ignore-scripts - name: Set up QEMU uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 #v4.1.0 - name: Set up Docker Buildx @@ -230,14 +292,18 @@ jobs: uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2 with: version: ${{ env.GORELEASER_VER }} - args: release --clean ${{ env.flags }} + args: release --config .goreleaser.generated.yaml --clean ${{ env.flags }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }} UPLOAD_DEBIAN_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }} UPLOAD_YUM_SECRET: ${{ secrets.PKG_UPLOAD_SECRET }} GPG_RPM_KEY_FILE: ${{ env.GPG_RPM_KEY_FILE }} - NFPM_NETBIRD_RPM_PASSPHRASE: ${{ secrets.GPG_RPM_PASSPHRASE }} + # One per nfpm id: GoReleaser looks the passphrase up as NFPM__PASSPHRASE. + NFPM_NETBIRD_RPM_AMD64_PASSPHRASE: ${{ secrets.GPG_RPM_PASSPHRASE }} + NFPM_NETBIRD_RPM_ARM64_PASSPHRASE: ${{ secrets.GPG_RPM_PASSPHRASE }} + NFPM_NETBIRD_RPM_ARM_PASSPHRASE: ${{ secrets.GPG_RPM_PASSPHRASE }} + NFPM_NETBIRD_RPM_386_PASSPHRASE: ${{ secrets.GPG_RPM_PASSPHRASE }} SKIP_PUBLISH: ${{ env.SKIP_PUBLISH }} SKIP_DOCKER_PUSH: ${{ env.SKIP_DOCKER_PUSH }} - name: Verify RPM signatures @@ -294,10 +360,12 @@ jobs: tag_and_push() { local src="$1" img_name tag dst variant="" img_name="${src%%:*}" - # Client variants share a repository, so keep their tag suffixes. + # Variants share a repository with their default image, so keep + # their tag suffixes. Order matters: the first matching pattern wins. case "$src" in *-rootless-ubi-amd64) variant="-rootless-ubi" ;; *-rootless-amd64) variant="-rootless" ;; + *-ubi-amd64) variant="-ubi" ;; esac for tag in $(resolve_tags); do dst="${img_name}:${tag}${variant}" @@ -346,7 +414,11 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 with: name: linux-packages - path: dist/netbird_linux** + path: | + dist/netbird_linux** + dist/netbird-pkcs11_linux** + dist/netbird-deb-32bit_linux** + dist/netbird-rpm-*_linux** retention-days: 7 - name: upload windows packages id: upload_windows_packages @@ -363,6 +435,24 @@ jobs: path: dist/netbird_darwin** retention-days: 7 + # Certify the UBI images in the Red Hat Ecosystem Catalog on stable tags. + # See redhat-certify.yml, which can also be run by hand for any released version. + redhat_certification: + name: "Red Hat" + needs: release + if: | + github.repository == 'netbirdio/netbird' && + startsWith(github.ref, 'refs/tags/v') && + !contains(github.ref_name, '-') + permissions: + contents: read + uses: ./.github/workflows/redhat-certify.yml + with: + component: all + version: ${{ github.ref_name }} + secrets: + PYXIS_API_TOKEN: ${{ secrets.PYXIS_API_TOKEN }} + release_ui: runs-on: ubuntu-latest outputs: @@ -417,12 +507,12 @@ jobs: run: git --no-pager diff --exit-code - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: '22' - name: Set up pnpm - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3.0.0 + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: version: 11 @@ -554,12 +644,12 @@ jobs: run: git --no-pager diff --exit-code - name: Set up Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' - name: Set up pnpm - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3.0.0 + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: version: 11 @@ -651,11 +741,11 @@ jobs: - name: check git status run: git --no-pager diff --exit-code - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: '22' - name: Set up pnpm - uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3.0.0 + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: version: 11 - name: Install wails3 CLI @@ -774,7 +864,7 @@ jobs: run: 7z x -o"${{ github.workspace }}/NSIS_Plugins" "${{ github.workspace }}/ShellExecAsUser_amd64-Unicode.7z" - name: Set up Go for wails3 CLI - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version-file: "go.mod" cache: false diff --git a/.github/workflows/reverse-proxy-e2e.yml b/.github/workflows/reverse-proxy-e2e.yml new file mode 100644 index 000000000..934a6d4c9 --- /dev/null +++ b/.github/workflows/reverse-proxy-e2e.yml @@ -0,0 +1,53 @@ +name: Reverse Proxy E2E + +on: + pull_request: + paths: + - '.github/workflows/reverse-proxy-e2e.yml' + - '**/*.go' + - 'e2e/**' + - 'shared/management/**' + - 'combined/Dockerfile*' + - 'proxy/Dockerfile*' + - '.dockerignore' + - 'go.mod' + - 'go.sum' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + reverse-proxy: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Install Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version-file: go.mod + + - name: Set up Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + - name: Cache Docker layers + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: /tmp/.reverse-proxy-buildx-cache + key: ${{ runner.os }}-reverse-proxy-e2e-${{ hashFiles('go.sum', 'combined/Dockerfile.multistage', 'proxy/Dockerfile.multistage') }} + restore-keys: | + ${{ runner.os }}-reverse-proxy-e2e- + + - name: Run reverse-proxy E2E + env: + NB_E2E_BUILDX_CACHE: /tmp/.reverse-proxy-buildx-cache + run: go test -tags e2e -timeout 25m -v ./e2e/reverseproxy/... diff --git a/.github/workflows/test-homebrew-cask.yml b/.github/workflows/test-homebrew-cask.yml new file mode 100644 index 000000000..a75951425 --- /dev/null +++ b/.github/workflows/test-homebrew-cask.yml @@ -0,0 +1,46 @@ +name: Test Homebrew cask + +on: + pull_request: + paths: + - "client/ui/netbird-ui.rb.tmpl" + - ".github/scripts/test-homebrew-cask.sh" + - ".github/workflows/test-homebrew-cask.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }} + cancel-in-progress: true + +jobs: + install-uninstall: + runs-on: macos-latest + timeout-minutes: 20 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Clone the Homebrew tap + run: git clone https://github.com/netbirdio/homebrew-tap.git .homebrew-cask-tap + + - name: Update Homebrew and install gomplate + # The runner image disables auto-update; the cask steps DSL needs Homebrew 6.0.20 or newer. + run: | + brew update + brew install gomplate + + - name: Install and uninstall the cask + run: .github/scripts/test-homebrew-cask.sh + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: homebrew-cask-results + path: ${{ runner.temp }}/homebrew-cask/results + if-no-files-found: ignore diff --git a/.github/workflows/test-infrastructure-files.yml b/.github/workflows/test-infrastructure-files.yml index 1313379ee..4875adc31 100644 --- a/.github/workflows/test-infrastructure-files.yml +++ b/.github/workflows/test-infrastructure-files.yml @@ -258,6 +258,18 @@ jobs: with: persist-credentials: false + - name: Verify enterprise Traefik tag validation + run: bash infrastructure_files/tests/test-getting-started-enterprise.sh + + - name: Verify community Traefik image validation + run: bash infrastructure_files/tests/test-getting-started.sh + + - name: Verify community Compose network scoping + run: bash infrastructure_files/tests/test-getting-started-networks.sh + + - name: Verify migration Compose network scoping + run: bash infrastructure_files/tests/test-getting-started-networks.sh --migration + - name: Verify fresh-install session cookie key hardening run: | grep -Fxq ' SESSION_COOKIE_ENCRYPTION_KEY=$(openssl rand -base64 32)' infrastructure_files/getting-started.sh diff --git a/.github/workflows/ui-translations.yml b/.github/workflows/ui-translations.yml index 7d3b12f2d..9ac524495 100644 --- a/.github/workflows/ui-translations.yml +++ b/.github/workflows/ui-translations.yml @@ -32,11 +32,12 @@ jobs: persist-credentials: false - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: "22" - # English (en) is the source of truth for translation keys; every other - # locale declared in _index.json must carry the exact same key set. + # English (en) is the source of truth for translation keys. Locales declared + # in _index.json fail on orphaned keys or placeholder mismatches; missing + # keys only warn, since they fall back to English at runtime. - name: Check translation key parity run: node client/ui/i18n/check-translations.mjs diff --git a/.gitignore b/.gitignore index dd7eea76f..5c01f6e60 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,7 @@ management/server/types/testdata/ # generated by chglog in the release workflow, embedded into the RPM changelog.yml + +# generated by rpm-provides.sh, the config GoReleaser actually runs +.goreleaser.generated.yaml .chglog.yml diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 19528f88e..41c330a64 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -40,6 +40,73 @@ builds: tags: - load_wgnt_from_rsrc + # deb and rpm packages target glibc distributions, so on amd64 and arm64, the only + # architectures with a PKCS#11 driver, they carry the PKCS#11 store, which links libc. + # Tarballs, the Alpine-based images and the other architectures keep the static build. + - id: netbird-pkcs11 + dir: client + binary: netbird + env: [CGO_ENABLED=0] + goos: + - linux + goarch: + - amd64 + - arm64 + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + tags: + - load_wgnt_from_rsrc + - pkcs11 + + # The 32-bit deb architectures, which have no PKCS#11 driver. + - id: netbird-deb-32bit + dir: client + binary: netbird + env: [CGO_ENABLED=0] + goos: + - linux + goarch: + - arm + - 386 + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + tags: + - load_wgnt_from_rsrc + + # Single-arch builds: nfpm provides is not templated, so the RPM splits per arch. The + # ones with a PKCS#11 driver carry the store like the deb build above. + - &netbird_rpm_build + id: netbird-rpm-amd64 + dir: client + binary: netbird + env: [CGO_ENABLED=0] + goos: [linux] + goarch: [amd64] + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + tags: + - load_wgnt_from_rsrc + - pkcs11 + + - <<: *netbird_rpm_build + id: netbird-rpm-arm64 + goarch: [arm64] + + - <<: *netbird_rpm_build + id: netbird-rpm-arm + goarch: [arm] + tags: + - load_wgnt_from_rsrc + + - <<: *netbird_rpm_build + id: netbird-rpm-386 + goarch: [386] + tags: + - load_wgnt_from_rsrc + - id: netbird-static dir: client binary: netbird @@ -168,6 +235,29 @@ builds: - -s -w -X main.Version={{.Version}} -X main.Commit={{.Commit}} -X main.BuildDate={{.CommitDate}} mod_timestamp: "{{ .CommitTimestamp }}" + - id: nblink + dir: client/link/cmd/nblink + env: [CGO_ENABLED=0] + binary: nblink + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + - arm + goarm: + - 7 + ignore: + - goos: windows + goarch: arm + - goos: darwin + goarch: arm + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} + mod_timestamp: "{{ .CommitTimestamp }}" + - id: netbird-idp-migrate dir: tools/idp-migrate env: @@ -190,6 +280,28 @@ builds: - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser mod_timestamp: "{{ .CommitTimestamp }}" + - id: netbird-mysql-migrate + dir: tools/mysql-migrate + env: + - CGO_ENABLED=1 + - >- + {{- if eq .Runtime.Goos "linux" }} + {{- if eq .Arch "arm64"}}CC=aarch64-linux-gnu-gcc{{- end }} + {{- if eq .Arch "arm"}}CC=arm-linux-gnueabihf-gcc{{- end }} + {{- end }} + binary: netbird-mysql-migrate + goos: + - linux + goarch: + - amd64 + - arm64 + - arm + goarm: + - 7 + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + universal_binaries: - id: netbird @@ -206,6 +318,19 @@ archives: builds: - netbird-idp-migrate name_template: "netbird-idp-migrate_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + - id: netbird-mysql-migrate + builds: + - netbird-mysql-migrate + name_template: "netbird-mysql-migrate_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + - id: nblink + builds: + - nblink + name_template: "nblink_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + # goreleaser archives as tar.gz everywhere unless told otherwise, and + # nblink is downloaded and extracted by hand on Windows too. + format_overrides: + - goos: windows + format: zip nfpms: - maintainer: Netbird @@ -216,24 +341,30 @@ nfpms: id: netbird_deb bindir: /usr/bin builds: - - netbird + - netbird-pkcs11 + - netbird-deb-32bit formats: - deb scripts: postinstall: "release_files/post_install.sh" preremove: "release_files/pre_remove.sh" - - maintainer: Netbird + - &netbird_rpm + maintainer: Netbird description: Netbird client. homepage: https://netbird.io/ license: BSD-3-Clause vendor: NetBird - id: netbird_rpm + id: netbird_rpm_amd64 bindir: /usr/bin - builds: - - netbird + ids: + - netbird-rpm-amd64 formats: - rpm + # Red Hat certification (RPM Version Handling) requires rpmbuild's ISA + # provide, which nfpm does not emit. The version is filled in by the release job. + provides: + - "netbird(x86-64) = @RPM_EVR@" # The client verifies TLS to management and signal against the system trust # store. Red Hat software certification (RPM Dependency Tracking) also # rejects packages that declare no dependencies at all. @@ -254,6 +385,9 @@ nfpms: - src: release_files/netbird.sysconfig dst: /etc/sysconfig/netbird type: config|noreplace + # May hold the PKCS#11 token PIN (NB_TPM_PIN), so readable by root alone. + file_info: + mode: 0600 scripts: postinstall: "release_files/post_install.sh" preremove: "release_files/pre_remove.sh" @@ -263,6 +397,27 @@ nfpms: packager: NetBird signature: key_file: '{{ if index .Env "GPG_RPM_KEY_FILE" }}{{ .Env.GPG_RPM_KEY_FILE }}{{ end }}' + + - <<: *netbird_rpm + id: netbird_rpm_arm64 + ids: + - netbird-rpm-arm64 + provides: + - "netbird(aarch-64) = @RPM_EVR@" + + - <<: *netbird_rpm + id: netbird_rpm_arm + ids: + - netbird-rpm-arm + provides: + - "netbird(armv6hl-32) = @RPM_EVR@" + + - <<: *netbird_rpm + id: netbird_rpm_386 + ids: + - netbird-rpm-386 + provides: + - "netbird(x86-32) = @RPM_EVR@" dockers_v2: - id: netbird disable: "{{ .Env.SKIP_DOCKER_PUSH }}" @@ -333,7 +488,7 @@ dockers_v2: RELEASE: "{{ .Timestamp }}" hooks: pre: - - cmd: 'sh client/collect-licenses.sh "{{ .ContextDir }}/licenses" amd64 arm64' + - cmd: 'sh release_files/collect-licenses.sh -t load_wgnt_from_rsrc "{{ .ContextDir }}/licenses" ./client amd64 arm64' env: - GOOS=linux - CGO_ENABLED=0 @@ -371,6 +526,41 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: relay-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-relay + images: + - netbirdio/relay + - ghcr.io/netbirdio/relay + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: relay/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l relay/LICENSE "{{ .ContextDir }}/licenses" ./relay amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=0 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: signal disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -393,6 +583,41 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: signal-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-signal + images: + - netbirdio/signal + - ghcr.io/netbirdio/signal + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: signal/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l signal/LICENSE "{{ .ContextDir }}/licenses" ./signal amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=0 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: management disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -415,6 +640,41 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: management-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-mgmt + images: + - netbirdio/management + - ghcr.io/netbirdio/management + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: management/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l management/LICENSE "{{ .ContextDir }}/licenses" ./management amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=1 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: upload disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -425,7 +685,7 @@ dockers_v2: tags: - "{{ .Version }}" - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}latest{{ end }}" - dockerfile: upload-server/Dockerfile + dockerfile: upload-server/Dockerfile.release platforms: - linux/amd64 - linux/arm64 @@ -459,6 +719,41 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: netbird-server-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-server + images: + - netbirdio/netbird-server + - ghcr.io/netbirdio/netbird-server + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: combined/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l combined/LICENSE "{{ .ContextDir }}/licenses" ./combined amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=1 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: netbird-proxy disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -481,6 +776,63 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: nblink + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - nblink + images: + - netbirdio/nblink + - ghcr.io/netbirdio/nblink + tags: + - "{{ .Version }}" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}latest{{ end }}" + dockerfile: client/Dockerfile-nblink + platforms: + - linux/amd64 + - linux/arm64 + - linux/arm + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "nblink" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" + - id: proxy-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-proxy + images: + - netbirdio/reverse-proxy + - ghcr.io/netbirdio/reverse-proxy + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: proxy/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l proxy/LICENSE -w "{{ .ContextDir }}/licenses" ./proxy/cmd/proxy amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=0 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" brews: - ids: @@ -513,7 +865,10 @@ uploads: - name: yum skip: "{{ .Env.SKIP_PUBLISH }}" ids: - - netbird_rpm + - netbird_rpm_amd64 + - netbird_rpm_arm64 + - netbird_rpm_arm + - netbird_rpm_386 mode: archive target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }} username: dev@wiretrustee.com diff --git a/LICENSE b/LICENSE index d922f155a..4d0551de8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,9 @@ -This BSD‑3‑Clause license applies to all parts of the repository except for the directories management/, signal/, relay/ and combined/. +This BSD-3-Clause license applies to all parts of the repository except for the directories management/, signal/, relay/ and combined/. Those directories are licensed under the GNU Affero General Public License version 3.0 (AGPLv3). See the respective LICENSE files inside each directory. BSD 3-Clause License -Copyright (c) 2022 NetBird GmbH & AUTHORS +Copyright (c) 2022-2026 NetBird GmbH & AUTHORS Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/LICENSES/BSD-3-Clause.txt b/LICENSES/BSD-3-Clause.txt index 7cba76dfd..5e9a2539c 100644 --- a/LICENSES/BSD-3-Clause.txt +++ b/LICENSES/BSD-3-Clause.txt @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2022 NetBird GmbH & AUTHORS +Copyright (c) 2022-2026 NetBird GmbH & AUTHORS Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.md b/README.md index 336332043..4dbfc7bd0 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,56 @@ export NETBIRD_DOMAIN=netbird.example.com; curl -fsSL https://github.com/netbird See a complete [architecture overview](https://docs.netbird.io/about-netbird/how-netbird-works#architecture) for details. +### Reporting bugs and requesting features + +NetBird uses a discussion-first workflow. Bug reports and feature requests start in +[Discussions](https://github.com/netbirdio/netbird/discussions), not as issues. + +| What you want to do | Where to go | +| --- | --- | +| Report a bug, regression, or unexpected behavior | [Issue Triage](https://github.com/netbirdio/netbird/discussions/new?category=issue-triage) | +| Request a feature or share an idea | [Ideas & Feature Requests](https://github.com/netbirdio/netbird/discussions/new?category=ideas-feature-requests) | +| Ask about setup, configuration, or self-hosting | [Q&A / Support](https://github.com/netbirdio/netbird/discussions/new?category=q-a-support) | +| Report a security vulnerability | [Security policy](https://github.com/netbirdio/netbird/security/policy), never a public thread | + +Our team and maintainers triage discussions, ask follow-up questions, check for duplicates, +and reproduce bugs. Validated reports are promoted to issues. This keeps the issue tracker a clear +answer to one question: what is the team working on. + +Please search existing discussions and issues first, including closed ones. If something similar +already exists, upvote it and add your details there instead of opening a duplicate. + +For bug reports, include your NetBird version, operating system, deployment type (Cloud, +self-hosted, Kubernetes, or Docker), reproduction steps, expected and actual behavior, and a debug +bundle where relevant: + +```shell +netbird version +netbird status -d -A +netbird debug for 1m -A -S -U +``` + +`-U` uploads the bundle and prints a file key you can paste instead of attaching the archive. +`-A` anonymizes the output, which matters on a public thread. It masks most identifying details +but is not full redaction, so read the bundle before posting it. Two levels are available: + +| Level | How to select | What it masks | +| --- | --- | --- | +| `default` | `-A` / `--anonymize` | Public IP addresses, IPv6 ULA addresses, MAC addresses, and domains other than `netbird.io`, `netbird.cloud`, `netbird.selfhosted`, and `netbird.stage`. IPv4 private, CGNAT, and link-local ranges are kept | +| `strict` | `--anonymize-level strict` (implies `-A`) | The above, plus IPv4 private, CGNAT, and link-local ranges, peer names in front of `netbird.cloud`, `netbird.selfhosted`, and `netbird.stage`, and WireGuard public keys. Labels under `netbird.io` are kept, since it only hosts infrastructure | + +See [collecting a debug bundle](https://docs.netbird.io/help/troubleshooting-client#debug-bundle) +and the [CLI reference](https://docs.netbird.io/get-started/cli#debug-for) for details. + +See [How to use Discussions, Issues, and Pull Requests](https://github.com/netbirdio/netbird/discussions/6075) +for the full workflow, or [SUPPORT.md](SUPPORT.md) for a shorter version. + +### Contributing + +Contributions are welcome. Read [CONTRIBUTING.md](CONTRIBUTING.md) first. NetBird works ticket +first, anything that changes behavior needs an issue the team has agreed on before you open a pull +request. + ### Community projects - [NetBird installer script](https://github.com/physk/netbird-installer) - [netbird-tui](https://github.com/n0pashkov/netbird-tui) - terminal UI for managing NetBird peers, routes, and settings diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 000000000..fade37286 --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,121 @@ +# Getting help with NetBird + +Where to go depends on what you need. If you are not sure, start with +[Q&A / Support](https://github.com/netbirdio/netbird/discussions/new?category=q-a-support) +and we will move it. + +## Before you post + +1. Search existing [discussions](https://github.com/netbirdio/netbird/discussions) and + [issues](https://github.com/netbirdio/netbird/issues), including closed ones. +2. Check the [documentation](https://docs.netbird.io) and the troubleshooting guides for + [clients](https://docs.netbird.io/help/troubleshooting-client) and + [self-hosted deployments](https://docs.netbird.io/selfhosted/troubleshooting). +3. Remove or anonymize sensitive information from logs, screenshots, and configuration. + +If a discussion already covers your problem, upvote it and add your details there rather than +opening a duplicate. Extra reproduction detail, affected versions, and deployment notes are +useful even on an existing thread. + +## Community support + +Free, for everyone. Covers the NetBird client, open source self-hosted deployments, and general +questions. + +| What you want to do | Where to go | +| --- | --- | +| Report a bug, regression, or unexpected behavior | [Issue Triage](https://github.com/netbirdio/netbird/discussions/new?category=issue-triage) | +| Request a feature or share an idea | [Ideas & Feature Requests](https://github.com/netbirdio/netbird/discussions/new?category=ideas-feature-requests) | +| Ask about setup, configuration, or self-hosting | [Q&A / Support](https://github.com/netbirdio/netbird/discussions/new?category=q-a-support) | +| Chat with the community | [Slack](https://docs.netbird.io/slack-url) | + +## Paid support + +For NetBird Cloud customers and commercial-license self-hosted deployments, covering the +dashboard, control plane, billing, and subscriptions, see +[reporting bugs and issues](https://docs.netbird.io/help/report-bug-issues). + +## Security + +Do not report security vulnerabilities in public issues or discussions, and do not post secrets, +private keys, internal hostnames, or sensitive logs. Use the +[security policy](https://github.com/netbirdio/netbird/security/policy). + +## What makes a report we can act on + +For a bug, the most useful reports include: + +- NetBird version, and component versions where applicable +- Operating system or environment +- Deployment type: NetBird Cloud, self-hosted, Kubernetes, Docker, or local development +- Current behavior and expected behavior +- The smallest set of steps that reproduces the problem +- Logs, status output, screenshots, or a debug bundle when relevant +- Whether this worked before, and the last known working version + +For client reports, these commands usually give us what we need: + +```shell +netbird version +netbird status -d -A +netbird debug for 1m -A -S -U +``` + +`-A` (`--anonymize`) replaces sensitive values consistently across every file in the bundle, so +it stays readable while masking most identifying details. It is not a guarantee of full redaction: +internal address ranges survive at the default level, and interface names, indexes, MTUs, and +flags are never anonymized. Read the bundle before posting it publicly. Two levels are +available: + +| Level | How to select | What it masks | +| --- | --- | --- | +| `default` | `-A` / `--anonymize`, or `--anonymize-level default` | Public IP addresses, IPv6 ULA addresses, MAC addresses, and domains other than `netbird.io`, `netbird.cloud`, `netbird.selfhosted`, and `netbird.stage`. IPv4 private, CGNAT, and link-local ranges are kept, and interface names are not anonymized | +| `strict` | `--anonymize-level strict` (implies `-A`) | The above, plus IPv4 private, CGNAT, and link-local ranges, peer names in front of `netbird.cloud`, `netbird.selfhosted`, and `netbird.stage`, and WireGuard public keys. Labels under `netbird.io` are kept, since it only hosts infrastructure | + +Use `strict` when internal addressing or peer naming is itself sensitive. Either way, private +keys and SSH keys are never included, and the packet capture (`capture.pcap`) is left out of +anonymized bundles because it holds raw decrypted packets. + +`-U` (`--upload-bundle`) uploads the bundle and returns a file key you can paste into the thread +instead of attaching an archive. Retention is controlled by the upload service; check its policy +before uploading, and configure cleanup for self-hosted deployments. + +For more detail, see [troubleshooting client issues](https://docs.netbird.io/help/troubleshooting-client), +which explains [what a debug bundle contains](https://docs.netbird.io/help/troubleshooting-client#debug-bundle), +and the [CLI reference](https://docs.netbird.io/get-started/cli#debug-for). + +Intermittent problems are still worth reporting. They just need enough detail to investigate: +trigger, frequency, timing, timestamps, and any related logs. + +For a feature request, describe the problem before the solution: what you are trying to +accomplish, who is affected and how often, why the current behavior or workaround is not enough, +and what you would like to see instead. + +## What happens after you post + +Our team, maintainers, or community members may ask for missing details, link related +threads, merge duplicates, move your post to a better category, or try to reproduce the problem. + +Not every discussion becomes an issue. Some are answered in Q&A, some turn out to be +configuration problems, and some need more information before engineering can act. A +well-answered discussion is still a useful outcome. + +When a report is confirmed and actionable, a maintainer opens a validated issue linked back to +the discussion, in whichever repository the fix belongs to. You do not need to know which +repository that is. Routing is part of triage. + +## A note on issues + +Issues in this repository are maintainer-curated work items. Every open issue is something a +maintainer or contributor can pick up and act on. Issues opened without a linked validated +discussion may be closed and redirected here. + +Maintainers can still open issues directly for work found internally, such as regressions caught +during development, planned maintenance, or release blockers. + +## Related reading + +- [How to use Discussions, Issues, and Pull Requests](https://github.com/netbirdio/netbird/discussions/6075) +- [Moving to a discussion-first approach](https://github.com/netbirdio/netbird/discussions/6074) +- [CONTRIBUTING.md](CONTRIBUTING.md) for opening pull requests +- [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) diff --git a/agent-network/README.md b/agent-network/README.md index 029ada299..35b9c6668 100644 --- a/agent-network/README.md +++ b/agent-network/README.md @@ -110,12 +110,12 @@ Two roles delegate Agent Network access without account-admin rights: read-only users, groups, peers, and account info (needed to build policies). Nothing else in the account. - **`usage_viewer`** — the regular User baseline plus read on - `agent_network.usage` (the aggregated usage and cost overview) and read-only - access to the resources the usage filters resolve against: users, groups, - peers, and the provider list (connection config redacted — no upstream URLs - or operator-supplied header values). No policies, and no account-wide - request-level access logs; like any caller, it still reads its own requests - through the self-scoped endpoints below. + `agent_network.usage` (the aggregated usage and cost overview) and + `agent_network.logs` (the account-wide request-level access logs, which can + contain captured prompts), and read-only access to the resources those + filters resolve against: users, groups, peers, and the provider list + (connection config redacted — no upstream URLs or operator-supplied header + values). No policies, guardrails, budgets, or settings. Every authenticated user, regardless of role, can read the caller-scoped self-service endpoint `GET /api/agent-network/agent-config` (the endpoint, providers, diff --git a/base62/base62.go b/base62/base62.go index efafbc768..1a02e98e2 100644 --- a/base62/base62.go +++ b/base62/base62.go @@ -3,56 +3,75 @@ package base62 import ( "fmt" "math" - "strings" ) const ( - alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - base = uint32(len(alphabet)) + alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + base = uint32(len(alphabet)) + maxBase62Digits = 6 // max number of digits required to encode MaxUint32 + ) +var ( + ErrEmptyString = fmt.Errorf("empty string") + ErrInvalidChar = fmt.Errorf("invalid character") + ErrOverflow = fmt.Errorf("integer overflow") +) + +// Fixed-size arrays have better performance and lower memory overhead compared to maps for small static sets of data +var charToIndex [123]int8 // Assuming ASCII from '\0' - 'z' + +func init() { + for i := range charToIndex { + charToIndex[i] = -1 + } + for i, c := range alphabet { + charToIndex[c] = int8(i) + } +} + // Encode encodes a uint32 value to a base62 string. -func Encode(num uint32) string { - if num == 0 { - return string(alphabet[0]) +// The returned string will be between 1-6 characters long. +func Encode(n uint32) string { + if n < base { + return string(alphabet[n]) + } + // avoid dynamic memory usage for small, fixed size data + buf := [maxBase62Digits]byte{} + idx := len(buf) + + for n > 0 { + idx-- + buf[idx] = alphabet[n%base] + n /= base } - var encoded strings.Builder - - for num > 0 { - remainder := num % base - encoded.WriteByte(alphabet[remainder]) - num /= base - } - - // Reverse the encoded string - encodedString := encoded.String() - reversed := reverse(encodedString) - return reversed + return string(buf[idx:]) } // Decode decodes a base62 string to a uint32 value. +// Returns an error if the input string is empty, contains invalid characters, +// or would result in integer overflow. func Decode(encoded string) (uint32, error) { + if len(encoded) == 0 { + return 0, ErrEmptyString + } var decoded uint32 - strLen := len(encoded) - - for i, char := range encoded { - index := strings.IndexRune(alphabet, char) + for _, char := range encoded { + index := int8(-1) + if int(char) < len(charToIndex) { + index = charToIndex[char] + } if index < 0 { - return 0, fmt.Errorf("invalid character: %c", char) + return 0, fmt.Errorf("%w: %c", ErrInvalidChar, char) + } + // Add overflow check when calculating the decoded value to prevent silent overflow of uint32 + if decoded > (math.MaxUint32-uint32(index))/base { + return 0, fmt.Errorf("%w: %s", ErrOverflow, encoded) } - decoded += uint32(index) * uint32(math.Pow(float64(base), float64(strLen-i-1))) + decoded = decoded*base + uint32(index) } return decoded, nil } - -// Reverse a string. -func reverse(s string) string { - runes := []rune(s) - for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { - runes[i], runes[j] = runes[j], runes[i] - } - return string(runes) -} diff --git a/base62/base62_test.go b/base62/base62_test.go index 00da2124a..f2ad06d6f 100644 --- a/base62/base62_test.go +++ b/base62/base62_test.go @@ -1,31 +1,67 @@ package base62 import ( + "errors" + "math" "testing" ) func TestEncodeDecode(t *testing.T) { - tests := []struct { - num uint32 + testCases := []struct { + input uint32 + expected string }{ - {0}, - {1}, - {42}, - {12345}, - {99999}, - {123456789}, + {0, "0"}, + {1, "1"}, + {5, "5"}, + {9, "9"}, + {10, "A"}, + {42, "g"}, + {61, "z"}, + {62, "10"}, + {'0', "m"}, + {'9', "v"}, + {'A', "13"}, + {'Z', "1S"}, + {'a', "1Z"}, + {'z', "1y"}, + {99999, "Q0t"}, + {12345, "3D7"}, + {123456789, "8M0kX"}, + {math.MaxUint32, "4gfFC3"}, } - for _, tt := range tests { - encoded := Encode(tt.num) + for _, tc := range testCases { + encoded := Encode(tc.input) + if encoded != tc.expected { + t.Errorf("Encode(%d) = %s; want %s", tc.input, encoded, tc.expected) + } decoded, err := Decode(encoded) - if err != nil { - t.Errorf("Decode error: %v", err) + t.Errorf("Expected error nil, got %v", err) } - if decoded != tt.num { - t.Errorf("Decode(%v) = %v, want %v", encoded, decoded, tt.num) + if decoded != tc.input { + t.Errorf("Decode(%v) = %v, want %v", encoded, decoded, tc.input) } } } + +// Decode handles empty string input with appropriate error +func TestDecodeEmptyString(t *testing.T) { + if _, err := Decode(""); !errors.Is(err, ErrEmptyString) { + t.Errorf("Expected error %v, got %v", ErrEmptyString, err) + } +} + +func TestDecodeOverflow(t *testing.T) { + if _, err := Decode("4gfFC4"); !errors.Is(err, ErrOverflow) { + t.Errorf("Expected error %v, got %v", ErrOverflow, err) + } +} + +func TestDecodeInvalid(t *testing.T) { + if _, err := Decode("/"); !errors.Is(err, ErrInvalidChar) { + t.Errorf("Expected error %v, got %v", ErrInvalidChar, err) + } +} diff --git a/client/Dockerfile-nblink b/client/Dockerfile-nblink new file mode 100644 index 000000000..63177a6c0 --- /dev/null +++ b/client/Dockerfile-nblink @@ -0,0 +1,34 @@ +# nblink runs unprivileged: no TUN device, no capabilities, no host networking. +# +# The image is built for platforms that assign an arbitrary UID at run time, so +# it must not depend on a passwd entry and its state directory has to be +# writable by the root group rather than by one named user. +FROM alpine:3.22 AS build + +# Group 0 owns the state directory so a container started as any UID in that +# group can still persist its identity. Mode 770 rather than g=u, because the +# directory holds the peer's keys and nothing outside owner and group should +# read them, including when the path is bind-mounted from a host. +RUN mkdir -p /tmp/state/nblink \ + && chgrp -R 0 /tmp/state/nblink \ + && chmod -R 770 /tmp/state/nblink + +FROM gcr.io/distroless/static-debian13:nonroot + +ARG TARGETPLATFORM +COPY ${TARGETPLATFORM}/nblink /usr/local/bin/nblink +# COPY of a directory copies its contents and creates the target with mode +# 755, which group 0 cannot write. Copying the parent makes nblink/ itself the +# copied entry, so its mode survives. +COPY --from=build --chown=65532:0 /tmp/state/ /var/lib/ + +# Numeric so the runtime can substitute its own UID without a passwd lookup. +USER 65532:0 + +# No NB_STATE_DIR by default: state stays in memory, so the image runs on a +# read-only root filesystem without a volume. Set NB_STATE_DIR=/var/lib/nblink +# and mount a volume there to keep the peer identity; a volume mounted over +# the directory starts out with its group-0 ownership. +ENV NB_LOG_LEVEL=info + +ENTRYPOINT ["/usr/local/bin/nblink"] diff --git a/client/android/client.go b/client/android/client.go index e47a1c13d..9705db8e0 100644 --- a/client/android/client.go +++ b/client/android/client.go @@ -104,8 +104,7 @@ type Client struct { stateChangeMu sync.Mutex stateChangeSubID string - eventSub *peer.EventSubscription - // Closed to stop the watch goroutines from delivering buffered items to a + // Closed to stop the watch goroutine from delivering buffered ticks to a // listener that has been removed or replaced. See stopStateChangeWatchLocked. stateChangeDone chan struct{} @@ -213,6 +212,7 @@ func (c *Client) Run(platformFiles PlatformFiles, urlOpener URLOpener, isAndroid connectClient := internal.NewConnectClient(ctx, cfg, c.recorder, internal.WithNetEvents(c.netMgr)) c.setState(cfg, cacheDir, cfgFile, connectClient) + connectClient.SetSyncResponsePersistence(true) // This path runs the interactive SSO flow, so reaching here means the peer // is authenticated again — release the latch Status() reports from. Clear // only once the fresh connect client is installed: until then Status() @@ -256,6 +256,7 @@ func (c *Client) RunWithoutLogin(platformFiles PlatformFiles, dns *DNSList, dnsR connectClient := internal.NewConnectClient(ctx, cfg, c.recorder, internal.WithNetEvents(c.netMgr)) c.setState(cfg, cacheDir, cfgFile, connectClient) + connectClient.SetSyncResponsePersistence(true) return connectClient.RunOnAndroid(c.tunAdapter, c.iFaceDiscover, c.networkChangeListener, slices.Clone(dns.items), dnsReadyListener, stateFile, cacheDir) } @@ -327,6 +328,19 @@ func (c *Client) NotifyNetworkChange() { // or "strict"; strict also anonymizes internal IP ranges, peer names, and // WireGuard public keys, and implies anonymize. func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool, anonymizeLevel string) (string, error) { + return c.debugBundle(platformFiles, anonymize, anonymizeLevel, true) +} + +// DebugBundleFile generates a debug bundle and returns the path of the zip in +// the cache directory instead of uploading it, so the app can hand the file to +// the user for inspection. The caller owns the file and removes it once done; +// the stale-bundle cleanup of later runs removes it only after a day. +// anonymize and anonymizeLevel behave as in DebugBundle. +func (c *Client) DebugBundleFile(platformFiles PlatformFiles, anonymize bool, anonymizeLevel string) (string, error) { + return c.debugBundle(platformFiles, anonymize, anonymizeLevel, false) +} + +func (c *Client) debugBundle(platformFiles PlatformFiles, anonymize bool, anonymizeLevel string, upload bool) (string, error) { cfg, cacheDir, cc := c.stateSnapshot() // If the engine hasn't been started, load config from disk @@ -342,6 +356,11 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool, anonym cacheDir = platformFiles.CacheDir() } + // Clear what an interrupted earlier run may have left in the cache before + // adding to it. Remote debug jobs write to the same directory, so anything + // younger than an hour is treated as possibly still in use. + debug.RemoveStaleBundles(cacheDir, time.Hour) + deps := debug.GeneratorDependencies{ InternalConfig: cfg, StatusRecorder: c.recorder, @@ -379,6 +398,9 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool, anonym if err != nil { return "", fmt.Errorf("generate debug bundle: %w", err) } + if !upload { + return debug.ExportBundle(path) + } defer func() { if err := os.Remove(path); err != nil { log.Errorf("failed to remove debug bundle file: %v", err) @@ -475,6 +497,7 @@ func (c *Client) Networks() *NetworkArray { routesMap := routeManager.GetClientRoutesWithNetID() v6Merged := route.V6ExitMergeSet(routesMap) resolvedDomains := c.recorder.GetResolvedDomainsStates() + activeRoutePeers := c.recorder.GetActiveRoutePeers() networkArray := &NetworkArray{ items: make([]Network, 0), @@ -488,7 +511,7 @@ func (c *Client) Networks() *NetworkArray { continue } - network := c.buildNetwork(id, routes, routeSelector.IsSelected(id), resolvedDomains, v6Merged) + network := c.buildNetwork(id, routes, routeSelector.IsSelected(id), resolvedDomains, v6Merged, activeRoutePeers) if network == nil { continue } @@ -497,14 +520,14 @@ func (c *Client) Networks() *NetworkArray { return networkArray } -func (c *Client) buildNetwork(id route.NetID, routes []*route.Route, selected bool, resolvedDomains map[domain.Domain]peer.ResolvedDomainInfo, v6Merged map[route.NetID]struct{}) *Network { +func (c *Client) buildNetwork(id route.NetID, routes []*route.Route, selected bool, resolvedDomains map[domain.Domain]peer.ResolvedDomainInfo, v6Merged map[route.NetID]struct{}, activeRoutePeers map[route.HAUniqueID]string) *Network { r := routes[0] netStr := r.Network.String() if r.IsDynamic() { netStr = r.Domains.SafeString() } - routePeer, err := c.findBestRoutePeer(routes) + routePeer, err := c.findBestRoutePeer(routes, activeRoutePeers) if err != nil { log.Errorf("could not get peer info for route %s: %v", id, err) return nil @@ -528,12 +551,9 @@ func (c *Client) buildNetwork(id route.NetID, routes []*route.Route, selected bo // findBestRoutePeer returns the peer actively routing traffic for the given // HA route group. Falls back to the first connected peer, then the first peer. -func (c *Client) findBestRoutePeer(routes []*route.Route) (peer.State, error) { - netStr := routes[0].Network.String() - - fullStatus := c.recorder.GetFullStatus() - for _, p := range fullStatus.Peers { - if _, ok := p.GetRoutes()[netStr]; ok { +func (c *Client) findBestRoutePeer(routes []*route.Route, activeRoutePeers map[route.HAUniqueID]string) (peer.State, error) { + if peerKey, ok := activeRoutePeers[routes[0].GetHAUniqueID()]; ok { + if p, err := c.recorder.GetPeer(peerKey); err == nil { return p, nil } } diff --git a/client/android/login.go b/client/android/login.go index 155c6eadd..b9ec21b39 100644 --- a/client/android/login.go +++ b/client/android/login.go @@ -194,12 +194,49 @@ func (a *Auth) login(urlOpener URLOpener, isAndroidTV bool) error { } func (a *Auth) foregroundGetTokenInfo(authClient *auth.Auth, urlOpener URLOpener, isAndroidTV bool) (*auth.TokenInfo, error) { - oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, isAndroidTV, profileLoginHint(a.cfgPath)) + return a.foregroundGetTokenInfoFlow(authClient, urlOpener, isAndroidTV, false) +} + +// foregroundGetTokenInfoFlow runs the interactive flow. sessionExtend tells the +// server the token will renew this peer's session rather than log a peer in, so +// it can rule out a silent authorization the IdP could answer from an unrelated +// account. See PKCEAuthorizationFlowRequest. +func (a *Auth) foregroundGetTokenInfoFlow(authClient *auth.Auth, urlOpener URLOpener, isAndroidTV bool, sessionExtend bool) (*auth.TokenInfo, error) { + hint := profileLoginHint(a.cfgPath) + + oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, isAndroidTV, sessionExtend, hint) if err != nil { return nil, fmt.Errorf("failed to get OAuth flow: %v", err) } - return runOAuthFlow(a.ctx, oAuthFlow, urlOpener, nil) + tokenInfo, err := runOAuthFlow(a.ctx, oAuthFlow, urlOpener, nil) + if err != nil { + return nil, err + } + + if tokenInfo.MatchesAccount(hint) { + return tokenInfo, nil + } + + // The IdP answered from a session belonging to another account. Retrying is + // what makes this recoverable: on a peer already registered the server would + // reject the token, and on a fresh one it would silently register the peer + // under the wrong account and bind the profile to it. + log.Infof("login returned an account other than the one this profile is bound to, retrying with an account prompt") + retryFlow := auth.RetryFlowForAccount(oAuthFlow) + if retryFlow == nil { + return tokenInfo, nil + } + + retryToken, err := runOAuthFlow(a.ctx, retryFlow, urlOpener, nil) + if err != nil { + return nil, err + } + if !retryToken.MatchesAccount(hint) { + log.Warnf("login still returned a different account after the prompt, continuing with it") + } + + return retryToken, nil } // profileLoginHint returns the stored account email for the profile at cfgPath. diff --git a/client/android/preferences.go b/client/android/preferences.go index 5ce31026c..3623de23f 100644 --- a/client/android/preferences.go +++ b/client/android/preferences.go @@ -46,7 +46,7 @@ func (p *Preferences) GetManagementURL() (string, error) { return p.configInput.ManagementURL, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return "", err } @@ -64,7 +64,7 @@ func (p *Preferences) GetAdminURL() (string, error) { return p.configInput.AdminURL, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return "", err } @@ -86,7 +86,7 @@ func (p *Preferences) HasPreSharedKey() (bool, error) { return *p.configInput.PreSharedKey != "", nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -112,7 +112,7 @@ func (p *Preferences) GetRosenpassEnabled() (bool, error) { return *p.configInput.RosenpassEnabled, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -133,7 +133,7 @@ func (p *Preferences) GetRosenpassPermissive() (bool, error) { return *p.configInput.RosenpassPermissive, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -149,7 +149,7 @@ func (p *Preferences) GetDisableClientRoutes() (bool, error) { return *p.configInput.DisableClientRoutes, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -170,7 +170,7 @@ func (p *Preferences) GetDisableServerRoutes() (bool, error) { return *p.configInput.DisableServerRoutes, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -188,7 +188,7 @@ func (p *Preferences) GetDisableDNS() (bool, error) { return *p.configInput.DisableDNS, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -206,7 +206,7 @@ func (p *Preferences) GetDisableFirewall() (bool, error) { return *p.configInput.DisableFirewall, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -227,7 +227,7 @@ func (p *Preferences) GetServerSSHAllowed() (bool, error) { return *p.configInput.ServerSSHAllowed, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -249,7 +249,7 @@ func (p *Preferences) GetEnableSSHRoot() (bool, error) { return *p.configInput.EnableSSHRoot, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -271,7 +271,7 @@ func (p *Preferences) GetEnableSSHSFTP() (bool, error) { return *p.configInput.EnableSSHSFTP, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -293,7 +293,7 @@ func (p *Preferences) GetEnableSSHLocalPortForwarding() (bool, error) { return *p.configInput.EnableSSHLocalPortForwarding, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -315,7 +315,7 @@ func (p *Preferences) GetEnableSSHRemotePortForwarding() (bool, error) { return *p.configInput.EnableSSHRemotePortForwarding, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -340,7 +340,7 @@ func (p *Preferences) GetBlockInbound() (bool, error) { return *p.configInput.BlockInbound, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -358,7 +358,7 @@ func (p *Preferences) GetDisableIPv6() (bool, error) { return *p.configInput.DisableIPv6, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -377,7 +377,7 @@ func (p *Preferences) GetRemoteJobsAllowed() (bool, error) { return *p.configInput.RemoteJobsAllowed, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } diff --git a/client/android/session.go b/client/android/session.go index d5da09c93..b2de8dadb 100644 --- a/client/android/session.go +++ b/client/android/session.go @@ -6,13 +6,8 @@ import ( "context" "fmt" - log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/auth" - "github.com/netbirdio/netbird/client/internal/auth/sessionwatch" - "github.com/netbirdio/netbird/client/internal/peer" - cProto "github.com/netbirdio/netbird/client/proto" ) // StateChangeListener receives client state notifications. @@ -21,16 +16,11 @@ import ( // changed: connection state, the run-loop status label (e.g. NeedsLogin) or // the session deadline. It mirrors the daemon's SubscribeStatus stream // trigger — on each signal the consumer pulls the fresh values via -// Status() / SessionExpiresAtUnix(). -// -// OnSessionExpiring forwards the engine's session-expiry warnings, fired at -// sessionwatch.WarningLead before the deadline and again at FinalWarningLead -// (finalWarning true). The second one is suppressed when the user dismissed -// the first via DismissSessionWarning. The daemon turns the same events into -// its tray notification. +// Status() / SessionExpiresAtUnix(). The engine arms no expiry-warning +// timers on Android; the app schedules the warnings from the deadline it +// reads here. type StateChangeListener interface { OnStateChanged() - OnSessionExpiring(expiresAtUnix int64, leadMinutes int64, finalWarning bool) } // Status returns the connect run-loop's status label — the same value the @@ -110,11 +100,11 @@ func (c *Client) SetStateChangeListener(listener StateChangeListener) { return } - // Both subscriptions are buffered (one pending tick, ten pending events), - // so unsubscribing is not enough to stop callbacks: the loops would drain - // what is already queued and deliver it to a listener the caller has - // already removed or replaced. Gate every callback on this registration's - // own signal, which is closed before unsubscribing. + // The subscription is buffered (one pending tick), so unsubscribing is + // not enough to stop callbacks: the loop would drain what is already + // queued and deliver it to a listener the caller has already removed or + // replaced. Gate every callback on this registration's own signal, which + // is closed before unsubscribing. done := make(chan struct{}) c.stateChangeDone = done @@ -133,9 +123,6 @@ func (c *Client) SetStateChangeListener(listener StateChangeListener) { listener.OnStateChanged() } }() - - c.eventSub = c.recorder.SubscribeToEvents() - go watchSessionWarnings(c.eventSub, listener, done) } // RemoveStateChangeListener unregisters the state notification listener. @@ -145,21 +132,6 @@ func (c *Client) RemoveStateChangeListener() { c.stopStateChangeWatchLocked() } -// DismissSessionWarning records the user's "Dismiss" on the first expiry -// warning and suppresses the final one for the current deadline. A refreshed -// deadline re-arms both. No-op while the engine is not running. -func (c *Client) DismissSessionWarning() { - cc := c.getConnectClient() - if cc == nil { - return - } - engine := cc.Engine() - if engine == nil { - return - } - engine.DismissSessionWarning() -} - // ExtendAuthSession runs the interactive SSO flow to obtain a fresh JWT and // asks the management server to extend the session deadline. The tunnel is // untouched: no resync, no reconnect. Async; the result arrives on the @@ -201,8 +173,8 @@ func (c *Client) CancelExtendAuthSession() { } func (c *Client) stopStateChangeWatchLocked() { - // Signal first, unsubscribe second: closing the channels only stops new - // items, and the loops would still hand whatever is buffered to a listener + // Signal first, unsubscribe second: closing the channel only stops new + // items, and the loop would still hand whatever is buffered to a listener // that is no longer registered. if c.stateChangeDone != nil { close(c.stateChangeDone) @@ -212,49 +184,6 @@ func (c *Client) stopStateChangeWatchLocked() { c.recorder.UnsubscribeFromStateChanges(c.stateChangeSubID) c.stateChangeSubID = "" } - if c.eventSub != nil { - // Closes the channel, which ends watchSessionWarnings. - c.recorder.UnsubscribeFromEvents(c.eventSub) - c.eventSub = nil - } -} - -// watchSessionWarnings forwards the engine's session-expiry warnings to the -// listener. The event stream also carries unrelated traffic — network-map -// updates on every sync, DNS and route errors — so everything but an -// AUTHENTICATION event carrying the session-warning marker is dropped. Exits -// when the subscription is closed by UnsubscribeFromEvents, or earlier when -// done is closed — the stream buffers up to ten events, and a deregistered -// listener must not receive the ones already queued. -func watchSessionWarnings(sub *peer.EventSubscription, listener StateChangeListener, done <-chan struct{}) { - for ev := range sub.Events() { - select { - case <-done: - return - default: - } - if ev.GetCategory() != cProto.SystemEvent_AUTHENTICATION { - continue - } - meta := ev.GetMetadata() - if meta[sessionwatch.MetaSessionWarning] != "true" { - // Other AUTHENTICATION events exist (e.g. a deadline rejected as - // out of range); they carry no warning marker. - continue - } - deadline, err := sessionwatch.ParseExpiresAt(meta[sessionwatch.MetaSessionExpiresAt]) - if err != nil { - log.Warnf("session warning event with unparsable deadline: %v", err) - continue - } - lead, err := sessionwatch.ParseLeadMinutes(meta[sessionwatch.MetaSessionLeadMinutes]) - if err != nil { - // Informational only — the deadline above is what drives the UI. - lead = 0 - } - listener.OnSessionExpiring(deadline.Unix(), int64(lead), - meta[sessionwatch.MetaSessionFinal] == "true") - } } func (c *Client) beginExtend() (context.Context, error) { @@ -293,11 +222,13 @@ func (c *Client) extendAuthSession(ctx context.Context, urlOpener URLOpener, isA } defer authClient.Close() - // Passing the config path makes the flow pick up the login_hint: an extend - // renews the session of the account already signed in, so it must not stop to - // offer a choice. + // Passing the config path makes the flow pick up the login_hint. That alone + // cannot keep the IdP on this profile's account though — a hint is only a + // suggestion, and a silent authorization is answered from whatever session the + // IdP already has, which need not be this peer's when several accounts are + // signed in. Marking the flow as an extend lets the server rule that out. a := NewAuthWithConfig(ctx, cfg, cfgPath) - tokenInfo, err := a.foregroundGetTokenInfo(authClient, urlOpener, isAndroidTV) + tokenInfo, err := a.foregroundGetTokenInfoFlow(authClient, urlOpener, isAndroidTV, true) if err != nil { return fmt.Errorf("interactive sso login failed: %v", err) } diff --git a/client/android/ssh_client.go b/client/android/ssh_client.go index 2822b6539..e05249e98 100644 --- a/client/android/ssh_client.go +++ b/client/android/ssh_client.go @@ -31,6 +31,8 @@ const ( // PasswordRequiredMarker tells Java to prompt for a password and retry. It is // a string because gomobile flattens errors to their message, so a sentinel // value would not survive the binding. +// +//nolint:gosec // G101 false positive: a sentinel marker, not a credential const PasswordRequiredMarker = "netbird-ssh-password-required" // HostKeyUnknownMarker tells Java to show the fingerprint and, on confirmation, @@ -467,7 +469,7 @@ func (s *SSHClient) requestJWTToken(cfg *profilemanager.Config, cfgPath string) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - flow, err := auth.NewOAuthFlow(ctx, cfg, false, true, profileLoginHint(cfgPath)) + flow, err := auth.NewOAuthFlow(ctx, cfg, false, true, profileLoginHint(cfgPath), false) if err != nil { return "", fmt.Errorf("create oauth flow: %w", err) } diff --git a/client/cmd/debug.go b/client/cmd/debug.go index 98fe53626..c4d5ad6d7 100644 --- a/client/cmd/debug.go +++ b/client/cmd/debug.go @@ -23,7 +23,10 @@ import ( "github.com/netbirdio/netbird/version" ) -const errCloseConnection = "Failed to close connection: %v" +const ( + errCloseConnection = "Failed to close connection: %v" + noUpDownFlag = "no-updown" +) var ( logFileCount uint32 @@ -257,13 +260,14 @@ func runForDuration(cmd *cobra.Command, args []string) error { } stateWasDown := stat.Status != string(internal.StatusConnected) && stat.Status != string(internal.StatusConnecting) + noUpDown, _ := cmd.Flags().GetBool(noUpDownFlag) initialLogLevel, err := client.GetLogLevel(cmd.Context(), &proto.GetLogLevelRequest{}) if err != nil { return fmt.Errorf("failed to get log level: %v", status.Convert(err).Message()) } - if stateWasDown { + if stateWasDown && !noUpDown { if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil { cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message()) } else { @@ -284,34 +288,20 @@ func runForDuration(cmd *cobra.Command, args []string) error { } needsRestoreUp := false - if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil { - cmd.PrintErrf("Failed to bring service down: %v\n", status.Convert(err).Message()) + if noUpDown { + enableSyncResponsePersistence(cmd, client) } else { - needsRestoreUp = !stateWasDown - cmd.Println("netbird down") + needsRestoreUp = restartDaemon(cmd, client, stateWasDown) } - time.Sleep(1 * time.Second) - - // Enable sync response persistence before bringing the service up - if _, err := client.SetSyncResponsePersistence(cmd.Context(), &proto.SetSyncResponsePersistenceRequest{ - Enabled: true, - }); err != nil { - cmd.PrintErrf("Failed to enable sync response persistence: %v\n", status.Convert(err).Message()) - } - - if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil { - cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message()) - } else { - needsRestoreUp = false - cmd.Println("netbird up") - } - - time.Sleep(3 * time.Second) - cpuProfilingStarted := false if _, err := client.StartCPUProfile(cmd.Context(), &proto.StartCPUProfileRequest{}); err != nil { - cmd.PrintErrf("Failed to start CPU profiling: %v\n", err) + if msg := status.Convert(err).Message(); strings.Contains(msg, "already in progress") { + cmd.PrintErrln("CPU profiling is already running (started with `netbird debug cpu start`). " + + "It is left running and is included in a bundle created after `netbird debug cpu stop`.") + } else { + cmd.PrintErrf("Failed to start CPU profiling: %v\n", msg) + } } else { cpuProfilingStarted = true defer func() { @@ -401,7 +391,7 @@ func runForDuration(cmd *cobra.Command, args []string) error { } } - if stateWasDown { + if stateWasDown && !noUpDown { if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil { cmd.PrintErrf("Failed to restore service down state: %v\n", status.Convert(err).Message()) } else { @@ -458,6 +448,47 @@ func setSyncResponsePersistence(cmd *cobra.Command, args []string) error { return nil } +// enableSyncResponsePersistence asks the daemon to keep the latest sync +// response so the bundle carries the network map. With a running daemon only +// syncs received after the call are kept. +func enableSyncResponsePersistence(cmd *cobra.Command, client proto.DaemonServiceClient) { + if _, err := client.SetSyncResponsePersistence(cmd.Context(), &proto.SetSyncResponsePersistenceRequest{ + Enabled: true, + }); err != nil { + cmd.PrintErrf("Failed to enable sync response persistence: %v\n", status.Convert(err).Message()) + } +} + +// restartDaemon cycles the daemon down and up with sync response persistence +// enabled so the bundle carries the network map. It reports whether the +// daemon was left down although it was running before, so the caller can +// bring it back up. +func restartDaemon(cmd *cobra.Command, client proto.DaemonServiceClient, stateWasDown bool) bool { + needsRestoreUp := false + if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil { + cmd.PrintErrf("Failed to bring service down: %v\n", status.Convert(err).Message()) + } else { + needsRestoreUp = !stateWasDown + cmd.Println("netbird down") + } + + time.Sleep(1 * time.Second) + + // Enable sync response persistence before bringing the service up + enableSyncResponsePersistence(cmd, client) + + if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil { + cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message()) + } else { + needsRestoreUp = false + cmd.Println("netbird up") + } + + time.Sleep(3 * time.Second) + + return needsRestoreUp +} + func waitForDurationOrCancel(ctx context.Context, duration time.Duration, cmd *cobra.Command) error { ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() @@ -546,4 +577,5 @@ func init() { forCmd.Flags().StringVar(&uploadBundleURLFlag, "upload-bundle-url", types.DefaultBundleURL, "Service URL to get an URL to upload the debug bundle") forCmd.Flags().BoolVar(&uploadBundleInsecureFlag, "upload-bundle-insecure", false, "Allow uploading to an http or untrusted-TLS upload server (self-hosted); requires root") forCmd.Flags().Bool("capture", false, "Capture packets during the debug duration and include in bundle") + forCmd.Flags().Bool(noUpDownFlag, false, "Keep the daemon running instead of bringing it down and up before collecting. The bundle only includes the network map if a sync arrives during the run") } diff --git a/client/cmd/debug_cpu.go b/client/cmd/debug_cpu.go new file mode 100644 index 000000000..a01b845cf --- /dev/null +++ b/client/cmd/debug_cpu.go @@ -0,0 +1,83 @@ +package cmd + +import ( + "fmt" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "google.golang.org/grpc/status" + + "github.com/netbirdio/netbird/client/proto" +) + +var debugCPUCmd = &cobra.Command{ + Use: "cpu", + Short: "Profile the daemon's CPU usage", + Long: `Starts and stops CPU profiling in the running daemon without restarting it. +The profile is included in the next debug bundle as cpu.prof. + +Profiling is not time limited: it keeps running, and keeps costing CPU, until +"netbird debug cpu stop" is run.`, +} + +var debugCPUStartCmd = &cobra.Command{ + Use: "start", + Short: "Start CPU profiling in the daemon", + Example: " netbird debug cpu start", + Args: cobra.NoArgs, + RunE: debugCPUStart, +} + +var debugCPUStopCmd = &cobra.Command{ + Use: "stop", + Short: "Stop CPU profiling in the daemon", + Long: `Stops CPU profiling. The captured profile stays in the daemon until the next +debug bundle is created, which includes it as cpu.prof.`, + Example: " netbird debug cpu stop && netbird debug bundle", + Args: cobra.NoArgs, + RunE: debugCPUStop, +} + +func debugCPUStart(cmd *cobra.Command, _ []string) error { + conn, err := getClient(cmd) + if err != nil { + return err + } + defer func() { + if err := conn.Close(); err != nil { + log.Errorf(errCloseConnection, err) + } + }() + + if _, err := proto.NewDaemonServiceClient(conn).StartCPUProfile(cmd.Context(), &proto.StartCPUProfileRequest{}); err != nil { + return fmt.Errorf("start CPU profiling: %v", status.Convert(err).Message()) + } + + cmd.Println("CPU profiling started and runs until stopped. Run `netbird debug cpu stop` and then `netbird debug bundle` to collect it.") + return nil +} + +func debugCPUStop(cmd *cobra.Command, _ []string) error { + conn, err := getClient(cmd) + if err != nil { + return err + } + defer func() { + if err := conn.Close(); err != nil { + log.Errorf(errCloseConnection, err) + } + }() + + if _, err := proto.NewDaemonServiceClient(conn).StopCPUProfile(cmd.Context(), &proto.StopCPUProfileRequest{}); err != nil { + return fmt.Errorf("stop CPU profiling: %v", status.Convert(err).Message()) + } + + cmd.Println("CPU profiling stopped. Run `netbird debug bundle` to include cpu.prof.") + return nil +} + +func init() { + debugCPUCmd.AddCommand(debugCPUStartCmd) + debugCPUCmd.AddCommand(debugCPUStopCmd) + debugCmd.AddCommand(debugCPUCmd) +} diff --git a/client/cmd/debug_cpu_test.go b/client/cmd/debug_cpu_test.go new file mode 100644 index 000000000..85fffd462 --- /dev/null +++ b/client/cmd/debug_cpu_test.go @@ -0,0 +1,164 @@ +package cmd + +import ( + "bytes" + "context" + "os/user" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/internal" + "github.com/netbirdio/netbird/client/internal/profilemanager" +) + +// startDebugTestDaemon starts an in-process daemon with an isolated profile +// directory and returns the address the CLI should dial. +func startDebugTestDaemon(t *testing.T) string { + t.Helper() + + tempDir := t.TempDir() + origDefaultProfileDir := profilemanager.DefaultConfigPathDir + origActiveProfileStatePath := profilemanager.ActiveProfileStatePath + origConfigDirOverride := profilemanager.ConfigDirOverride + origDaemonAddr := daemonAddr + t.Cleanup(func() { + profilemanager.DefaultConfigPathDir = origDefaultProfileDir + profilemanager.ActiveProfileStatePath = origActiveProfileStatePath + profilemanager.ConfigDirOverride = origConfigDirOverride + daemonAddr = origDaemonAddr + }) + + profilemanager.DefaultConfigPathDir = tempDir + profilemanager.ActiveProfileStatePath = tempDir + "/active_profile.json" + profilemanager.ConfigDirOverride = tempDir + + currUser, err := user.Current() + require.NoError(t, err) + sm := profilemanager.ServiceManager{} + created, err := sm.AddProfile("test1", currUser.Username) + require.NoError(t, err) + require.NoError(t, sm.SetActiveProfileState(&profilemanager.ActiveProfileState{ + ID: created.ID, + Username: currUser.Username, + })) + + ctx, cancel := context.WithCancel(internal.CtxInitState(context.Background())) + srv, lis := startClientDaemon(t, ctx, "", tempDir+"/config.json") + t.Cleanup(func() { + cancel() + srv.Stop() + }) + + return "tcp://" + lis.Addr().String() +} + +// runDebugCmd runs `netbird debug ` against the daemon at addr and +// returns everything the command printed. +func runDebugCmd(addr string, args ...string) (string, error) { + daemonAddr = addr + var out bytes.Buffer + rootCmd.SetOut(&out) + rootCmd.SetErr(&out) + rootCmd.SetArgs(append(append([]string{"debug"}, args...), "--daemon-addr", addr, "--log-file", "")) + err := rootCmd.Execute() + rootCmd.SetOut(nil) + rootCmd.SetErr(nil) + rootCmd.SetArgs(nil) + resetFlags(rootCmd) + return out.String(), err +} + +// resetFlags puts every flag of the command and its subcommands back to its +// default so a value parsed in one run does not leak into the next in-process +// execution. +func resetFlags(cmd *cobra.Command) { + reset := func(f *pflag.Flag) { + // Set appends to a slice flag and would parse the "[a,b]" default + // text as elements, so slices are replaced instead. + if sv, ok := f.Value.(pflag.SliceValue); ok { + var def []string + if trimmed := strings.Trim(f.DefValue, "[]"); trimmed != "" { + def = strings.Split(trimmed, ",") + } + _ = sv.Replace(def) + } else { + _ = f.Value.Set(f.DefValue) + } + f.Changed = false + } + cmd.Flags().VisitAll(reset) + cmd.PersistentFlags().VisitAll(reset) + // Commands pin their writers to the buffer of the run that first used + // them, so a later run would print into the old buffer. + cmd.SetOut(nil) + cmd.SetErr(nil) + for _, sub := range cmd.Commands() { + resetFlags(sub) + } +} + +// TestResetFlagsSliceDefault guards against Set("[]") on slice flags, which +// stores a literal "[]" element instead of the empty default. +func TestResetFlagsSliceDefault(t *testing.T) { + cmd := &cobra.Command{Use: "x"} + var env, withDefault []string + cmd.Flags().StringSliceVar(&env, "env", nil, "") + cmd.Flags().StringSliceVar(&withDefault, "names", []string{"a", "b"}, "") + require.NoError(t, cmd.Flags().Parse([]string{"--env", "K=V", "--names", "c"})) + + resetFlags(cmd) + + assert.Empty(t, env, "slice flag with no default must reset to empty") + assert.Equal(t, []string{"a", "b"}, withDefault, "slice flag must reset to its default") +} + +func TestDebugCPUStartStop(t *testing.T) { + addr := startDebugTestDaemon(t) + + run := func(args ...string) error { + _, err := runDebugCmd(addr, append([]string{"cpu"}, args...)...) + return err + } + + require.Error(t, run("stop"), "stop without a running profile must fail") + require.NoError(t, run("start")) + assert.Error(t, run("start"), "second start must be rejected while profiling") + require.NoError(t, run("stop")) + assert.Error(t, run("stop"), "second stop must be rejected") + assert.NoError(t, run("start"), "profiling can be started again after a stop") + assert.NoError(t, run("stop")) +} + +// TestDebugForKeepsRunningCPUProfile covers `debug for` started while a +// profile from `debug cpu start` is running: it must say so, leave the +// profile alone, and still create the bundle. +func TestDebugForKeepsRunningCPUProfile(t *testing.T) { + addr := startDebugTestDaemon(t) + + _, err := runDebugCmd(addr, "cpu", "start") + require.NoError(t, err) + + out, err := runDebugCmd(addr, "for", "1s", "-S=false", "--no-updown") + require.NoError(t, err, "output: %s", out) + assert.Contains(t, out, "CPU profiling is already running", "the conflict must be explained") + assert.NotContains(t, out, "rpc error", "the raw RPC error must not reach the user") + assert.Contains(t, out, "Local file:", "the bundle must still be created") + + _, err = runDebugCmd(addr, "cpu", "stop") + assert.NoError(t, err, "the profile started by the user must still be running") +} + +func TestDebugForNoUpDown(t *testing.T) { + addr := startDebugTestDaemon(t) + + out, err := runDebugCmd(addr, "for", "1s", "-S=false", "--no-updown") + require.NoError(t, err, "output: %s", out) + assert.NotContains(t, out, "netbird down", "--no-updown must not bring the daemon down") + assert.NotContains(t, out, "netbird up", "--no-updown must not bring the daemon up") + assert.Contains(t, out, "Local file:", "the bundle must still be created") +} diff --git a/client/cmd/forwarding_rules.go b/client/cmd/forwarding_rules.go deleted file mode 100644 index b3052746a..000000000 --- a/client/cmd/forwarding_rules.go +++ /dev/null @@ -1,98 +0,0 @@ -package cmd - -import ( - "fmt" - "sort" - - "github.com/spf13/cobra" - "google.golang.org/grpc/status" - - "github.com/netbirdio/netbird/client/proto" -) - -var forwardingRulesCmd = &cobra.Command{ - Use: "forwarding", - Short: "List forwarding rules", - Long: `Commands to list forwarding rules.`, -} - -var forwardingRulesListCmd = &cobra.Command{ - Use: "list", - Aliases: []string{"ls"}, - Short: "List forwarding rules", - Example: " netbird forwarding list", - Long: "Commands to list forwarding rules.", - RunE: listForwardingRules, -} - -func listForwardingRules(cmd *cobra.Command, _ []string) error { - conn, err := getClient(cmd) - if err != nil { - return err - } - defer conn.Close() - - client := proto.NewDaemonServiceClient(conn) - resp, err := client.ForwardingRules(cmd.Context(), &proto.EmptyRequest{}) - if err != nil { - return fmt.Errorf("failed to list network: %v", status.Convert(err).Message()) - } - - if len(resp.GetRules()) == 0 { - cmd.Println("No forwarding rules available.") - return nil - } - - printForwardingRules(cmd, resp.GetRules()) - return nil -} - -func printForwardingRules(cmd *cobra.Command, rules []*proto.ForwardingRule) { - cmd.Println("Available forwarding rules:") - - // Sort rules by translated address - sort.Slice(rules, func(i, j int) bool { - if rules[i].GetTranslatedAddress() != rules[j].GetTranslatedAddress() { - return rules[i].GetTranslatedAddress() < rules[j].GetTranslatedAddress() - } - if rules[i].GetProtocol() != rules[j].GetProtocol() { - return rules[i].GetProtocol() < rules[j].GetProtocol() - } - - return getFirstPort(rules[i].GetDestinationPort()) < getFirstPort(rules[j].GetDestinationPort()) - }) - - var lastIP string - for _, rule := range rules { - dPort := portToString(rule.GetDestinationPort()) - tPort := portToString(rule.GetTranslatedPort()) - if lastIP != rule.GetTranslatedAddress() { - lastIP = rule.GetTranslatedAddress() - cmd.Printf("\nTranslated peer: %s\n", rule.GetTranslatedHostname()) - } - - cmd.Printf(" Local %s/%s to %s:%s\n", rule.GetProtocol(), dPort, rule.GetTranslatedAddress(), tPort) - } -} - -func getFirstPort(portInfo *proto.PortInfo) int { - switch v := portInfo.PortSelection.(type) { - case *proto.PortInfo_Port: - return int(v.Port) - case *proto.PortInfo_Range_: - return int(v.Range.GetStart()) - default: - return 0 - } -} - -func portToString(translatedPort *proto.PortInfo) string { - switch v := translatedPort.PortSelection.(type) { - case *proto.PortInfo_Port: - return fmt.Sprintf("%d", v.Port) - case *proto.PortInfo_Range_: - return fmt.Sprintf("%d-%d", v.Range.GetStart(), v.Range.GetEnd()) - default: - return "No port specified" - } -} diff --git a/client/cmd/login.go b/client/cmd/login.go index 11867be09..764fdd5b1 100644 --- a/client/cmd/login.go +++ b/client/cmd/login.go @@ -9,8 +9,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/term" - "google.golang.org/grpc/codes" - gstatus "google.golang.org/grpc/status" "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/auth" @@ -145,10 +143,7 @@ func doDaemonLogin(ctx context.Context, cmd *cobra.Command, providedSetupKey str err = WithBackOff(func() error { var backOffErr error loginResp, backOffErr = client.Login(ctx, &loginRequest) - if s, ok := gstatus.FromError(backOffErr); ok && (s.Code() == codes.InvalidArgument || - s.Code() == codes.PermissionDenied || - s.Code() == codes.NotFound || - s.Code() == codes.Unimplemented) { + if terminalLoginError(backOffErr) { loginErr = backOffErr return nil } @@ -327,10 +322,28 @@ func doForegroundLogin(ctx context.Context, cmd *cobra.Command, setupKey string, } - config, err := profilemanager.ReadConfig(configFilePath) + config, err := profilemanager.ReadConfigOrDefault(configFilePath) if err != nil { return fmt.Errorf("read config file %s: %v", configFilePath, err) } + // Reading a config does not provision one: this login is about to dial + // management with the profile's identity, so mint the keys if the profile + // has none yet and put them on disk — a key that stayed in memory would + // come back different on the next run and register a second peer. + // + // Before the MDM overlay below, on purpose: the file must keep the + // profile's own values. The overlay is runtime-only and re-derived on + // every load, so persisting it would turn an enforced management URL or + // pre-shared key into one the user appears to own once the policy is + // withdrawn. + if generated, err := config.EnsureIdentity(); err != nil { + return fmt.Errorf("ensure profile identity: %v", err) + } else if generated { + if err := profilemanager.WriteOutConfig(configFilePath, config); err != nil { + return fmt.Errorf("write out config file %s: %v", configFilePath, err) + } + } + // CLI standalone login: profilemanager no longer auto-applies MDM, // so layer in the OS-native policy here. Desktop builds construct // a Loader with no fetcher — the build-tagged loadPlatform reads @@ -412,11 +425,44 @@ func foregroundGetTokenInfo(ctx context.Context, cmd *cobra.Command, config *pro hint = profileState.Email } - oAuthFlow, err := auth.NewOAuthFlow(ctx, config, util.HasGraphicalSession(), false, hint) + oAuthFlow, err := auth.NewOAuthFlow(ctx, config, util.HasGraphicalSession(), false, hint, false) if err != nil { return nil, err } + tokenInfo, err := runInteractiveFlow(cmd, oAuthFlow) + if err != nil { + return nil, err + } + + if tokenInfo.MatchesAccount(hint) { + return tokenInfo, nil + } + + // The IdP answered from a session belonging to another account. Retrying is + // what makes this recoverable: on a peer already registered the server would + // reject the token, and on a fresh one it would silently register the peer + // under the wrong account and bind the profile to it. + cmd.Println("The login returned a different account than this profile uses. Asking to sign in again.") + retryFlow := auth.RetryFlowForAccount(oAuthFlow) + if retryFlow == nil { + return tokenInfo, nil + } + + retryToken, err := runInteractiveFlow(cmd, retryFlow) + if err != nil { + return nil, err + } + if !retryToken.MatchesAccount(hint) { + log.Warnf("login still returned a different account after the prompt, continuing with it") + } + + return retryToken, nil +} + +// runInteractiveFlow requests the authorization info, shows the URL to the user +// and blocks until the token comes back. +func runInteractiveFlow(cmd *cobra.Command, oAuthFlow auth.OAuthFlow) (*auth.TokenInfo, error) { flowInfo, err := oAuthFlow.RequestAuthInfo(context.TODO()) if err != nil { return nil, fmt.Errorf("getting a request OAuth flow info failed: %v", err) diff --git a/client/cmd/posture.go b/client/cmd/posture.go new file mode 100644 index 000000000..b0870233d --- /dev/null +++ b/client/cmd/posture.go @@ -0,0 +1,30 @@ +package cmd + +import ( + "github.com/spf13/cobra" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/certproof" +) + +var postureCmd = &cobra.Command{ + Use: "posture", + Short: "Posture helpers invoked by the NetBird daemon", + Hidden: true, +} + +var postureCertProofCmd = &cobra.Command{ + Use: "cert-proof", + Short: "Answer certificate posture challenges from the calling user's keychain", + Long: "Reads a certificate challenge set as JSON on stdin and writes the proofs as JSON on stdout.\n" + + "The daemon launches this in the console user's desktop session, because a login keychain\n" + + "cannot be reached from a root daemon. Not intended to be run by hand.", + Hidden: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + // Proofs travel on stdout, so every log line has to go elsewhere. + log.SetOutput(cmd.ErrOrStderr()) + return certproof.RunHelper(cmd.Context(), cmd.InOrStdin(), cmd.OutOrStdout()) + }, +} diff --git a/client/cmd/root.go b/client/cmd/root.go index be6479440..51179f77c 100644 --- a/client/cmd/root.go +++ b/client/cmd/root.go @@ -20,6 +20,8 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + gstatus "google.golang.org/grpc/status" "github.com/netbirdio/netbird/client/anonymize" daddr "github.com/netbirdio/netbird/client/internal/daemonaddr" @@ -175,16 +177,16 @@ func init() { rootCmd.AddCommand(versionCmd) rootCmd.AddCommand(sshCmd) rootCmd.AddCommand(networksCMD) - rootCmd.AddCommand(forwardingRulesCmd) rootCmd.AddCommand(debugCmd) rootCmd.AddCommand(profileCmd) rootCmd.AddCommand(exposeCmd) + rootCmd.AddCommand(postureCmd) + + postureCmd.AddCommand(postureCertProofCmd) networksCMD.AddCommand(routesListCmd) networksCMD.AddCommand(routesSelectCmd, routesDeselectCmd) - forwardingRulesCmd.AddCommand(forwardingRulesListCmd) - debugCmd.AddCommand(debugBundleCmd) debugCmd.AddCommand(logCmd) logCmd.AddCommand(logLevelCmd) @@ -285,6 +287,43 @@ func DialClientGRPCServer(ctx context.Context, addr string) (*grpc.ClientConn, e return grpc.DialContext(ctx, target, opts...) } +// terminalLoginError reports whether a Login failure is final, so the backoff +// cycle stops and the caller is told what the daemon said instead of "login +// backoff cycle failed" thirty seconds later. Retrying cannot change any of +// these answers: the request is malformed, the caller is not allowed, the +// target does not exist, a precondition on the daemon refuses it (the +// update-settings kill switch, an MDM-managed field), or the method is not +// implemented. +// +// Both `netbird up` and `netbird login` run Login through the backoff, and +// they each carried their own copy of this list — which is how one of them +// ended up retrying a refusal the other treated as final. +func terminalLoginError(err error) bool { + // A successful Login reaches here with a nil error, and that is not a + // terminal failure. Handled explicitly rather than left to + // gstatus.FromError, which answers (nil, true) for a nil error and leans on + // Status.Code tolerating a nil receiver to come back as codes.OK. + if err == nil { + return false + } + + s, ok := gstatus.FromError(err) + if !ok { + return false + } + + switch s.Code() { + case codes.InvalidArgument, + codes.PermissionDenied, + codes.NotFound, + codes.FailedPrecondition, + codes.Unimplemented: + return true + default: + return false + } +} + // WithBackOff execute function in backoff cycle. func WithBackOff(bf func() error) error { return backoff.RetryNotify(bf, CLIBackOffSettings, func(err error, duration time.Duration) { diff --git a/client/cmd/service.go b/client/cmd/service.go index 7410d60ea..2a558e6d5 100644 --- a/client/cmd/service.go +++ b/client/cmd/service.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "runtime" + "slices" "strings" "sync" @@ -25,6 +26,30 @@ var serviceCmd = &cobra.Command{ const defaultJSONSocket = "unix:///var/run/netbird-http.sock" +// forbiddenServiceEnvVars are the environment variables the service is never +// registered with, keyed in upper case since these are Windows names. Each one +// decides where the daemon resolves something it then uses with the privileges +// of the account it runs under — LocalSystem on Windows, root elsewhere: the +// executables it runs (PATH, PATHEXT, COMSPEC, SystemRoot, windir) or the +// directory it writes temporary files in (TEMP, TMP). The daemon needs none of +// them, and the utilities it shells out to are resolved by absolute path. +var forbiddenServiceEnvVars = map[string]struct{}{ + "PATH": {}, + "PATHEXT": {}, + "SYSTEMROOT": {}, + "WINDIR": {}, + "COMSPEC": {}, + "TEMP": {}, + "TMP": {}, +} + +// forbiddenServiceEnvPrefixes are the dynamic-loader families, refused whole +// rather than by name: LD_PRELOAD, DYLD_INSERT_LIBRARIES and their siblings all +// reach the loader of the process, the set differs per platform and libc, and +// new members arrive with new OS releases. Listing them one by one is a list +// that is wrong the moment it is written. +var forbiddenServiceEnvPrefixes = []string{"LD_", "DYLD_"} + var ( serviceName string serviceEnvVars []string @@ -127,8 +152,33 @@ func parseServiceEnvVars(envVars []string) (map[string]string, error) { return nil, fmt.Errorf("empty environment variable key in: %s", env) } + if isForbiddenServiceEnvVar(key) { + return nil, fmt.Errorf("environment variable %s cannot be set on the service: it decides where the service resolves the executables, libraries or temporary files it uses", key) + } + envMap[key] = value } return envMap, nil } + +// isForbiddenServiceEnvVar reports whether name is one the service must not be +// registered with. +// +// The names are matched case-insensitively only on Windows, where they are the +// same variable however they are spelled. Elsewhere the environment is +// case-sensitive, so Path and PATH are two different variables and only the +// exact spelling is the one the loader reads. +func isForbiddenServiceEnvVar(name string) bool { + if runtime.GOOS == "windows" { + name = strings.ToUpper(name) + } + + if _, forbidden := forbiddenServiceEnvVars[name]; forbidden { + return true + } + + return slices.ContainsFunc(forbiddenServiceEnvPrefixes, func(prefix string) bool { + return strings.HasPrefix(name, prefix) + }) +} diff --git a/client/cmd/service_params.go b/client/cmd/service_params.go index 750b22ae6..6e2dbec40 100644 --- a/client/cmd/service_params.go +++ b/client/cmd/service_params.go @@ -14,6 +14,7 @@ import ( "github.com/netbirdio/netbird/client/configs" "github.com/netbirdio/netbird/client/internal/daemonaddr" + "github.com/netbirdio/netbird/client/internal/elevate" "github.com/netbirdio/netbird/util" ) @@ -43,10 +44,33 @@ func serviceParamsPath() string { // loadServiceParams reads saved service parameters from disk. // Returns nil with no error if the file does not exist. +// +// The file is read by an elevated install and decides the arguments and the +// environment of the service it then registers, so it is used only when its +// ownership and permissions are the ones saveServiceParams leaves behind. That +// restricted ACL is applied when the file is written, which is not necessarily +// before it is first read, so this is checked rather than assumed. A file that +// fails the check is treated as absent, and the install proceeds with its +// defaults. func loadServiceParams() (*serviceParams, error) { path := serviceParamsPath() - data, err := os.ReadFile(path) + // Resolve links first so the checks apply to the file that is actually read. + // Since the check covers every directory above it as well, nobody who fails + // it can swap the file between here and the read below. + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil //nolint:nilnil + } + return nil, fmt.Errorf("resolve service params %s: %w", path, err) + } + + if err := elevate.CheckOnlyOwnerWritable(resolved); err != nil { + return nil, fmt.Errorf("refusing to read service params from %s: %w", resolved, err) + } + + data, err := os.ReadFile(resolved) if err != nil { if os.IsNotExist(err) { return nil, nil //nolint:nilnil @@ -182,10 +206,16 @@ func applyServiceParams(cmd *cobra.Command, params *serviceParams) { // If --service-env was explicitly set to empty, all saved env vars are cleared. // If --service-env was not set, saved env vars are used entirely. func applyServiceEnvParams(cmd *cobra.Command, params *serviceParams) { + // A forbidden name explicitly passed on the command line is an error the + // operator is told about, but one restored from a file written by an older + // version is dropped: an install that refuses to run would leave the host + // without a daemon over a variable nobody is asking for any more. + saved := dropForbiddenServiceEnvVars(cmd, params.ServiceEnvVars) + if !cmd.Flags().Changed("service-env") { - if len(params.ServiceEnvVars) > 0 { + if len(saved) > 0 { // No explicit env vars: rebuild serviceEnvVars from saved params. - serviceEnvVars = envMapToSlice(params.ServiceEnvVars) + serviceEnvVars = envMapToSlice(saved) } return } @@ -204,13 +234,13 @@ func applyServiceEnvParams(cmd *cobra.Command, params *serviceParams) { return } - if len(params.ServiceEnvVars) == 0 { + if len(saved) == 0 { return } // Merge saved values underneath explicit ones. - merged := make(map[string]string, len(params.ServiceEnvVars)+len(explicit)) - maps.Copy(merged, params.ServiceEnvVars) + merged := make(map[string]string, len(saved)+len(explicit)) + maps.Copy(merged, saved) maps.Copy(merged, explicit) // explicit wins on conflict serviceEnvVars = envMapToSlice(merged) } @@ -233,6 +263,20 @@ var resetParamsCmd = &cobra.Command{ }, } +// dropForbiddenServiceEnvVars returns the saved entries that may still be +// registered on the service, reporting every one it leaves behind. +func dropForbiddenServiceEnvVars(cmd *cobra.Command, saved map[string]string) map[string]string { + kept := make(map[string]string, len(saved)) + for key, value := range saved { + if isForbiddenServiceEnvVar(key) { + cmd.PrintErrf("Warning: ignoring saved service environment variable %s: it decides where the service resolves the executables, libraries or temporary files it uses\n", key) + continue + } + kept[key] = value + } + return kept +} + // envMapToSlice converts a map of env vars to a KEY=VALUE slice. func envMapToSlice(m map[string]string) []string { s := make([]string, 0, len(m)) diff --git a/client/cmd/service_params_test.go b/client/cmd/service_params_test.go index 94f98a0ce..1f83374cb 100644 --- a/client/cmd/service_params_test.go +++ b/client/cmd/service_params_test.go @@ -9,6 +9,7 @@ import ( "go/token" "os" "path/filepath" + "runtime" "strings" "testing" @@ -353,6 +354,59 @@ func TestApplyServiceEnvParams_NotChanged(t *testing.T) { assert.Equal(t, map[string]string{"FROM_SAVED": "val"}, result) } +func TestParseServiceEnvVars_RejectsForbiddenNames(t *testing.T) { + for _, env := range []string{"PATH=C:\\somewhere", "LD_PRELOAD=/tmp/lib.so", "DYLD_FALLBACK_LIBRARY_PATH=/tmp"} { + _, err := parseServiceEnvVars([]string{"KEEP=me", env}) + require.Errorf(t, err, "%s selects what the service resolves and must be refused", env) + } +} + +func TestIsForbiddenServiceEnvVar(t *testing.T) { + // The loader families are matched by prefix, so a name nobody has heard of + // yet is refused too. + for _, name := range []string{ + "PATH", "PATHEXT", "COMSPEC", "SYSTEMROOT", "WINDIR", "TEMP", "TMP", + "LD_PRELOAD", "LD_AUDIT", "DYLD_INSERT_LIBRARIES", "DYLD_FALLBACK_FRAMEWORK_PATH", + } { + assert.Truef(t, isForbiddenServiceEnvVar(name), "%s must be refused", name) + } + + // The prefix must not swallow names that merely start with the same letters. + for _, name := range []string{"NB_LOG_LEVEL", "NB_WG_DEBUG", "HTTPS_PROXY", "LDAP_URL", "DYLDX"} { + assert.Falsef(t, isForbiddenServiceEnvVar(name), "%s has no reason to be refused", name) + } + + // On Windows a variable is the same one however it is spelled; elsewhere + // Path and PATH are two variables and only the exact one is read. + if runtime.GOOS == "windows" { + assert.True(t, isForbiddenServiceEnvVar("Path")) + assert.True(t, isForbiddenServiceEnvVar("ld_preload")) + } else { + assert.False(t, isForbiddenServiceEnvVar("Path")) + assert.False(t, isForbiddenServiceEnvVar("ld_preload")) + } +} + +func TestApplyServiceEnvParams_DropsForbiddenSavedNames(t *testing.T) { + origServiceEnvVars := serviceEnvVars + t.Cleanup(func() { serviceEnvVars = origServiceEnvVars }) + + serviceEnvVars = nil + + cmd := &cobra.Command{} + cmd.Flags().StringSlice("service-env", nil, "") + + saved := &serviceParams{ + ServiceEnvVars: map[string]string{"PATH": "C:\\attacker", "NB_LOG_FORMAT": "json"}, + } + + applyServiceEnvParams(cmd, saved) + + result, err := parseServiceEnvVars(serviceEnvVars) + require.NoError(t, err, "a saved PATH must be dropped rather than fail the install") + assert.Equal(t, map[string]string{"NB_LOG_FORMAT": "json"}, result) +} + func TestApplyServiceEnvParams_ExplicitEmptyClears(t *testing.T) { origServiceEnvVars := serviceEnvVars t.Cleanup(func() { serviceEnvVars = origServiceEnvVars }) diff --git a/client/cmd/service_params_trust_test.go b/client/cmd/service_params_trust_test.go new file mode 100644 index 000000000..1cf564445 --- /dev/null +++ b/client/cmd/service_params_trust_test.go @@ -0,0 +1,57 @@ +//go:build !windows && !ios && !android + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/configs" +) + +// The Windows equivalent of this is the ACL check in +// elevate.CheckOnlyOwnerWritable, covered by that package's own tests; here the +// point is that loadServiceParams asks the question at all. +func TestLoadServiceParams_RefusesWorldWritableFile(t *testing.T) { + tmpDir := t.TempDir() + + original := configs.StateDir + t.Cleanup(func() { configs.StateDir = original }) + configs.StateDir = tmpDir + + path := filepath.Join(tmpDir, serviceParamsFile) + require.NoError(t, os.WriteFile(path, []byte(`{"log_level":"debug"}`), 0o666)) + // WriteFile is subject to the umask, so set the bits that matter explicitly. + require.NoError(t, os.Chmod(path, 0o666)) + + params, err := loadServiceParams() + require.Error(t, err, "a service.json anyone can rewrite must not be trusted") + assert.Nil(t, params) + + require.NoError(t, os.Chmod(path, 0o600)) + params, err = loadServiceParams() + require.NoError(t, err) + require.NotNil(t, params) + assert.Equal(t, "debug", params.LogLevel) +} + +func TestLoadServiceParams_RefusesWorldWritableDirectory(t *testing.T) { + tmpDir := t.TempDir() + stateDir := filepath.Join(tmpDir, "state") + require.NoError(t, os.Mkdir(stateDir, 0o777)) + require.NoError(t, os.Chmod(stateDir, 0o777)) + + original := configs.StateDir + t.Cleanup(func() { configs.StateDir = original }) + configs.StateDir = stateDir + + require.NoError(t, os.WriteFile(filepath.Join(stateDir, serviceParamsFile), []byte(`{}`), 0o600)) + + params, err := loadServiceParams() + require.Error(t, err, "a service.json in a directory anyone can replace entries in must not be trusted") + assert.Nil(t, params) +} diff --git a/client/cmd/testutil_test.go b/client/cmd/testutil_test.go index 328a15454..46bf31837 100644 --- a/client/cmd/testutil_test.go +++ b/client/cmd/testutil_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" - "go.uber.org/mock/gomock" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" + "go.uber.org/mock/gomock" "google.golang.org/grpc" "github.com/netbirdio/netbird/management/server/integrations/integrated_validator/validator" @@ -28,7 +28,6 @@ import ( mgmt "github.com/netbirdio/netbird/management/server" "github.com/netbirdio/netbird/management/server/activity" "github.com/netbirdio/netbird/management/server/groups" - "github.com/netbirdio/netbird/management/server/integrations/port_forwarding" "github.com/netbirdio/netbird/management/server/permissions" "github.com/netbirdio/netbird/management/server/settings" "github.com/netbirdio/netbird/management/server/store" @@ -124,9 +123,9 @@ func startManagement(t *testing.T, config *config.Config, testFile string) (*grp updateManager := update_channel.NewPeersUpdateManager(metrics) requestBuffer := mgmt.NewAccountRequestBuffer(ctx, store) - networkMapController := controller.NewController(ctx, store, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.cloud", port_forwarding.NewControllerMock(), manager.NewEphemeralManager(store, peersmanager), config, nil) + networkMapController := controller.NewController(ctx, store, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.cloud", manager.NewEphemeralManager(store, peersmanager), config, nil) - accountManager, err := mgmt.BuildManager(ctx, config, store, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManagerMock, false, cacheStore) + accountManager, err := mgmt.BuildManager(ctx, config, store, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, settingsMockManager, permissionsManagerMock, false, cacheStore) if err != nil { t.Fatal(err) } diff --git a/client/cmd/up.go b/client/cmd/up.go index 6013f4b2f..8bf28418f 100644 --- a/client/cmd/up.go +++ b/client/cmd/up.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "net" "net/netip" "runtime" "strings" @@ -26,16 +25,11 @@ import ( "github.com/netbirdio/netbird/client/proto" "github.com/netbirdio/netbird/client/server" "github.com/netbirdio/netbird/client/system" + "github.com/netbirdio/netbird/shared/externalipmap" "github.com/netbirdio/netbird/shared/management/domain" "github.com/netbirdio/netbird/util" ) -const ( - invalidInputType int = iota - ipInputType - interfaceInputType -) - const ( dnsLabelsFlag = "extra-dns-labels" @@ -106,7 +100,7 @@ func upFunc(cmd *cobra.Command, args []string) error { return fmt.Errorf("failed initializing log %v", err) } - err = validateNATExternalIPs(natExternalIPs) + err = externalipmap.Validate(natExternalIPs) if err != nil { return err } @@ -357,9 +351,17 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command, pm *profilemanager // set the new config req := setupSetConfigReq(customDNSAddressConverted, cmd, activeProf.ID.String(), username.Username) if _, err := client.SetConfig(ctx, req); err != nil { - if st, ok := gstatus.FromError(err); ok && st.Code() == codes.Unavailable { - log.Warnf("setConfig method is not available in the daemon: %s", st.Message()) - } else { + switch reason, refused := refusedSettingsUpdate(err); { + case refused: + // Failing here is the point: carrying on would connect while + // silently dropping the settings the caller asked for, since + // nothing further down the line applies them. + return fmt.Errorf("the daemon refused the settings update: %s", reason) + case gstatus.Code(err) == codes.Unavailable: + // The daemon cannot serve the method at all, which is what this + // code means; an older daemon without it lands here. + log.Warnf("the daemon did not apply the settings update: %s", gstatus.Convert(err).Message()) + default: return daemonCallError("call service setConfig method", err) } } @@ -400,10 +402,7 @@ func doDaemonUp(ctx context.Context, cmd *cobra.Command, client proto.DaemonServ err = WithBackOff(func() error { var backOffErr error loginResp, backOffErr = client.Login(ctx, loginRequest) - if s, ok := gstatus.FromError(backOffErr); ok && (s.Code() == codes.InvalidArgument || - s.Code() == codes.PermissionDenied || - s.Code() == codes.NotFound || - s.Code() == codes.Unimplemented) { + if terminalLoginError(backOffErr) { loginErr = backOffErr return nil } @@ -482,6 +481,22 @@ func setVNCSetConfigFields(req *proto.SetConfigRequest, cmd *cobra.Command) { } } +// refusedSettingsUpdate reports whether err is the daemon refusing the settings +// a request carried — the update-settings kill switch, or a field an MDM policy +// manages — and returns the reason it gave. +// +// The distinction that matters is against codes.Unavailable, which means the +// daemon cannot serve the call: that one is worth a warning, because an older +// daemon without the method lands there and the rest of `netbird up` still +// works. A refusal is not, because the settings would be silently dropped. +func refusedSettingsUpdate(err error) (string, bool) { + st, ok := gstatus.FromError(err) + if !ok || st.Code() != codes.FailedPrecondition { + return "", false + } + return st.Message(), true +} + func setupSetConfigReq(customDNSAddressConverted []byte, cmd *cobra.Command, profileName, username string) *proto.SetConfigRequest { var req proto.SetConfigRequest req.ProfileName = profileName @@ -847,36 +862,6 @@ func setupLoginRequest(providedSetupKey string, customDNSAddressConverted []byte return &loginRequest, nil } -func validateNATExternalIPs(list []string) error { - for _, element := range list { - if element == "" { - return fmt.Errorf("empty string is not a valid input for %s", externalIPMapFlag) - } - - subElements := strings.Split(element, "/") - if len(subElements) > 2 { - return fmt.Errorf("%s is not a valid input for %s. it should be formatted as \"String\" or \"String/String\"", element, externalIPMapFlag) - } - - if len(subElements) == 1 && !isValidIP(subElements[0]) { - return fmt.Errorf("%s is not a valid input for %s. it should be formatted as \"IP\" or \"IP/IP\", or \"IP/Interface Name\"", element, externalIPMapFlag) - } - - last := 0 - for _, singleElement := range subElements { - inputType, err := validateElement(singleElement) - if err != nil { - return fmt.Errorf("%s is not a valid input for %s. it should be an IP string or a network name", singleElement, externalIPMapFlag) - } - if last == interfaceInputType && inputType == interfaceInputType { - return fmt.Errorf("%s is not a valid input for %s. it should not contain two interface names", element, externalIPMapFlag) - } - last = inputType - } - } - return nil -} - func parseInterfaceName(name string) error { if runtime.GOOS != "darwin" { return nil @@ -889,39 +874,6 @@ func parseInterfaceName(name string) error { return fmt.Errorf("invalid interface name %s. Please use the prefix utun followed by a number on MacOS. e.g., utun1 or utun199", name) } -func validateElement(element string) (int, error) { - if isValidIP(element) { - return ipInputType, nil - } - validIface, err := isValidInterface(element) - if err != nil { - return invalidInputType, fmt.Errorf("unable to validate the network interface name, error: %s", err) - } - - if validIface { - return interfaceInputType, nil - } - - return interfaceInputType, fmt.Errorf("invalid IP or network interface name not found") -} - -func isValidIP(ip string) bool { - return net.ParseIP(ip) != nil -} - -func isValidInterface(name string) (bool, error) { - netInterfaces, err := net.Interfaces() - if err != nil { - return false, err - } - for _, iface := range netInterfaces { - if iface.Name == name { - return true, nil - } - } - return false, nil -} - func parseCustomDNSAddress(modified bool) ([]byte, error) { var parsed []byte if modified { diff --git a/client/cmd/up_setconfig_refusal_test.go b/client/cmd/up_setconfig_refusal_test.go new file mode 100644 index 000000000..fdf580102 --- /dev/null +++ b/client/cmd/up_setconfig_refusal_test.go @@ -0,0 +1,85 @@ +package cmd + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + gstatus "google.golang.org/grpc/status" +) + +// A refused settings update has to fail `netbird up`, or a caller that asked +// for a setting the daemon will not apply connects as if it had been applied. +// The daemon being unable to serve the call is the case that stays a warning. +func TestRefusedSettingsUpdate(t *testing.T) { + tests := []struct { + name string + err error + wantRefused bool + }{ + { + name: "the kill switch refused the change", + err: gstatus.Errorf(codes.FailedPrecondition, "update settings are disabled, you cannot use this feature without update settings enabled"), + wantRefused: true, + }, + { + name: "an MDM policy manages the field", + err: gstatus.Errorf(codes.FailedPrecondition, "fields managed by MDM policy: managementURL"), + wantRefused: true, + }, + { + name: "the daemon cannot serve the call", + err: gstatus.Errorf(codes.Unavailable, "connection refused"), + wantRefused: false, + }, + { + name: "any other RPC failure", + err: gstatus.Errorf(codes.Internal, "boom"), + wantRefused: false, + }, + { + name: "not a status error at all", + err: errors.New("boom"), + wantRefused: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reason, refused := refusedSettingsUpdate(tt.err) + require.Equal(t, tt.wantRefused, refused) + if tt.wantRefused { + require.Equal(t, gstatus.Convert(tt.err).Message(), reason, "the daemon's reason must reach the caller") + } + }) + } +} + +// Both `netbird up` and `netbird login` drive Login through the backoff cycle, +// and a final answer has to stop it: retrying a refusal only replaces the +// daemon's reason with "login backoff cycle failed" thirty seconds later. +func TestTerminalLoginError(t *testing.T) { + tests := []struct { + name string + err error + wantTerminal bool + }{ + {name: "settings refused by the kill switch", err: gstatus.Errorf(codes.FailedPrecondition, "update settings are disabled"), wantTerminal: true}, + {name: "field managed by MDM", err: gstatus.Errorf(codes.FailedPrecondition, "fields managed by MDM policy: managementURL"), wantTerminal: true}, + {name: "caller not allowed", err: gstatus.Errorf(codes.PermissionDenied, "nope"), wantTerminal: true}, + {name: "malformed request", err: gstatus.Errorf(codes.InvalidArgument, "nope"), wantTerminal: true}, + {name: "profile not found", err: gstatus.Errorf(codes.NotFound, "nope"), wantTerminal: true}, + {name: "method missing on an older daemon", err: gstatus.Errorf(codes.Unimplemented, "nope"), wantTerminal: true}, + {name: "daemon unreachable, worth retrying", err: gstatus.Errorf(codes.Unavailable, "connection refused"), wantTerminal: false}, + {name: "transient internal failure", err: gstatus.Errorf(codes.Internal, "boom"), wantTerminal: false}, + {name: "not a status error", err: errors.New("boom"), wantTerminal: false}, + {name: "no error at all, the login succeeded", err: nil, wantTerminal: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.wantTerminal, terminalLoginError(tt.err)) + }) + } +} diff --git a/client/collect-licenses.sh b/client/collect-licenses.sh deleted file mode 100644 index 7dfabada9..000000000 --- a/client/collect-licenses.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/sh -set -eu - -if [ "$#" -lt 2 ]; then - printf '%s\n' "usage: $0 OUTPUT_DIRECTORY GOARCH..." >&2 - exit 2 -fi - -repo_root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -output_name=$(basename "$1") -if [ -z "$output_name" ] || [ "$output_name" = "." ] || - [ "$output_name" = ".." ] || [ "$output_name" = "/" ]; then - printf '%s\n' "OUTPUT_DIRECTORY must name a directory" >&2 - exit 2 -fi -output_parent=$(CDPATH= cd -- "$(dirname "$1")" && pwd) -output="$output_parent/$output_name" -shift -modules=$(mktemp "${TMPDIR:-/tmp}/netbird-client-licenses.modules.XXXXXX") -sorted_modules=$(mktemp "${TMPDIR:-/tmp}/netbird-client-licenses.sorted.XXXXXX") -trap 'rm -f "$modules" "$sorted_modules"' EXIT HUP INT TERM - -if [ -e "$output" ] || [ -L "$output" ]; then - printf 'output directory already exists: %s\n' "$output" >&2 - exit 1 -fi -mkdir "$output" -mkdir "$output/third_party" - -cp "$repo_root/LICENSE" "$output/BSD-3-Clause.txt" - -cd "$repo_root" -for arch in "$@"; do - GOOS=${GOOS:-linux} GOARCH="$arch" CGO_ENABLED=${CGO_ENABLED:-0} \ - go list -deps -f '{{with .Module}}{{if .Replace}}{{.Replace.Path}}{{"\t"}}{{.Replace.Version}}{{"\t"}}{{.Replace.Dir}}{{else}}{{.Path}}{{"\t"}}{{.Version}}{{"\t"}}{{.Dir}}{{end}}{{end}}' -tags load_wgnt_from_rsrc ./client >>"$modules" -done -LC_ALL=C sort -u "$modules" >"$sorted_modules" - -goroot=$(go env GOROOT) -for term in LICENSE PATENTS; do - if [ ! -f "$goroot/$term" ]; then - printf 'missing Go standard-library term: %s\n' "$goroot/$term" >&2 - exit 1 - fi - cp "$goroot/$term" "$output/Go-$term" -done - -while IFS=' ' read -r module version module_dir; do - [ -n "$module" ] || continue - [ "$module" = "github.com/netbirdio/netbird" ] && continue - - if [ -z "$version" ] || [ ! -d "$module_dir" ]; then - printf 'cannot collect terms for module %s at version %s\n' "$module" "$version" >&2 - exit 1 - fi - - destination="$output/third_party/$module/$version" - mkdir -p "$destination" - printf 'module: %s\nversion: %s\n' "$module" "$version" >"$destination/MODULE" - - found=false - for term in \ - "$module_dir"/LICENSE* "$module_dir"/License* "$module_dir"/license* \ - "$module_dir"/LICENCE* "$module_dir"/Licence* "$module_dir"/licence* \ - "$module_dir"/COPYING* "$module_dir"/Copying* "$module_dir"/copying* \ - "$module_dir"/NOTICE* "$module_dir"/Notice* "$module_dir"/notice* \ - "$module_dir"/PATENTS* "$module_dir"/Patents* "$module_dir"/patents*; do - [ -f "$term" ] || continue - cp "$term" "$destination/" - found=true - done - - if [ "$found" = false ]; then - printf 'no root license terms found for module %s at %s\n' "$module" "$module_dir" >&2 - exit 1 - fi -done <"$sorted_modules" diff --git a/client/embed/embed.go b/client/embed/embed.go index 5a3d540ec..208e71817 100644 --- a/client/embed/embed.go +++ b/client/embed/embed.go @@ -54,6 +54,9 @@ type Client struct { jwtToken string connect *internal.ConnectClient recorder *peer.Status + // inMemoryState is set when no StatePath was given in netstack mode, + // where the client changes nothing on the host that needs restoring. + inMemoryState bool } // Options configures a new Client. @@ -78,7 +81,8 @@ type Options struct { NoUserspace bool // ConfigPath is the path to the netbird config file. If empty, the config will be stored in memory and not persisted. ConfigPath string - // StatePath is the path to the netbird state file + // StatePath is the path to the netbird state file. When empty in netstack + // mode, state is kept in memory; with NoUserspace the default path is used. StatePath string // DisableClientRoutes disables the client routes DisableClientRoutes bool @@ -114,6 +118,9 @@ type Options struct { MTU *uint16 // DNSLabels defines additional DNS labels configured in the peer. DNSLabels []string + // NATExternalIPs maps external IPs to internal IPs or interfaces for ICE + // candidates, same format as the agent's --external-ip-map flag. + NATExternalIPs []string // Performance configures the tunnel's buffer pool cap and batch size. Performance Performance } @@ -193,7 +200,6 @@ func New(opts Options) (*Client, error) { } if opts.StatePath != "" { - // TODO: Disable state if path not provided if err := os.Setenv("NB_DNS_STATE_FILE", opts.StatePath); err != nil { return nil, fmt.Errorf("setenv: %w", err) } @@ -221,6 +227,7 @@ func New(opts Options) (*Client, error) { WireguardPort: opts.WireguardPort, MTU: opts.MTU, DNSLabels: parsedLabels, + NATExternalIPs: opts.NATExternalIPs, } if opts.ConfigPath != "" { config, err = profilemanager.UpdateOrCreateConfig(input) @@ -261,6 +268,9 @@ func New(opts Options) (*Client, error) { jwtToken: opts.JWTToken, config: config, recorder: peer.NewRecorder(config.ManagementURL.String()), + // Without a path the default would be the installed agent's state + // file, which this client must neither read nor overwrite. + inMemoryState: opts.StatePath == "" && !opts.NoUserspace, }, nil } @@ -292,7 +302,11 @@ func (c *Client) Start(startCtx context.Context) error { if err, _ := authClient.Login(ctx, c.setupKey, c.jwtToken); err != nil { return fmt.Errorf("login: %w", err) } - client := internal.NewConnectClient(ctx, c.config, c.recorder) + var connectOpts []internal.ConnectClientOption + if c.inMemoryState { + connectOpts = append(connectOpts, internal.WithInMemoryState()) + } + client := internal.NewConnectClient(ctx, c.config, c.recorder, connectOpts...) client.SetSyncResponsePersistence(true) // either startup error (permanent backoff err) or nil err (successful engine up) diff --git a/client/embed/embed_test.go b/client/embed/embed_test.go index 4ff5c9978..a818af055 100644 --- a/client/embed/embed_test.go +++ b/client/embed/embed_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "go.uber.org/mock/gomock" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "google.golang.org/grpc" "github.com/netbirdio/netbird/management/internals/controllers/network_map/controller" @@ -21,7 +21,6 @@ import ( nbcache "github.com/netbirdio/netbird/management/server/cache" "github.com/netbirdio/netbird/management/server/groups" "github.com/netbirdio/netbird/management/server/integrations/integrated_validator/validator" - "github.com/netbirdio/netbird/management/server/integrations/port_forwarding" "github.com/netbirdio/netbird/management/server/job" "github.com/netbirdio/netbird/management/server/permissions" "github.com/netbirdio/netbird/management/server/settings" @@ -146,8 +145,8 @@ func startManagement(t *testing.T, signalAddr string) string { updateManager := update_channel.NewPeersUpdateManager(metrics) requestBuffer := mgmt.NewAccountRequestBuffer(context.Background(), testStore) - networkMapController := controller.NewController(context.Background(), testStore, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.selfhosted", port_forwarding.NewControllerMock(), manager.NewEphemeralManager(testStore, peersManager), cfg, nil) - accountManager, err := mgmt.BuildManager(context.Background(), cfg, testStore, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false, cacheStore) + networkMapController := controller.NewController(context.Background(), testStore, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.selfhosted", manager.NewEphemeralManager(testStore, peersManager), cfg, nil) + accountManager, err := mgmt.BuildManager(context.Background(), cfg, testStore, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, settingsMockManager, permissionsManager, false, cacheStore) require.NoError(t, err) secretsManager, err := nbgrpc.NewTimeBasedAuthSecretsManager(updateManager, cfg.TURNConfig, cfg.Relay, settingsMockManager, groupsManager) diff --git a/client/firewall/iptables/dnat_linux.go b/client/firewall/iptables/dnat_linux.go index eca8386c0..f118c9dfe 100644 --- a/client/firewall/iptables/dnat_linux.go +++ b/client/firewall/iptables/dnat_linux.go @@ -8,177 +8,11 @@ import ( "strconv" "strings" - "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" - nberrors "github.com/netbirdio/netbird/client/errors" firewall "github.com/netbirdio/netbird/client/firewall/manager" ) -func (r *family) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - ruleID := rule.ID() - if _, exists := r.rules[ruleID+dnatSuffix]; exists { - return rule, nil - } - - toDestination := rule.TranslatedAddress.String() - switch { - case len(rule.TranslatedPort.Values) == 0: - // no translated port, use original port - case len(rule.TranslatedPort.Values) == 1: - toDestination += fmt.Sprintf(":%d", rule.TranslatedPort.Values[0]) - case rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2: - // need the "/originalport" suffix to avoid dnat port randomization - toDestination += fmt.Sprintf(":%d-%d/%d", rule.TranslatedPort.Values[0], rule.TranslatedPort.Values[1], rule.DestinationPort.Values[0]) - default: - return nil, fmt.Errorf("invalid translated port: %v", rule.TranslatedPort) - } - - proto := strings.ToLower(string(rule.Protocol)) - - rules := make(map[firewall.RuleID]ruleInfo, 3) - - // DNAT rule - dnatRule := []string{ - "!", "-i", r.wgIface.Name(), - "-p", proto, - "-j", "DNAT", - "--to-destination", toDestination, - } - dnatRule = append(dnatRule, applyPort("--dport", &rule.DestinationPort)...) - rules[ruleID+dnatSuffix] = ruleInfo{ - table: tableNat, - chain: chainRTRdr, - rule: dnatRule, - } - - // SNAT rule - snatRule := []string{ - "-o", r.wgIface.Name(), - "-p", proto, - "-d", rule.TranslatedAddress.String(), - "-j", "MASQUERADE", - } - snatRule = append(snatRule, applyPort("--dport", &rule.TranslatedPort)...) - rules[ruleID+snatSuffix] = ruleInfo{ - table: tableNat, - chain: chainRTNAT, - rule: snatRule, - } - - // Forward filtering rule, if fwd policy is DROP - forwardRule := []string{ - "-o", r.wgIface.Name(), - "-p", proto, - "-d", rule.TranslatedAddress.String(), - "-j", "ACCEPT", - } - forwardRule = append(forwardRule, applyPort("--dport", &rule.TranslatedPort)...) - rules[ruleID+fwdSuffix] = ruleInfo{ - table: tableFilter, - chain: chainRTFwdOut, - rule: forwardRule, - } - - for key, ruleInfo := range rules { - if err := r.iptablesClient.Append(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { - r.cleanupFailedDNATAdd(rules) - return nil, fmt.Errorf("add rule %s: %w", key, err) - } - r.rules[key] = ruleInfo.rule - } - - if err := r.ipFwdState.RequestForwarding(r.v6); err != nil { - r.cleanupFailedDNATAdd(rules) - return nil, fmt.Errorf("enable forwarding: %w", err) - } - - r.updateState() - return rule, nil -} - -// cleanupFailedDNATAdd removes the bookkeeping written by a partially applied -// AddDNATRule before rolling back the kernel rules, so no entries remain that -// never got a forwarding refcount. rollbackRules re-adds entries it failed to -// remove from the kernel. -func (r *family) cleanupFailedDNATAdd(rules map[firewall.RuleID]ruleInfo) { - for key := range rules { - delete(r.rules, key) - } - if err := r.rollbackRules(rules); err != nil { - log.Errorf("rollback failed: %v", err) - } -} - -func (r *family) rollbackRules(rules map[firewall.RuleID]ruleInfo) error { - var merr *multierror.Error - for key, ruleInfo := range rules { - if err := r.iptablesClient.DeleteIfExists(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("rollback rule %s: %w", key, err)) - // On rollback error, add to rules map for next cleanup - r.rules[key] = ruleInfo.rule - } - } - if merr != nil { - r.updateState() - } - return nberrors.FormatErrorOrNil(merr) -} - -func (r *family) DeleteDNATRule(rule firewall.Rule) error { - ruleID := rule.ID() - - _, hadDNAT := r.rules[ruleID+dnatSuffix] - _, hadSNAT := r.rules[ruleID+snatSuffix] - _, hadFWD := r.rules[ruleID+fwdSuffix] - if !hadDNAT && !hadSNAT && !hadFWD { - return nil - } - - var merr *multierror.Error - if dnatRule, exists := r.rules[ruleID+dnatSuffix]; exists { - if err := r.iptablesClient.Delete(tableNat, chainRTRdr, dnatRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete DNAT rule: %w", err)) - } else { - delete(r.rules, ruleID+dnatSuffix) - } - } - - if snatRule, exists := r.rules[ruleID+snatSuffix]; exists { - if err := r.iptablesClient.Delete(tableNat, chainRTNAT, snatRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete SNAT rule: %w", err)) - } else { - delete(r.rules, ruleID+snatSuffix) - } - } - - if fwdRule, exists := r.rules[ruleID+fwdSuffix]; exists { - if err := r.iptablesClient.Delete(tableFilter, chainRTFwdOut, fwdRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete forward rule: %w", err)) - } else { - delete(r.rules, ruleID+fwdSuffix) - } - } - - // Release the refcount only once all rules are gone from the kernel. On - // partial failure the failed entries stay in r.rules so a retry can remove - // them and release then. - if merr == nil { - r.releaseForwarding() - } - - r.updateState() - - return nberrors.FormatErrorOrNil(merr) -} - -// releaseForwarding drops one IP forwarding reference, logging any error. -func (r *family) releaseForwarding() { - if err := r.ipFwdState.ReleaseForwarding(r.v6); err != nil { - log.Errorf("release IP forwarding: %v", err) - } -} - func (r *family) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) diff --git a/client/firewall/iptables/dnat_refcount_linux_test.go b/client/firewall/iptables/dnat_refcount_linux_test.go deleted file mode 100644 index 40ebc6cc3..000000000 --- a/client/firewall/iptables/dnat_refcount_linux_test.go +++ /dev/null @@ -1,240 +0,0 @@ -//go:build privileged - -package iptables - -import ( - "net/netip" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - fw "github.com/netbirdio/netbird/client/firewall/manager" - "github.com/netbirdio/netbird/client/iface" - "github.com/netbirdio/netbird/client/iface/wgaddr" -) - -func iptRefcountIfaceV4() *iFaceMock { - return &iFaceMock{ - NameFunc: func() string { return "wt-refcount" }, - AddressFunc: func() wgaddr.Address { - return wgaddr.Address{ - IP: netip.MustParseAddr("10.20.0.1"), - Network: netip.MustParsePrefix("10.20.0.0/24"), - } - }, - } -} - -func iptRefcountIfaceDual() *iFaceMock { - return &iFaceMock{ - NameFunc: func() string { return "wt-refcount" }, - AddressFunc: func() wgaddr.Address { - return wgaddr.Address{ - IP: netip.MustParseAddr("10.20.0.1"), - Network: netip.MustParsePrefix("10.20.0.0/24"), - IPv6: netip.MustParseAddr("fd00::1"), - IPv6Net: netip.MustParsePrefix("fd00::/64"), - } - }, - } -} - -func newIptRefcountManager(t *testing.T, dual bool) *Manager { - t.Helper() - var ifMock *iFaceMock - if dual { - ifMock = iptRefcountIfaceDual() - } else { - ifMock = iptRefcountIfaceV4() - } - m, err := Create(ifMock, iface.DefaultMTU) - require.NoError(t, err, "create manager") - require.NoError(t, m.Init(nil), "init manager") - t.Cleanup(func() { - require.NoError(t, m.Close(nil), "close manager") - }) - return m -} - -func iptDnatV4(port uint16) fw.ForwardRule { - return fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{port}}, - TranslatedAddress: netip.MustParseAddr("10.20.0.2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - } -} - -func iptDnatV6(port uint16) fw.ForwardRule { - return fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{port}}, - TranslatedAddress: netip.MustParseAddr("fd00::2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - } -} - -// TestIptablesRouting_RepeatedEnableSingleReference verifies that EnableRouting -// (called on every network-map update) holds at most one reference per family -// and a single DisableRouting drops both back to zero. -func TestIptablesRouting_RepeatedEnableSingleReference(t *testing.T) { - m := newIptRefcountManager(t, true) - state := m.family4.ipFwdState - - require.NoError(t, m.EnableRouting(), "first enable") - require.NoError(t, m.EnableRouting(), "second enable") - require.NoError(t, m.EnableRouting(), "third enable") - v4, v6 := state.Counts() - assert.Equal(t, 1, v4, "repeated enable holds a single v4 reference") - assert.Equal(t, 1, v6, "repeated enable holds a single v6 reference") - - require.NoError(t, m.DisableRouting(), "disable") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "single disable releases the v4 reference") - assert.Equal(t, 0, v6, "single disable releases the v6 reference") -} - -// TestIptablesRouting_DisableKeepsDNATReference verifies that an unpaired -// DisableRouting does not release references held by active DNAT rules. -func TestIptablesRouting_DisableKeepsDNATReference(t *testing.T) { - m := newIptRefcountManager(t, true) - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(iptDnatV6(9095)) - require.NoError(t, err, "add v6 dnat") - - require.NoError(t, m.DisableRouting(), "unpaired disable") - _, v6 := state.Counts() - assert.Equal(t, 1, v6, "DNAT-held reference survives unpaired DisableRouting") - - require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat") - _, v6 = state.Counts() - assert.Equal(t, 0, v6, "delete releases the DNAT reference") -} - -// TestIptablesDNAT_RefcountBalancedV4 covers a Balanced Add/Delete pair on v4. -func TestIptablesDNAT_RefcountBalancedV4(t *testing.T) { - m := newIptRefcountManager(t, false) - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(iptDnatV4(7081)) - require.NoError(t, err, "add v4 dnat 1") - v4, v6 := state.Counts() - assert.Equal(t, 1, v4, "v4 refcount after first add") - assert.Equal(t, 0, v6, "v6 refcount unchanged") - - r2, err := m.AddDNATRule(iptDnatV4(7082)) - require.NoError(t, err, "add v4 dnat 2") - v4, v6 = state.Counts() - assert.Equal(t, 2, v4, "v4 refcount after second add") - assert.Equal(t, 0, v6, "v6 refcount unchanged") - - require.NoError(t, m.DeleteDNATRule(r1)) - v4, v6 = state.Counts() - assert.Equal(t, 1, v4, "v4 refcount after first delete") - assert.Equal(t, 0, v6, "v6 refcount unchanged") - - require.NoError(t, m.DeleteDNATRule(r2)) - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "v4 refcount after second delete") - assert.Equal(t, 0, v6, "v6 refcount unchanged") -} - -// TestIptablesDNAT_RefcountBalancedV6 checks the v6 path increments v6 only and -// decrements back to zero. -func TestIptablesDNAT_RefcountBalancedV6(t *testing.T) { - m := newIptRefcountManager(t, true) - require.NotNil(t, m.family6, "v6 family") - require.Same(t, m.family4.ipFwdState, m.family6.ipFwdState, "shared state") - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(iptDnatV6(9081)) - require.NoError(t, err, "add v6 dnat 1") - v4, v6 := state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 1, v6, "v6 refcount after first add") - - r2, err := m.AddDNATRule(iptDnatV6(9082)) - require.NoError(t, err, "add v6 dnat 2") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "v4 refcount unchanged") - assert.Equal(t, 2, v6, "v6 refcount after second add") - - require.NoError(t, m.DeleteDNATRule(r1)) - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "v4 refcount unchanged") - assert.Equal(t, 1, v6, "v6 refcount after first delete") - - require.NoError(t, m.DeleteDNATRule(r2)) - v4, v6 = state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 0, v6, "v6 refcount after second delete") -} - -// TestIptablesDNAT_DuplicateAddNoLeak verifies the duplicate-rule path returns -// without bumping the refcount. -func TestIptablesDNAT_DuplicateAddNoLeak(t *testing.T) { - m := newIptRefcountManager(t, true) - state := m.family4.ipFwdState - - rule := iptDnatV4(7083) - r1, err := m.AddDNATRule(rule) - require.NoError(t, err) - v4, _ := state.Counts() - assert.Equal(t, 1, v4) - - _, err = m.AddDNATRule(rule) - require.NoError(t, err, "duplicate add") - v4, _ = state.Counts() - assert.Equal(t, 1, v4, "duplicate add must not increment") - - require.NoError(t, m.DeleteDNATRule(r1)) - v4, _ = state.Counts() - assert.Equal(t, 0, v4, "single delete must drop to zero") -} - -// TestIptablesDNAT_DeleteMissingNoUnderflow verifies Delete on an unknown rule -// neither errors nor releases the refcount. -func TestIptablesDNAT_DeleteMissingNoUnderflow(t *testing.T) { - m := newIptRefcountManager(t, true) - state := m.family4.ipFwdState - - phantom := iptDnatV4(7099) - require.NoError(t, m.DeleteDNATRule(&phantom), "delete missing v4") - v4, v6 := state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 0, v6) - - phantom6 := iptDnatV6(9099) - require.NoError(t, m.DeleteDNATRule(&phantom6), "delete missing v6") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 0, v6) - - r1, err := m.AddDNATRule(iptDnatV4(7100)) - require.NoError(t, err) - v4, _ = state.Counts() - assert.Equal(t, 1, v4, "real add still increments after phantom delete") - require.NoError(t, m.DeleteDNATRule(r1)) -} - -// TestIptablesDNAT_DoubleDeleteNoUnderflow verifies a second Delete on the same -// rule is a no-op. -func TestIptablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) { - m := newIptRefcountManager(t, true) - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(iptDnatV6(9083)) - require.NoError(t, err) - _, v6 := state.Counts() - assert.Equal(t, 1, v6) - - require.NoError(t, m.DeleteDNATRule(r1), "first delete") - _, v6 = state.Counts() - assert.Equal(t, 0, v6) - - require.NoError(t, m.DeleteDNATRule(r1), "second delete must be no-op") - _, v6 = state.Counts() - assert.Equal(t, 0, v6, "double delete must not underflow") -} diff --git a/client/firewall/iptables/family_linux.go b/client/firewall/iptables/family_linux.go index c5ed8cc20..2ac860a0a 100644 --- a/client/firewall/iptables/family_linux.go +++ b/client/firewall/iptables/family_linux.go @@ -24,6 +24,7 @@ const ( tableFilter = "filter" tableNat = "nat" tableMangle = "mangle" + tableRaw = "raw" // chainACLInput is the peer ACL chain that holds installed // peer-filtering rules. @@ -34,6 +35,7 @@ const ( mangleForwardKey chainKey = "MANGLE-FORWARD" chainInput = "INPUT" + chainOutput = "OUTPUT" chainPostrouting = "POSTROUTING" chainPrerouting = "PREROUTING" chainForward = "FORWARD" @@ -54,10 +56,6 @@ const ( markManglePost = "mark-mangle-post" matchSet = "--match-set" - dnatSuffix firewall.RuleID = "_dnat" - snatSuffix firewall.RuleID = "_snat" - fwdSuffix firewall.RuleID = "_fwd" - // ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation. ipv4TCPHeaderSize = 40 // ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation. diff --git a/client/firewall/iptables/filter_linux.go b/client/firewall/iptables/filter_linux.go index dc606da2d..30cd81018 100644 --- a/client/firewall/iptables/filter_linux.go +++ b/client/firewall/iptables/filter_linux.go @@ -81,15 +81,6 @@ func (r *family) hasRule(id nbid.RuleID) bool { return ok } -// hasDNATRule reports whether this family owns the DNAT rule set for -// the given user id. DNAT rules live in r.rules under the well-known -// "_dnat" key; the lookup here is used by Manager.DeleteDNATRule -// to pick the right family. -func (r *family) hasDNATRule(id firewall.RuleID) bool { - _, ok := r.rules[id+dnatSuffix] - return ok -} - // DeleteFilterRule removes a previously installed filter rule. The // rule's stored chain/table identify where to delete from; source set // references are recovered from the spec via findSets and dropped diff --git a/client/firewall/iptables/manager_linux.go b/client/firewall/iptables/manager_linux.go index 49b88f1ea..a566909c8 100644 --- a/client/firewall/iptables/manager_linux.go +++ b/client/firewall/iptables/manager_linux.go @@ -25,9 +25,8 @@ type Manager struct { wgIface iFaceMapper - ipv4Client *iptables.IPTables - family4 *family - rawSupported bool + ipv4Client *iptables.IPTables + family4 *family // IPv6 counterparts, nil when no v6 overlay ipv6Client *iptables.IPTables @@ -108,10 +107,6 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error { return err } - if err := m.initNoTrackChain(); err != nil { - log.Warnf("raw table not available, notrack rules will be disabled: %v", err) - } - // Trust after all fatal init steps so a later failure doesn't leave the // interface in firewalld's trusted zone without a corresponding Close. if err := firewalld.TrustInterface(m.wgIface.Name()); err != nil { @@ -285,10 +280,6 @@ func (m *Manager) Close(stateManager *statemanager.Manager) error { var merr *multierror.Error - if err := m.cleanupNoTrackChain(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("cleanup notrack chain: %w", err)) - } - if m.hasIPv6() { if err := m.family6.Reset(); err != nil { merr = multierror.Append(merr, fmt.Errorf("reset v6 family: %w", err)) @@ -332,31 +323,6 @@ func (m *Manager) DisableRouting() error { return m.family4.ipFwdState.ReleaseRouting() } -// AddDNATRule adds a DNAT rule -func (m *Manager) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - m.mutex.Lock() - defer m.mutex.Unlock() - - if rule.TranslatedAddress.Is6() { - if !m.hasIPv6() { - return nil, fmt.Errorf("add DNAT rule: %w", firewall.ErrIPv6NotInitialized) - } - return m.family6.AddDNATRule(rule) - } - return m.family4.AddDNATRule(rule) -} - -// DeleteDNATRule deletes a DNAT rule -func (m *Manager) DeleteDNATRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if m.hasIPv6() && !m.family4.hasDNATRule(rule.ID()) { - return m.family6.DeleteDNATRule(rule) - } - return m.family4.DeleteDNATRule(rule) -} - // UpdateSet updates the set with the given prefixes func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { m.mutex.Lock() @@ -440,134 +406,6 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } -const ( - chainNameRaw = "NETBIRD-RAW" - chainOutput = "OUTPUT" - tableRaw = "raw" -) - -// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic. -// This prevents conntrack from tracking WireGuard proxy traffic on loopback, which -// can interfere with MASQUERADE rules (e.g., from container runtimes like Podman/netavark). -// -// Traffic flows that need NOTRACK: -// -// 1. Egress: WireGuard -> fake endpoint (before eBPF rewrite) -// src=127.0.0.1:wgPort -> dst=127.0.0.1:fakePort -// Matched by: sport=wgPort -// -// 2. Egress: Proxy -> WireGuard (via raw socket) -// src=127.0.0.1:fakePort -> dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 3. Ingress: Packets to WireGuard -// dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 4. Ingress: Packets to proxy (after eBPF rewrite) -// dst=127.0.0.1:proxyPort -// Matched by: dport=proxyPort -// -// Rules are cleaned up when the firewall manager is closed. -func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if !m.rawSupported { - return fmt.Errorf("raw table not available") - } - - wgPortStr := fmt.Sprintf("%d", wgPort) - proxyPortStr := fmt.Sprintf("%d", proxyPort) - - // Egress rules: match outgoing loopback UDP packets - outputRuleSport := []string{"-o", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--sport", wgPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, outputRuleSport...); err != nil { - return fmt.Errorf("add output sport notrack rule: %w", err) - } - - outputRuleDport := []string{"-o", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, outputRuleDport...); err != nil { - return fmt.Errorf("add output dport notrack rule: %w", err) - } - - // Ingress rules: match incoming loopback UDP packets - preroutingRuleWg := []string{"-i", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, preroutingRuleWg...); err != nil { - return fmt.Errorf("add prerouting wg notrack rule: %w", err) - } - - preroutingRuleProxy := []string{"-i", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", proxyPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, preroutingRuleProxy...); err != nil { - return fmt.Errorf("add prerouting proxy notrack rule: %w", err) - } - - log.Debugf("set up ebpf proxy notrack rules for ports %d,%d", proxyPort, wgPort) - return nil -} - -func (m *Manager) initNoTrackChain() error { - if err := m.cleanupNoTrackChain(); err != nil { - log.Debugf("cleanup notrack chain: %v", err) - } - - if err := m.ipv4Client.NewChain(tableRaw, chainNameRaw); err != nil { - return fmt.Errorf("create chain: %w", err) - } - - jumpRule := []string{"-j", chainNameRaw} - - if err := m.ipv4Client.InsertUnique(tableRaw, chainOutput, 1, jumpRule...); err != nil { - if delErr := m.ipv4Client.DeleteChain(tableRaw, chainNameRaw); delErr != nil { - log.Debugf("delete orphan chain: %v", delErr) - } - return fmt.Errorf("add output jump rule: %w", err) - } - - if err := m.ipv4Client.InsertUnique(tableRaw, chainPrerouting, 1, jumpRule...); err != nil { - if delErr := m.ipv4Client.DeleteIfExists(tableRaw, chainOutput, jumpRule...); delErr != nil { - log.Debugf("delete output jump rule: %v", delErr) - } - if delErr := m.ipv4Client.DeleteChain(tableRaw, chainNameRaw); delErr != nil { - log.Debugf("delete orphan chain: %v", delErr) - } - return fmt.Errorf("add prerouting jump rule: %w", err) - } - - m.rawSupported = true - return nil -} - -func (m *Manager) cleanupNoTrackChain() error { - exists, err := m.ipv4Client.ChainExists(tableRaw, chainNameRaw) - if err != nil { - if !m.rawSupported { - return nil - } - return fmt.Errorf("check chain exists: %w", err) - } - if !exists { - return nil - } - - jumpRule := []string{"-j", chainNameRaw} - - if err := m.ipv4Client.DeleteIfExists(tableRaw, chainOutput, jumpRule...); err != nil { - return fmt.Errorf("remove output jump rule: %w", err) - } - - if err := m.ipv4Client.DeleteIfExists(tableRaw, chainPrerouting, jumpRule...); err != nil { - return fmt.Errorf("remove prerouting jump rule: %w", err) - } - - if err := m.ipv4Client.ClearAndDeleteChain(tableRaw, chainNameRaw); err != nil { - return fmt.Errorf("clear and delete chain: %w", err) - } - - m.rawSupported = false - return nil -} - func getConntrackEstablished() []string { return []string{"-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"} } diff --git a/client/firewall/iptables/manager_linux_test.go b/client/firewall/iptables/manager_linux_test.go index 9f53352e1..8435bf6a5 100644 --- a/client/firewall/iptables/manager_linux_test.go +++ b/client/firewall/iptables/manager_linux_test.go @@ -497,16 +497,6 @@ func TestIptablesCloseRemovesAllState(t *testing.T) { require.NoError(t, manager.AddNatRule(pair), "add nat rule") require.NoError(t, manager.EnableRouting(), "enable routing") - // A DNAT redirect, which also holds a forwarding reference. - dnat := fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{8080}}, - TranslatedAddress: netip.MustParseAddr("10.20.0.44"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - } - _, err = manager.AddDNATRule(dnat) - require.NoError(t, err, "add dnat rule") - require.NotEqual(t, before, snapshotIptables(t, ipv4Client), "the manager must have installed state") // Everything above stays in place, so Close is what has to remove it. diff --git a/client/firewall/manager/firewall.go b/client/firewall/manager/firewall.go index 97a94d0f5..f8de1e2b5 100644 --- a/client/firewall/manager/firewall.go +++ b/client/firewall/manager/firewall.go @@ -172,12 +172,6 @@ type Manager interface { DisableRouting() error - // AddDNATRule adds outbound DNAT rule for forwarding external traffic to the NetBird network. - AddDNATRule(ForwardRule) (Rule, error) - - // DeleteDNATRule deletes the outbound DNAT rule. - DeleteDNATRule(Rule) error - // UpdateSet updates the set with the given prefixes UpdateSet(hash Set, prefixes []netip.Prefix) error @@ -192,10 +186,6 @@ type Manager interface { // RemoveOutputDNAT removes an OUTPUT chain DNAT rule. RemoveOutputDNAT(localAddr netip.Addr, protocol Protocol, originalPort, translatedPort uint16) error - - // SetupEBPFProxyNoTrack creates static notrack rules for eBPF proxy loopback traffic. - // This prevents conntrack from interfering with WireGuard proxy communication. - SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error } // GenKey builds the rule id for this pair from the given format. diff --git a/client/firewall/manager/forward_rule.go b/client/firewall/manager/forward_rule.go deleted file mode 100644 index c2e9e5c60..000000000 --- a/client/firewall/manager/forward_rule.go +++ /dev/null @@ -1,27 +0,0 @@ -package manager - -import ( - "fmt" - "net/netip" -) - -// ForwardRule todo figure out better place to this to avoid circular imports -type ForwardRule struct { - Protocol Protocol - DestinationPort Port - TranslatedAddress netip.Addr - TranslatedPort Port -} - -func (r ForwardRule) ID() RuleID { - id := fmt.Sprintf("%s;%s;%s;%s", - r.Protocol, - r.DestinationPort.String(), - r.TranslatedAddress.String(), - r.TranslatedPort.String()) - return RuleID(id) -} - -func (r ForwardRule) String() string { - return fmt.Sprintf("protocol: %s, destinationPort: %s, translatedAddress: %s, translatedPort: %s", r.Protocol, r.DestinationPort.String(), r.TranslatedAddress.String(), r.TranslatedPort.String()) -} diff --git a/client/firewall/nftables/dnat_linux.go b/client/firewall/nftables/dnat_linux.go index 8eae694a2..c179d60cc 100644 --- a/client/firewall/nftables/dnat_linux.go +++ b/client/firewall/nftables/dnat_linux.go @@ -9,332 +9,11 @@ import ( "github.com/google/nftables" "github.com/google/nftables/binaryutil" "github.com/google/nftables/expr" - "github.com/google/nftables/xt" - "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" - nberrors "github.com/netbirdio/netbird/client/errors" firewall "github.com/netbirdio/netbird/client/firewall/manager" ) -func (r *family) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - ruleID := rule.ID() - if _, exists := r.rules[ruleID+dnatSuffix]; exists { - return rule, nil - } - - protoNum, err := r.af.protoNum(rule.Protocol) - if err != nil { - return nil, fmt.Errorf("convert protocol to number: %w", err) - } - - // Request forwarding before queueing rules: addDnatRedirect/addDnatMasq - // buffer netlink messages on r.conn that the next caller's Flush would - // commit if we returned without flushing them ourselves. - if err := r.ipFwdState.RequestForwarding(r.isV6()); err != nil { - return nil, fmt.Errorf("enable forwarding: %w", err) - } - - if err := r.addDnatRedirect(rule, protoNum, ruleID); err != nil { - r.releaseForwarding() - return nil, err - } - - if err := r.addDnatMasq(rule, protoNum, ruleID); err != nil { - r.releaseForwarding() - delete(r.rules, ruleID+dnatSuffix) - return nil, err - } - - // Unlike iptables, there's no point in adding "out" rules in the forward chain here as our policy is ACCEPT. - // To overcome DROP policies in other chains, we'd have to add rules to the chains there. - // We also cannot just add "oif accept" there and filter in our own table as we don't know what is supposed to be allowed. - // TODO: find chains with drop policies and add rules there - - if err := r.conn.Flush(); err != nil { - r.releaseForwarding() - delete(r.rules, ruleID+dnatSuffix) - delete(r.rules, ruleID+snatSuffix) - return nil, fmt.Errorf("flush rules: %w", err) - } - - return &rule, nil -} - -func (r *family) addDnatRedirect(rule firewall.ForwardRule, protoNum uint8, ruleID firewall.RuleID) error { - dnatExprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{protoNum}, - }, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseTransportHeader, - Offset: 2, - Len: 2, - }, - } - portExprs, err := r.applyPort(&rule.DestinationPort, false) - if err != nil { - return fmt.Errorf("apply destination port: %w", err) - } - dnatExprs = append(dnatExprs, portExprs...) - - // shifted translated port is not supported in nftables, so we hand this over to xtables - if rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2 { - if rule.TranslatedPort.Values[0] != rule.DestinationPort.Values[0] || - rule.TranslatedPort.Values[1] != rule.DestinationPort.Values[1] { - return r.addXTablesRedirect(dnatExprs, ruleID, rule) - } - } - - additionalExprs, regProtoMin, regProtoMax, err := r.handleTranslatedPort(rule) - if err != nil { - return err - } - dnatExprs = append(dnatExprs, additionalExprs...) - - dnatExprs = append(dnatExprs, - &expr.NAT{ - Type: expr.NATTypeDestNAT, - Family: uint32(r.af.tableFamily), - RegAddrMin: 1, - RegProtoMin: regProtoMin, - RegProtoMax: regProtoMax, - }, - ) - - dnatRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingRdr], - Exprs: dnatExprs, - UserData: []byte(ruleID + dnatSuffix), - } - r.conn.AddRule(dnatRule) - r.rules[ruleID+dnatSuffix] = dnatRule - - return nil -} - -func (r *family) handleTranslatedPort(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - switch { - case rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2: - return r.handlePortRange(rule) - case len(rule.TranslatedPort.Values) == 0: - return r.handleAddressOnly(rule) - case len(rule.TranslatedPort.Values) == 1: - return r.handleSinglePort(rule) - default: - return nil, 0, 0, fmt.Errorf("invalid translated port: %v", rule.TranslatedPort) - } -} - -func (r *family) handlePortRange(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - exprs := []expr.Any{ - &expr.Immediate{ - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - &expr.Immediate{ - Register: 2, - Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[0]), - }, - &expr.Immediate{ - Register: 3, - Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[1]), - }, - } - return exprs, 2, 3, nil -} - -func (r *family) handleAddressOnly(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - exprs := []expr.Any{ - &expr.Immediate{ - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - } - return exprs, 0, 0, nil -} - -func (r *family) handleSinglePort(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - exprs := []expr.Any{ - &expr.Immediate{ - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - &expr.Immediate{ - Register: 2, - Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[0]), - }, - } - return exprs, 2, 0, nil -} - -func (r *family) addXTablesRedirect(dnatExprs []expr.Any, ruleID firewall.RuleID, rule firewall.ForwardRule) error { - dnatExprs = append(dnatExprs, - &expr.Counter{}, - &expr.Target{ - Name: "DNAT", - Rev: 2, - Info: &xt.NatRange2{ - NatRange: xt.NatRange{ - Flags: uint(xt.NatRangeMapIPs | xt.NatRangeProtoSpecified | xt.NatRangeProtoOffset), - MinIP: rule.TranslatedAddress.AsSlice(), - MaxIP: rule.TranslatedAddress.AsSlice(), - MinPort: rule.TranslatedPort.Values[0], - MaxPort: rule.TranslatedPort.Values[1], - }, - BasePort: rule.DestinationPort.Values[0], - }, - }, - ) - - natTable := &nftables.Table{ - Name: tableNat, - Family: r.af.tableFamily, - } - dnatRule := &nftables.Rule{ - Table: natTable, - Chain: &nftables.Chain{ - Name: chainNameNatPrerouting, - Table: natTable, - Type: nftables.ChainTypeNAT, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityNATDest, - }, - Exprs: dnatExprs, - UserData: []byte(ruleID + dnatSuffix), - } - r.conn.AddRule(dnatRule) - r.rules[ruleID+dnatSuffix] = dnatRule - - return nil -} - -func (r *family) addDnatMasq(rule firewall.ForwardRule, protoNum uint8, ruleID firewall.RuleID) error { - portExprs, err := r.applyPort(&rule.TranslatedPort, false) - if err != nil { - return fmt.Errorf("apply translated port: %w", err) - } - - masqExprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{protoNum}, - }, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: r.af.dstAddrOffset, - Len: r.af.addrLen, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - } - - masqExprs = append(masqExprs, portExprs...) - masqExprs = append(masqExprs, &expr.Masq{}) - - masqRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingNat], - Exprs: masqExprs, - UserData: []byte(ruleID + snatSuffix), - } - r.conn.AddRule(masqRule) - r.rules[ruleID+snatSuffix] = masqRule - - return nil -} - -func (r *family) DeleteDNATRule(rule firewall.Rule) error { - ruleID := rule.ID() - - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - var merr *multierror.Error - var needsFlush bool - var found bool - - if dnatRule, exists := r.rules[ruleID+dnatSuffix]; exists { - found = true - if dnatRule.Handle == 0 { - log.Warnf("dnat rule %s has no handle, removing stale entry", ruleID+dnatSuffix) - delete(r.rules, ruleID+dnatSuffix) - } else if err := r.conn.DelRule(dnatRule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete dnat rule: %w", err)) - } else { - needsFlush = true - } - } - - if masqRule, exists := r.rules[ruleID+snatSuffix]; exists { - found = true - if masqRule.Handle == 0 { - log.Warnf("snat rule %s has no handle, removing stale entry", ruleID+snatSuffix) - delete(r.rules, ruleID+snatSuffix) - } else if err := r.conn.DelRule(masqRule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete snat rule: %w", err)) - } else { - needsFlush = true - } - } - - if needsFlush { - if err := r.conn.Flush(); err != nil { - merr = multierror.Append(merr, fmt.Errorf(flushError, err)) - } - } - - if merr != nil { - return nberrors.FormatErrorOrNil(merr) - } - - delete(r.rules, ruleID+dnatSuffix) - delete(r.rules, ruleID+snatSuffix) - - // Release once, only if the rule was present and removed. - if found { - r.releaseForwarding() - } - - return nil -} - -// releaseForwarding drops one IP forwarding reference, logging any error. -func (r *family) releaseForwarding() { - if err := r.ipFwdState.ReleaseForwarding(r.isV6()); err != nil { - log.Errorf("release IP forwarding: %v", err) - } -} - -// isV6 reports whether this family handles the IPv6 table. -func (r *family) isV6() bool { - return r.af.tableFamily == nftables.TableFamilyIPv6 -} - func (r *family) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) diff --git a/client/firewall/nftables/dnat_refcount_linux_test.go b/client/firewall/nftables/dnat_refcount_linux_test.go deleted file mode 100644 index cdc24e77f..000000000 --- a/client/firewall/nftables/dnat_refcount_linux_test.go +++ /dev/null @@ -1,249 +0,0 @@ -//go:build privileged - -package nftables - -import ( - "net/netip" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - fw "github.com/netbirdio/netbird/client/firewall/manager" - "github.com/netbirdio/netbird/client/iface" - "github.com/netbirdio/netbird/client/iface/wgaddr" -) - -func nftRefcountIfaceV4() *iFaceMock { - return &iFaceMock{ - NameFunc: func() string { return "wt-refcount" }, - AddressFunc: func() wgaddr.Address { - return wgaddr.Address{ - IP: netip.MustParseAddr("100.96.0.1"), - Network: netip.MustParsePrefix("100.96.0.0/16"), - } - }, - } -} - -func nftRefcountIfaceDual() *iFaceMock { - return &iFaceMock{ - NameFunc: func() string { return "wt-refcount" }, - AddressFunc: func() wgaddr.Address { - return wgaddr.Address{ - IP: netip.MustParseAddr("100.96.0.1"), - Network: netip.MustParsePrefix("100.96.0.0/16"), - IPv6: netip.MustParseAddr("fd00::1"), - IPv6Net: netip.MustParsePrefix("fd00::/64"), - } - }, - } -} - -func newNftRefcountManager(t *testing.T, dual bool) *Manager { - t.Helper() - if check() != NFTABLES { - t.Skip("nftables not supported on this system") - } - var ifMock *iFaceMock - if dual { - ifMock = nftRefcountIfaceDual() - } else { - ifMock = nftRefcountIfaceV4() - } - m, err := Create(ifMock, iface.DefaultMTU) - require.NoError(t, err, "create manager") - require.NoError(t, m.Init(nil), "init manager") - t.Cleanup(func() { - require.NoError(t, m.Close(nil), "close manager") - }) - return m -} - -func dnatV4(port uint16) fw.ForwardRule { - return fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{port}}, - TranslatedAddress: netip.MustParseAddr("100.96.0.2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - } -} - -func dnatV6(port uint16) fw.ForwardRule { - return fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{port}}, - TranslatedAddress: netip.MustParseAddr("fd00::2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - } -} - -// TestNftablesDNAT_RefcountBalancedV4 verifies that Add/Delete pairs leave the -// v4 refcount at zero. -func TestNftablesDNAT_RefcountBalancedV4(t *testing.T) { - m := newNftRefcountManager(t, false) - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(dnatV4(8081)) - require.NoError(t, err, "add v4 dnat 1") - v4, v6 := state.Counts() - assert.Equal(t, 1, v4, "v4 refcount after first add") - assert.Equal(t, 0, v6, "v6 refcount unchanged") - - r2, err := m.AddDNATRule(dnatV4(8082)) - require.NoError(t, err, "add v4 dnat 2") - v4, v6 = state.Counts() - assert.Equal(t, 2, v4, "v4 refcount after second add") - assert.Equal(t, 0, v6, "v6 refcount unchanged") - - require.NoError(t, m.DeleteDNATRule(r1), "delete v4 dnat 1") - v4, v6 = state.Counts() - assert.Equal(t, 1, v4, "v4 refcount after first delete") - assert.Equal(t, 0, v6, "v6 refcount unchanged") - - require.NoError(t, m.DeleteDNATRule(r2), "delete v4 dnat 2") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "v4 refcount after second delete") - assert.Equal(t, 0, v6, "v6 refcount unchanged") -} - -// TestNftablesDNAT_RefcountBalancedV6 verifies the v6 path increments v6 only -// and decrements back to zero on Delete. -func TestNftablesDNAT_RefcountBalancedV6(t *testing.T) { - m := newNftRefcountManager(t, true) - require.NotNil(t, m.family6, "v6 family") - require.Same(t, m.family4.ipFwdState, m.family6.ipFwdState, "shared state") - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(dnatV6(9091)) - require.NoError(t, err, "add v6 dnat 1") - v4, v6 := state.Counts() - assert.Equal(t, 0, v4, "v4 refcount unchanged") - assert.Equal(t, 1, v6, "v6 refcount after first add") - - r2, err := m.AddDNATRule(dnatV6(9092)) - require.NoError(t, err, "add v6 dnat 2") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 2, v6, "v6 refcount after second add") - - require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat 1") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "v4 refcount unchanged") - assert.Equal(t, 1, v6, "v6 refcount after first delete") - - require.NoError(t, m.DeleteDNATRule(r2), "delete v6 dnat 2") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 0, v6, "v6 refcount after second delete") -} - -// TestNftablesDNAT_DuplicateAddNoLeak verifies that a duplicate Add (same -// ForwardRule) does not double-increment the refcount. -func TestNftablesDNAT_DuplicateAddNoLeak(t *testing.T) { - m := newNftRefcountManager(t, true) - state := m.family4.ipFwdState - - rule := dnatV4(8083) - r1, err := m.AddDNATRule(rule) - require.NoError(t, err, "add v4 dnat") - v4, _ := state.Counts() - assert.Equal(t, 1, v4) - - // duplicate add: same rule ID, must be a no-op for the refcount. - _, err = m.AddDNATRule(rule) - require.NoError(t, err, "duplicate add") - v4, _ = state.Counts() - assert.Equal(t, 1, v4, "duplicate add must not increment") - - require.NoError(t, m.DeleteDNATRule(r1), "delete v4 dnat") - v4, _ = state.Counts() - assert.Equal(t, 0, v4, "single delete must drop to zero") -} - -// TestNftablesDNAT_DeleteMissingNoUnderflow verifies deleting a rule that was -// never added does not underflow the refcount. -func TestNftablesDNAT_DeleteMissingNoUnderflow(t *testing.T) { - m := newNftRefcountManager(t, true) - state := m.family4.ipFwdState - - // Construct a Rule reference for something never added. The router stores - // rules by ID(), and DeleteDNATRule looks them up in r.rules; a missing - // entry must be a no-op rather than calling Release. - phantom := dnatV4(8099) - require.NoError(t, m.DeleteDNATRule(&phantom), "delete missing v4 dnat") - v4, v6 := state.Counts() - assert.Equal(t, 0, v4, "v4 refcount unaffected by missing delete") - assert.Equal(t, 0, v6, "v6 refcount unaffected") - - phantom6 := dnatV6(9099) - require.NoError(t, m.DeleteDNATRule(&phantom6), "delete missing v6 dnat") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 0, v6, "v6 refcount unaffected by missing delete") - - // And after a phantom delete, a real add still results in count=1. - r1, err := m.AddDNATRule(dnatV4(8100)) - require.NoError(t, err, "add v4 dnat after phantom delete") - v4, _ = state.Counts() - assert.Equal(t, 1, v4, "real add still increments after phantom delete") - require.NoError(t, m.DeleteDNATRule(r1)) -} - -// TestNftablesRouting_RepeatedEnableSingleReference verifies that EnableRouting -// (called on every network-map update) holds at most one reference per family -// and a single DisableRouting drops both back to zero. -func TestNftablesRouting_RepeatedEnableSingleReference(t *testing.T) { - m := newNftRefcountManager(t, true) - state := m.family4.ipFwdState - - require.NoError(t, m.EnableRouting(), "first enable") - require.NoError(t, m.EnableRouting(), "second enable") - require.NoError(t, m.EnableRouting(), "third enable") - v4, v6 := state.Counts() - assert.Equal(t, 1, v4, "repeated enable holds a single v4 reference") - assert.Equal(t, 1, v6, "repeated enable holds a single v6 reference") - - require.NoError(t, m.DisableRouting(), "disable") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "single disable releases the v4 reference") - assert.Equal(t, 0, v6, "single disable releases the v6 reference") -} - -// TestNftablesRouting_DisableKeepsDNATReference verifies that an unpaired -// DisableRouting does not release references held by active DNAT rules. -func TestNftablesRouting_DisableKeepsDNATReference(t *testing.T) { - m := newNftRefcountManager(t, true) - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(dnatV6(9095)) - require.NoError(t, err, "add v6 dnat") - - require.NoError(t, m.DisableRouting(), "unpaired disable") - _, v6 := state.Counts() - assert.Equal(t, 1, v6, "DNAT-held reference survives unpaired DisableRouting") - - require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat") - _, v6 = state.Counts() - assert.Equal(t, 0, v6, "delete releases the DNAT reference") -} - -// TestNftablesDNAT_DoubleDeleteNoUnderflow verifies that deleting the same rule -// twice does not underflow the refcount (the second delete is a no-op). -func TestNftablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) { - m := newNftRefcountManager(t, true) - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(dnatV6(9093)) - require.NoError(t, err) - _, v6 := state.Counts() - assert.Equal(t, 1, v6) - - require.NoError(t, m.DeleteDNATRule(r1), "first delete") - _, v6 = state.Counts() - assert.Equal(t, 0, v6) - - require.NoError(t, m.DeleteDNATRule(r1), "second delete must be no-op") - _, v6 = state.Counts() - assert.Equal(t, 0, v6, "double delete must not underflow") -} diff --git a/client/firewall/nftables/family_linux.go b/client/firewall/nftables/family_linux.go index 7a5df3ed7..4169c9d2d 100644 --- a/client/firewall/nftables/family_linux.go +++ b/client/firewall/nftables/family_linux.go @@ -24,7 +24,6 @@ const ( tableRaw = "raw" tableSecurity = "security" - chainNameNatPrerouting = "PREROUTING" chainNameRoutingFw = "netbird-rt-fwd" chainNameRoutingNat = "netbird-rt-postrouting" chainNameRoutingRdr = "netbird-rt-redirect" @@ -47,9 +46,6 @@ const ( userDataAcceptForwardRuleOif = "frwacceptoif" userDataAcceptInputRule = "inputaccept" - dnatSuffix firewall.RuleID = "_dnat" - snatSuffix firewall.RuleID = "_snat" - // ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation. ipv4TCPHeaderSize = 40 // ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation. @@ -167,10 +163,6 @@ func (r *family) Reset() error { merr = multierror.Append(merr, err) } - if err := r.removeNatPreroutingRules(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove filter prerouting rules: %w", err)) - } - return nberrors.FormatErrorOrNil(merr) } diff --git a/client/firewall/nftables/filter_linux.go b/client/firewall/nftables/filter_linux.go index ebd238063..bb3ac1dfe 100644 --- a/client/firewall/nftables/filter_linux.go +++ b/client/firewall/nftables/filter_linux.go @@ -197,11 +197,6 @@ func (r *family) hasRule(id firewall.RuleID) bool { return ok } -func (r *family) hasDNATRule(id firewall.RuleID) bool { - _, ok := r.rules[id+dnatSuffix] - return ok -} - // DeleteFilterRule removes a previously installed filter rule. Source // set references are recovered from the stored rule's expressions via // findSets and dropped from the shared refcounter. diff --git a/client/firewall/nftables/manager_linux.go b/client/firewall/nftables/manager_linux.go index dbd5e4fa2..75405e213 100644 --- a/client/firewall/nftables/manager_linux.go +++ b/client/firewall/nftables/manager_linux.go @@ -12,7 +12,6 @@ import ( "github.com/google/nftables/expr" "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" nberrors "github.com/netbirdio/netbird/client/errors" firewall "github.com/netbirdio/netbird/client/firewall/manager" @@ -55,9 +54,6 @@ type Manager struct { // IPv6 counterpart, nil when no v6 overlay. family6 *family - notrackOutputChain *nftables.Chain - notrackPreroutingChain *nftables.Chain - extMonitor *externalChainMonitor } @@ -170,10 +166,6 @@ func (m *Manager) initFirewall() (err error) { } } - if err := m.initNoTrackChains(workTable); err != nil { - log.Warnf("raw priority chains not available, notrack rules will be disabled: %v", err) - } - return nil } @@ -260,7 +252,7 @@ func (m *Manager) DeleteFilterRule(rule firewall.Rule) error { m.mutex.Lock() defer m.mutex.Unlock() - fam, err := m.familyForRuleID(rule.ID(), (*family).hasRule, false) + fam, err := m.familyForRuleID(rule.ID(), (*family).hasRule) if err != nil { return err } @@ -268,11 +260,8 @@ func (m *Manager) DeleteFilterRule(rule firewall.Rule) error { } // familyForRuleID picks the family holding the rule with the given id, using -// the supplied lookup. With refresh set, a miss in both cached maps reloads -// the NAT/DNAT rule maps from the kernel once and re-checks before falling -// back to the v4 family. Filter rules are tracked only in memory and have no -// kernel-backed reload, so their callers pass refresh as false. -func (m *Manager) familyForRuleID(id firewall.RuleID, has func(*family, firewall.RuleID) bool, refresh bool) (*family, error) { +// the supplied lookup, and falls back to the v4 family on a miss. +func (m *Manager) familyForRuleID(id firewall.RuleID, has func(*family, firewall.RuleID) bool) (*family, error) { if has(m.family4, id) { return m.family4, nil } @@ -282,18 +271,6 @@ func (m *Manager) familyForRuleID(id firewall.RuleID, has func(*family, firewall if has(m.family6, id) { return m.family6, nil } - if !refresh { - return m.family4, nil - } - if err := m.family4.refreshRulesMap(); err != nil { - return nil, fmt.Errorf("refresh v4 rules: %w", err) - } - if err := m.family6.refreshRulesMap(); err != nil { - return nil, fmt.Errorf("refresh v6 rules: %w", err) - } - if has(m.family6, id) && !has(m.family4, id) { - return m.family6, nil - } return m.family4, nil } @@ -455,39 +432,9 @@ func (m *Manager) Flush() error { } } - if err := m.refreshNoTrackChains(); err != nil { - log.Errorf("failed to refresh notrack chains: %v", err) - } - return nil } -// AddDNATRule adds a DNAT rule -func (m *Manager) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - m.mutex.Lock() - defer m.mutex.Unlock() - - if rule.TranslatedAddress.Is6() { - if !m.hasIPv6() { - return nil, fmt.Errorf("add DNAT rule: %w", firewall.ErrIPv6NotInitialized) - } - return m.family6.AddDNATRule(rule) - } - return m.family4.AddDNATRule(rule) -} - -// DeleteDNATRule deletes a DNAT rule -func (m *Manager) DeleteDNATRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - r, err := m.familyForRuleID(rule.ID(), (*family).hasDNATRule, true) - if err != nil { - return err - } - return r.DeleteDNATRule(rule) -} - // UpdateSet updates the set with the given prefixes func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { m.mutex.Lock() @@ -571,176 +518,6 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } -const ( - chainNameRawOutput = "netbird-raw-out" - chainNameRawPrerouting = "netbird-raw-pre" -) - -// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic. -// This prevents conntrack from tracking WireGuard proxy traffic on loopback, which -// can interfere with MASQUERADE rules (e.g., from container runtimes like Podman/netavark). -// -// Traffic flows that need NOTRACK: -// -// 1. Egress: WireGuard -> fake endpoint (before eBPF rewrite) -// src=127.0.0.1:wgPort -> dst=127.0.0.1:fakePort -// Matched by: sport=wgPort -// -// 2. Egress: Proxy -> WireGuard (via raw socket) -// src=127.0.0.1:fakePort -> dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 3. Ingress: Packets to WireGuard -// dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 4. Ingress: Packets to proxy (after eBPF rewrite) -// dst=127.0.0.1:proxyPort -// Matched by: dport=proxyPort -// -// Rules are cleaned up when the firewall manager is closed. -func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if m.notrackOutputChain == nil || m.notrackPreroutingChain == nil { - return fmt.Errorf("notrack chains not initialized") - } - - proxyPortBytes := binaryutil.BigEndian.PutUint16(proxyPort) - wgPortBytes := binaryutil.BigEndian.PutUint16(wgPort) - loopback := []byte{127, 0, 0, 1} - - // Egress rules: match outgoing loopback UDP packets - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackOutputChain.Table, - Chain: m.notrackOutputChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // sport=wgPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackOutputChain.Table, - Chain: m.notrackOutputChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // dport=wgPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - - // Ingress rules: match incoming loopback UDP packets - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackPreroutingChain.Table, - Chain: m.notrackPreroutingChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // dport=wgPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackPreroutingChain.Table, - Chain: m.notrackPreroutingChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: proxyPortBytes}, // dport=proxyPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf("flush notrack rules: %w", err) - } - - log.Debugf("set up ebpf proxy notrack rules for ports %d,%d", proxyPort, wgPort) - return nil -} - -func (m *Manager) initNoTrackChains(table *nftables.Table) error { - m.notrackOutputChain = m.rConn.AddChain(&nftables.Chain{ - Name: chainNameRawOutput, - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookOutput, - Priority: nftables.ChainPriorityRaw, - }) - - m.notrackPreroutingChain = m.rConn.AddChain(&nftables.Chain{ - Name: chainNameRawPrerouting, - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityRaw, - }) - - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf("flush chain creation: %w", err) - } - - return nil -} - -func (m *Manager) refreshNoTrackChains() error { - chains, err := m.rConn.ListChainsOfTableFamily(nftables.TableFamilyIPv4) - if err != nil { - return fmt.Errorf("list chains: %w", err) - } - - tableName := getTableName() - for _, c := range chains { - if c.Table.Name != tableName { - continue - } - switch c.Name { - case chainNameRawOutput: - m.notrackOutputChain = c - case chainNameRawPrerouting: - m.notrackPreroutingChain = c - } - } - - return nil -} - func (m *Manager) createWorkTable() (*nftables.Table, error) { return m.createWorkTableFamily(nftables.TableFamilyIPv4) } diff --git a/client/firewall/nftables/manager_linux_test.go b/client/firewall/nftables/manager_linux_test.go index 0ca56409e..4d6eec3c1 100644 --- a/client/firewall/nftables/manager_linux_test.go +++ b/client/firewall/nftables/manager_linux_test.go @@ -378,18 +378,6 @@ func TestNftablesManagerCompatibilityWithIptables(t *testing.T) { err = manager.AddNatRule(pair) require.NoError(t, err, "failed to add NAT rule") - dnatRule, err := manager.AddDNATRule(fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{8080}}, - TranslatedAddress: netip.MustParseAddr("100.96.0.2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - }) - require.NoError(t, err, "failed to add DNAT rule") - - t.Cleanup(func() { - require.NoError(t, manager.DeleteDNATRule(dnatRule), "failed to delete DNAT rule") - }) - stdout, stderr = runIptablesSave(t) verifyIptablesOutput(t, stdout, stderr) } @@ -453,18 +441,6 @@ func TestNftablesManagerIPv6CompatibilityWithIp6tables(t *testing.T) { }) require.NoError(t, err, "add v6 NAT rule") - dnatRule, err := manager.AddDNATRule(fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{8080}}, - TranslatedAddress: netip.MustParseAddr("fd00::2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - }) - require.NoError(t, err, "add v6 DNAT rule") - - t.Cleanup(func() { - require.NoError(t, manager.DeleteDNATRule(dnatRule), "delete v6 DNAT rule") - }) - stdout, stderr := runIptablesSave(t) verifyIptablesOutput(t, stdout, stderr) diff --git a/client/firewall/nftables/routing_linux.go b/client/firewall/nftables/routing_linux.go index 4115c94bd..e98471e8f 100644 --- a/client/firewall/nftables/routing_linux.go +++ b/client/firewall/nftables/routing_linux.go @@ -192,7 +192,7 @@ func (r *family) addPostroutingRules() { Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade), }, - // We need to exclude the loopback interface as this changes the ebpf proxy port + // We need to exclude the loopback interface as this changes the wg proxy port &expr.Meta{ Key: expr.MetaKeyOIFNAME, Register: 1, @@ -459,41 +459,6 @@ func (r *family) RemoveAllLegacyRouteRules() error { return nberrors.FormatErrorOrNil(merr) } -func (r *family) removeNatPreroutingRules() error { - table := &nftables.Table{ - Name: tableNat, - Family: r.af.tableFamily, - } - chain := &nftables.Chain{ - Name: chainNameNatPrerouting, - Table: table, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityNATDest, - Type: nftables.ChainTypeNAT, - } - rules, err := r.conn.GetRules(table, chain) - if err != nil { - return fmt.Errorf("get rules from nat table: %w", err) - } - - var merr *multierror.Error - - // Delete rules that have our UserData suffix - for _, rule := range rules { - if len(rule.UserData) == 0 || !strings.HasSuffix(string(rule.UserData), string(dnatSuffix)) { - continue - } - if err := r.conn.DelRule(rule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete rule %s: %w", rule.UserData, err)) - } - } - - if err := r.conn.Flush(); err != nil { - merr = multierror.Append(merr, fmt.Errorf(flushError, err)) - } - return nberrors.FormatErrorOrNil(merr) -} - func (r *family) RemoveNatRule(pair firewall.RouterPair) error { if err := r.refreshRulesMap(); err != nil { return fmt.Errorf(refreshRulesMapError, err) diff --git a/client/firewall/uspfilter/filter.go b/client/firewall/uspfilter/filter.go index 5e1366c1f..0c73400f1 100644 --- a/client/firewall/uspfilter/filter.go +++ b/client/firewall/uspfilter/filter.go @@ -879,12 +879,6 @@ func (m *Manager) resetState() { } } -// SetupEBPFProxyNoTrack is not supported by the userspace firewall: eBPF isn't -// used in userspace mode, so this should never be called. -func (m *Manager) SetupEBPFProxyNoTrack(uint16, uint16) error { - return errNotSupported -} - // UpdateSet updates the rule destinations associated with the given set // by merging the existing prefixes with the new ones, then deduplicating. func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { diff --git a/client/firewall/uspfilter/interface_allower_windows.go b/client/firewall/uspfilter/interface_allower_windows.go index 7f525e28c..4cd0fe969 100644 --- a/client/firewall/uspfilter/interface_allower_windows.go +++ b/client/firewall/uspfilter/interface_allower_windows.go @@ -9,6 +9,7 @@ import ( log "github.com/sirupsen/logrus" nberrors "github.com/netbirdio/netbird/client/errors" + "github.com/netbirdio/netbird/client/internal/wincmd" ) type action string @@ -91,7 +92,7 @@ func manageFirewallRule(ruleName string, action action, extraArgs ...string) err if action == addRule { args = append(args, extraArgs...) } - netshCmd := GetSystem32Command("netsh") + netshCmd := wincmd.System32("netsh") cmd := exec.Command(netshCmd, args...) cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true} return cmd.Run() @@ -100,7 +101,7 @@ func manageFirewallRule(ruleName string, action action, extraArgs ...string) err func isWindowsFirewallReachable() bool { args := []string{"advfirewall", "show", "allprofiles", "state"} - netshCmd := GetSystem32Command("netsh") + netshCmd := wincmd.System32("netsh") cmd := exec.Command(netshCmd, args...) cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true} @@ -117,23 +118,10 @@ func isWindowsFirewallReachable() bool { func isFirewallRuleActive(ruleName string) bool { args := []string{"advfirewall", "firewall", "show", "rule", "name=" + ruleName} - netshCmd := GetSystem32Command("netsh") + netshCmd := wincmd.System32("netsh") cmd := exec.Command(netshCmd, args...) cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true} _, err := cmd.Output() return err == nil } - -// GetSystem32Command checks if a command can be found in the system path and returns it. In case it can't find it -// in the path it will return the full path of a command assuming C:\windows\system32 as the base path. -func GetSystem32Command(command string) string { - _, err := exec.LookPath(command) - if err == nil { - return command - } - - log.Tracef("Command %s not found in PATH, using C:\\windows\\system32\\%s.exe path", command, command) - - return "C:\\windows\\system32\\" + command + ".exe" -} diff --git a/client/firewall/uspfilter/nat.go b/client/firewall/uspfilter/nat.go index 06312aabf..49c26766a 100644 --- a/client/firewall/uspfilter/nat.go +++ b/client/firewall/uspfilter/nat.go @@ -486,16 +486,6 @@ func incrementalUpdate(oldChecksum uint16, oldBytes, newBytes []byte) uint16 { return ^uint16(sum) } -// AddDNATRule adds outbound DNAT rule for forwarding external traffic to NetBird network. -func (m *Manager) AddDNATRule(firewall.ForwardRule) (firewall.Rule, error) { - return nil, errNotSupported -} - -// DeleteDNATRule deletes outbound DNAT rule. -func (m *Manager) DeleteDNATRule(firewall.Rule) error { - return errNotSupported -} - // addPortRedirection adds a port redirection rule. func (m *Manager) addPortRedirection(targetIP netip.Addr, protocol gopacket.LayerType, originalPort, translatedPort uint16) error { m.portDNATMutex.Lock() diff --git a/client/grpc/dialer_generic.go b/client/grpc/dialer_generic.go index 737787223..72b3538ba 100644 --- a/client/grpc/dialer_generic.go +++ b/client/grpc/dialer_generic.go @@ -6,12 +6,9 @@ import ( "context" "fmt" "net" - "os/user" + "os" "runtime" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - log "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -45,18 +42,13 @@ func WithSweeper(sweeper Sweeper) grpc.DialOption { } func dialContext(ctx context.Context, addr string) (net.Conn, error) { - if runtime.GOOS == "linux" { - currentUser, err := user.Current() - if err != nil { - return nil, status.Errorf(codes.FailedPrecondition, "failed to get current user: %v", err) - } - - // the custom dialer requires root permissions which are not required for use cases run as non-root - if currentUser.Uid != "0" { - log.Debug("Not running as root, using standard dialer") - dialer := &net.Dialer{} - return dialer.DialContext(ctx, "tcp", addr) - } + // The custom dialer requires root permissions which are not required for + // use cases run as non-root. The effective UID is read directly: a passwd + // lookup fails without cgo for a UID that has no entry, as on OpenShift. + if runtime.GOOS == "linux" && os.Geteuid() != 0 { + log.Debug("Not running as root, using standard dialer") + dialer := &net.Dialer{} + return dialer.DialContext(ctx, "tcp", addr) } conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr) diff --git a/client/iface/bind/ice_bind_test.go b/client/iface/bind/ice_bind_test.go index 0b8db7640..8dd82d7df 100644 --- a/client/iface/bind/ice_bind_test.go +++ b/client/iface/bind/ice_bind_test.go @@ -2,17 +2,22 @@ package bind import ( "fmt" + "io" "net" "net/netip" "sync" + "sync/atomic" "testing" "time" + "github.com/pion/transport/v3" "github.com/pion/transport/v3/stdnet" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" + "golang.zx2c4.com/wireguard/device" + "golang.zx2c4.com/wireguard/tun/netstack" "github.com/netbirdio/netbird/client/iface/wgaddr" ) @@ -326,3 +331,86 @@ func listenUDP(t *testing.T, network, addr string) *net.UDPConn { require.NoError(t, err) return conn } + +type countingNet struct { + transport.Net + calls atomic.Int64 + ifaces []*transport.Interface + before func() +} + +func (n *countingNet) Interfaces() ([]*transport.Interface, error) { + n.calls.Add(1) + if n.before != nil { + n.before() + } + return n.ifaces, nil +} + +func TestICEBind_OpenDoesNotDiscoverInterfaces(t *testing.T) { + n := newCountingNet() + iceBind := NewICEBind(n, testAddress(), 1280) + + for i := 0; i < 2; i++ { + _, port, err := iceBind.Open(0) + require.NoError(t, err) + require.NotZero(t, port, "the bind listens on a port") + assert.Equal(t, int64(0), n.calls.Load(), "opening the bind must not enumerate interfaces, attempt %d", i+1) + require.NoError(t, iceBind.Close()) + } + + _, _, err := iceBind.Open(0) + require.NoError(t, err) + t.Cleanup(func() { _ = iceBind.Close() }) + + mux, err := iceBind.GetICEMux() + require.NoError(t, err) + assert.NotEmpty(t, mux.SingleSocketUDPMux.GetListenAddresses(), "listen addresses come from the discovered interfaces") + assert.Equal(t, int64(1), n.calls.Load(), "interfaces are discovered when the listen addresses are asked for") +} + +func TestICEBind_DeviceUpDoesNotWaitOnInterfaceDiscovery(t *testing.T) { + tunDev, _, err := netstack.CreateNetTUN([]netip.Addr{netip.MustParseAddr("100.64.0.1")}, nil, 1280) + require.NoError(t, err) + + var dev *device.Device + n := newCountingNet() + n.before = func() { + _ = dev.IpcGetOperation(io.Discard) + } + iceBind := NewICEBind(n, testAddress(), 1280) + dev = device.NewDevice(tunDev, iceBind, device.NewLogger(device.LogLevelSilent, "")) + + upDone := make(chan error, 1) + go func() { + upDone <- dev.Up() + }() + + select { + case err := <-upDone: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("device Up did not return: interface discovery ran under the device lock and waited on the device itself") + } + t.Cleanup(dev.Close) + + assert.Equal(t, int64(0), n.calls.Load(), "bringing the device up must not enumerate interfaces") + + mux, err := iceBind.GetICEMux() + require.NoError(t, err) + assert.NotEmpty(t, mux.SingleSocketUDPMux.GetListenAddresses(), "discovery that talks to the device succeeds once the device is up") + assert.Equal(t, int64(1), n.calls.Load(), "interfaces are discovered outside the device lock") +} + +func newCountingNet() *countingNet { + ifc := transport.NewInterface(net.Interface{Index: 1, MTU: 1500, Name: "eth0", Flags: net.FlagUp | net.FlagMulticast}) + ifc.AddAddress(&net.IPNet{IP: net.IPv4(192, 168, 1, 10), Mask: net.CIDRMask(24, 32)}) + return &countingNet{ifaces: []*transport.Interface{ifc}} +} + +func testAddress() wgaddr.Address { + return wgaddr.Address{ + IP: netip.MustParseAddr("100.64.0.1"), + Network: netip.MustParsePrefix("100.64.0.0/10"), + } +} diff --git a/client/iface/configurer/allowedips.go b/client/iface/configurer/allowedips.go new file mode 100644 index 000000000..08a766ee0 --- /dev/null +++ b/client/iface/configurer/allowedips.go @@ -0,0 +1,201 @@ +//go:build (linux && !android) || freebsd + +package configurer + +import ( + "net" + "net/netip" + "slices" + "sync" + + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +// allowedIPStore mirrors the allowed IPs configured on each peer of a device. +// +// A configurer is the only writer of its device's peer set, so the mirror is authoritative +// by construction. It spares the paths that have to rewrite one peer's allowed IPs a full +// device dump just to recover prefixes the process already configured itself. +// +// An allowed IP belongs to exactly one peer: configuring a prefix on a peer takes it away +// from whichever peer held it before, and the configurer leaves that handover to the device +// rather than removing the prefix from the previous holder itself. The store tracks the +// owner of each prefix and performs the same handover, so rewriting one peer's list never +// takes a prefix back from the peer that owns it now. +// +// Its own lock guards the map alone, not the device write it accompanies. Consistency +// between the two rests on the caller serializing every configurer call, which WGIface +// does with its mutex; two unserialized writers would interleave a device write with the +// record of a different one. +// +// An operator reconfiguring the device out of band, through `wg set` or the UAPI socket, +// is the one way the mirror can still go stale. A peer missing from it falls back to the +// device, which reseats that peer's prefixes and their ownership; a peer that is present +// does not, so one recorded from empty while the device already held prefixes keeps only +// what was recorded, and the next endpoint removal drops the rest. +type allowedIPStore struct { + mu sync.RWMutex + peers map[wgtypes.Key][]netip.Prefix + owners map[netip.Prefix]wgtypes.Key +} + +func newAllowedIPStore() *allowedIPStore { + return &allowedIPStore{ + peers: make(map[wgtypes.Key][]netip.Prefix), + owners: make(map[netip.Prefix]wgtypes.Key), + } +} + +// get returns the prefixes recorded for a peer, and whether the peer is known at all. +// The caller receives a copy and may retain or modify it freely. +func (s *allowedIPStore) get(key wgtypes.Key) ([]netip.Prefix, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + prefixes, ok := s.peers[key] + if !ok { + return nil, false + } + return slices.Clone(prefixes), true +} + +// set replaces the prefixes recorded for a peer. +func (s *allowedIPStore) set(key wgtypes.Key, prefixes []netip.Prefix) { + s.mu.Lock() + defer s.mu.Unlock() + + k := key + s.releaseLocked(k) + + normalized := normalizePrefixes(prefixes) + for _, prefix := range normalized { + s.claimLocked(k, prefix) + } + s.peers[k] = normalized +} + +// add records prefixes on a peer without dropping the ones already there, matching the +// union semantics of a peer update that does not replace its allowed IPs. It records the +// peer if it is not known yet, so it belongs to the operations that create a peer on the +// device rather than to the update-only ones. +func (s *allowedIPStore) add(key wgtypes.Key, prefixes []netip.Prefix) { + s.mu.Lock() + defer s.mu.Unlock() + + s.mergeLocked(key, prefixes) +} + +// addExisting is add for an update-only device operation. Such an operation is a silent +// no-op when the peer is absent, so recording a peer here would leave the store claiming +// prefixes the device never took, and the peer would then be recreated by the next endpoint +// removal, stealing those allowed IPs from the peer that legitimately holds them. +func (s *allowedIPStore) addExisting(key wgtypes.Key, prefixes []netip.Prefix) { + s.mu.Lock() + defer s.mu.Unlock() + + k := key + if _, ok := s.peers[k]; !ok { + return + } + s.mergeLocked(k, prefixes) +} + +// ensure records a peer with no prefixes unless it is already known. A device operation +// that is not update-only creates the peer when it is absent, so it has to be recorded even +// when it configures nothing else; otherwise the peer exists on the device while the store +// treats it as unknown, and a prefix later handed over to it is not accounted for. +func (s *allowedIPStore) ensure(key wgtypes.Key) { + s.mu.Lock() + defer s.mu.Unlock() + + k := key + if _, ok := s.peers[k]; !ok { + s.peers[k] = nil + } +} + +// forget drops every prefix recorded for a peer. +func (s *allowedIPStore) forget(key wgtypes.Key) { + s.mu.Lock() + defer s.mu.Unlock() + + k := key + s.releaseLocked(k) + delete(s.peers, k) +} + +// reset drops every peer, mirroring a device reconfiguration that replaces the peer set. +func (s *allowedIPStore) reset() { + s.mu.Lock() + defer s.mu.Unlock() + + s.peers = make(map[wgtypes.Key][]netip.Prefix) + s.owners = make(map[netip.Prefix]wgtypes.Key) +} + +// mergeLocked unions normalized prefixes into a peer and transfers their ownership. +// The caller must hold s.mu for writing. +func (s *allowedIPStore) mergeLocked(k wgtypes.Key, prefixes []netip.Prefix) { + merged := s.peers[k] + for _, prefix := range prefixes { + prefix = normalizePrefix(prefix) + s.claimLocked(k, prefix) + if !slices.Contains(merged, prefix) { + merged = append(merged, prefix) + } + } + s.peers[k] = merged +} + +// claimLocked hands a prefix over to a peer, taking it from its previous owner the way the +// device does when the same prefix is configured on a second peer. +func (s *allowedIPStore) claimLocked(k wgtypes.Key, prefix netip.Prefix) { + if owner, ok := s.owners[prefix]; ok && owner != k { + s.peers[owner] = slices.DeleteFunc(s.peers[owner], func(p netip.Prefix) bool { + return p == prefix + }) + } + s.owners[prefix] = k +} + +// releaseLocked drops a peer's claim on every prefix it currently holds. +func (s *allowedIPStore) releaseLocked(k wgtypes.Key) { + for _, prefix := range s.peers[k] { + if s.owners[prefix] == k { + delete(s.owners, prefix) + } + } +} + +// ipNetsToPrefixes converts addresses read back from a device. Unmap keeps a v4-mapped v6 +// address comparable to the plain v4 prefix the configurer was given. +func ipNetsToPrefixes(ipNets []net.IPNet) []netip.Prefix { + prefixes := make([]netip.Prefix, 0, len(ipNets)) + for _, ipNet := range ipNets { + addr, ok := netip.AddrFromSlice(ipNet.IP) + if !ok { + continue + } + + ones, maskBits := ipNet.Mask.Size() + // A device may report a v4 prefix as a v4-mapped address. Align the address form with + // the mask rather than unmapping on sight: a 32 bit mask always describes v4, while a + // 128 bit mask describes v4 only when it covers the mapped prefix, so a genuine v6 + // prefix inside the mapped range stays v6 instead of being dropped as invalid. + if addr.Is4In6() { + switch { + case maskBits == 32: + addr = addr.Unmap() + case maskBits == 128 && ones >= 96: + addr, ones = addr.Unmap(), ones-96 + } + } + + prefix := netip.PrefixFrom(addr, ones) + if !prefix.IsValid() { + continue + } + prefixes = append(prefixes, prefix.Masked()) + } + return prefixes +} diff --git a/client/iface/configurer/allowedips_test.go b/client/iface/configurer/allowedips_test.go new file mode 100644 index 000000000..405609e29 --- /dev/null +++ b/client/iface/configurer/allowedips_test.go @@ -0,0 +1,192 @@ +//go:build (linux && !android) || freebsd + +package configurer + +import ( + "net" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +// The store keys on the parsed key, so the tests use two distinct ones rather than names. +var ( + testPeer = wgtypes.Key{1} + otherPeer = wgtypes.Key{2} +) + +func TestAllowedIPStoreUnknownPeer(t *testing.T) { + s := newAllowedIPStore() + + prefixes, ok := s.get(testPeer) + assert.False(t, ok, "an unconfigured peer must be reported as unknown, not as one without prefixes") + assert.Nil(t, prefixes, "an unknown peer has no prefixes") +} + +func TestAllowedIPStoreAddUnions(t *testing.T) { + s := newAllowedIPStore() + overlay := netip.MustParsePrefix("100.64.0.1/32") + routed := netip.MustParsePrefix("10.20.0.0/16") + + s.set(testPeer, []netip.Prefix{overlay}) + // A peer update does not replace allowed IPs, and a repeated prefix must not be doubled. + s.add(testPeer, []netip.Prefix{overlay, routed}) + + prefixes, ok := s.get(testPeer) + require.True(t, ok, "peer must be known after set") + assert.Equal(t, []netip.Prefix{overlay, routed}, prefixes, "add must union rather than replace") +} + +func TestAllowedIPStoreGetReturnsCopy(t *testing.T) { + s := newAllowedIPStore() + overlay := netip.MustParsePrefix("100.64.0.1/32") + s.set(testPeer, []netip.Prefix{overlay}) + + prefixes, ok := s.get(testPeer) + require.True(t, ok, "peer must be known after set") + prefixes[0] = netip.MustParsePrefix("0.0.0.0/0") + + stored, _ := s.get(testPeer) + assert.Equal(t, []netip.Prefix{overlay}, stored, "a caller mutating the returned slice must not corrupt the store") +} + +func TestAllowedIPStoreForgetAndReset(t *testing.T) { + s := newAllowedIPStore() + s.set(testPeer, []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}) + s.set(otherPeer, []netip.Prefix{netip.MustParsePrefix("100.64.0.2/32")}) + + s.forget(testPeer) + _, ok := s.get(testPeer) + assert.False(t, ok, "a forgotten peer must be unknown") + _, ok = s.get(otherPeer) + assert.True(t, ok, "forgetting one peer must not touch the others") + + s.reset() + _, ok = s.get(otherPeer) + assert.False(t, ok, "reset must drop every peer") +} + +func TestIPNetsToPrefixes(t *testing.T) { + tests := []struct { + name string + ipNet net.IPNet + want string + }{ + { + name: "v4", + ipNet: net.IPNet{IP: net.IP{10, 20, 0, 0}, Mask: net.CIDRMask(16, 32)}, + want: "10.20.0.0/16", + }, + { + name: "v4 mapped under a 128 bit mask", + ipNet: net.IPNet{IP: net.ParseIP("10.20.0.0"), Mask: net.CIDRMask(112, 128)}, + want: "10.20.0.0/16", + }, + { + name: "v6", + ipNet: net.IPNet{IP: net.ParseIP("fd00::"), Mask: net.CIDRMask(64, 128)}, + want: "fd00::/64", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := ipNetsToPrefixes([]net.IPNet{tc.ipNet}) + require.Len(t, got, 1, "the address must be converted, not dropped") + assert.Equal(t, tc.want, got[0].String(), "converted prefix") + }) + } +} + +func TestIPNetsToPrefixesRoundTrip(t *testing.T) { + prefixes := []netip.Prefix{ + netip.MustParsePrefix("100.64.0.1/32"), + netip.MustParsePrefix("10.20.0.0/16"), + netip.MustParsePrefix("fd00::/64"), + } + + assert.Equal(t, prefixes, ipNetsToPrefixes(prefixesToIPNets(prefixes)), + "prefixes handed to a device must come back unchanged") +} + +func TestAllowedIPStoreNormalizesMappedPrefixes(t *testing.T) { + s := newAllowedIPStore() + v4 := netip.MustParsePrefix("10.20.0.0/16") + mapped := netip.PrefixFrom(netip.AddrFrom16(v4.Addr().As16()), 112) + + s.set(testPeer, []netip.Prefix{mapped}) + // A v4 rule only matches a v4-mapped address once it has been unmapped, so the store must + // hold the plain form and recognise the two spellings as the same prefix. + s.add(testPeer, []netip.Prefix{v4}) + + prefixes, ok := s.get(testPeer) + require.True(t, ok, "peer must be known after set") + assert.Equal(t, []netip.Prefix{v4}, prefixes, "a mapped prefix must be stored unmapped and not duplicated") +} + +func TestAllowedIPStoreAddExistingDoesNotCreate(t *testing.T) { + s := newAllowedIPStore() + routed := netip.MustParsePrefix("10.20.0.0/16") + + // An update-only device operation on an absent peer is a silent no-op, so nothing may be + // recorded for a peer the store does not already know. + s.addExisting(testPeer, []netip.Prefix{routed}) + _, ok := s.get(testPeer) + assert.False(t, ok, "addExisting must not record an unknown peer") + + overlay := netip.MustParsePrefix("100.64.0.1/32") + s.set(testPeer, []netip.Prefix{overlay}) + s.addExisting(testPeer, []netip.Prefix{routed}) + + prefixes, _ := s.get(testPeer) + assert.Equal(t, []netip.Prefix{overlay, routed}, prefixes, "addExisting must union onto a known peer") +} + +func TestAllowedIPStoreHandsPrefixOverToTheNewOwner(t *testing.T) { + s := newAllowedIPStore() + routed := netip.MustParsePrefix("10.20.0.0/16") + other := otherPeer + + s.set(testPeer, []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32"), routed}) + s.set(other, []netip.Prefix{netip.MustParsePrefix("100.64.0.2/32")}) + + // The device takes an allowed IP away from its previous holder when it is configured on + // another peer, so the store must do the same rather than list it under both. + s.addExisting(other, []netip.Prefix{routed}) + + previous, _ := s.get(testPeer) + assert.NotContains(t, previous, routed, "the previous owner must lose the prefix") + current, _ := s.get(other) + assert.Contains(t, current, routed, "the new owner must hold the prefix") +} + +func TestAllowedIPStoreForgetReleasesOwnership(t *testing.T) { + s := newAllowedIPStore() + routed := netip.MustParsePrefix("10.20.0.0/16") + + s.set(testPeer, []netip.Prefix{routed}) + s.forget(testPeer) + s.set(otherPeer, []netip.Prefix{routed}) + + // A forgotten peer must not be resurrected as a key in the peer map by a later claim. + _, ok := s.get(testPeer) + assert.False(t, ok, "the forgotten peer must stay unknown") + current, _ := s.get(otherPeer) + assert.Equal(t, []netip.Prefix{routed}, current, "the new owner must hold the prefix") +} + +func TestIPNetsToPrefixesKeepsV6InTheMappedRange(t *testing.T) { + // ::ffff:0:0/64 reads as v4-mapped but is a genuine v6 prefix: unmapping it would leave a + // v4 address under a 64 bit mask, which is invalid, and the allowed IP would be dropped. + got := ipNetsToPrefixes([]net.IPNet{{ + IP: net.ParseIP("::ffff:0:0"), + Mask: net.CIDRMask(64, 128), + }}) + + require.Len(t, got, 1, "the prefix must be converted, not dropped") + assert.False(t, got[0].Addr().Is4(), "a v6 prefix in the mapped range must not become v4") + assert.Equal(t, 64, got[0].Bits(), "the prefix length must survive the conversion") +} diff --git a/client/iface/configurer/common.go b/client/iface/configurer/common.go index 10162d703..02d2a7e6a 100644 --- a/client/iface/configurer/common.go +++ b/client/iface/configurer/common.go @@ -3,29 +3,48 @@ package configurer import ( "net" "net/netip" - - "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) -// buildPresharedKeyConfig creates a wgtypes.Config for setting a preshared key on a peer. -// This is a shared helper used by both kernel and userspace configurers. -func buildPresharedKeyConfig(peerKey wgtypes.Key, psk wgtypes.Key, updateOnly bool) wgtypes.Config { - return wgtypes.Config{ - Peers: []wgtypes.PeerConfig{{ - PublicKey: peerKey, - PresharedKey: &psk, - UpdateOnly: updateOnly, - }}, - } -} - +// prefixesToIPNets converts prefixes on their way to a device. It is the only place that +// conversion happens, so it also normalizes: the device is then given the same form the +// store records, and a v4-mapped prefix cannot reach net.IPNet, which prints such an +// address as v4 while taking the length from its 16 byte mask and so turns +// ::ffff:10.1.2.3/64 into 10.1.2.3/0 — an allowed IP matching every v4 address. func prefixesToIPNets(prefixes []netip.Prefix) []net.IPNet { ipNets := make([]net.IPNet, len(prefixes)) for i, prefix := range prefixes { + normalized := normalizePrefix(prefix) ipNets[i] = net.IPNet{ - IP: prefix.Addr().AsSlice(), // Convert netip.Addr to net.IP - Mask: net.CIDRMask(prefix.Bits(), prefix.Addr().BitLen()), // Create subnet mask + IP: normalized.Addr().AsSlice(), + Mask: net.CIDRMask(normalized.Bits(), normalized.Addr().BitLen()), } } return ipNets } + +// normalizePrefix puts a prefix into the form the store recognises it by. It clears the +// host bits, which a device does on its own, so a caller passing 10.20.0.1/16 still matches +// the 10.20.0.0/16 read back from the device; and it unmaps a v4-mapped prefix so that it +// compares equal to, and marshals like, the plain v4 prefix for the same network. +// +// Masking comes first because it also decides the address family: only a prefix at least 96 +// bits long keeps the mapped marker through the mask, so a shorter prefix inside the mapped +// range is a genuine v6 prefix and unmapping it would yield an invalid v4 prefix. +func normalizePrefix(prefix netip.Prefix) netip.Prefix { + masked := prefix.Masked() + + addr := masked.Addr() + if !addr.Is4In6() { + return masked + } + return netip.PrefixFrom(addr.Unmap(), masked.Bits()-96) +} + +// normalizePrefixes returns a normalized copy without changing the caller's slice. +func normalizePrefixes(prefixes []netip.Prefix) []netip.Prefix { + normalized := make([]netip.Prefix, len(prefixes)) + for i, prefix := range prefixes { + normalized[i] = normalizePrefix(prefix) + } + return normalized +} diff --git a/client/iface/configurer/common_test.go b/client/iface/configurer/common_test.go new file mode 100644 index 000000000..487c1a734 --- /dev/null +++ b/client/iface/configurer/common_test.go @@ -0,0 +1,81 @@ +package configurer + +import ( + "net" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizePrefix(t *testing.T) { + v4 := netip.MustParsePrefix("10.20.0.0/16") + v6 := netip.MustParsePrefix("fd00::/64") + + assert.Equal(t, v4, normalizePrefix(v4), "a plain v4 prefix is unchanged") + assert.Equal(t, v6, normalizePrefix(v6), "a real v6 prefix is unchanged") + assert.Equal(t, v4, normalizePrefix(netip.PrefixFrom(netip.AddrFrom16(v4.Addr().As16()), 112)), + "a mapped prefix under a 128 bit mask becomes plain v4") + // A prefix shorter than /96 inside the mapped range is a genuine v6 prefix. Unmapping it + // would pair a v4 address with a v6 sized mask, which is invalid, and the store would then + // record a zero prefix that can never recreate the allowed IP. + for _, tc := range []string{"::ffff:0:0/64", "::ffff:1.2.3.4/80", "::ffff:1.2.3.4/95"} { + got := normalizePrefix(netip.MustParsePrefix(tc)) + assert.True(t, got.IsValid(), "%s must normalize to a valid prefix", tc) + assert.False(t, got.Addr().Is4(), "%s must stay v6", tc) + } +} + +func TestNormalizePrefixClearsHostBits(t *testing.T) { + // A device stores a prefix masked, so a caller passing host bits must still match what a + // device fallback seeded, otherwise that prefix could never be removed by value. + assert.Equal(t, netip.MustParsePrefix("10.20.0.0/16"), + normalizePrefix(netip.MustParsePrefix("10.20.0.1/16")), "host bits must be cleared") + assert.Equal(t, netip.MustParsePrefix("fd00::/64"), + normalizePrefix(netip.MustParsePrefix("fd00::1/64")), "host bits must be cleared for v6") +} + +func TestPrefixesToIPNetsNormalizes(t *testing.T) { + // net.IPNet prints a v4-mapped address as v4 but takes the length from its 16 byte + // mask, so an unnormalized ::ffff:10.1.2.3/64 reaches a userspace device as 10.1.2.3/0, + // an allowed IP that matches every v4 address. + tests := []struct { + name string + given string + want string + }{ + {name: "mapped below /96", given: "::ffff:10.1.2.3/64", want: "::/64"}, + {name: "mapped at /112", given: "::ffff:10.1.2.3/112", want: "10.1.0.0/16"}, + {name: "host bits are cleared", given: "10.20.0.1/16", want: "10.20.0.0/16"}, + {name: "v6 is untouched", given: "fd00::1/64", want: "fd00::/64"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := prefixesToIPNets([]netip.Prefix{netip.MustParsePrefix(tc.given)}) + require.Len(t, got, 1, "the prefix must be converted, not dropped") + assert.Equal(t, tc.want, got[0].String(), "what the device is given") + assert.NotEqual(t, 0, mustOnes(t, got[0]), "a device must never be given a zero length allowed IP") + }) + } +} + +func TestPrefixesToIPNetsAgreesWithTheStore(t *testing.T) { + for _, given := range []string{"::ffff:10.1.2.3/64", "::ffff:10.1.2.3/112", "10.20.0.1/16", "fd00::1/64"} { + prefix := netip.MustParsePrefix(given) + + toDevice := prefixesToIPNets([]netip.Prefix{prefix}) + recorded := normalizePrefix(prefix) + + assert.Equal(t, recorded.String(), toDevice[0].String(), + "%s must reach the device in the form the store records", given) + } +} + +func mustOnes(t *testing.T, ipNet net.IPNet) int { + t.Helper() + + ones, _ := ipNet.Mask.Size() + return ones +} diff --git a/client/iface/configurer/kernel_unix.go b/client/iface/configurer/kernel_unix.go index da69c2a35..afe1300d5 100644 --- a/client/iface/configurer/kernel_unix.go +++ b/client/iface/configurer/kernel_unix.go @@ -3,9 +3,12 @@ package configurer import ( + "errors" "fmt" "net" "net/netip" + "slices" + "sync" "time" log "github.com/sirupsen/logrus" @@ -15,19 +18,45 @@ import ( "github.com/netbirdio/netbird/monotime" ) -type KernelConfigurer struct { - deviceName string - statsCache *statsCache +var errConfigurerClosed = errors.New("configurer closed") + +type wgClient interface { + Device(name string) (*wgtypes.Device, error) + ConfigureDevice(name string, cfg wgtypes.Config) error + Close() error } +type KernelConfigurer struct { + deviceName string + statsCache *statsCache + allowedIPs *allowedIPStore + newWgctrlFn func() (wgClient, error) + + clientMu sync.Mutex + client wgClient + closed bool +} + +// NewKernelConfigurer creates a configurer with an empty allowed IP mirror +// and a statistics cache for the named kernel device. func NewKernelConfigurer(deviceName string) *KernelConfigurer { + return newKernelConfigurer(deviceName, func() (wgClient, error) { + return wgctrl.New() + }) +} + +func newKernelConfigurer(deviceName string, newWgctrlFn func() (wgClient, error)) *KernelConfigurer { c := &KernelConfigurer{ - deviceName: deviceName, + deviceName: deviceName, + allowedIPs: newAllowedIPStore(), + newWgctrlFn: newWgctrlFn, } c.statsCache = newStatsCache(statsCacheTTL, c.fetchStats) return c } +// ConfigureInterface sets the device key, port and firewall mark, replacing all peers. +// The allowed IP mirror is reset only after the device accepts the configuration. func (c *KernelConfigurer) ConfigureInterface(privateKey string, port int) error { log.Debugf("adding Wireguard private key") key, err := wgtypes.ParseKey(privateKey) @@ -46,6 +75,8 @@ func (c *KernelConfigurer) ConfigureInterface(privateKey string, port int) error if err != nil { return fmt.Errorf(`received error "%w" while configuring interface %s with port %d`, err, c.deviceName, port) } + + c.allowedIPs.reset() return nil } @@ -58,9 +89,20 @@ func (c *KernelConfigurer) SetPresharedKey(peerKey string, psk wgtypes.Key, upda } cfg := buildPresharedKeyConfig(parsedPeerKey, psk, updateOnly) - return c.configure(cfg) + if err := c.configure(cfg); err != nil { + return err + } + + // Without updateOnly this creates the peer when it is absent, so the store has to + // know about it even though no allowed IP was configured. + if !updateOnly { + c.allowedIPs.ensure(parsedPeerKey) + } + return nil } +// UpdatePeer creates or updates a peer, merging allowed IPs with its existing set. +// Prefixes assigned to this peer are transferred from their previous owners. func (c *KernelConfigurer) UpdatePeer(peerKey string, allowedIps []netip.Prefix, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { @@ -83,19 +125,23 @@ func (c *KernelConfigurer) UpdatePeer(peerKey string, allowedIps []netip.Prefix, if err != nil { return fmt.Errorf(`received error "%w" while updating peer on interface %s with settings: allowed ips %s, endpoint %s`, err, c.deviceName, allowedIps, endpoint.String()) } + + c.allowedIPs.add(peerKeyParsed, allowedIps) return nil } +// RemoveEndpointAddress clears the endpoint of a peer while keeping it configured. +// Neither the netlink API nor the userspace one can clear an endpoint in place, so the peer +// is removed and re-added with the allowed IPs it already had. func (c *KernelConfigurer) RemoveEndpointAddress(peerKey string) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return err } - // Get the existing peer to preserve its allowed IPs - existingPeer, err := c.getPeer(c.deviceName, peerKey) + allowedIPs, err := c.peerAllowedIPs(peerKeyParsed) if err != nil { - return fmt.Errorf("get peer: %w", err) + return err } removePeerCfg := wgtypes.PeerConfig{ @@ -104,26 +150,27 @@ func (c *KernelConfigurer) RemoveEndpointAddress(peerKey string) error { } if err := c.configure(wgtypes.Config{Peers: []wgtypes.PeerConfig{removePeerCfg}}); err != nil { - return fmt.Errorf(`error removing peer %s from interface %s: %w`, peerKey, c.deviceName, err) + return fmt.Errorf("remove peer %s from interface %s: %w", peerKey, c.deviceName, err) } - //Re-add the peer without the endpoint but same AllowedIPs reAddPeerCfg := wgtypes.PeerConfig{ PublicKey: peerKeyParsed, - AllowedIPs: existingPeer.AllowedIPs, + AllowedIPs: prefixesToIPNets(allowedIPs), ReplaceAllowedIPs: true, } if err := c.configure(wgtypes.Config{Peers: []wgtypes.PeerConfig{reAddPeerCfg}}); err != nil { + c.allowedIPs.forget(peerKeyParsed) return fmt.Errorf( - `error re-adding peer %s to interface %s with allowed IPs %v: %w`, - peerKey, c.deviceName, existingPeer.AllowedIPs, err, + "re-add peer %s to interface %s with allowed IPs %v: %w", + peerKey, c.deviceName, allowedIPs, err, ) } return nil } +// RemovePeer removes a peer and forgets its allowed IPs after a successful device write. func (c *KernelConfigurer) RemovePeer(peerKey string) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { @@ -142,15 +189,13 @@ func (c *KernelConfigurer) RemovePeer(peerKey string) error { if err != nil { return fmt.Errorf(`received error "%w" while removing peer %s from interface %s`, err, peerKey, c.deviceName) } + + c.allowedIPs.forget(peerKeyParsed) return nil } +// AddAllowedIP adds a prefix to an existing peer; an absent peer is a silent no-op. func (c *KernelConfigurer) AddAllowedIP(peerKey string, allowedIP netip.Prefix) error { - ipNet := net.IPNet{ - IP: allowedIP.Addr().AsSlice(), - Mask: net.CIDRMask(allowedIP.Bits(), allowedIP.Addr().BitLen()), - } - peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return err @@ -159,7 +204,7 @@ func (c *KernelConfigurer) AddAllowedIP(peerKey string, allowedIP netip.Prefix) PublicKey: peerKeyParsed, UpdateOnly: true, ReplaceAllowedIPs: false, - AllowedIPs: []net.IPNet{ipNet}, + AllowedIPs: prefixesToIPNets([]netip.Prefix{allowedIP}), } config := wgtypes.Config{ @@ -169,69 +214,75 @@ func (c *KernelConfigurer) AddAllowedIP(peerKey string, allowedIP netip.Prefix) if err != nil { return fmt.Errorf(`received error "%w" while adding allowed Ip to peer on interface %s with settings: allowed ips %s`, err, c.deviceName, allowedIP) } + + c.allowedIPs.addExisting(peerKeyParsed, []netip.Prefix{allowedIP}) return nil } +// RemoveAllowedIP removes a prefix while preserving the peer's other allowed IPs. +// A prefix not assigned to the peer is a no-op. func (c *KernelConfigurer) RemoveAllowedIP(peerKey string, allowedIP netip.Prefix) error { - ipNet := net.IPNet{ - IP: allowedIP.Addr().AsSlice(), - Mask: net.CIDRMask(allowedIP.Bits(), allowedIP.Addr().BitLen()), - } - peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return fmt.Errorf("parse peer key: %w", err) } - existingPeer, err := c.getPeer(c.deviceName, peerKey) + currentAllowedIPs, err := c.peerAllowedIPs(peerKeyParsed) if err != nil { - return fmt.Errorf("get peer: %w", err) + return err } - newAllowedIPs := existingPeer.AllowedIPs - - for i, existingAllowedIP := range existingPeer.AllowedIPs { - if existingAllowedIP.String() == ipNet.String() { - newAllowedIPs = append(existingPeer.AllowedIPs[:i], existingPeer.AllowedIPs[i+1:]...) //nolint:gocritic - break - } + idx := slices.Index(currentAllowedIPs, normalizePrefix(allowedIP)) + if idx < 0 { + return nil } + newAllowedIPs := slices.Delete(currentAllowedIPs, idx, idx+1) peer := wgtypes.PeerConfig{ PublicKey: peerKeyParsed, UpdateOnly: true, ReplaceAllowedIPs: true, - AllowedIPs: newAllowedIPs, + AllowedIPs: prefixesToIPNets(newAllowedIPs), } config := wgtypes.Config{ Peers: []wgtypes.PeerConfig{peer}, } - err = c.configure(config) - if err != nil { + if err := c.configure(config); err != nil { return fmt.Errorf("remove allowed IP %s on interface %s: %w", allowedIP, c.deviceName, err) } + + c.allowedIPs.set(peerKeyParsed, newAllowedIPs) return nil } -func (c *KernelConfigurer) getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) { - wg, err := wgctrl.New() - if err != nil { - return wgtypes.Peer{}, fmt.Errorf("wgctl: %w", err) +// peerAllowedIPs returns the allowed IPs configured for a peer, reading them from the device +// only for a peer the store has not seen. Dumping the device costs a netlink round trip +// proportional to the whole network map, and this runs on every relay and ICE transition. +func (c *KernelConfigurer) peerAllowedIPs(peerKey wgtypes.Key) ([]netip.Prefix, error) { + if prefixes, ok := c.allowedIPs.get(peerKey); ok { + return prefixes, nil } - defer func() { - err = wg.Close() - if err != nil { - log.Errorf("Got error while closing wgctl: %v", err) - } - }() - wgDevice, err := wg.Device(ifaceName) + existingPeer, err := c.getPeer(peerKey) if err != nil { - return wgtypes.Peer{}, fmt.Errorf("get device %s: %w", ifaceName, err) + return nil, fmt.Errorf("get peer: %w", err) + } + + prefixes := ipNetsToPrefixes(existingPeer.AllowedIPs) + c.allowedIPs.set(peerKey, prefixes) + return prefixes, nil +} + +// getPeer scans the device for one peer. wgtypes.Key is an array, so the comparison is a +// plain equality: Key.String would base64 encode into a fresh allocation for every peer. +func (c *KernelConfigurer) getPeer(peerPubKey wgtypes.Key) (wgtypes.Peer, error) { + wgDevice, err := c.device() + if err != nil { + return wgtypes.Peer{}, err } for _, peer := range wgDevice.Peers { - if peer.PublicKey.String() == peerPubKey { + if peer.PublicKey == peerPubKey { return peer, nil } } @@ -239,37 +290,73 @@ func (c *KernelConfigurer) getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, } func (c *KernelConfigurer) configure(config wgtypes.Config) error { - wg, err := wgctrl.New() + c.clientMu.Lock() + defer c.clientMu.Unlock() + + client, err := c.clientLocked() if err != nil { return err } - defer func() { - if err := wg.Close(); err != nil { - log.Errorf("Failed to close wgctrl client: %v", err) - } - }() - - return wg.ConfigureDevice(c.deviceName, config) + if err := client.ConfigureDevice(c.deviceName, config); err != nil { + c.discardLocked() + return err + } + return nil } -func (c *KernelConfigurer) Close() { +func (c *KernelConfigurer) device() (*wgtypes.Device, error) { + c.clientMu.Lock() + defer c.clientMu.Unlock() + + client, err := c.clientLocked() + if err != nil { + return nil, err + } + wgDevice, err := client.Device(c.deviceName) + if err != nil { + c.discardLocked() + return nil, fmt.Errorf("get device %s: %w", c.deviceName, err) + } + return wgDevice, nil } -func (c *KernelConfigurer) FullStats() (*Stats, error) { - wg, err := wgctrl.New() +func (c *KernelConfigurer) clientLocked() (wgClient, error) { + if c.closed { + return nil, errConfigurerClosed + } + if c.client != nil { + return c.client, nil + } + client, err := c.newWgctrlFn() if err != nil { return nil, fmt.Errorf("wgctl: %w", err) } - defer func() { - err = wg.Close() - if err != nil { - log.Errorf("Got error while closing wgctl: %v", err) - } - }() + c.client = client + return client, nil +} - wgDevice, err := wg.Device(c.deviceName) +func (c *KernelConfigurer) discardLocked() { + if c.client == nil { + return + } + if err := c.client.Close(); err != nil { + log.Warnf("failed to close wgctrl client: %v", err) + } + c.client = nil +} + +func (c *KernelConfigurer) Close() { + c.clientMu.Lock() + defer c.clientMu.Unlock() + + c.closed = true + c.discardLocked() +} + +func (c *KernelConfigurer) FullStats() (*Stats, error) { + wgDevice, err := c.device() if err != nil { - return nil, fmt.Errorf("get device %s: %w", c.deviceName, err) + return nil, err } fullStats := &Stats{ DeviceName: wgDevice.Name, @@ -305,23 +392,12 @@ func (c *KernelConfigurer) LastActivities() map[string]monotime.Time { } func (c *KernelConfigurer) fetchStats() (map[string]WGStats, error) { + wgDevice, err := c.device() + if err != nil { + return nil, err + } + stats := make(map[string]WGStats) - wg, err := wgctrl.New() - if err != nil { - return nil, fmt.Errorf("wgctl: %w", err) - } - defer func() { - err = wg.Close() - if err != nil { - log.Errorf("Got error while closing wgctl: %v", err) - } - }() - - wgDevice, err := wg.Device(c.deviceName) - if err != nil { - return nil, fmt.Errorf("get device %s: %w", c.deviceName, err) - } - for _, peer := range wgDevice.Peers { stats[peer.PublicKey.String()] = WGStats{ LastHandshake: peer.LastHandshakeTime, @@ -331,3 +407,13 @@ func (c *KernelConfigurer) fetchStats() (map[string]WGStats, error) { } return stats, nil } + +func buildPresharedKeyConfig(peerKey wgtypes.Key, psk wgtypes.Key, updateOnly bool) wgtypes.Config { + return wgtypes.Config{ + Peers: []wgtypes.PeerConfig{{ + PublicKey: peerKey, + PresharedKey: &psk, + UpdateOnly: updateOnly, + }}, + } +} diff --git a/client/iface/configurer/kernel_unix_test.go b/client/iface/configurer/kernel_unix_test.go new file mode 100644 index 000000000..8daa15a25 --- /dev/null +++ b/client/iface/configurer/kernel_unix_test.go @@ -0,0 +1,167 @@ +//go:build (linux && !android) || freebsd + +package configurer + +import ( + "errors" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +type fakeWGClient struct { + configureErr error + deviceErr error + calls int + closed bool +} + +type fakeWGClientFactory struct { + openErr error + clients []*fakeWGClient +} + +func (f *fakeWGClient) Device(string) (*wgtypes.Device, error) { + f.calls++ + if f.deviceErr != nil { + return nil, f.deviceErr + } + return &wgtypes.Device{}, nil +} + +func (f *fakeWGClient) ConfigureDevice(string, wgtypes.Config) error { + f.calls++ + return f.configureErr +} + +func (f *fakeWGClient) Close() error { + f.closed = true + return nil +} + +func (f *fakeWGClientFactory) open() (wgClient, error) { + if f.openErr != nil { + return nil, f.openErr + } + client := &fakeWGClient{} + f.clients = append(f.clients, client) + return client, nil +} + +func TestKernelConfigurer_ReusesClientAcrossRequests(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + require.NoError(t, c.RemovePeer(peerKey)) + require.NoError(t, c.RemovePeer(peerKey)) + _, err := c.FullStats() + require.NoError(t, err) + + require.Len(t, factory.clients, 1, "all requests should share one client") + assert.Equal(t, 3, factory.clients[0].calls, "every request should go through the shared client") + assert.False(t, factory.clients[0].closed, "a healthy client should stay open") +} + +func TestKernelConfigurer_ReopensClientAfterFailedConfigure(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + require.NoError(t, c.RemovePeer(peerKey)) + require.Len(t, factory.clients, 1) + + factory.clients[0].configureErr = errors.New("netlink down") + require.Error(t, c.RemovePeer(peerKey)) + assert.True(t, factory.clients[0].closed, "the failed client should be closed") + assert.Len(t, factory.clients, 1, "a failure must not open a client on its own") + + require.NoError(t, c.RemovePeer(peerKey)) + require.Len(t, factory.clients, 2, "the next request should open a fresh client") + assert.Equal(t, 1, factory.clients[1].calls, "the request should run on the fresh client") + assert.False(t, factory.clients[1].closed, "the fresh client should stay open") +} + +func TestKernelConfigurer_ReopensClientAfterFailedDeviceQuery(t *testing.T) { + c, factory := newFakeKernelConfigurer() + + _, err := c.FullStats() + require.NoError(t, err) + require.Len(t, factory.clients, 1) + + factory.clients[0].deviceErr = errors.New("netlink down") + _, err = c.FullStats() + require.Error(t, err) + assert.True(t, factory.clients[0].closed, "the failed client should be closed") + + _, err = c.FullStats() + require.NoError(t, err) + assert.Len(t, factory.clients, 2, "the next request should open a fresh client") +} + +func TestKernelConfigurer_RetriesOpenOnNextRequest(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + factory.openErr = errors.New("no netlink") + require.Error(t, c.RemovePeer(peerKey)) + assert.Empty(t, factory.clients, "a failed open leaves no client behind") + + factory.openErr = nil + require.NoError(t, c.RemovePeer(peerKey)) + assert.Len(t, factory.clients, 1, "the next request should open the client") +} + +func TestKernelConfigurer_CloseRejectsFurtherRequests(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + require.NoError(t, c.RemovePeer(peerKey)) + require.Len(t, factory.clients, 1) + + c.Close() + c.Close() + assert.True(t, factory.clients[0].closed, "Close should release the client") + + require.ErrorIs(t, c.RemovePeer(peerKey), errConfigurerClosed) + _, err := c.FullStats() + require.ErrorIs(t, err, errConfigurerClosed) + assert.Len(t, factory.clients, 1, "a closed configurer must not open a client") +} + +func TestKernelConfigurer_ConcurrentRequestsShareOneClient(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + const workers = 16 + errs := make(chan error, workers) + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + errs <- c.RemovePeer(peerKey) + }() + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + + require.Len(t, factory.clients, 1, "concurrent requests should share one client") + assert.Equal(t, workers, factory.clients[0].calls, "every request should reach the shared client") +} + +func newFakeKernelConfigurer() (*KernelConfigurer, *fakeWGClientFactory) { + factory := &fakeWGClientFactory{} + return newKernelConfigurer("wt0", factory.open), factory +} + +func testPeerKey(t *testing.T) string { + t.Helper() + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + return key.PublicKey().String() +} diff --git a/client/iface/configurer/usp.go b/client/iface/configurer/usp.go index 2be1b861e..1b0965f0d 100644 --- a/client/iface/configurer/usp.go +++ b/client/iface/configurer/usp.go @@ -1,18 +1,16 @@ package configurer import ( - "encoding/base64" - "encoding/hex" "fmt" + "math" "net" "net/netip" "os" "runtime" - "strconv" - "strings" "time" log "github.com/sirupsen/logrus" + wgconn "golang.zx2c4.com/wireguard/conn" "golang.zx2c4.com/wireguard/device" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" @@ -21,19 +19,6 @@ import ( "github.com/netbirdio/netbird/monotime" ) -const ( - privateKey = "private_key" - ipcKeyLastHandshakeTimeSec = "last_handshake_time_sec" - ipcKeyTxBytes = "tx_bytes" - ipcKeyRxBytes = "rx_bytes" - allowedIP = "allowed_ip" - endpoint = "endpoint" - fwmark = "fwmark" - listenPort = "listen_port" - publicKey = "public_key" - presharedKey = "preshared_key" -) - var ErrAllowedIPNotFound = fmt.Errorf("allowed IP not found") type WGUSPConfigurer struct { @@ -45,17 +30,14 @@ type WGUSPConfigurer struct { uapiListener net.Listener } +// NewUSPConfigurer creates a userspace configurer and starts its UAPI listener. func NewUSPConfigurer(device *device.Device, deviceName string, activityRecorder *bind.ActivityRecorder) *WGUSPConfigurer { - wgCfg := &WGUSPConfigurer{ - device: device, - deviceName: deviceName, - activityRecorder: activityRecorder, - } - wgCfg.statsCache = newStatsCache(statsCacheTTL, wgCfg.fetchStats) + wgCfg := NewUSPConfigurerNoUAPI(device, deviceName, activityRecorder) wgCfg.startUAPI() return wgCfg } +// NewUSPConfigurerNoUAPI creates a userspace configurer without a UAPI listener. func NewUSPConfigurerNoUAPI(device *device.Device, deviceName string, activityRecorder *bind.ActivityRecorder) *WGUSPConfigurer { wgCfg := &WGUSPConfigurer{ device: device, @@ -66,21 +48,28 @@ func NewUSPConfigurerNoUAPI(device *device.Device, deviceName string, activityRe return wgCfg } +// ConfigureInterface sets the device key, port and firewall mark, replacing all peers. func (c *WGUSPConfigurer) ConfigureInterface(privateKey string, port int) error { log.Debugf("adding Wireguard private key") key, err := wgtypes.ParseKey(privateKey) if err != nil { return err } - fwmark := getFwmark() - config := wgtypes.Config{ - PrivateKey: &key, - ReplacePeers: true, - FirewallMark: &fwmark, - ListenPort: &port, + if port < 0 || port > math.MaxUint16 { + return fmt.Errorf("invalid listen port %d", port) } - return c.device.IpcSet(toWgUserspaceString(config)) + if err := c.device.SetPrivateKey(device.NoisePrivateKey(key)); err != nil { + return fmt.Errorf("set private key: %w", err) + } + if err := c.device.SetListenPort(uint16(port)); err != nil { + return fmt.Errorf("set listen port %d: %w", port, err) + } + c.device.RemoveAllPeers() + if err := c.device.BindSetMark(uint32(getFwmark())); err != nil { + return fmt.Errorf("set fwmark: %w", err) + } + return nil } // SetPresharedKey sets the preshared key for a peer. @@ -91,231 +80,178 @@ func (c *WGUSPConfigurer) SetPresharedKey(peerKey string, psk wgtypes.Key, updat return err } - cfg := buildPresharedKeyConfig(parsedPeerKey, psk, updateOnly) - return c.device.IpcSet(toWgUserspaceString(cfg)) + presharedKey := device.NoisePresharedKey(psk) + return c.device.ConfigurePeer(device.NoisePublicKey(parsedPeerKey), device.PeerConfig{ + PresharedKey: &presharedKey, + UpdateOnly: updateOnly, + }) } +// UpdatePeer creates or updates a peer, merging allowed IPs with its existing set. func (c *WGUSPConfigurer) UpdatePeer(peerKey string, allowedIps []netip.Prefix, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return err } - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - ReplaceAllowedIPs: false, - // don't replace allowed ips, wg will handle duplicated peer IP - AllowedIPs: prefixesToIPNets(allowedIps), - PersistentKeepaliveInterval: &keepAlive, - PresharedKey: preSharedKey, - Endpoint: endpoint, + + prefixes, err := validPrefixes(allowedIps) + if err != nil { + return err } - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, + cfg := device.PeerConfig{ + PersistentKeepalive: &keepAlive, + AllowedIPs: prefixes, } - if ipcErr := c.device.IpcSet(toWgUserspaceString(config)); ipcErr != nil { - return ipcErr + var addrPort netip.AddrPort + if endpoint != nil { + addr, ok := netip.AddrFromSlice(endpoint.IP) + if !ok { + return fmt.Errorf("parse endpoint address %v", endpoint.IP) + } + addr = addr.Unmap() + addrPort = netip.AddrPortFrom(addr, uint16(endpoint.Port)) + cfg.Endpoint = &bind.Endpoint{AddrPort: netip.AddrPortFrom(addr.WithZone(endpoint.Zone), addrPort.Port())} + } + + if preSharedKey != nil { + psk := device.NoisePresharedKey(*preSharedKey) + cfg.PresharedKey = &psk + } + + if err := c.device.ConfigurePeer(device.NoisePublicKey(peerKeyParsed), cfg); err != nil { + return fmt.Errorf("configure peer: %w", err) } if endpoint != nil { - addr, err := netip.ParseAddr(endpoint.IP.String()) - if err != nil { - return fmt.Errorf("failed to parse endpoint address: %w", err) - } - addrPort := netip.AddrPortFrom(addr.Unmap(), uint16(endpoint.Port)) c.activityRecorder.UpsertAddress(peerKey, addrPort) } return nil } +// RemoveEndpointAddress clears the endpoint of a peer while keeping it configured. +// The session is reset and the keepalive is switched off until the next endpoint arrives. func (c *WGUSPConfigurer) RemoveEndpointAddress(peerKey string) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return fmt.Errorf("parse peer key: %w", err) } - ipcStr, err := c.device.IpcGet() - if err != nil { - return fmt.Errorf("get IPC config: %w", err) + peer := c.device.LookupPeer(device.NoisePublicKey(peerKeyParsed)) + if peer == nil { + return ErrPeerNotFound } - // Parse current status to get allowed IPs for the peer - stats, err := parseStatus(c.deviceName, ipcStr) - if err != nil { - return fmt.Errorf("parse IPC config: %w", err) - } + peer.ClearEndpoint() + peer.ZeroAndFlushAll() - var allowedIPs []net.IPNet - found := false - for _, peer := range stats.Peers { - if peer.PublicKey == peerKey { - allowedIPs = peer.AllowedIPs - found = true - break - } + var noKeepAlive time.Duration + if err := c.device.ConfigurePeer(device.NoisePublicKey(peerKeyParsed), device.PeerConfig{ + PersistentKeepalive: &noKeepAlive, + UpdateOnly: true, + }); err != nil { + return fmt.Errorf("disable keepalive: %w", err) } - if !found { - return fmt.Errorf("peer %s not found", peerKey) - } - - // remove the peer from the WireGuard configuration - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - Remove: true, - } - - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, - } - if ipcErr := c.device.IpcSet(toWgUserspaceString(config)); ipcErr != nil { - return fmt.Errorf("failed to remove peer: %s", ipcErr) - } - - // Build the peer config - peer = wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - ReplaceAllowedIPs: true, - AllowedIPs: allowedIPs, - } - - config = wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, - } - - if err := c.device.IpcSet(toWgUserspaceString(config)); err != nil { - return fmt.Errorf("remove endpoint address: %w", err) - } - return nil } +// RemovePeer removes a peer and clears its activity record. func (c *WGUSPConfigurer) RemovePeer(peerKey string) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return err } - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - Remove: true, - } - - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, - } - ipcErr := c.device.IpcSet(toWgUserspaceString(config)) - + c.device.RemovePeer(device.NoisePublicKey(peerKeyParsed)) c.activityRecorder.Remove(peerKey) - return ipcErr + return nil } +// AddAllowedIP adds a prefix to an existing peer; an absent peer is a silent no-op. func (c *WGUSPConfigurer) AddAllowedIP(peerKey string, allowedIP netip.Prefix) error { - ipNet := net.IPNet{ - IP: allowedIP.Addr().AsSlice(), - Mask: net.CIDRMask(allowedIP.Bits(), allowedIP.Addr().BitLen()), - } - peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return err } - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - UpdateOnly: true, - ReplaceAllowedIPs: false, - AllowedIPs: []net.IPNet{ipNet}, + if !allowedIP.IsValid() { + return fmt.Errorf("invalid allowed IP %v", allowedIP) } - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, + peer := c.device.LookupPeer(device.NoisePublicKey(peerKeyParsed)) + if peer == nil { + return nil } - return c.device.IpcSet(toWgUserspaceString(config)) + peer.AddAllowedIP(normalizePrefix(allowedIP)) + return nil } +// RemoveAllowedIP removes a prefix while preserving the peer's other allowed IPs. +// It returns ErrAllowedIPNotFound if the prefix is not assigned to the peer. func (c *WGUSPConfigurer) RemoveAllowedIP(peerKey string, allowedIP netip.Prefix) error { - ipc, err := c.device.IpcGet() - if err != nil { - return err - } - peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { - return err + return fmt.Errorf("parse peer key: %w", err) } - hexKey := hex.EncodeToString(peerKeyParsed[:]) - - lines := strings.Split(ipc, "\n") - - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - UpdateOnly: true, - ReplaceAllowedIPs: true, - AllowedIPs: []net.IPNet{}, - } - - foundPeer := false - removedAllowedIP := false - ip := allowedIP.String() - - for _, line := range lines { - line = strings.TrimSpace(line) - - // If we're within the details of the found peer and encounter another public key, - // this means we're starting another peer's details. So, reset the flag. - if strings.HasPrefix(line, "public_key=") && foundPeer { - foundPeer = false - } - - // Identify the peer with the specific public key - if line == fmt.Sprintf("public_key=%s", hexKey) { - foundPeer = true - } - - // If we're within the details of the found peer and find the specific allowed IP, skip this line - if foundPeer && line == "allowed_ip="+ip { - removedAllowedIP = true - continue - } - - // Append the line to the output string - if foundPeer && strings.HasPrefix(line, "allowed_ip=") { - allowedIPStr := strings.TrimPrefix(line, "allowed_ip=") - _, ipNet, err := net.ParseCIDR(allowedIPStr) - if err != nil { - return err - } - peer.AllowedIPs = append(peer.AllowedIPs, *ipNet) - } - } - - if !removedAllowedIP { + if !allowedIP.IsValid() { return ErrAllowedIPNotFound } - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, + + peer := c.device.LookupPeer(device.NoisePublicKey(peerKeyParsed)) + if peer == nil { + return ErrPeerNotFound } - return c.device.IpcSet(toWgUserspaceString(config)) + + if !peer.RemoveAllowedIP(normalizePrefix(allowedIP)) { + return ErrAllowedIPNotFound + } + return nil } func (c *WGUSPConfigurer) FullStats() (*Stats, error) { - ipcStr, err := c.device.IpcGet() - if err != nil { - return nil, fmt.Errorf("IpcGet failed: %w", err) + peers := c.device.Peers() + stats := &Stats{ + DeviceName: c.deviceName, + PublicKey: wgtypes.Key(c.device.PublicKey()).String(), + ListenPort: int(c.device.ListenPort()), + FWMark: int(c.device.FirewallMark()), + Peers: make([]Peer, 0, len(peers)), } - - return parseStatus(c.deviceName, ipcStr) + for _, peer := range peers { + stats.Peers = append(stats.Peers, peerStats(peer)) + } + return stats, nil } func (c *WGUSPConfigurer) LastActivities() map[string]monotime.Time { return c.activityRecorder.GetLastActivities() } +func (c *WGUSPConfigurer) GetStats() (map[string]WGStats, error) { + return c.statsCache.get() +} + +func (c *WGUSPConfigurer) Close() { + if c.uapiListener != nil { + err := c.uapiListener.Close() + if err != nil { + log.Errorf("failed to close uapi listener: %v", err) + } + } + + if runtime.GOOS == "linux" { + sockPath := "/var/run/wireguard/" + c.deviceName + ".sock" + if _, statErr := os.Stat(sockPath); statErr == nil { + _ = os.Remove(sockPath) + } + } +} + // startUAPI starts the UAPI listener for managing the WireGuard interface via external tool -func (t *WGUSPConfigurer) startUAPI() { +func (c *WGUSPConfigurer) startUAPI() { var err error - t.uapiListener, err = openUAPI(t.deviceName) + c.uapiListener, err = openUAPI(c.deviceName) if err != nil { log.Errorf("failed to open uapi listener: %v", err) return @@ -329,175 +265,61 @@ func (t *WGUSPConfigurer) startUAPI() { return } go func() { - t.device.IpcHandle(uapiConn) + c.device.IpcHandle(uapiConn) }() } - }(t.uapiListener) + }(c.uapiListener) } -func (t *WGUSPConfigurer) Close() { - if t.uapiListener != nil { - err := t.uapiListener.Close() - if err != nil { - log.Errorf("failed to close uapi listener: %v", err) +func (c *WGUSPConfigurer) fetchStats() (map[string]WGStats, error) { + peers := c.device.Peers() + stats := make(map[string]WGStats, len(peers)) + for _, peer := range peers { + stats[wgtypes.Key(peer.PublicKey()).String()] = WGStats{ + LastHandshake: peer.LastHandshake(), + TxBytes: int64(peer.TxBytes()), + RxBytes: int64(peer.RxBytes()), } } - - if runtime.GOOS == "linux" { - sockPath := "/var/run/wireguard/" + t.deviceName + ".sock" - if _, statErr := os.Stat(sockPath); statErr == nil { - _ = os.Remove(sockPath) - } - } -} - -func (t *WGUSPConfigurer) GetStats() (map[string]WGStats, error) { - return t.statsCache.get() -} - -func (t *WGUSPConfigurer) fetchStats() (map[string]WGStats, error) { - ipc, err := t.device.IpcGet() - if err != nil { - return nil, fmt.Errorf("ipc get: %w", err) - } - - return parseTransfers(ipc) -} - -func parseTransfers(ipc string) (map[string]WGStats, error) { - stats := make(map[string]WGStats) - var ( - currentKey string - currentStats WGStats - hasPeer bool - ) - lines := strings.Split(ipc, "\n") - for _, line := range lines { - line = strings.TrimSpace(line) - - // If we're within the details of the found peer and encounter another public key, - // this means we're starting another peer's details. So, stop. - if strings.HasPrefix(line, "public_key=") { - peerID := strings.TrimPrefix(line, "public_key=") - h, err := hex.DecodeString(peerID) - if err != nil { - return nil, fmt.Errorf("decode peerID: %w", err) - } - currentKey = base64.StdEncoding.EncodeToString(h) - currentStats = WGStats{} // Reset stats for the new peer - hasPeer = true - stats[currentKey] = currentStats - continue - } - - if !hasPeer { - continue - } - - key := strings.SplitN(line, "=", 2) - if len(key) != 2 { - continue - } - switch key[0] { - case ipcKeyLastHandshakeTimeSec: - hs, err := toLastHandshake(key[1]) - if err != nil { - return nil, err - } - currentStats.LastHandshake = hs - stats[currentKey] = currentStats - case ipcKeyRxBytes: - rxBytes, err := toBytes(key[1]) - if err != nil { - return nil, fmt.Errorf("parse rx_bytes: %w", err) - } - currentStats.RxBytes = rxBytes - stats[currentKey] = currentStats - case ipcKeyTxBytes: - TxBytes, err := toBytes(key[1]) - if err != nil { - return nil, fmt.Errorf("parse tx_bytes: %w", err) - } - currentStats.TxBytes = TxBytes - stats[currentKey] = currentStats - } - } - return stats, nil } -func toWgUserspaceString(wgCfg wgtypes.Config) string { - var sb strings.Builder - if wgCfg.PrivateKey != nil { - hexKey := hex.EncodeToString(wgCfg.PrivateKey[:]) - sb.WriteString(fmt.Sprintf("private_key=%s\n", hexKey)) +func peerStats(peer *device.Peer) Peer { + stats := Peer{ + PublicKey: wgtypes.Key(peer.PublicKey()).String(), + AllowedIPs: prefixesToIPNets(peer.AllowedIPs()), + TxBytes: int64(peer.TxBytes()), + RxBytes: int64(peer.RxBytes()), + LastHandshake: peer.LastHandshake(), + PresharedKey: [32]byte(peer.PresharedKey()), } - - if wgCfg.ListenPort != nil { - sb.WriteString(fmt.Sprintf("listen_port=%d\n", *wgCfg.ListenPort)) + if addrPort, ok := endpointAddrPort(peer.Endpoint()); ok { + stats.Endpoint = net.UDPAddr{IP: addrPort.Addr().AsSlice(), Port: int(addrPort.Port())} } - - if wgCfg.ReplacePeers { - sb.WriteString("replace_peers=true\n") - } - - if wgCfg.FirewallMark != nil { - sb.WriteString(fmt.Sprintf("fwmark=%d\n", *wgCfg.FirewallMark)) - } - - for _, p := range wgCfg.Peers { - hexKey := hex.EncodeToString(p.PublicKey[:]) - sb.WriteString(fmt.Sprintf("public_key=%s\n", hexKey)) - - if p.Remove { - sb.WriteString("remove=true\n") - } - - if p.UpdateOnly { - sb.WriteString("update_only=true\n") - } - - if p.PresharedKey != nil { - preSharedHexKey := hex.EncodeToString(p.PresharedKey[:]) - sb.WriteString(fmt.Sprintf("preshared_key=%s\n", preSharedHexKey)) - } - - if p.Endpoint != nil { - sb.WriteString(fmt.Sprintf("endpoint=%s\n", p.Endpoint.String())) - } - - if p.PersistentKeepaliveInterval != nil { - sb.WriteString(fmt.Sprintf("persistent_keepalive_interval=%d\n", int(p.PersistentKeepaliveInterval.Seconds()))) - } - - if p.ReplaceAllowedIPs { - sb.WriteString("replace_allowed_ips=true\n") - } - - for _, aip := range p.AllowedIPs { - sb.WriteString(fmt.Sprintf("allowed_ip=%s\n", aip.String())) - } - } - return sb.String() + return stats } -func toLastHandshake(stringVar string) (time.Time, error) { - sec, err := strconv.ParseInt(stringVar, 10, 64) +func endpointAddrPort(endpoint wgconn.Endpoint) (netip.AddrPort, bool) { + if endpoint == nil { + return netip.AddrPort{}, false + } + if ep, ok := endpoint.(*bind.Endpoint); ok { + return ep.AddrPort, true + } + addrPort, err := netip.ParseAddrPort(endpoint.DstToString()) if err != nil { - return time.Time{}, fmt.Errorf("parse handshake sec: %w", err) + return netip.AddrPort{}, false } - - // If sec is 0 (Unix epoch), return zero time instead - // This indicates no handshake has occurred - if sec == 0 { - return time.Time{}, nil - } - - return time.Unix(sec, 0), nil + return addrPort, true } -func toBytes(s string) (int64, error) { - return strconv.ParseInt(s, 10, 64) +func validPrefixes(prefixes []netip.Prefix) ([]netip.Prefix, error) { + for _, prefix := range prefixes { + if !prefix.IsValid() { + return nil, fmt.Errorf("invalid allowed IP %v", prefix) + } + } + return normalizePrefixes(prefixes), nil } func getFwmark() int { @@ -506,138 +328,3 @@ func getFwmark() int { } return 0 } - -func hexToWireguardKey(hexKey string) (wgtypes.Key, error) { - // Decode hex string to bytes - keyBytes, err := hex.DecodeString(hexKey) - if err != nil { - return wgtypes.Key{}, fmt.Errorf("failed to decode hex key: %w", err) - } - - // Check if we have the right number of bytes (WireGuard keys are 32 bytes) - if len(keyBytes) != 32 { - return wgtypes.Key{}, fmt.Errorf("invalid key length: expected 32 bytes, got %d", len(keyBytes)) - } - - // Convert to wgtypes.Key - var key wgtypes.Key - copy(key[:], keyBytes) - - return key, nil -} - -func parseStatus(deviceName, ipcStr string) (*Stats, error) { - stats := &Stats{DeviceName: deviceName} - var currentPeer *Peer - for _, line := range strings.Split(strings.TrimSpace(ipcStr), "\n") { - if line == "" { - continue - } - parts := strings.SplitN(line, "=", 2) - if len(parts) != 2 { - continue - } - key := parts[0] - val := parts[1] - - switch key { - case privateKey: - key, err := hexToWireguardKey(val) - if err != nil { - log.Errorf("failed to parse private key: %v", err) - continue - } - stats.PublicKey = key.PublicKey().String() - case publicKey: - // Save previous peer - if currentPeer != nil { - stats.Peers = append(stats.Peers, *currentPeer) - } - key, err := hexToWireguardKey(val) - if err != nil { - log.Errorf("failed to parse public key: %v", err) - continue - } - currentPeer = &Peer{ - PublicKey: key.String(), - } - case listenPort: - if port, err := strconv.Atoi(val); err == nil { - stats.ListenPort = port - } - case fwmark: - if fwmark, err := strconv.Atoi(val); err == nil { - stats.FWMark = fwmark - } - case endpoint: - if currentPeer == nil { - continue - } - - host, portStr, err := net.SplitHostPort(val) - if err != nil { - log.Errorf("failed to parse endpoint: %v", err) - continue - } - port, err := strconv.Atoi(portStr) - if err != nil { - log.Errorf("failed to parse endpoint port: %v", err) - continue - } - currentPeer.Endpoint = net.UDPAddr{ - IP: net.ParseIP(host), - Port: port, - } - case allowedIP: - if currentPeer == nil { - continue - } - _, ipnet, err := net.ParseCIDR(val) - if err == nil { - currentPeer.AllowedIPs = append(currentPeer.AllowedIPs, *ipnet) - } - case ipcKeyTxBytes: - if currentPeer == nil { - continue - } - rxBytes, err := toBytes(val) - if err != nil { - continue - } - currentPeer.TxBytes = rxBytes - case ipcKeyRxBytes: - if currentPeer == nil { - continue - } - rxBytes, err := toBytes(val) - if err != nil { - continue - } - currentPeer.RxBytes = rxBytes - - case ipcKeyLastHandshakeTimeSec: - if currentPeer == nil { - continue - } - - ts, err := toLastHandshake(val) - if err != nil { - continue - } - currentPeer.LastHandshake = ts - case presharedKey: - if currentPeer == nil { - continue - } - if val != "" && val != "0000000000000000000000000000000000000000000000000000000000000000" { - if pskKey, err := hexToWireguardKey(val); err == nil { - currentPeer.PresharedKey = [32]byte(pskKey) - } - } - } - } - if currentPeer != nil { - stats.Peers = append(stats.Peers, *currentPeer) - } - return stats, nil -} diff --git a/client/iface/configurer/usp_allowedips_test.go b/client/iface/configurer/usp_allowedips_test.go new file mode 100644 index 000000000..effb360e9 --- /dev/null +++ b/client/iface/configurer/usp_allowedips_test.go @@ -0,0 +1,252 @@ +package configurer + +import ( + "net" + "net/netip" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + wgconn "golang.zx2c4.com/wireguard/conn" + wgdevice "golang.zx2c4.com/wireguard/device" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/client/iface/bind" +) + +// newTestUSPConfigurer builds a configurer over a real wireguard-go device backed by an +// in-memory TUN. The device stays down, so no socket is opened and no privileges are needed. +func newTestUSPConfigurer(t *testing.T) *WGUSPConfigurer { + t.Helper() + + dev := wgdevice.NewDevice(newIdleTUN(), wgconn.NewDefaultBind(), wgdevice.NewLogger(wgdevice.LogLevelSilent, "")) + t.Cleanup(dev.Close) + + c := NewUSPConfigurerNoUAPI(dev, "wgtest0", bind.NewActivityRecorder()) + + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate device private key") + require.NoError(t, c.ConfigureInterface(key.String(), 0), "configure test device") + + return c +} + +// seedPeers adds count peers, each with a /32 overlay address, and returns their public keys. +func seedPeers(t *testing.T, c *WGUSPConfigurer, count int) []string { + t.Helper() + + keys := make([]string, 0, count) + for i := 0; i < count; i++ { + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + pub := priv.PublicKey().String() + + addr := netip.PrefixFrom(netip.AddrFrom4([4]byte{100, 64, byte(i >> 8), byte(i)}), 32) + require.NoError(t, c.UpdatePeer(pub, []netip.Prefix{addr}, 25*time.Second, nil, nil), "add peer") + keys = append(keys, pub) + } + return keys +} + +func peerAllowedIPs(t *testing.T, c *WGUSPConfigurer, peerKey string) []string { + t.Helper() + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + + for _, p := range stats.Peers { + if p.PublicKey != peerKey { + continue + } + got := make([]string, 0, len(p.AllowedIPs)) + for _, ipNet := range p.AllowedIPs { + got = append(got, ipNet.String()) + } + return got + } + t.Fatalf("peer %s not found on device", peerKey) + return nil +} + +// TestRemoveEndpointAddressPreservesRoutedAllowedIPs covers the prefixes the route manager +// attaches to a routing peer through AddAllowedIP. Those are not known to the peer.Conn that +// triggers the endpoint removal, so dropping them here would silently blackhole every route +// behind that peer on each relay or ICE disconnect. +func TestRemoveEndpointAddressPreservesRoutedAllowedIPs(t *testing.T) { + c := newTestUSPConfigurer(t) + peerKey := seedPeers(t, c, 3)[1] + + routed := []netip.Prefix{ + netip.MustParsePrefix("10.20.0.0/16"), + netip.MustParsePrefix("192.168.7.0/24"), + } + for _, prefix := range routed { + require.NoError(t, c.AddAllowedIP(peerKey, prefix), "add routed prefix") + } + + before := peerAllowedIPs(t, c, peerKey) + require.Len(t, before, 3, "peer should hold its overlay address plus both routed prefixes") + + require.NoError(t, c.RemoveEndpointAddress(peerKey), "remove endpoint address") + + assert.ElementsMatch(t, before, peerAllowedIPs(t, c, peerKey), + "allowed IPs must survive the endpoint removal unchanged") +} + +// TestRemoveEndpointAddressDoesNotScaleWithPeerCount is the regression guard for the actual +// defect: clearing one peer's endpoint used to dump and parse the whole device, so its cost +// grew with the size of the network map. On a routing peer with thousands of peers that dump +// runs on every relay and ICE transition, under the interface lock. +func TestRemoveEndpointAddressDoesNotScaleWithPeerCount(t *testing.T) { + measure := func(peerCount int) float64 { + c := newTestUSPConfigurer(t) + peerKey := seedPeers(t, c, peerCount)[peerCount/2] + + return testing.AllocsPerRun(5, func() { + require.NoError(t, c.RemoveEndpointAddress(peerKey), "remove endpoint address") + }) + } + + small := measure(64) + large := measure(1024) + + assert.Less(t, large, small*2, + "clearing one endpoint allocated %.0f objects with 1024 peers against %.0f with 64: the cost still scales with the peer count", + large, small) +} + +func TestRemoveAllowedIPKeepsTheOtherPrefixes(t *testing.T) { + c := newTestUSPConfigurer(t) + peerKey := seedPeers(t, c, 3)[0] + routed := netip.MustParsePrefix("10.20.0.0/16") + require.NoError(t, c.AddAllowedIP(peerKey, routed), "add routed prefix") + require.NoError(t, c.AddAllowedIP(peerKey, netip.MustParsePrefix("192.168.7.0/24")), "add routed prefix") + + require.NoError(t, c.RemoveAllowedIP(peerKey, routed), "remove routed prefix") + + assert.ElementsMatch(t, []string{"100.64.0.0/32", "192.168.7.0/24"}, peerAllowedIPs(t, c, peerKey), + "only the removed prefix should be gone") + + assert.ErrorIs(t, c.RemoveAllowedIP(peerKey, routed), ErrAllowedIPNotFound, + "removing a prefix that is no longer configured must be reported") +} + +// TestAddAllowedIPOnAbsentPeerDoesNotResurrectIt covers the lazy connection window documented +// in #6863: AddAllowedIP is update-only, a silent no-op when the peer is absent, and clearing +// the endpoint of an absent peer must not create one either. +func TestAddAllowedIPOnAbsentPeerDoesNotResurrectIt(t *testing.T) { + c := newTestUSPConfigurer(t) + seedPeers(t, c, 2) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + absent := priv.PublicKey().String() + + require.NoError(t, c.AddAllowedIP(absent, netip.MustParsePrefix("10.20.0.0/16")), + "update-only add on an absent peer is a silent no-op") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + require.Len(t, stats.Peers, 2, "the absent peer must not have been created by AddAllowedIP") + + assert.ErrorIs(t, c.RemoveEndpointAddress(absent), ErrPeerNotFound, + "clearing the endpoint of a peer the device does not have must fail") + + stats, err = c.FullStats() + require.NoError(t, err, "read device stats") + assert.Len(t, stats.Peers, 2, "no peer may be created while clearing an endpoint") +} + +// TestRemoveEndpointAddressDoesNotStealAPrefixFromAnotherPeer covers WireGuard's rule that an +// allowed IP belongs to exactly one peer: configuring a prefix on a peer takes it away from +// whichever peer held it before. UpdatePeer relies on that rule rather than removing the prefix +// from the previous holder itself, so a prefix handed over between peers must not come back. +func TestRemoveEndpointAddressDoesNotStealAPrefixFromAnotherPeer(t *testing.T) { + c := newTestUSPConfigurer(t) + keys := seedPeers(t, c, 2) + peerA, peerB := keys[0], keys[1] + routed := netip.MustParsePrefix("10.20.0.0/16") + + require.NoError(t, c.AddAllowedIP(peerA, routed), "give the prefix to A") + require.Contains(t, peerAllowedIPs(t, c, peerA), routed.String(), "A must hold the prefix") + + // The route moves to B. The device takes it away from A on its own. + require.NoError(t, c.AddAllowedIP(peerB, routed), "hand the prefix over to B") + require.Contains(t, peerAllowedIPs(t, c, peerB), routed.String(), "B must hold the prefix") + require.NotContains(t, peerAllowedIPs(t, c, peerA), routed.String(), "the device must have taken it from A") + + require.NoError(t, c.RemoveEndpointAddress(peerA), "clear A's endpoint") + + assert.NotContains(t, peerAllowedIPs(t, c, peerA), routed.String(), + "clearing A's endpoint must not take the prefix back from B") + assert.Contains(t, peerAllowedIPs(t, c, peerB), routed.String(), + "B must still hold the prefix") +} + +// TestPresharedKeyCreatedPeerTakesPartInPrefixHandover covers a peer created by a preshared +// key write rather than by a peer update. Rosenpass applies a peer's first key without +// updateOnly, which creates the peer on the device, so a store that ignored that operation +// would treat the peer as unknown and would not account for a prefix later handed over to it. +func TestPresharedKeyCreatedPeerTakesPartInPrefixHandover(t *testing.T) { + c := newTestUSPConfigurer(t) + peerA := seedPeers(t, c, 1)[0] + routed := netip.MustParsePrefix("10.20.0.0/16") + require.NoError(t, c.AddAllowedIP(peerA, routed), "give the prefix to A") + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerB := priv.PublicKey().String() + + psk, err := wgtypes.GenerateKey() + require.NoError(t, err, "generate preshared key") + require.NoError(t, c.SetPresharedKey(peerB, psk, false), "a first key creates the peer") + + require.NoError(t, c.AddAllowedIP(peerB, routed), "hand the prefix over to B") + require.Contains(t, peerAllowedIPs(t, c, peerB), routed.String(), "B must hold the prefix") + + require.NoError(t, c.RemoveEndpointAddress(peerA), "clear A's endpoint") + + assert.NotContains(t, peerAllowedIPs(t, c, peerA), routed.String(), + "clearing A's endpoint must not take the prefix back from B") + assert.Contains(t, peerAllowedIPs(t, c, peerB), routed.String(), "B must still hold the prefix") +} + +// TestUpdatePeerDoesNotWidenAMappedPrefixOnTheDevice is the end to end form of the +// conversion: a v4-mapped prefix must not reach the device as a zero length allowed IP, +// which would route every v4 address to that peer. +func TestUpdatePeerDoesNotWidenAMappedPrefixOnTheDevice(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + + mapped := netip.MustParsePrefix("::ffff:10.1.2.3/112") + require.NoError(t, c.UpdatePeer(peerKey, []netip.Prefix{mapped}, 25*time.Second, nil, nil), "add peer") + + onDevice := peerAllowedIPs(t, c, peerKey) + assert.NotContains(t, onDevice, "0.0.0.0/0", "the device must not be given a catch-all allowed IP") + assert.Equal(t, []string{"10.1.0.0/16"}, onDevice, "the device holds the normalized prefix") +} + +// TestUpdatePeerWithAnUnusableEndpointTouchesNothing pins the ordering: the endpoint is +// parsed before the device is configured, so a failure cannot leave a half-configured peer +// on the device. +func TestUpdatePeerWithAnUnusableEndpointTouchesNothing(t *testing.T) { + c := newTestUSPConfigurer(t) + seedPeers(t, c, 2) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + + // A three byte address has no textual form netip can parse back. + endpoint := &net.UDPAddr{IP: net.IP{1, 2, 3}, Port: 51820} + require.Error(t, c.UpdatePeer(peerKey, []netip.Prefix{netip.MustParsePrefix("10.30.0.0/16")}, + 25*time.Second, endpoint, nil), "an unusable endpoint must fail the update") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + assert.Len(t, stats.Peers, 2, "the peer must not have reached the device") +} diff --git a/client/iface/configurer/usp_test.go b/client/iface/configurer/usp_test.go index e32491c54..48b5ad1f0 100644 --- a/client/iface/configurer/usp_test.go +++ b/client/iface/configurer/usp_test.go @@ -2,88 +2,222 @@ package configurer import ( "encoding/hex" + "net" + "net/netip" + "os" + "strings" "testing" + "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + wgdevice "golang.zx2c4.com/wireguard/device" + "golang.zx2c4.com/wireguard/tun" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) -var ipcFixture = ` -private_key=e84b5a6d2717c1003a13b431570353dbaca9146cf150c5f8575680feba52027a -listen_port=12912 -public_key=b85996fecc9c7f1fc6d2572a76eda11d59bcd20be8e543b15ce4bd85a8e75a33 -preshared_key=188515093e952f5f22e865cef3012e72f8b5f0b598ac0309d5dacce3b70fcf52 -allowed_ip=192.168.4.4/32 -endpoint=[abcd:23::33%2]:51820 -public_key=58402e695ba1772b1cc9309755f043251ea77fdcf10fbe63989ceb7e19321376 -tx_bytes=38333 -rx_bytes=2224 -allowed_ip=192.168.4.6/32 -persistent_keepalive_interval=111 -endpoint=182.122.22.19:3233 -public_key=662e14fd594556f522604703340351258903b64f35553763f19426ab2a515c58 -endpoint=5.152.198.39:51820 -allowed_ip=192.168.4.10/32 -allowed_ip=192.168.4.11/32 -tx_bytes=1212111 -rx_bytes=1929999999 -protocol_version=1 -errno=0 +// idleTUN never reports an up event, so the wireguard-go device stays down: no socket is +// opened, no peer goroutine starts, and the configuration state is all the tests observe. +type idleTUN struct { + events chan tun.Event + closed chan struct{} +} -` +func newIdleTUN() *idleTUN { + return &idleTUN{events: make(chan tun.Event), closed: make(chan struct{})} +} -func Test_parseTransfers(t *testing.T) { - tests := []struct { - name string - peerKey string - want WGStats - }{ - { - name: "single", - peerKey: "b85996fecc9c7f1fc6d2572a76eda11d59bcd20be8e543b15ce4bd85a8e75a33", - want: WGStats{ - TxBytes: 0, - RxBytes: 0, - }, - }, - { - name: "multiple", - peerKey: "58402e695ba1772b1cc9309755f043251ea77fdcf10fbe63989ceb7e19321376", - want: WGStats{ - TxBytes: 38333, - RxBytes: 2224, - }, - }, - { - name: "lastpeer", - peerKey: "662e14fd594556f522604703340351258903b64f35553763f19426ab2a515c58", - want: WGStats{ - TxBytes: 1212111, - RxBytes: 1929999999, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - res, err := hex.DecodeString(tt.peerKey) - require.NoError(t, err) +func (t *idleTUN) File() *os.File { return nil } - key, err := wgtypes.NewKey(res) - require.NoError(t, err) +func (t *idleTUN) Read([][]byte, []int, int) (int, error) { + <-t.closed + return 0, os.ErrClosed +} - stats, err := parseTransfers(ipcFixture) - if err != nil { - require.NoError(t, err) - return - } +func (t *idleTUN) Write(bufs [][]byte, _ int) (int, error) { return len(bufs), nil } - stat, ok := stats[key.String()] - if !ok { - require.True(t, ok) - return - } +func (t *idleTUN) MTU() (int, error) { return 1420, nil } - require.Equal(t, tt.want, stat) - }) +func (t *idleTUN) Name() (string, error) { return "idle", nil } + +func (t *idleTUN) Events() <-chan tun.Event { return t.events } + +func (t *idleTUN) Close() error { + close(t.closed) + close(t.events) + return nil +} + +func (t *idleTUN) BatchSize() int { return 1 } + +func TestConfigureInterfaceAppliesKeyAndPortAndReplacesPeers(t *testing.T) { + c := newTestUSPConfigurer(t) + seedPeers(t, c, 2) + + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate device private key") + require.NoError(t, c.ConfigureInterface(key.String(), 51820), "reconfigure the interface") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + assert.Equal(t, key.PublicKey().String(), stats.PublicKey, "device public key") + assert.Equal(t, 51820, stats.ListenPort, "listen port") + assert.Empty(t, stats.Peers, "configuring the interface replaces every peer") +} + +func TestConfigureInterfaceRejectsAnInvalidPort(t *testing.T) { + c := newTestUSPConfigurer(t) + + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate device private key") + assert.Error(t, c.ConfigureInterface(key.String(), 70000), "a port above 65535 must be rejected") +} + +func TestGetStatsIsKeyedByPeerKey(t *testing.T) { + c := newTestUSPConfigurer(t) + keys := seedPeers(t, c, 3) + + stats, err := c.GetStats() + require.NoError(t, err, "read stats") + require.Len(t, stats, 3, "one entry per peer") + for _, key := range keys { + entry, ok := stats[key] + require.True(t, ok, "peer %s must be reported", key) + assert.Zero(t, entry.TxBytes, "no traffic was sent") + assert.Zero(t, entry.RxBytes, "no traffic was received") + assert.True(t, entry.LastHandshake.IsZero(), "no handshake happened") } } + +func TestFullStatsMatchesTheDeviceDump(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + psk, err := wgtypes.GenerateKey() + require.NoError(t, err, "generate preshared key") + endpoint := &net.UDPAddr{IP: net.IPv4(192, 0, 2, 10), Port: 51820} + prefixes := []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32"), netip.MustParsePrefix("10.20.0.0/16")} + require.NoError(t, c.UpdatePeer(peerKey, prefixes, 25*time.Second, endpoint, &psk), "add peer") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + require.Len(t, stats.Peers, 1, "one peer on the device") + peer := stats.Peers[0] + assert.Equal(t, peerKey, peer.PublicKey, "peer key") + assert.Equal(t, "192.0.2.10:51820", peer.Endpoint.String(), "endpoint") + assert.Equal(t, [32]byte(psk), peer.PresharedKey, "preshared key") + assert.Equal(t, []string{"100.64.0.1/32", "10.20.0.0/16"}, ipNetStrings(peer.AllowedIPs), "allowed IPs") + + block := uapiPeerBlock(t, c, peerKey) + assert.Contains(t, block, "endpoint=192.0.2.10:51820", "the dump must show the endpoint") + assert.Contains(t, block, "preshared_key="+hex.EncodeToString(psk[:]), "the dump must show the preshared key") + assert.Contains(t, block, "persistent_keepalive_interval=25", "the dump must show the keepalive") + assert.Contains(t, block, "allowed_ip=100.64.0.1/32", "the dump must show the first prefix") + assert.Contains(t, block, "allowed_ip=10.20.0.0/16", "the dump must show the second prefix") +} + +func TestUpdatePeerKeepsTheEndpointZone(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + // The idle device opens no socket, so the zone name does not have to exist on the host. + endpoint := &net.UDPAddr{IP: net.ParseIP("fe80::1"), Port: 51820, Zone: "eth0"} + prefixes := []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")} + require.NoError(t, c.UpdatePeer(peerKey, prefixes, 25*time.Second, endpoint, nil), "add peer") + + assert.Contains(t, uapiPeerBlock(t, c, peerKey), "endpoint=[fe80::1%eth0]:51820", + "a link-local endpoint must keep its zone") +} + +func TestRemoveEndpointAddressKeepsThePeerAndStopsTheKeepalive(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + endpoint := &net.UDPAddr{IP: net.IPv4(192, 0, 2, 10), Port: 51820} + prefixes := []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")} + require.NoError(t, c.UpdatePeer(peerKey, prefixes, 25*time.Second, endpoint, nil), "add peer") + before := c.device.LookupPeer(wgdevice.NoisePublicKey(priv.PublicKey())) + require.NotNil(t, before, "the peer must be on the device") + + require.NoError(t, c.RemoveEndpointAddress(peerKey), "remove endpoint address") + + assert.Same(t, before, c.device.LookupPeer(wgdevice.NoisePublicKey(priv.PublicKey())), + "the peer object must survive the endpoint removal") + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + require.Len(t, stats.Peers, 1, "the peer stays configured") + assert.Nil(t, stats.Peers[0].Endpoint.IP, "the endpoint must be gone") + assert.Equal(t, []string{"100.64.0.1/32"}, ipNetStrings(stats.Peers[0].AllowedIPs), "allowed IPs survive") + assert.Contains(t, uapiPeerBlock(t, c, peerKey), "persistent_keepalive_interval=0", + "an endpoint-less peer must not keep sending keepalives") + + require.NoError(t, c.UpdatePeer(peerKey, prefixes, 25*time.Second, endpoint, nil), "restore the endpoint") + block := uapiPeerBlock(t, c, peerKey) + assert.Contains(t, block, "endpoint=192.0.2.10:51820", "the endpoint must be back") + assert.Contains(t, block, "persistent_keepalive_interval=25", "the keepalive must be back") +} + +func TestUpdatePeerRejectsAnInvalidPrefix(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + + require.Error(t, c.UpdatePeer(peerKey, []netip.Prefix{{}}, 25*time.Second, nil, nil), "an invalid prefix must be rejected") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + assert.Empty(t, stats.Peers, "the peer must not have reached the device") +} + +func TestRemoveAllowedIPOnAbsentPeerReportsThePeer(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + + assert.ErrorIs(t, c.RemoveAllowedIP(priv.PublicKey().String(), netip.MustParsePrefix("10.20.0.0/16")), ErrPeerNotFound, + "removing a prefix from a peer the device does not have must report the peer") +} + +func ipNetStrings(ipNets []net.IPNet) []string { + out := make([]string, 0, len(ipNets)) + for _, ipNet := range ipNets { + out = append(out, ipNet.String()) + } + return out +} + +// uapiPeerBlock reads the device back through the UAPI dump, so a typed write is checked +// against an independent view of the device. +func uapiPeerBlock(t *testing.T, c *WGUSPConfigurer, peerKey string) []string { + t.Helper() + + parsed, err := wgtypes.ParseKey(peerKey) + require.NoError(t, err, "parse peer key") + wanted := "public_key=" + hex.EncodeToString(parsed[:]) + + dump, err := c.device.IpcGet() + require.NoError(t, err, "dump device") + + var block []string + inBlock := false + for _, line := range strings.Split(strings.TrimSpace(dump), "\n") { + if strings.HasPrefix(line, "public_key=") { + inBlock = line == wanted + continue + } + if inBlock { + block = append(block, line) + } + } + require.NotEmpty(t, block, "peer %s must appear in the dump", peerKey) + return block +} diff --git a/client/iface/device/device_kernel_unix.go b/client/iface/device/device_kernel_unix.go index 3c429fb96..ee965dd24 100644 --- a/client/iface/device/device_kernel_unix.go +++ b/client/iface/device/device_kernel_unix.go @@ -32,6 +32,7 @@ type TunKernelDevice struct { link *wgLink udpMuxConn net.PacketConn udpMux *udpmux.UniversalUDPMuxDefault + configurer WGConfigurer } func NewKernelDevice(name string, address wgaddr.Address, wgPort int, key string, mtu uint16, transportNet transport.Net) *TunKernelDevice { @@ -68,13 +69,18 @@ func (t *TunKernelDevice) Create() (WGConfigurer, error) { return nil, fmt.Errorf("set mtu: %w", err) } - configurer := configurer.NewKernelConfigurer(t.name) + cfg := configurer.NewKernelConfigurer(t.name) - if err := configurer.ConfigureInterface(t.key, t.wgPort); err != nil { + if err := cfg.ConfigureInterface(t.key, t.wgPort); err != nil { + cfg.Close() return nil, fmt.Errorf("error configuring interface: %s", err) } - return configurer, nil + if t.configurer != nil { + t.configurer.Close() + } + t.configurer = cfg + return cfg, nil } func (t *TunKernelDevice) Up() (*udpmux.UniversalUDPMuxDefault, error) { @@ -132,6 +138,10 @@ func (t *TunKernelDevice) Close() error { closErr = err } + if t.configurer != nil { + t.configurer.Close() + } + if t.udpMux != nil { if err := t.udpMux.Close(); err != nil { log.Debugf("failed to close udp mux: %s", err) diff --git a/client/iface/iface.go b/client/iface/iface.go index 247f421a2..f6006fa87 100644 --- a/client/iface/iface.go +++ b/client/iface/iface.go @@ -51,7 +51,6 @@ func ValidateMTU(mtu uint16) error { type wgProxyFactory interface { GetProxy() wgproxy.Proxy - GetProxyPort() uint16 Free() error } @@ -81,12 +80,6 @@ func (w *WGIface) GetProxy() wgproxy.Proxy { return w.wgProxyFactory.GetProxy() } -// GetProxyPort returns the proxy port used by the WireGuard proxy. -// Returns 0 if no proxy port is used (e.g., for userspace WireGuard). -func (w *WGIface) GetProxyPort() uint16 { - return w.wgProxyFactory.GetProxyPort() -} - // GetBind returns the EndpointManager userspace bind mode. func (w *WGIface) GetBind() device.EndpointManager { w.mu.Lock() diff --git a/client/iface/iface_close_test.go b/client/iface/iface_close_test.go index 171e15d0a..ea3115ec0 100644 --- a/client/iface/iface_close_test.go +++ b/client/iface/iface_close_test.go @@ -52,7 +52,6 @@ func (f *fakeTunDevice) Close() error { type fakeProxyFactory struct{} func (fakeProxyFactory) GetProxy() wgproxy.Proxy { return nil } -func (fakeProxyFactory) GetProxyPort() uint16 { return 0 } func (fakeProxyFactory) Free() error { return nil } // TestWGIface_CloseReleasesMutexBeforeTunClose guards against a deadlock diff --git a/client/iface/iface_destroy_windows.go b/client/iface/iface_destroy_windows.go index 0bfa4e211..54c0014c4 100644 --- a/client/iface/iface_destroy_windows.go +++ b/client/iface/iface_destroy_windows.go @@ -6,27 +6,14 @@ import ( "fmt" "os/exec" - log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/client/internal/wincmd" ) func (w *WGIface) Destroy() error { - netshCmd := GetSystem32Command("netsh") + netshCmd := wincmd.System32("netsh") out, err := exec.Command(netshCmd, "interface", "set", "interface", w.Name(), "admin=disable").CombinedOutput() if err != nil { return fmt.Errorf("failed to remove interface %s: %w - %s", w.Name(), err, out) } return nil } - -// GetSystem32Command checks if a command can be found in the system path and returns it. In case it can't find it -// in the path it will return the full path of a command assuming C:\windows\system32 as the base path. -func GetSystem32Command(command string) string { - _, err := exec.LookPath(command) - if err == nil { - return command - } - - log.Tracef("Command %s not found in PATH, using C:\\windows\\system32\\%s.exe path", command, command) - - return "C:\\windows\\system32\\" + command + ".exe" -} diff --git a/client/iface/iface_test.go b/client/iface/iface_test.go index 89c8cd16e..690a0f66f 100644 --- a/client/iface/iface_test.go +++ b/client/iface/iface_test.go @@ -40,14 +40,18 @@ func init() { peerPubKey = peerPrivateKey.PublicKey().String() } +// testIFaceBlackList mirrors the prefixes profilemanager.DefaultInterfaceBlacklist +// carries for the overlay interface. These tests create their own utun device, and +// stdnet's filter probes with wgctrl every interface it is not told to skip, which +// on a userspace WireGuard platform reaches the UAPI socket of this same process. +// Declared here rather than imported because profilemanager imports this package. +var testIFaceBlackList = []string{"wt", "utun", "tun0"} + func TestWGIface_UpdateAddr(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+4) addr := "100.64.0.1/8" wgPort := 33100 - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -127,10 +131,7 @@ func getIfaceAddrs(ifaceName string) ([]net.Addr, error) { func Test_CreateInterface(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+1) wgIP := "10.99.99.1/32" - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, Address: wgaddr.MustParseWGAddress(wgIP), @@ -170,10 +171,7 @@ func Test_Close(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+2) wgIP := "10.99.99.2/32" wgPort := 33100 - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -215,10 +213,7 @@ func TestRecreation(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+2) wgIP := "10.99.99.2/32" wgPort := 33100 - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -288,10 +283,7 @@ func Test_ConfigureInterface(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+3) wgIP := "10.99.99.5/30" wgPort := 33100 - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, Address: wgaddr.MustParseWGAddress(wgIP), @@ -343,10 +335,7 @@ func Test_ConfigureInterface(t *testing.T) { func Test_UpdatePeer(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+4) wgIP := "10.99.99.9/30" - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -413,10 +402,7 @@ func Test_UpdatePeer(t *testing.T) { func Test_RemovePeer(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+4) wgIP := "10.99.99.13/30" - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -477,10 +463,7 @@ func Test_ConnectPeers(t *testing.T) { peer2wgPort := 33200 keepAlive := 1 * time.Second - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) guid := fmt.Sprintf("{%s}", uuid.New().String()) device.CustomWindowsGUIDString = strings.ToLower(guid) @@ -516,10 +499,7 @@ func Test_ConnectPeers(t *testing.T) { guid = fmt.Sprintf("{%s}", uuid.New().String()) device.CustomWindowsGUIDString = strings.ToLower(guid) - newNet, err = stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet = stdnet.NewNet(context.Background(), testIFaceBlackList, nil) optsPeer2 := WGIFaceOpts{ IFaceName: peer2ifaceName, @@ -568,11 +548,14 @@ func Test_ConnectPeers(t *testing.T) { if err != nil { t.Fatal(err) } - // The peers use userspace WireGuard (stdnet transport). A tight busy-loop - // here starves the wireguard-go goroutines that process the handshake, so - // poll on a ticker instead and yield the CPU between checks. WireGuard also - // only retries a lost handshake initiation every REKEY_TIMEOUT (5s), which - // is why the overall wait can occasionally stretch to tens of seconds. + // On Linux with the kernel module both peers are kernel devices, elsewhere + // they run on wireguard-go. A tight busy-loop here would starve the + // wireguard-go goroutines that process the handshake, so poll on a ticker + // instead and yield the CPU between checks. WireGuard also only retries a + // lost handshake initiation every REKEY_TIMEOUT (5s), which is why the + // overall wait can occasionally stretch to tens of seconds. Each side sends + // its first initiation when its peer is configured, and the first one leaves + // before the other device knows the peer, so that one is always wasted. timeout := 30 * time.Second timeoutChannel := time.After(timeout) ticker := time.NewTicker(500 * time.Millisecond) @@ -590,13 +573,26 @@ func Test_ConnectPeers(t *testing.T) { select { case <-timeoutChannel: - t.Fatalf("waiting for peer handshake timeout after %s", timeout.String()) + // The counters tell whether initiations were sent at all, whether they + // arrived, and whether only one direction is working. + t.Fatalf("waiting for peer handshake timeout after %s\n%s\n%s", timeout.String(), + describePeer(peer1ifaceName, peer2Key.PublicKey().String()), + describePeer(peer2ifaceName, peer1Key.PublicKey().String())) case <-ticker.C: } } } +func describePeer(ifaceName, peerPubKey string) string { + peer, err := getPeer(ifaceName, peerPubKey) + if err != nil { + return fmt.Sprintf("%s: peer %s: %v", ifaceName, peerPubKey, err) + } + return fmt.Sprintf("%s: peer %s endpoint=%v tx=%d rx=%d last_handshake=%v", + ifaceName, peerPubKey, peer.Endpoint, peer.TransmitBytes, peer.ReceiveBytes, peer.LastHandshakeTime) +} + func getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) { wg, err := wgctrl.New() if err != nil { diff --git a/client/iface/udpmux/mux.go b/client/iface/udpmux/mux.go index c5d2de4a5..2f7287b53 100644 --- a/client/iface/udpmux/mux.go +++ b/client/iface/udpmux/mux.go @@ -47,6 +47,7 @@ type SingleSocketUDPMux struct { // for UDP connection listen at unspecified address localAddrsForUnspecified []net.Addr + unspecified bool } const maxAddrSize = 512 @@ -157,6 +158,9 @@ func NewSingleSocketUDPMux(params Params) *SingleSocketUDPMux { if params.Logger == nil { params.Logger = getLogger() } + if params.Net == nil { + params.Net = stdnet.NewNet(context.Background(), nil, nil) + } mux := &SingleSocketUDPMux{ addressMap: map[string][]*udpMuxedConn{}, @@ -165,6 +169,7 @@ func NewSingleSocketUDPMux(params Params) *SingleSocketUDPMux { connsIPv6: make(map[string]*udpMuxedConn), candidateConnMap: make(map[string]*udpMuxedConn), closedChan: make(chan struct{}, 1), + unspecified: isUnspecified(params.UDPConn.LocalAddr()), pool: &sync.Pool{ New: func() interface{} { // big enough buffer to fit both packet and address @@ -173,7 +178,6 @@ func NewSingleSocketUDPMux(params Params) *SingleSocketUDPMux { }, } - mux.updateLocalAddresses() return mux } @@ -199,13 +203,6 @@ func (m *SingleSocketUDPMux) updateLocalAddresses() { m.params.Logger.Errorf("LocalAddr expected IPV4 or IPV6, got %T", m.params.UDPConn.LocalAddr()) } if len(networks) > 0 { - if m.params.Net == nil { - var err error - if m.params.Net, err = stdnet.NewNet(context.Background(), nil); err != nil { - m.params.Logger.Errorf("failed to get create network: %v", err) - } - } - ips, err := localInterfaces(m.params.Net, m.params.InterfaceFilter, nil, networks, true) if err == nil { for _, ip := range ips { @@ -244,10 +241,7 @@ func (m *SingleSocketUDPMux) GetListenAddresses() []net.Addr { // creates the connection if an existing one can't be found func (m *SingleSocketUDPMux) GetConn(ufrag string, addr net.Addr, candidateID string) (net.PacketConn, error) { // don't check addr for mux using unspecified address - m.mu.Lock() - lenLocalAddrs := len(m.localAddrsForUnspecified) - m.mu.Unlock() - if lenLocalAddrs == 0 && m.params.UDPConn.LocalAddr().String() != addr.String() { + if !m.unspecified && m.params.UDPConn.LocalAddr().String() != addr.String() { return nil, fmt.Errorf("invalid address %s", addr.String()) } @@ -511,3 +505,8 @@ func getLogger() logging.LeveledLogger { //fac.Writer = log.StandardLogger().Writer() return fac.NewLogger("ice") } + +func isUnspecified(addr net.Addr) bool { + udpAddr, ok := addr.(*net.UDPAddr) + return ok && udpAddr.IP.IsUnspecified() +} diff --git a/client/iface/udpmux/mux_test.go b/client/iface/udpmux/mux_test.go new file mode 100644 index 000000000..2e2770282 --- /dev/null +++ b/client/iface/udpmux/mux_test.go @@ -0,0 +1,99 @@ +package udpmux + +import ( + "fmt" + "net" + "sync" + "sync/atomic" + "testing" + + "github.com/pion/transport/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type countingNet struct { + transport.Net + calls atomic.Int64 + ifaces []*transport.Interface +} + +func (n *countingNet) Interfaces() ([]*transport.Interface, error) { + n.calls.Add(1) + return n.ifaces, nil +} + +func TestNewSingleSocketUDPMux_DoesNotDiscoverInterfaces(t *testing.T) { + conn := listenUDP(t, net.IPv4zero) + n := &countingNet{ifaces: []*transport.Interface{testInterface("eth0", net.IPv4(192, 168, 1, 10))}} + + mux := NewSingleSocketUDPMux(Params{UDPConn: conn, Net: n}) + t.Cleanup(func() { _ = mux.Close() }) + + assert.Equal(t, int64(0), n.calls.Load(), "constructing the mux must not enumerate interfaces") + + addrs := mux.GetListenAddresses() + assert.Equal(t, int64(1), n.calls.Load(), "listen addresses are discovered when they are asked for") + + port := conn.LocalAddr().(*net.UDPAddr).Port + require.Len(t, addrs, 1, "one address per discovered interface IP") + assert.Equal(t, fmt.Sprintf("192.168.1.10:%d", port), addrs[0].String(), "the discovered IP is paired with the socket port") +} + +func TestSingleSocketUDPMux_GetConnWithoutListenAddresses(t *testing.T) { + t.Run("unspecified socket accepts any address", func(t *testing.T) { + conn := listenUDP(t, net.IPv4zero) + n := &countingNet{} + mux := NewSingleSocketUDPMux(Params{UDPConn: conn, Net: n}) + t.Cleanup(func() { _ = mux.Close() }) + + port := conn.LocalAddr().(*net.UDPAddr).Port + pc, err := mux.GetConn("ufrag", &net.UDPAddr{IP: net.IPv4(192, 168, 1, 10), Port: port}, "candidate") + require.NoError(t, err) + assert.NotNil(t, pc, "a connection is created for a local address of an unspecified socket") + assert.Equal(t, int64(0), n.calls.Load(), "GetConn must not depend on a prior discovery") + }) + + t.Run("specific socket rejects a foreign address", func(t *testing.T) { + conn := listenUDP(t, net.IPv4(127, 0, 0, 1)) + mux := NewSingleSocketUDPMux(Params{UDPConn: conn}) + t.Cleanup(func() { _ = mux.Close() }) + + _, err := mux.GetConn("ufrag", &net.UDPAddr{IP: net.IPv4(192, 168, 1, 10), Port: 1234}, "candidate") + assert.Error(t, err, "an address the socket is not bound to must be rejected") + + pc, err := mux.GetConn("ufrag", conn.LocalAddr(), "candidate") + require.NoError(t, err) + assert.NotNil(t, pc, "the bound address is accepted") + }) +} + +func listenUDP(t *testing.T, ip net.IP) *net.UDPConn { + t.Helper() + conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: ip, Port: 0}) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + return conn +} + +func testInterface(name string, ip net.IP) *transport.Interface { + ifc := transport.NewInterface(net.Interface{Index: 1, MTU: 1500, Name: name, Flags: net.FlagUp | net.FlagMulticast}) + ifc.AddAddress(&net.IPNet{IP: ip, Mask: net.CIDRMask(24, 32)}) + return ifc +} + +func TestSingleSocketUDPMux_ConcurrentListenAddressesWithDefaultNet(t *testing.T) { + conn := listenUDP(t, net.IPv4zero) + mux := NewSingleSocketUDPMux(Params{UDPConn: conn}) + t.Cleanup(func() { _ = mux.Close() }) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + assert.NotEmpty(t, mux.GetListenAddresses(), "every caller gets at least the socket address") + }() + } + wg.Wait() +} diff --git a/client/iface/wgproxy/ebpf/portlookup.go b/client/iface/wgproxy/ebpf/portlookup.go deleted file mode 100644 index fce8f1507..000000000 --- a/client/iface/wgproxy/ebpf/portlookup.go +++ /dev/null @@ -1,32 +0,0 @@ -package ebpf - -import ( - "fmt" - "net" -) - -var ( - portRangeStart = 3128 - portRangeEnd = portRangeStart + 100 -) - -type portLookup struct { -} - -func (pl portLookup) searchFreePort() (int, error) { - for i := portRangeStart; i <= portRangeEnd; i++ { - if pl.tryToBind(i) == nil { - return i, nil - } - } - return 0, fmt.Errorf("failed to bind free port for eBPF proxy") -} - -func (pl portLookup) tryToBind(port int) error { - l, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port)) - if err != nil { - return err - } - _ = l.Close() - return nil -} diff --git a/client/iface/wgproxy/ebpf/portlookup_test.go b/client/iface/wgproxy/ebpf/portlookup_test.go deleted file mode 100644 index a2e92fc79..000000000 --- a/client/iface/wgproxy/ebpf/portlookup_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package ebpf - -import ( - "fmt" - "net" - "testing" -) - -func Test_portLookup_searchFreePort(t *testing.T) { - pl := portLookup{} - _, err := pl.searchFreePort() - if err != nil { - t.Fatal(err) - } -} - -func Test_portLookup_on_allocated(t *testing.T) { - pl := portLookup{} - - portRangeStart = 4128 - portRangeEnd = portRangeStart + 100 - - allocatedPort, err := allocatePort(portRangeStart) - if err != nil { - t.Fatal(err) - } - defer allocatedPort.Close() - - fp, err := pl.searchFreePort() - if err != nil { - t.Fatal(err) - } - - if fp != (portRangeStart + 1) { - t.Errorf("invalid free port, expected: %d, got: %d", portRangeStart+1, fp) - } -} - -func allocatePort(port int) (net.PacketConn, error) { - c, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port)) - if err != nil { - return nil, err - } - return c, err -} diff --git a/client/iface/wgproxy/ebpf/proxy.go b/client/iface/wgproxy/ebpf/proxy.go deleted file mode 100644 index 91c741c0d..000000000 --- a/client/iface/wgproxy/ebpf/proxy.go +++ /dev/null @@ -1,243 +0,0 @@ -//go:build linux && !android - -package ebpf - -import ( - "context" - "fmt" - "net" - "sync" - - "github.com/hashicorp/go-multierror" - "github.com/pion/transport/v3" - log "github.com/sirupsen/logrus" - - nberrors "github.com/netbirdio/netbird/client/errors" - "github.com/netbirdio/netbird/client/iface/bufsize" - "github.com/netbirdio/netbird/client/iface/wgproxy/rawsocket" - "github.com/netbirdio/netbird/client/internal/ebpf" - ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager" - nbnet "github.com/netbirdio/netbird/client/net" -) - -const ( - loopbackAddr = "127.0.0.1" -) - -// WGEBPFProxy definition for proxy with EBPF support -type WGEBPFProxy struct { - localWGListenPort int - proxyPort int - mtu uint16 - - ebpfManager ebpfMgr.Manager - relayedConnStore map[uint16]net.Conn - relayedConnMutex sync.Mutex - - lastUsedPort uint16 - rawConnIPv4 net.PacketConn - rawConnIPv6 net.PacketConn - conn transport.UDPConn - - ctx context.Context - ctxCancel context.CancelFunc -} - -// NewWGEBPFProxy create new WGEBPFProxy instance -func NewWGEBPFProxy(wgPort int, mtu uint16) *WGEBPFProxy { - log.Debugf("instantiate ebpf proxy") - wgProxy := &WGEBPFProxy{ - localWGListenPort: wgPort, - mtu: mtu, - ebpfManager: ebpf.GetEbpfManagerInstance(), - relayedConnStore: make(map[uint16]net.Conn), - } - return wgProxy -} - -// Listen load ebpf program and listen the proxy -func (p *WGEBPFProxy) Listen() error { - pl := portLookup{} - proxyPort, err := pl.searchFreePort() - if err != nil { - return err - } - p.proxyPort = proxyPort - - // Prepare IPv4 raw socket (required) - p.rawConnIPv4, err = rawsocket.PrepareSenderRawSocketIPv4() - if err != nil { - return err - } - - // Prepare IPv6 raw socket (optional) - p.rawConnIPv6, err = rawsocket.PrepareSenderRawSocketIPv6() - if err != nil { - log.Warnf("failed to prepare IPv6 raw socket, continuing with IPv4 only: %v", err) - } - - err = p.ebpfManager.LoadWgProxy(proxyPort, p.localWGListenPort) - if err != nil { - if closeErr := p.rawConnIPv4.Close(); closeErr != nil { - log.Warnf("failed to close IPv4 raw socket: %v", closeErr) - } - if p.rawConnIPv6 != nil { - if closeErr := p.rawConnIPv6.Close(); closeErr != nil { - log.Warnf("failed to close IPv6 raw socket: %v", closeErr) - } - } - return err - } - - addr := net.UDPAddr{ - Port: proxyPort, - IP: net.ParseIP(loopbackAddr), - } - - p.ctx, p.ctxCancel = context.WithCancel(context.Background()) - - conn, err := nbnet.ListenUDP("udp", &addr) - if err != nil { - if cErr := p.Free(); cErr != nil { - log.Errorf("Failed to close the wgproxy: %s", cErr) - } - return err - } - p.conn = conn - - go p.proxyToRemote() - log.Infof("local wg proxy listening on: %d", proxyPort) - return nil -} - -// AddRelayedConn add new relayed connection for the proxy -func (p *WGEBPFProxy) AddRelayedConn(relayedConn net.Conn) (*net.UDPAddr, error) { - wgEndpointPort, err := p.storeRelayedConn(relayedConn) - if err != nil { - return nil, err - } - - log.Infof("relayed conn added to wg proxy store: %s, endpoint port: :%d", relayedConn.RemoteAddr(), wgEndpointPort) - - wgEndpoint := &net.UDPAddr{ - IP: net.ParseIP(loopbackAddr), - Port: int(wgEndpointPort), - } - return wgEndpoint, nil -} - -// Free resources except the remoteConns will be keep open. -func (p *WGEBPFProxy) Free() error { - log.Debugf("free up ebpf wg proxy") - if p.ctx != nil && p.ctx.Err() != nil { - //nolint - return nil - } - - p.ctxCancel() - - var result *multierror.Error - if p.conn != nil { - if err := p.conn.Close(); err != nil { - result = multierror.Append(result, err) - } - } - - if err := p.ebpfManager.FreeWGProxy(); err != nil { - result = multierror.Append(result, err) - } - - if p.rawConnIPv4 != nil { - if err := p.rawConnIPv4.Close(); err != nil { - result = multierror.Append(result, err) - } - } - - if p.rawConnIPv6 != nil { - if err := p.rawConnIPv6.Close(); err != nil { - result = multierror.Append(result, err) - } - } - return nberrors.FormatErrorOrNil(result) -} - -// GetProxyPort returns the proxy listening port. -func (p *WGEBPFProxy) GetProxyPort() uint16 { - return uint16(p.proxyPort) -} - -// proxyToRemote read messages from local WireGuard interface and forward it to remote conn -// From this go routine has only one instance. -func (p *WGEBPFProxy) proxyToRemote() { - buf := make([]byte, p.mtu+bufsize.WGBufferOverhead) - for p.ctx.Err() == nil { - if err := p.readAndForwardPacket(buf); err != nil { - if p.ctx.Err() != nil { - return - } - log.Errorf("failed to proxy packet to remote conn: %s", err) - } - } -} - -func (p *WGEBPFProxy) readAndForwardPacket(buf []byte) error { - n, addr, err := p.conn.ReadFromUDP(buf) - if err != nil { - return fmt.Errorf("failed to read UDP packet from WG: %w", err) - } - - p.relayedConnMutex.Lock() - conn, ok := p.relayedConnStore[uint16(addr.Port)] - p.relayedConnMutex.Unlock() - if !ok { - if p.ctx.Err() == nil { - log.Debugf("relayed conn not found by port because conn already has been closed: %d", addr.Port) - } - return nil - } - - if _, err := conn.Write(buf[:n]); err != nil { - return fmt.Errorf("forward local WG packet (%d) to remote relayed conn: %w", addr.Port, err) - } - return nil -} - -func (p *WGEBPFProxy) storeRelayedConn(relayedConn net.Conn) (uint16, error) { - p.relayedConnMutex.Lock() - defer p.relayedConnMutex.Unlock() - - np, err := p.nextFreePort() - if err != nil { - return np, err - } - p.relayedConnStore[np] = relayedConn - return np, nil -} - -func (p *WGEBPFProxy) removeRelayedConn(relayedConnID uint16) { - p.relayedConnMutex.Lock() - defer p.relayedConnMutex.Unlock() - - _, ok := p.relayedConnStore[relayedConnID] - if ok { - log.Debugf("remove relayed conn from store by port: %d", relayedConnID) - } - delete(p.relayedConnStore, relayedConnID) -} - -func (p *WGEBPFProxy) nextFreePort() (uint16, error) { - if len(p.relayedConnStore) == 65535 { - return 0, fmt.Errorf("reached maximum relayed connection numbers") - } -generatePort: - if p.lastUsedPort == 65535 { - p.lastUsedPort = 1 - } else { - p.lastUsedPort++ - } - - if _, ok := p.relayedConnStore[p.lastUsedPort]; ok { - goto generatePort - } - return p.lastUsedPort, nil -} diff --git a/client/iface/wgproxy/ebpf/proxy_test.go b/client/iface/wgproxy/ebpf/proxy_test.go deleted file mode 100644 index 228c06c9b..000000000 --- a/client/iface/wgproxy/ebpf/proxy_test.go +++ /dev/null @@ -1,56 +0,0 @@ -//go:build linux && !android - -package ebpf - -import ( - "testing" -) - -func TestWGEBPFProxy_connStore(t *testing.T) { - wgProxy := NewWGEBPFProxy(1, 1280) - - p, _ := wgProxy.storeRelayedConn(nil) - if p != 1 { - t.Errorf("invalid initial port: %d", wgProxy.lastUsedPort) - } - - numOfConns := 10 - for i := 0; i < numOfConns; i++ { - p, _ = wgProxy.storeRelayedConn(nil) - } - if p != uint16(numOfConns)+1 { - t.Errorf("invalid last used port: %d, expected: %d", p, numOfConns+1) - } - if len(wgProxy.relayedConnStore) != numOfConns+1 { - t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.relayedConnStore), numOfConns+1) - } -} - -func TestWGEBPFProxy_portCalculation_overflow(t *testing.T) { - wgProxy := NewWGEBPFProxy(1, 1280) - - _, _ = wgProxy.storeRelayedConn(nil) - wgProxy.lastUsedPort = 65535 - p, _ := wgProxy.storeRelayedConn(nil) - - if len(wgProxy.relayedConnStore) != 2 { - t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.relayedConnStore), 2) - } - - if p != 2 { - t.Errorf("invalid last used port: %d, expected: %d", p, 2) - } -} - -func TestWGEBPFProxy_portCalculation_maxConn(t *testing.T) { - wgProxy := NewWGEBPFProxy(1, 1280) - - for i := 0; i < 65535; i++ { - _, _ = wgProxy.storeRelayedConn(nil) - } - - _, err := wgProxy.storeRelayedConn(nil) - if err == nil { - t.Errorf("invalid relayed conn store calculation") - } -} diff --git a/client/iface/wgproxy/factory_kernel.go b/client/iface/wgproxy/factory_kernel.go index 7821df3de..0b2329b96 100644 --- a/client/iface/wgproxy/factory_kernel.go +++ b/client/iface/wgproxy/factory_kernel.go @@ -8,11 +8,13 @@ import ( log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/client/iface/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/iface/wgproxy/loopback" udpProxy "github.com/netbirdio/netbird/client/iface/wgproxy/udp" ) const ( + envDisableKernelWGProxy = "NB_DISABLE_KERNEL_WG_PROXY" + // envDisableEBPFWGProxy is a deprecated alias for envDisableKernelWGProxy. envDisableEBPFWGProxy = "NB_DISABLE_EBPF_WG_PROXY" ) @@ -20,7 +22,7 @@ type KernelFactory struct { wgPort int mtu uint16 - ebpfProxy *ebpf.WGEBPFProxy + loopbackProxy *loopback.Proxy } func NewKernelFactory(wgPort int, mtu uint16) *KernelFactory { @@ -29,55 +31,56 @@ func NewKernelFactory(wgPort int, mtu uint16) *KernelFactory { mtu: mtu, } - if isEBPFDisabled() { + if isKernelProxyDisabled() { log.Infof("WireGuard Proxy Factory will produce UDP proxy") - log.Infof("eBPF WireGuard proxy is disabled via %s environment variable", envDisableEBPFWGProxy) return f } - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, mtu) - if err := ebpfProxy.Listen(); err != nil { + loopbackProxy := loopback.NewProxy(wgPort, mtu) + if err := loopbackProxy.Listen(); err != nil { log.Infof("WireGuard Proxy Factory will produce UDP proxy") - log.Warnf("failed to initialize ebpf proxy, fallback to user space proxy: %s", err) + log.Warnf("failed to initialize loopback proxy, fallback to user space proxy: %s", err) return f } - log.Infof("WireGuard Proxy Factory will produce eBPF proxy") - f.ebpfProxy = ebpfProxy + log.Infof("WireGuard Proxy Factory will produce loopback proxy") + f.loopbackProxy = loopbackProxy return f } func (w *KernelFactory) GetProxy() Proxy { - if w.ebpfProxy == nil { + if w.loopbackProxy == nil { return udpProxy.NewWGUDPProxy(w.wgPort, w.mtu) } - return ebpf.NewProxyWrapper(w.ebpfProxy) -} - -// GetProxyPort returns the eBPF proxy port, or 0 if eBPF is not active. -func (w *KernelFactory) GetProxyPort() uint16 { - if w.ebpfProxy == nil { - return 0 - } - return w.ebpfProxy.GetProxyPort() + return loopback.NewProxyWrapper(w.loopbackProxy) } func (w *KernelFactory) Free() error { - if w.ebpfProxy == nil { + if w.loopbackProxy == nil { return nil } - return w.ebpfProxy.Free() + return w.loopbackProxy.Free() } -func isEBPFDisabled() bool { - val := os.Getenv(envDisableEBPFWGProxy) +func isKernelProxyDisabled() bool { + env := envDisableKernelWGProxy + val := os.Getenv(env) + if val == "" { + env = envDisableEBPFWGProxy + val = os.Getenv(env) + } if val == "" { return false } + disabled, err := strconv.ParseBool(val) if err != nil { - log.Warnf("failed to parse %s: %v", envDisableEBPFWGProxy, err) + log.Warnf("failed to parse %s: %v", env, err) return false } + + if disabled { + log.Infof("kernel WireGuard proxy is disabled via %s", env) + } return disabled } diff --git a/client/iface/wgproxy/factory_usp.go b/client/iface/wgproxy/factory_usp.go index bbd67e076..a1b1c34d7 100644 --- a/client/iface/wgproxy/factory_usp.go +++ b/client/iface/wgproxy/factory_usp.go @@ -24,11 +24,6 @@ func (w *USPFactory) GetProxy() Proxy { return proxyBind.NewProxyBind(w.bind, w.mtu) } -// GetProxyPort returns 0 as userspace WireGuard doesn't use a separate proxy port. -func (w *USPFactory) GetProxyPort() uint16 { - return 0 -} - func (w *USPFactory) Free() error { return nil } diff --git a/client/iface/wgproxy/loopback/addr.go b/client/iface/wgproxy/loopback/addr.go new file mode 100644 index 000000000..52feee295 --- /dev/null +++ b/client/iface/wgproxy/loopback/addr.go @@ -0,0 +1,70 @@ +//go:build linux && !android + +package loopback + +import ( + "fmt" + "net/netip" +) + +// Peer endpoints live in the upper half of 127.0.0.0/8. Everything in that +// range is delivered to the loopback device without any address or route being +// configured, and staying out of 127.0.0.0/9 keeps well-known squatters such as +// 127.0.0.53 (systemd-resolved) and 127.0.1.1 out of the way. +const ( + addrRangeBase uint32 = 0x7f800000 // 127.128.0.0 + addrRangeSize uint32 = 1 << 23 // /9 + addrRangePrefix = "127.128.0.0/9" +) + +// allocator hands out one loopback address per relayed connection. The address +// is the peer's identity: WireGuard sends to it, and the proxy recovers which +// peer a packet belongs to from the destination address. +type allocator struct { + cursor uint32 +} + +// next returns the first free address at or after the cursor, wrapping once. +// inUse reports whether an address is already handed out. +func (a *allocator) next(inUse func(netip.Addr) bool) (netip.Addr, error) { + for i := uint32(0); i < addrRangeSize; i++ { + a.cursor = (a.cursor + 1) % addrRangeSize + addr := addrFromOffset(a.cursor) + if !addr.IsValid() { + continue + } + if inUse(addr) { + continue + } + return addr, nil + } + return netip.Addr{}, fmt.Errorf("no free endpoint address in %s", addrRangePrefix) +} + +// addrFromOffset maps an offset in the range to an address, skipping the .0 and +// .255 hosts. They are unremarkable on loopback, but tools and firewall rules +// tend to treat them as network and broadcast addresses. +func addrFromOffset(offset uint32) netip.Addr { + last := offset & 0xff + if last == 0 || last == 0xff { + return netip.Addr{} + } + + v := addrRangeBase + offset + return netip.AddrFrom4([4]byte{ + byte(v >> 24), + byte(v >> 16), + byte(v >> 8), + byte(v), + }) +} + +// inRange reports whether addr is one this proxy could have handed out. +func inRange(addr netip.Addr) bool { + if !addr.Is4() { + return false + } + b := addr.As4() + v := uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]) + return v >= addrRangeBase && v < addrRangeBase+addrRangeSize && b[3] != 0 && b[3] != 0xff +} diff --git a/client/iface/wgproxy/loopback/addr_test.go b/client/iface/wgproxy/loopback/addr_test.go new file mode 100644 index 000000000..3755b7269 --- /dev/null +++ b/client/iface/wgproxy/loopback/addr_test.go @@ -0,0 +1,114 @@ +//go:build linux && !android + +package loopback + +import ( + "net/netip" + "testing" +) + +func TestAllocatorHandsOutDistinctAddresses(t *testing.T) { + var a allocator + taken := make(map[netip.Addr]bool) + + for i := 0; i < 1000; i++ { + addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] }) + if err != nil { + t.Fatalf("allocate %d: %v", i, err) + } + if taken[addr] { + t.Fatalf("address %s handed out twice", addr) + } + if !inRange(addr) { + t.Fatalf("address %s outside %s", addr, addrRangePrefix) + } + taken[addr] = true + } +} + +func TestAllocatorSkipsNetworkAndBroadcastHosts(t *testing.T) { + var a allocator + taken := make(map[netip.Addr]bool) + + // enough allocations to walk past a .255/.0 boundary + for i := 0; i < 600; i++ { + addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] }) + if err != nil { + t.Fatalf("allocate %d: %v", i, err) + } + last := addr.As4()[3] + if last == 0 || last == 255 { + t.Fatalf("address %s ends in .%d", addr, last) + } + taken[addr] = true + } +} + +func TestAllocatorReusesReleasedAddresses(t *testing.T) { + var a allocator + taken := make(map[netip.Addr]bool) + inUse := func(candidate netip.Addr) bool { return taken[candidate] } + alloc := func() netip.Addr { + t.Helper() + addr, err := a.next(inUse) + if err != nil { + t.Fatalf("allocate: %v", err) + } + taken[addr] = true + return addr + } + + first := alloc() + second := alloc() + delete(taken, first) + + // The cursor only moves forward, so a released address comes back after a + // wrap. Park the cursor near the end of the range instead of allocating + // 2^23 addresses: the next call takes the last usable address, and the one + // after that wraps past the skipped .255 and .0 hosts to the released one. + a.cursor = addrRangeSize - 3 + last := alloc() + if want := netip.MustParseAddr("127.255.255.254"); last != want { + t.Fatalf("expected the last usable address %s before the wrap, got %s", want, last) + } + + if reused := alloc(); reused != first { + t.Fatalf("expected the released address %s after the wrap, got %s", first, reused) + } + + // second is still held, so the allocator must step over it. + if next := alloc(); next == second { + t.Fatalf("allocator handed out %s while it was still in use", second) + } +} + +func TestInRange(t *testing.T) { + tests := []struct { + addr string + want bool + }{ + {"127.128.0.1", true}, + {"127.255.255.254", true}, + {"127.128.0.0", false}, // network host, never handed out + {"127.128.5.255", false}, // broadcast host, never handed out + {"127.127.255.255", false}, // below the range, where 127.0.0.53 and friends live + {"127.0.0.1", false}, + {"127.0.0.53", false}, + {"127.0.1.1", false}, + {"128.0.0.1", false}, + {"10.0.0.1", false}, + } + + for _, tc := range tests { + addr := netip.MustParseAddr(tc.addr) + if got := inRange(addr); got != tc.want { + t.Errorf("inRange(%s) = %v, want %v", tc.addr, got, tc.want) + } + } +} + +func TestInRangeIgnoresIPv6(t *testing.T) { + if inRange(netip.MustParseAddr("::1")) { + t.Error("inRange(::1) = true, want false") + } +} diff --git a/client/iface/wgproxy/loopback/proxy.go b/client/iface/wgproxy/loopback/proxy.go new file mode 100644 index 000000000..f9364c766 --- /dev/null +++ b/client/iface/wgproxy/loopback/proxy.go @@ -0,0 +1,291 @@ +//go:build linux && !android + +package loopback + +import ( + "context" + "fmt" + "net" + "net/netip" + "sync" + "syscall" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + "golang.org/x/net/ipv4" + "golang.org/x/sys/unix" + + nberrors "github.com/netbirdio/netbird/client/errors" + "github.com/netbirdio/netbird/client/iface/bufsize" + "github.com/netbirdio/netbird/client/iface/wgproxy/rawsocket" +) + +const ( + loopbackDevice = "lo" + + portRangeStart = 3128 + portRangeEnd = portRangeStart + 100 +) + +// Proxy forwards packets between relayed connections and a local kernel +// WireGuard instance. Every relayed peer gets its own loopback address as its +// WireGuard endpoint, so a single socket serves all of them: the destination +// address of an incoming packet identifies the peer. +type Proxy struct { + localWGListenPort int + mtu uint16 + proxyPort int + + conn *net.UDPConn + packetConn *ipv4.PacketConn + loIndex int + rawConnIPv4 net.PacketConn + rawConnIPv6 net.PacketConn + + relayedConnMutex sync.Mutex + relayedConnStore map[netip.Addr]net.Conn + addrs allocator + + ctx context.Context + ctxCancel context.CancelFunc +} + +// NewProxy creates a proxy for the WireGuard instance listening on wgPort. +func NewProxy(wgPort int, mtu uint16) *Proxy { + log.Debugf("instantiate loopback wg proxy") + return &Proxy{ + localWGListenPort: wgPort, + mtu: mtu, + relayedConnStore: make(map[netip.Addr]net.Conn), + } +} + +// Listen opens the shared socket and starts forwarding WireGuard packets to the +// relayed connections. +func (p *Proxy) Listen() error { + rawConnIPv4, err := rawsocket.PrepareSenderRawSocketIPv4() + if err != nil { + return fmt.Errorf("prepare IPv4 raw socket: %w", err) + } + p.rawConnIPv4 = rawConnIPv4 + + p.rawConnIPv6, err = rawsocket.PrepareSenderRawSocketIPv6() + if err != nil { + log.Warnf("failed to prepare IPv6 raw socket, continuing with IPv4 only: %v", err) + } + + loopback, err := net.InterfaceByName(loopbackDevice) + if err != nil { + if freeErr := p.Free(); freeErr != nil { + log.Errorf("failed to free the wgproxy: %s", freeErr) + } + return fmt.Errorf("look up %s: %w", loopbackDevice, err) + } + p.loIndex = loopback.Index + + if err := p.listen(); err != nil { + if freeErr := p.Free(); freeErr != nil { + log.Errorf("failed to free the wgproxy: %s", freeErr) + } + return err + } + + p.ctx, p.ctxCancel = context.WithCancel(context.Background()) + + go p.proxyToRemote() + log.Infof("local wg proxy listening on %s:%d", addrRangePrefix, p.proxyPort) + return nil +} + +// listen binds the shared socket on the first free port of the range. The bind +// has to be a wildcard one to receive every peer address in the range, so it is +// restricted to the loopback device: without that the port would be reachable +// on every interface. +func (p *Proxy) listen() error { + var lastErr error + for port := portRangeStart; port <= portRangeEnd; port++ { + err := p.listenOn(port) + if err == nil { + p.proxyPort = port + return nil + } + lastErr = err + } + return fmt.Errorf("bind proxy port in range %d-%d: %w", portRangeStart, portRangeEnd, lastErr) +} + +func (p *Proxy) listenOn(proxyPort int) error { + lc := net.ListenConfig{ + Control: func(_, _ string, c syscall.RawConn) error { + var sockErr error + if err := c.Control(func(fd uintptr) { + if err := unix.SetsockoptString(int(fd), unix.SOL_SOCKET, unix.SO_BINDTODEVICE, loopbackDevice); err != nil { + sockErr = fmt.Errorf("bind to %s: %w", loopbackDevice, err) + return + } + }); err != nil { + return fmt.Errorf("control socket: %w", err) + } + return sockErr + }, + } + + conn, err := lc.ListenPacket(context.Background(), "udp4", fmt.Sprintf(":%d", proxyPort)) + if err != nil { + return fmt.Errorf("listen on :%d: %w", proxyPort, err) + } + + udpConn, ok := conn.(*net.UDPConn) + if !ok { + if closeErr := conn.Close(); closeErr != nil { + log.Errorf("failed to close proxy conn: %s", closeErr) + } + return fmt.Errorf("unexpected conn type %T", conn) + } + + packetConn := ipv4.NewPacketConn(udpConn) + // the destination address carries the peer identity, the interface index is + // checked on receive as a second line of defense behind SO_BINDTODEVICE + if err := packetConn.SetControlMessage(ipv4.FlagDst|ipv4.FlagInterface, true); err != nil { + if closeErr := udpConn.Close(); closeErr != nil { + log.Errorf("failed to close proxy conn: %s", closeErr) + } + return fmt.Errorf("request destination address: %w", err) + } + + p.conn = udpConn + p.packetConn = packetConn + return nil +} + +// AddRelayedConn assigns an endpoint address to the relayed connection and +// returns the address WireGuard should send to, along with the key the +// connection is stored under. +func (p *Proxy) AddRelayedConn(relayedConn net.Conn) (*net.UDPAddr, netip.Addr, error) { + addr, err := p.storeRelayedConn(relayedConn) + if err != nil { + return nil, netip.Addr{}, err + } + + log.Infof("relayed conn added to wg proxy store: %s, endpoint address: %s", relayedConn.RemoteAddr(), addr) + + return &net.UDPAddr{ + IP: addr.AsSlice(), + Port: p.proxyPort, + }, addr, nil +} + +// Free releases the proxy resources. The relayed connections are left open. +func (p *Proxy) Free() error { + log.Debugf("free up loopback wg proxy") + if p.ctx != nil && p.ctx.Err() != nil { + //nolint + return nil + } + + if p.ctxCancel != nil { + p.ctxCancel() + } + + var result *multierror.Error + if p.conn != nil { + if err := p.conn.Close(); err != nil { + result = multierror.Append(result, err) + } + } + + if p.rawConnIPv4 != nil { + if err := p.rawConnIPv4.Close(); err != nil { + result = multierror.Append(result, err) + } + } + + if p.rawConnIPv6 != nil { + if err := p.rawConnIPv6.Close(); err != nil { + result = multierror.Append(result, err) + } + } + return nberrors.FormatErrorOrNil(result) +} + +// proxyToRemote reads packets from the local WireGuard instance and forwards +// them to the relayed connection the destination address belongs to. +func (p *Proxy) proxyToRemote() { + buf := make([]byte, p.mtu+bufsize.WGBufferOverhead) + for p.ctx.Err() == nil { + if err := p.readAndForwardPacket(buf); err != nil { + if p.ctx.Err() != nil { + return + } + log.Errorf("failed to proxy packet to remote conn: %s", err) + } + } +} + +func (p *Proxy) readAndForwardPacket(buf []byte) error { + n, cm, _, err := p.packetConn.ReadFrom(buf) + if err != nil { + return fmt.Errorf("read UDP packet from WG: %w", err) + } + + if cm == nil { + return fmt.Errorf("no control message on packet") + } + + if cm.IfIndex != p.loIndex { + log.Tracef("dropping packet received on interface %d instead of %s", cm.IfIndex, loopbackDevice) + return nil + } + + dst, ok := netip.AddrFromSlice(cm.Dst.To4()) + if !ok || !inRange(dst) { + log.Tracef("dropping packet for unexpected destination %s", cm.Dst) + return nil + } + + p.relayedConnMutex.Lock() + conn, ok := p.relayedConnStore[dst] + p.relayedConnMutex.Unlock() + if !ok { + if p.ctx.Err() == nil { + log.Debugf("relayed conn not found by address because conn already has been closed: %s", dst) + } + return nil + } + + if _, err := conn.Write(buf[:n]); err != nil { + return fmt.Errorf("forward local WG packet (%s) to remote relayed conn: %w", dst, err) + } + return nil +} + +func (p *Proxy) storeRelayedConn(relayedConn net.Conn) (netip.Addr, error) { + p.relayedConnMutex.Lock() + defer p.relayedConnMutex.Unlock() + + addr, err := p.addrs.next(func(a netip.Addr) bool { + _, ok := p.relayedConnStore[a] + return ok + }) + if err != nil { + return netip.Addr{}, err + } + + p.relayedConnStore[addr] = relayedConn + return addr, nil +} + +// removeRelayedConn releases an endpoint address. It only removes the entry +// while it still belongs to relayedConn, so a late release cannot take an +// address away from the peer it was handed to next. +func (p *Proxy) removeRelayedConn(addr netip.Addr, relayedConn net.Conn) { + p.relayedConnMutex.Lock() + defer p.relayedConnMutex.Unlock() + + if stored, ok := p.relayedConnStore[addr]; !ok || stored != relayedConn { + return + } + + log.Debugf("remove relayed conn from store by address: %s", addr) + delete(p.relayedConnStore, addr) +} diff --git a/client/iface/wgproxy/loopback/proxy_privileged_test.go b/client/iface/wgproxy/loopback/proxy_privileged_test.go new file mode 100644 index 000000000..6314fe2de --- /dev/null +++ b/client/iface/wgproxy/loopback/proxy_privileged_test.go @@ -0,0 +1,196 @@ +//go:build linux && !android && privileged + +package loopback + +import ( + "context" + "net" + "strconv" + "testing" + "time" +) + +const testWGPort = 51862 + +// relayEnd stands in for a relayed connection: the proxy writes what it read +// from WireGuard into it, and the test reads it back out here. +func relayEnd(t *testing.T) (proxySide net.Conn, testSide *net.UDPConn) { + t.Helper() + + testSide, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")}) + if err != nil { + t.Fatalf("relay listener: %v", err) + } + t.Cleanup(func() { + if err := testSide.Close(); err != nil { + t.Logf("close relay listener: %v", err) + } + }) + + proxySide, err = net.Dial("udp", testSide.LocalAddr().String()) + if err != nil { + t.Fatalf("relay conn: %v", err) + } + t.Cleanup(func() { + if err := proxySide.Close(); err != nil { + t.Logf("close relay conn: %v", err) + } + }) + + return proxySide, testSide +} + +// TestProxyDemuxesByDestinationAddress is the core of the design: one socket +// serves every peer, and the destination address decides which relayed +// connection a WireGuard packet belongs to. +func TestProxyDemuxesByDestinationAddress(t *testing.T) { + proxy := NewProxy(testWGPort, 1280) + if err := proxy.Listen(); err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { + if err := proxy.Free(); err != nil { + t.Errorf("free proxy: %v", err) + } + }() + + const peers = 3 + endpoints := make([]*net.UDPAddr, 0, peers) + readers := make([]*net.UDPConn, 0, peers) + for i := 0; i < peers; i++ { + proxySide, testSide := relayEnd(t) + endpoint, _, err := proxy.AddRelayedConn(proxySide) + if err != nil { + t.Fatalf("add relayed conn %d: %v", i, err) + } + if endpoint.Port != proxy.proxyPort { + t.Errorf("peer %d endpoint port = %d, want the shared proxy port %d", i, endpoint.Port, proxy.proxyPort) + } + endpoints = append(endpoints, endpoint) + readers = append(readers, testSide) + } + + // every peer must have its own address, otherwise they are indistinguishable + seen := make(map[string]bool, peers) + for i, endpoint := range endpoints { + if seen[endpoint.IP.String()] { + t.Fatalf("peer %d reuses endpoint address %s", i, endpoint.IP) + } + seen[endpoint.IP.String()] = true + } + + wgSock, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")}) + if err != nil { + t.Fatalf("wg socket: %v", err) + } + defer func() { + if err := wgSock.Close(); err != nil { + t.Logf("close wg socket: %v", err) + } + }() + + for i, endpoint := range endpoints { + payload := []byte{byte(i), 'p', 'k', 't'} + if _, err := wgSock.WriteTo(payload, endpoint); err != nil { + t.Fatalf("write to peer %d endpoint %s: %v", i, endpoint, err) + } + + buf := make([]byte, 1500) + if err := readers[i].SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set read deadline: %v", err) + } + n, _, err := readers[i].ReadFrom(buf) + if err != nil { + t.Fatalf("peer %d did not receive its packet: %v", i, err) + } + if string(buf[:n]) != string(payload) { + t.Errorf("peer %d got %q, want %q", i, buf[:n], payload) + } + + // no other peer may see it + for j, other := range readers { + if j == i { + continue + } + if err := other.SetReadDeadline(time.Now().Add(100 * time.Millisecond)); err != nil { + t.Fatalf("set read deadline: %v", err) + } + if _, _, err := other.ReadFrom(buf); err == nil { + t.Errorf("packet for peer %d also delivered to peer %d", i, j) + } + } + } +} + +// TestProxyDropsPacketsOutsideTheRange guards the wildcard bind: anything that +// is not addressed to a handed-out endpoint must not reach a relayed peer. +func TestProxyDropsPacketsOutsideTheRange(t *testing.T) { + proxy := NewProxy(testWGPort+1, 1280) + if err := proxy.Listen(); err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { + if err := proxy.Free(); err != nil { + t.Errorf("free proxy: %v", err) + } + }() + + proxySide, testSide := relayEnd(t) + if _, _, err := proxy.AddRelayedConn(proxySide); err != nil { + t.Fatalf("add relayed conn: %v", err) + } + + sender, err := net.Dial("udp", net.JoinHostPort("127.0.0.1", strconv.Itoa(proxy.proxyPort))) + if err != nil { + t.Fatalf("sender: %v", err) + } + defer func() { + if err := sender.Close(); err != nil { + t.Logf("close sender: %v", err) + } + }() + + if _, err := sender.Write([]byte("stray")); err != nil { + t.Fatalf("write stray packet: %v", err) + } + + buf := make([]byte, 1500) + if err := testSide.SetReadDeadline(time.Now().Add(500 * time.Millisecond)); err != nil { + t.Fatalf("set read deadline: %v", err) + } + if _, _, err := testSide.ReadFrom(buf); err == nil { + t.Error("packet addressed to 127.0.0.1 was forwarded to a relayed peer") + } +} + +// A wrapper that is closed before it starts forwarding still has to give its +// endpoint address back, otherwise the range leaks an address per attempt. +func TestClosingBeforeWorkReleasesTheAddress(t *testing.T) { + proxy := NewProxy(testWGPort+2, 1280) + if err := proxy.Listen(); err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { + if err := proxy.Free(); err != nil { + t.Errorf("free proxy: %v", err) + } + }() + + proxySide, _ := relayEnd(t) + wrapper := NewProxyWrapper(proxy) + if err := wrapper.AddRelayedConn(context.Background(), nil, proxySide); err != nil { + t.Fatalf("add relayed conn: %v", err) + } + + if got := len(proxy.relayedConnStore); got != 1 { + t.Fatalf("store holds %d entries after adding one conn, want 1", got) + } + + if err := wrapper.CloseConn(); err != nil { + t.Fatalf("close conn: %v", err) + } + + if got := len(proxy.relayedConnStore); got != 0 { + t.Errorf("store holds %d entries after close, want 0", got) + } +} diff --git a/client/iface/wgproxy/ebpf/wrapper.go b/client/iface/wgproxy/loopback/wrapper.go similarity index 84% rename from client/iface/wgproxy/ebpf/wrapper.go rename to client/iface/wgproxy/loopback/wrapper.go index f75e21aa6..a9cb1ab59 100644 --- a/client/iface/wgproxy/ebpf/wrapper.go +++ b/client/iface/wgproxy/loopback/wrapper.go @@ -1,6 +1,6 @@ //go:build linux && !android -package ebpf +package loopback import ( "context" @@ -8,6 +8,7 @@ import ( "fmt" "io" "net" + "net/netip" "sync" "github.com/google/gopacket" @@ -95,13 +96,14 @@ func NewPacketHeaders(localWGListenPort int, endpoint *net.UDPAddr) (*PacketHead // ProxyWrapper help to keep the remoteConn instance for net.Conn.Close function call type ProxyWrapper struct { - wgeBPFProxy *WGEBPFProxy + proxy *Proxy remoteConn net.Conn ctx context.Context cancel context.CancelFunc wgRelayedEndpointAddr *net.UDPAddr + peerAddr netip.Addr headers *PacketHeaders headerCurrentUsed *PacketHeaders rawConn net.PacketConn @@ -113,36 +115,44 @@ type ProxyWrapper struct { closeListener *listener.CloseListener } -func NewProxyWrapper(proxy *WGEBPFProxy) *ProxyWrapper { +func NewProxyWrapper(proxy *Proxy) *ProxyWrapper { return &ProxyWrapper{ - wgeBPFProxy: proxy, + proxy: proxy, pausedCond: sync.NewCond(&sync.Mutex{}), closeListener: listener.NewCloseListener(), } } func (p *ProxyWrapper) AddRelayedConn(ctx context.Context, _ *net.UDPAddr, remoteConn net.Conn) error { - addr, err := p.wgeBPFProxy.AddRelayedConn(remoteConn) + addr, peerAddr, err := p.proxy.AddRelayedConn(remoteConn) if err != nil { return fmt.Errorf("add relayed conn: %w", err) } - headers, err := NewPacketHeaders(p.wgeBPFProxy.localWGListenPort, addr) + // the endpoint address is otherwise only released by the forwarding + // goroutine, which never starts when the setup below fails + release := func() { p.proxy.removeRelayedConn(peerAddr, remoteConn) } + + headers, err := NewPacketHeaders(p.proxy.localWGListenPort, addr) if err != nil { + release() return fmt.Errorf("create packet sender: %w", err) } // Check if required raw connection is available - if !headers.isIPv4 && p.wgeBPFProxy.rawConnIPv6 == nil { + if !headers.isIPv4 && p.proxy.rawConnIPv6 == nil { + release() return errIPv6ConnNotAvailable } - if headers.isIPv4 && p.wgeBPFProxy.rawConnIPv4 == nil { + if headers.isIPv4 && p.proxy.rawConnIPv4 == nil { + release() return errIPv4ConnNotAvailable } p.remoteConn = remoteConn p.ctx, p.cancel = context.WithCancel(ctx) p.wgRelayedEndpointAddr = addr + p.peerAddr = peerAddr p.headers = headers p.rawConn = p.selectRawConn(headers) return nil @@ -193,18 +203,18 @@ func (p *ProxyWrapper) RedirectAs(endpoint *net.UDPAddr) { return } - header, err := NewPacketHeaders(p.wgeBPFProxy.localWGListenPort, endpoint) + header, err := NewPacketHeaders(p.proxy.localWGListenPort, endpoint) if err != nil { log.Errorf("failed to create packet headers: %s", err) return } // Check if required raw connection is available - if !header.isIPv4 && p.wgeBPFProxy.rawConnIPv6 == nil { + if !header.isIPv4 && p.proxy.rawConnIPv6 == nil { log.Error(errIPv6ConnNotAvailable) return } - if header.isIPv4 && p.wgeBPFProxy.rawConnIPv4 == nil { + if header.isIPv4 && p.proxy.rawConnIPv4 == nil { log.Error(errIPv4ConnNotAvailable) return } @@ -240,6 +250,10 @@ func (p *ProxyWrapper) CloseConn() error { p.closeListener.SetCloseListener(nil) + // releases the endpoint address for a wrapper that was never started, and + // is a no-op once the forwarding goroutine has released it + p.proxy.removeRelayedConn(p.peerAddr, p.remoteConn) + p.pausedCond.L.Lock() p.paused = false p.pausedCond.Signal() @@ -252,9 +266,9 @@ func (p *ProxyWrapper) CloseConn() error { } func (p *ProxyWrapper) proxyToLocal(ctx context.Context) { - defer p.wgeBPFProxy.removeRelayedConn(uint16(p.wgRelayedEndpointAddr.Port)) + defer p.proxy.removeRelayedConn(p.peerAddr, p.remoteConn) - buf := make([]byte, p.wgeBPFProxy.mtu+bufsize.WGBufferOverhead) + buf := make([]byte, p.proxy.mtu+bufsize.WGBufferOverhead) for { n, err := p.readFromRemote(ctx, buf) if err != nil { @@ -286,7 +300,7 @@ func (p *ProxyWrapper) readFromRemote(ctx context.Context, buf []byte) (int, err } p.closeListener.Notify() if !errors.Is(err, io.EOF) { - log.Errorf("failed to read from relayed conn (endpoint: :%d): %s", p.wgRelayedEndpointAddr.Port, err) + log.Errorf("failed to read from relayed conn (endpoint: %s): %s", p.wgRelayedEndpointAddr, err) } return 0, err } @@ -314,7 +328,7 @@ func (p *ProxyWrapper) sendPkg(data []byte, header *PacketHeaders) error { func (p *ProxyWrapper) selectRawConn(header *PacketHeaders) net.PacketConn { if header.isIPv4 { - return p.wgeBPFProxy.rawConnIPv4 + return p.proxy.rawConnIPv4 } - return p.wgeBPFProxy.rawConnIPv6 + return p.proxy.rawConnIPv6 } diff --git a/client/iface/wgproxy/proxy_linux_test.go b/client/iface/wgproxy/proxy_linux_test.go index e34dd3b6b..88d4588a5 100644 --- a/client/iface/wgproxy/proxy_linux_test.go +++ b/client/iface/wgproxy/proxy_linux_test.go @@ -9,25 +9,25 @@ import ( "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/wgaddr" bindproxy "github.com/netbirdio/netbird/client/iface/wgproxy/bind" - "github.com/netbirdio/netbird/client/iface/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/iface/wgproxy/loopback" "github.com/netbirdio/netbird/client/iface/wgproxy/udp" ) func seedProxies() ([]proxyInstance, error) { pl := make([]proxyInstance, 0) - ebpfProxy := ebpf.NewWGEBPFProxy(51831, 1280) - if err := ebpfProxy.Listen(); err != nil { - return nil, fmt.Errorf("failed to initialize ebpf proxy: %s", err) + loopbackProxy := loopback.NewProxy(51831, 1280) + if err := loopbackProxy.Listen(); err != nil { + return nil, fmt.Errorf("failed to initialize loopback proxy: %s", err) } - pEbpf := proxyInstance{ - name: "ebpf kernel proxy", - proxy: ebpf.NewProxyWrapper(ebpfProxy), + pLoopback := proxyInstance{ + name: "loopback kernel proxy", + proxy: loopback.NewProxyWrapper(loopbackProxy), wgPort: 51831, - closeFn: ebpfProxy.Free, + closeFn: loopbackProxy.Free, } - pl = append(pl, pEbpf) + pl = append(pl, pLoopback) pUDP := proxyInstance{ name: "udp kernel proxy", @@ -42,18 +42,18 @@ func seedProxies() ([]proxyInstance, error) { func seedProxyForProxyCloseByRemoteConn() ([]proxyInstance, error) { pl := make([]proxyInstance, 0) - ebpfProxy := ebpf.NewWGEBPFProxy(51831, 1280) - if err := ebpfProxy.Listen(); err != nil { - return nil, fmt.Errorf("failed to initialize ebpf proxy: %s", err) + loopbackProxy := loopback.NewProxy(51831, 1280) + if err := loopbackProxy.Listen(); err != nil { + return nil, fmt.Errorf("failed to initialize loopback proxy: %s", err) } - pEbpf := proxyInstance{ - name: "ebpf kernel proxy", - proxy: ebpf.NewProxyWrapper(ebpfProxy), + pLoopback := proxyInstance{ + name: "loopback kernel proxy", + proxy: loopback.NewProxyWrapper(loopbackProxy), wgPort: 51831, - closeFn: ebpfProxy.Free, + closeFn: loopbackProxy.Free, } - pl = append(pl, pEbpf) + pl = append(pl, pLoopback) pUDP := proxyInstance{ name: "udp kernel proxy", diff --git a/client/iface/wgproxy/redirect_test.go b/client/iface/wgproxy/redirect_test.go index f0d59cc64..47f571f2b 100644 --- a/client/iface/wgproxy/redirect_test.go +++ b/client/iface/wgproxy/redirect_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/netbirdio/netbird/client/iface/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/iface/wgproxy/loopback" "github.com/netbirdio/netbird/client/iface/wgproxy/udp" ) @@ -198,20 +198,20 @@ func testRedirectAs(t *testing.T, proxy Proxy, wgPort int, nbAddr, p2pEndpoint * } } -// TestRedirectAs_eBPF_IPv4 tests RedirectAs with eBPF proxy using IPv4 addresses -func TestRedirectAs_eBPF_IPv4(t *testing.T) { +// TestRedirectAs_Loopback_IPv4 tests RedirectAs with the loopback proxy using IPv4 addresses +func TestRedirectAs_Loopback_IPv4(t *testing.T) { wgPort := 51850 - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280) - if err := ebpfProxy.Listen(); err != nil { - t.Fatalf("failed to initialize ebpf proxy: %v", err) + loopbackProxy := loopback.NewProxy(wgPort, 1280) + if err := loopbackProxy.Listen(); err != nil { + t.Fatalf("failed to initialize loopback proxy: %v", err) } defer func() { - if err := ebpfProxy.Free(); err != nil { - t.Errorf("failed to free ebpf proxy: %v", err) + if err := loopbackProxy.Free(); err != nil { + t.Errorf("failed to free loopback proxy: %v", err) } }() - proxy := ebpf.NewProxyWrapper(ebpfProxy) + proxy := loopback.NewProxyWrapper(loopbackProxy) // NetBird UDP address of the remote peer nbAddr := &net.UDPAddr{ @@ -227,20 +227,20 @@ func TestRedirectAs_eBPF_IPv4(t *testing.T) { testRedirectAs(t, proxy, wgPort, nbAddr, p2pEndpoint) } -// TestRedirectAs_eBPF_IPv6 tests RedirectAs with eBPF proxy using IPv6 addresses -func TestRedirectAs_eBPF_IPv6(t *testing.T) { +// TestRedirectAs_Loopback_IPv6 tests RedirectAs with the loopback proxy using IPv6 addresses +func TestRedirectAs_Loopback_IPv6(t *testing.T) { wgPort := 51851 - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280) - if err := ebpfProxy.Listen(); err != nil { - t.Fatalf("failed to initialize ebpf proxy: %v", err) + loopbackProxy := loopback.NewProxy(wgPort, 1280) + if err := loopbackProxy.Listen(); err != nil { + t.Fatalf("failed to initialize loopback proxy: %v", err) } defer func() { - if err := ebpfProxy.Free(); err != nil { - t.Errorf("failed to free ebpf proxy: %v", err) + if err := loopbackProxy.Free(); err != nil { + t.Errorf("failed to free loopback proxy: %v", err) } }() - proxy := ebpf.NewProxyWrapper(ebpfProxy) + proxy := loopback.NewProxyWrapper(loopbackProxy) // NetBird UDP address of the remote peer nbAddr := &net.UDPAddr{ @@ -259,17 +259,17 @@ func TestRedirectAs_eBPF_IPv6(t *testing.T) { // TestRedirectAs_Multiple_Switches tests switching between multiple endpoints func TestRedirectAs_Multiple_Switches(t *testing.T) { wgPort := 51856 - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280) - if err := ebpfProxy.Listen(); err != nil { - t.Fatalf("failed to initialize ebpf proxy: %v", err) + loopbackProxy := loopback.NewProxy(wgPort, 1280) + if err := loopbackProxy.Listen(); err != nil { + t.Fatalf("failed to initialize loopback proxy: %v", err) } defer func() { - if err := ebpfProxy.Free(); err != nil { - t.Errorf("failed to free ebpf proxy: %v", err) + if err := loopbackProxy.Free(); err != nil { + t.Errorf("failed to free loopback proxy: %v", err) } }() - proxy := ebpf.NewProxyWrapper(ebpfProxy) + proxy := loopback.NewProxyWrapper(loopbackProxy) ctx := context.Background() diff --git a/client/installer.nsis b/client/installer.nsis index eb2d7d5bd..304d38864 100644 --- a/client/installer.nsis +++ b/client/installer.nsis @@ -2,7 +2,7 @@ !define COMP_NAME "Netbird" !define WEB_SITE "Netbird.io" !define VERSION $%APPVER% -!define COPYRIGHT "Netbird Authors, 2022" +!define COPYRIGHT "Netbird Authors, 2022-2026" !define DESCRIPTION "Connect your devices into a secure WireGuard-based overlay network with SSO, MFA, and granular access controls." !define INSTALLER_NAME "netbird-installer.exe" !define MAIN_APP_EXE "Netbird" diff --git a/client/internal/auth/account_match_test.go b/client/internal/auth/account_match_test.go new file mode 100644 index 000000000..b879cf9ab --- /dev/null +++ b/client/internal/auth/account_match_test.go @@ -0,0 +1,113 @@ +package auth + +import ( + "encoding/base64" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTokenInfoMatchesAccount(t *testing.T) { + tests := []struct { + name string + token TokenInfo + hint string + match bool + }{ + { + name: "same account", + token: TokenInfo{EmailClaim: "user@example.com"}, + hint: "user@example.com", + match: true, + }, + { + name: "different account", + token: TokenInfo{EmailClaim: "other@example.com"}, + hint: "user@example.com", + match: false, + }, + { + name: "case differences are the same account", + token: TokenInfo{EmailClaim: "User@Example.com"}, + hint: "user@example.com", + match: true, + }, + { + name: "no hint leaves the choice to the IdP", + token: TokenInfo{EmailClaim: "other@example.com"}, + hint: "", + match: true, + }, + { + name: "token without an email claim is not judged", + token: TokenInfo{EmailClaim: ""}, + hint: "user@example.com", + match: true, + }, + { + name: "name fallback does not trigger matching", + token: TokenInfo{Email: "Some One"}, + hint: "user@example.com", + match: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.match, tc.token.MatchesAccount(tc.hint)) + }) + } +} + +func TestParseEmailFromIDToken(t *testing.T) { + tests := []struct { + name string + claims map[string]interface{} + wantValue string + wantFromEmail bool + wantErr bool + }{ + { + name: "email claim", + claims: map[string]interface{}{"email": "user@example.com", "name": "Some One"}, + wantValue: "user@example.com", + wantFromEmail: true, + }, + { + name: "name fallback", + claims: map[string]interface{}{"name": "Some One"}, + wantValue: "Some One", + }, + { + name: "neither claim present", + claims: map[string]interface{}{"sub": "abc"}, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + value, fromEmailClaim, err := parseEmailFromIDToken(idTokenWithClaims(t, tc.claims)) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantValue, value) + assert.Equal(t, tc.wantFromEmail, fromEmailClaim) + }) + } +} + +func TestRetryFlowForAccountUnsupportedFlow(t *testing.T) { + assert.Nil(t, RetryFlowForAccount(&DeviceAuthorizationFlow{})) +} + +func idTokenWithClaims(t *testing.T, claims map[string]interface{}) string { + t.Helper() + payload, err := json.Marshal(claims) + require.NoError(t, err) + return "header." + base64.RawURLEncoding.EncodeToString(payload) + ".signature" +} diff --git a/client/internal/auth/auth.go b/client/internal/auth/auth.go index 240f7efe7..ff2a8c5e1 100644 --- a/client/internal/auth/auth.go +++ b/client/internal/auth/auth.go @@ -103,7 +103,7 @@ func (a *Auth) IsSSOSupported(ctx context.Context) (bool, error) { err := a.withRetry(ctx, func(client *mgm.GrpcClient) error { // Try PKCE flow first - _, err := a.getPKCEFlow(client) + _, err := a.getPKCEFlow(client, false) if err == nil { supportsSSO = true return nil @@ -136,9 +136,13 @@ func (a *Auth) IsSSOSupported(ctx context.Context) (bool, error) { return supportsSSO, err } -// GetOAuthFlow returns an OAuth flow (PKCE or Device) using the existing management connection -// This avoids creating a new connection to the management server -func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool, hint string) (OAuthFlow, error) { +// GetOAuthFlow returns an OAuth flow (PKCE or Device) using the existing management connection. +// This avoids creating a new connection to the management server. +// +// sessionExtend marks the flow as renewing an existing peer's session rather than +// logging one in; the server needs it to rule out a silent authorization that the +// IdP could answer from another account. See PKCEAuthorizationFlowRequest. +func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool, sessionExtend bool, hint string) (OAuthFlow, error) { var flow OAuthFlow err := a.withRetry(ctx, func(client *mgm.GrpcClient) error { @@ -153,7 +157,7 @@ func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool, hint stri } // Try PKCE flow first - pkceFlow, err := a.getPKCEFlow(client) + pkceFlow, err := a.getPKCEFlow(client, sessionExtend) if err != nil { // If PKCE not supported, try Device flow if s, ok := status.FromError(err); ok && (s.Code() == codes.NotFound || s.Code() == codes.Unimplemented) { @@ -240,8 +244,8 @@ func (a *Auth) Login(ctx context.Context, setupKey string, jwtToken string) (err } // getPKCEFlow retrieves PKCE authorization flow configuration and creates a flow instance -func (a *Auth) getPKCEFlow(client *mgm.GrpcClient) (*PKCEAuthorizationFlow, error) { - protoFlow, err := client.GetPKCEAuthorizationFlow() +func (a *Auth) getPKCEFlow(client *mgm.GrpcClient, sessionExtend bool) (*PKCEAuthorizationFlow, error) { + protoFlow, err := client.GetPKCEAuthorizationFlow(sessionExtend) if err != nil { if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound { log.Warnf("server couldn't find pkce flow, contact admin: %v", err) diff --git a/client/internal/auth/device_flow.go b/client/internal/auth/device_flow.go index 3592e589d..c2b8d34b8 100644 --- a/client/internal/auth/device_flow.go +++ b/client/internal/auth/device_flow.go @@ -308,10 +308,13 @@ func (d *DeviceAuthorizationFlow) WaitToken(ctx context.Context, info AuthFlowIn // callers store to send back as the login_hint. Without it a client // driven through the device flow — Android TV and tvOS — never binds // an account to its profile and every later login goes out blind. - if email, err := parseEmailFromIDToken(tokenInfo.IDToken); err != nil { + if email, fromEmailClaim, err := parseEmailFromIDToken(tokenInfo.IDToken); err != nil { log.Warnf("failed to parse email from ID token: %v", err) } else { tokenInfo.Email = email + if fromEmailClaim { + tokenInfo.EmailClaim = email + } } log.Infof("device flow: user authorization confirmed after %d polls in %s", polls, time.Since(start).Round(time.Second)) diff --git a/client/internal/auth/oauth.go b/client/internal/auth/oauth.go index 91329c98b..54ecb5d86 100644 --- a/client/internal/auth/oauth.go +++ b/client/internal/auth/oauth.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "runtime" + "strings" log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" @@ -25,6 +26,14 @@ type HTTPClient interface { Do(req *http.Request) (*http.Response, error) } +// accountPromptForcer is implemented by the PKCE flow only. The device code +// flow has no equivalent: RFC 8628 defines no prompt parameter, and the user +// confirms the code on a page that shows which account signs in, so a silent +// wrong-account answer is not the failure mode there. +type accountPromptForcer interface { + ForceAccountPrompt() +} + // AuthFlowInfo holds information for the OAuth 2.0 authorization flow type AuthFlowInfo struct { //nolint:revive DeviceCode string `json:"device_code"` @@ -49,6 +58,23 @@ type TokenInfo struct { ExpiresIn int `json:"expires_in"` UseIDToken bool `json:"-"` Email string `json:"-"` + EmailClaim string `json:"-"` +} + +// MatchesAccount reports whether the token belongs to the account a profile is +// bound to. A hint the IdP could not have acted on — no hint stored, or a token +// that carried no email claim — is reported as a match: the check exists to catch a +// login answered from the wrong account, not to block one it cannot judge. +// +// The comparison is case-insensitive. Local-parts are case-sensitive per RFC +// 5321, but no IdP in practice issues two accounts differing only in case, and +// an IdP that echoes a differently-cased address would otherwise fail every +// login. +func (t TokenInfo) MatchesAccount(hint string) bool { + if hint == "" || t.EmailClaim == "" { + return true + } + return strings.EqualFold(t.EmailClaim, hint) } // GetTokenToUse returns either the access or id token based on UseIDToken field @@ -63,19 +89,22 @@ func shouldUseDeviceFlow(force bool, isUnixDesktopClient bool) bool { return force || (runtime.GOOS == "linux" || runtime.GOOS == "freebsd") && !isUnixDesktopClient } -// NewOAuthFlow initializes and returns the appropriate OAuth flow based on the management configuration +// NewOAuthFlow initializes and returns the appropriate OAuth flow based on the management configuration. // // It starts by initializing the PKCE.If this process fails, it resorts to the Device Code Flow, // and if that also fails, the authentication process is deemed unsuccessful // -// On Linux distros without desktop environment support, it only tries to initialize the Device Code Flow -// forceDeviceCodeFlow can be used to skip PKCE and go directly to Device Code Flow (e.g., for Android TV) -func NewOAuthFlow(ctx context.Context, config *profilemanager.Config, isUnixDesktopClient bool, forceDeviceCodeFlow bool, hint string) (OAuthFlow, error) { +// On Linux distros without desktop environment support, it only tries to initialize the Device Code Flow. +// forceDeviceCodeFlow can be used to skip PKCE and go directly to Device Code Flow (e.g., for Android TV). +// +// sessionExtend marks the flow as renewing an existing peer's session rather than +// logging one in. See PKCEAuthorizationFlowRequest for what the server makes of it. +func NewOAuthFlow(ctx context.Context, config *profilemanager.Config, isUnixDesktopClient bool, forceDeviceCodeFlow bool, hint string, sessionExtend bool) (OAuthFlow, error) { if shouldUseDeviceFlow(forceDeviceCodeFlow, isUnixDesktopClient) { return authenticateWithDeviceCodeFlow(ctx, config, hint) } - pkceFlow, err := authenticateWithPKCEFlow(ctx, config, hint) + pkceFlow, err := authenticateWithPKCEFlow(ctx, config, hint, sessionExtend) if err != nil { log.Debugf("failed to initialize pkce authentication with error: %v\n", err) log.Debug("falling back to device code flow") @@ -85,14 +114,14 @@ func NewOAuthFlow(ctx context.Context, config *profilemanager.Config, isUnixDesk } // authenticateWithPKCEFlow initializes the Proof Key for Code Exchange flow auth flow -func authenticateWithPKCEFlow(ctx context.Context, config *profilemanager.Config, hint string) (OAuthFlow, error) { +func authenticateWithPKCEFlow(ctx context.Context, config *profilemanager.Config, hint string, sessionExtend bool) (OAuthFlow, error) { authClient, err := NewAuth(ctx, config.PrivateKey, config.ManagementURL, config) if err != nil { return nil, fmt.Errorf("failed to create auth client: %v", err) } defer authClient.Close() - pkceFlowInfo, err := authClient.getPKCEFlow(authClient.client) + pkceFlowInfo, err := authClient.getPKCEFlow(authClient.client, sessionExtend) if err != nil { return nil, fmt.Errorf("getting pkce authorization flow info failed with error: %v", err) } @@ -129,3 +158,21 @@ func authenticateWithDeviceCodeFlow(ctx context.Context, config *profilemanager. return deviceFlowInfo, nil } + +// RetryFlowForAccount returns a flow that asks the IdP to re-authenticate, for +// a login answered with an account other than the one hinted. Returns nil when +// the flow cannot ask — the caller then proceeds with the token it has. +// +// Proceeding rather than failing is deliberate. The hint is an email that may +// simply have changed since it was stored, and refusing the login would lock a +// user out of their own profile over a rename. The retry gives the account a +// chance to be corrected; the server still rejects a token that does not own +// the peer. +func RetryFlowForAccount(flow OAuthFlow) OAuthFlow { + forcer, ok := flow.(accountPromptForcer) + if !ok { + return nil + } + forcer.ForceAccountPrompt() + return flow +} diff --git a/client/internal/auth/pkce_flow.go b/client/internal/auth/pkce_flow.go index be64cc6a8..ebb0d4f1f 100644 --- a/client/internal/auth/pkce_flow.go +++ b/client/internal/auth/pkce_flow.go @@ -87,10 +87,11 @@ func validatePKCEConfig(config *PKCEAuthProviderConfig) error { // PKCEAuthorizationFlow implements the OAuthFlow interface for // the Authorization Code Flow with PKCE. type PKCEAuthorizationFlow struct { - providerConfig PKCEAuthProviderConfig - state string - codeVerifier string - oAuthConfig *oauth2.Config + providerConfig PKCEAuthProviderConfig + state string + codeVerifier string + oAuthConfig *oauth2.Config + forceAccountPrompt bool } // NewPKCEAuthorizationFlow returns new PKCE authorization code flow. @@ -153,11 +154,16 @@ func (p *PKCEAuthorizationFlow) RequestAuthInfo(ctx context.Context) (AuthFlowIn oauth2.SetAuthURLParam("code_challenge", codeChallenge), oauth2.SetAuthURLParam("audience", p.providerConfig.Audience), } + forceAccountPrompt := p.forceAccountPrompt + p.forceAccountPrompt = false + if !p.providerConfig.DisablePromptLogin { - switch p.providerConfig.LoginFlag { - case common.LoginFlagPromptLogin: + switch { + case forceAccountPrompt: params = append(params, oauth2.SetAuthURLParam("prompt", "login")) - case common.LoginFlagMaxAge0: + case p.providerConfig.LoginFlag == common.LoginFlagPromptLogin: + params = append(params, oauth2.SetAuthURLParam("prompt", "login")) + case p.providerConfig.LoginFlag == common.LoginFlagMaxAge0: params = append(params, oauth2.SetAuthURLParam("max_age", "0")) } } @@ -178,6 +184,20 @@ func (p *PKCEAuthorizationFlow) SetLoginHint(hint string) { p.providerConfig.LoginHint = hint } +// ForceAccountPrompt makes the next authorization request ask the IdP to +// re-authenticate instead of answering from the session it already holds. Used +// to retry a login that came back for an account other than the one hinted. +// +// The next RequestAuthInfo consumes the flag, so a flow that outlives its retry +// goes back to the configured behaviour instead of re-authenticating forever. +// +// DisablePromptLogin still wins: it is set for IdPs that break on prompt=login, +// where retrying with it would replace a wrong-account login with one that +// cannot complete at all. +func (p *PKCEAuthorizationFlow) ForceAccountPrompt() { + p.forceAccountPrompt = true +} + // WaitToken waits for the OAuth token in the PKCE Authorization Flow. // It starts an HTTP server to receive the OAuth token callback and waits for the token or an error. // Once the token is received, it is converted to TokenInfo and validated before returning. @@ -310,49 +330,48 @@ func (p *PKCEAuthorizationFlow) parseOAuthToken(token *oauth2.Token) (TokenInfo, return TokenInfo{}, fmt.Errorf("authentication failed: invalid access token - %w", err) } - email, err := parseEmailFromIDToken(tokenInfo.IDToken) + email, fromEmailClaim, err := parseEmailFromIDToken(tokenInfo.IDToken) if err != nil { log.Warnf("failed to parse email from ID token: %v", err) } else { tokenInfo.Email = email + if fromEmailClaim { + tokenInfo.EmailClaim = email + } } return tokenInfo, nil } // parseEmailFromIDToken extracts the email (or name) claim from an ID token -// without verifying its signature. The value is best-effort and used only as a -// UX convenience (login hint prefill and display); it never drives an -// authorization decision. The authoritative identity is established server-side -// from the signature-verified token. -func parseEmailFromIDToken(token string) (string, error) { +// without verifying its signature. The value is best-effort: it prefills the +// login hint and is displayed. Account matching (see MatchesAccount) only uses +// it when it came from the email claim, which fromEmailClaim reports. It never +// grants anything — the authoritative identity is established server-side from +// the signature-verified token. +func parseEmailFromIDToken(token string) (value string, fromEmailClaim bool, err error) { parts := strings.Split(token, ".") if len(parts) < 2 { - return "", fmt.Errorf("invalid token format") + return "", false, fmt.Errorf("invalid token format") } data, err := base64.RawURLEncoding.DecodeString(parts[1]) if err != nil { - return "", fmt.Errorf("failed to decode payload: %w", err) + return "", false, fmt.Errorf("failed to decode payload: %w", err) } var claims map[string]interface{} if err := json.Unmarshal(data, &claims); err != nil { - return "", fmt.Errorf("json unmarshal error: %w", err) + return "", false, fmt.Errorf("json unmarshal error: %w", err) } - var email string - if emailValue, ok := claims["email"].(string); ok { - email = emailValue - } else { - val, ok := claims["name"].(string) - if ok { - email = val - } else { - return "", fmt.Errorf("email or name field not found in token payload") - } + if email, ok := claims["email"].(string); ok { + return email, true, nil + } + if name, ok := claims["name"].(string); ok { + return name, false, nil } - return email, nil + return "", false, fmt.Errorf("email or name field not found in token payload") } func createCodeChallenge(codeVerifier string) string { diff --git a/client/internal/auth/pkce_flow_test.go b/client/internal/auth/pkce_flow_test.go index c487c13df..ccbca10e9 100644 --- a/client/internal/auth/pkce_flow_test.go +++ b/client/internal/auth/pkce_flow_test.go @@ -76,6 +76,32 @@ func TestPromptLogin(t *testing.T) { } } +func TestForceAccountPromptAppliesOnlyToTheRetry(t *testing.T) { + config := PKCEAuthProviderConfig{ + ClientID: "test-client-id", + Audience: "test-audience", + TokenEndpoint: "https://test-token-endpoint.com/token", + Scope: "openid email profile", + AuthorizationEndpoint: "https://test-auth-endpoint.com/authorize", + RedirectURLs: []string{"http://127.0.0.1:33992/"}, + UseIDToken: true, + LoginFlag: mgm.LoginFlagNone, + } + pkce, err := NewPKCEAuthorizationFlow(config) + require.NoError(t, err) + + pkce.ForceAccountPrompt() + + retry, err := pkce.RequestAuthInfo(context.Background()) + require.NoError(t, err) + require.Contains(t, retry.VerificationURIComplete, "prompt=login") + + next, err := pkce.RequestAuthInfo(context.Background()) + require.NoError(t, err) + require.NotContains(t, next.VerificationURIComplete, "prompt=login", + "the forced prompt outlived the retry it was armed for") +} + func TestIsPortInExcludedRange(t *testing.T) { tests := []struct { name string diff --git a/client/internal/auth/sessionwatch/watcher.go b/client/internal/auth/sessionwatch/watcher.go index 3c235b085..050c75545 100644 --- a/client/internal/auth/sessionwatch/watcher.go +++ b/client/internal/auth/sessionwatch/watcher.go @@ -4,12 +4,20 @@ // T-WarningLead notification and a dismiss-gated T-FinalWarningLead // fallback dialog. // +// The deadline is an absolute wall-clock instant, so the watcher compares +// it against the wall clock on a ticker rather than arming a relative +// timer for it. A relative timer runs on the monotonic clock, which does +// not advance while a device is suspended: it fires once that much awake +// time has passed, which can be long after the deadline, and nothing +// re-evaluates when the device wakes up. Polling makes every tick after a +// resume (or after an NTP correction) see the real remaining time. +// // The watcher is idempotent: Update may be called as often as the network // map snapshots arrive. Repeating the same deadline is a no-op; a new -// deadline reschedules the timers and arms a fresh warning cycle. +// deadline starts a fresh warning cycle. // -// Warning firing is edge-detected. Each unique deadline value fires each -// warning callback at most once. +// Warning firing is edge-detected. Each unique deadline value publishes +// each warning at most once. package sessionwatch import ( @@ -28,9 +36,15 @@ const ( // maxDeadlineHorizon caps how far in the future an accepted deadline // can sit. A timestamp beyond this is almost certainly a protocol - // glitch, and silently arming a 100-year timer would hide the bug. + // glitch, and silently tracking a 100-year deadline would hide the bug. maxDeadlineHorizon = 10 * 365 * 24 * time.Hour + // defaultEvalInterval is how often the tracked deadline is compared + // against the wall clock. The leads are minutes, so a coarse tick costs + // nothing in accuracy and keeps the wakeup cheap on battery-powered + // devices. + defaultEvalInterval = 10 * time.Second + // WarningLead is how far before expiry the first (interactive) // warning fires. Drives the T-10 OS notification with // Extend/Dismiss actions. @@ -92,18 +106,23 @@ type StatusRecorder interface { // fallback T-FinalWarningLead dialog (suppressed when the user dismissed // the first one for the same deadline). Safe for concurrent use. type Watcher struct { - lead time.Duration - finalLead time.Duration + lead time.Duration + finalLead time.Duration + interval time.Duration + deadlineOnly bool // record the deadline, leave the warnings to the caller mu sync.Mutex current time.Time - timer *time.Timer - finalTimer *time.Timer - firedAt time.Time // deadline value the T-WarningLead callback last fired against - finalFiredAt time.Time // deadline value the T-FinalWarningLead callback last fired against - dismissedAt time.Time // deadline value the user dismissed via Dismiss(); gates fireFinal + firedAt time.Time // deadline value the T-WarningLead warning last published for + finalFiredAt time.Time // deadline value the T-FinalWarningLead warning last published for + dismissedAt time.Time // deadline value the user dismissed via Dismiss(); gates the final warning + announcedAt time.Time // deadline value the recorder has been told about; gates publishing closed bool recorder StatusRecorder + nowFn func() time.Time + stop chan struct{} // closed to stop the evaluation loop; nil while it is not running + done chan struct{} // closed by the loop on its way out + wake chan struct{} // buffered nudge asking the loop to evaluate before its next tick } // New returns a watcher with the package defaults WarningLead and @@ -115,27 +134,40 @@ func New(recorder StatusRecorder) *Watcher { } // NewWithLeads returns a watcher with custom lead times. Useful for tests. -// final must be strictly less than lead; otherwise both timers fire in the -// wrong order or simultaneously and the UI flow breaks. A zero final lead -// disables the final-warning timer entirely (see armTimerLocked) so a -// millisecond-scale deadline doesn't flush both timers in one tick. +// final must be strictly less than lead; otherwise the final warning takes +// over the whole warning window and the interactive notification never +// shows. A zero final lead disables the final warning entirely (see +// evaluate), leaving the interactive one as the only warning. func NewWithLeads(lead, final time.Duration, recorder StatusRecorder) *Watcher { return &Watcher{ lead: lead, finalLead: final, + interval: defaultEvalInterval, recorder: recorder, + nowFn: time.Now, } } +// NewDeadlineOnly returns a watcher that validates and records deadlines +// but publishes no warnings about them, and runs no evaluation loop to +// decide. Used where the deadline is handed on to something that schedules +// the warnings itself, such as the Android app. +func NewDeadlineOnly(recorder StatusRecorder) *Watcher { + w := New(recorder) + w.deadlineOnly = true + return w +} + // Update sets the latest deadline. Pass the zero time to clear (e.g. when // a Sync push from the server omits the field because login expiration // was disabled). // -// Same-value updates are no-ops. A different non-zero value cancels any -// pending timer, resets the "already fired" guards, and — when the -// deadline lies in the future — arms fresh warning timers. A deadline -// already in the past (within maxPastHorizon) is recorded as-is with no -// timers: the session has expired and consumers render it that way. +// Same-value updates are no-ops. A different non-zero value resets the +// "already fired" guards and starts a fresh warning cycle, evaluated +// immediately so a deadline that already sits inside a warning window +// warns without waiting for the next tick. A deadline already in the past +// (within maxPastHorizon) is recorded as-is and warns nothing: the session +// has expired and consumers render it that way. // // Returns one of the sentinel Err* values when the deadline fails the // sanity checks (pre-epoch, far future, or past beyond maxPastHorizon). @@ -155,7 +187,7 @@ func (w *Watcher) Update(deadline time.Time) error { return nil } - now := time.Now() + now := w.nowFn() switch { case deadline.Before(time.Unix(0, 0)): w.clearLocked() @@ -173,18 +205,22 @@ func (w *Watcher) Update(deadline time.Time) error { return nil } - w.stopTimerLocked() w.current = deadline - // Reset every per-deadline guard so a refreshed deadline arms a fresh + // Reset every per-deadline guard so a refreshed deadline starts a fresh // warning cycle: both edge triggers and the user Dismiss decision // (the user agreed to the old deadline expiring; a new deadline // restarts the contract). w.firedAt = time.Time{} w.finalFiredAt = time.Time{} w.dismissedAt = time.Time{} + w.announcedAt = time.Time{} - if deadline.After(now) { - w.armTimerLocked(deadline) + // Poll every accepted deadline, including one that reads as already + // expired: the clock may be running ahead of real time and get + // corrected later, and evaluate ignores a deadline that has genuinely + // passed anyway. + if !w.deadlineOnly { + w.startPollLocked() } recorder := w.recorder w.mu.Unlock() @@ -192,6 +228,28 @@ func (w *Watcher) Update(deadline time.Time) error { recorder.SetSessionExpiresAt(deadline) } log.Infof("auth session deadline set to: %s (in %s)", deadline.Format(time.RFC3339), time.Until(deadline).Round(time.Second)) + + // Open the gate only once the recorder knows the new deadline, so a + // warning that refers to it can never reach consumers before the state + // change itself. A tick landing in between finds the gate shut. + w.mu.Lock() + if w.closed || !w.current.Equal(deadline) { + w.mu.Unlock() + return nil + } + w.announcedAt = deadline + wake := w.wake + w.mu.Unlock() + + // Hand the evaluation to the loop rather than running it here, so a + // deadline that already sits inside a warning window is published at + // once without this goroutine ever touching the recorder. + if wake != nil { + select { + case wake <- struct{}{}: + default: + } + } return nil } @@ -204,9 +262,9 @@ func (w *Watcher) Deadline() time.Time { } // Dismiss records the user's "Dismiss" action against the current deadline -// and suppresses the upcoming final-warning callback for that deadline. -// Idempotent: repeated calls are no-ops. A subsequent Update with a fresh -// deadline resets the dismissal so the final-warning cycle re-arms. +// and suppresses the final warning for that deadline. Idempotent: repeated +// calls are no-ops. A subsequent Update with a fresh deadline resets the +// dismissal so the final-warning cycle starts over. // // No-op when the watcher holds no deadline or has been closed. func (w *Watcher) Dismiss() { @@ -219,35 +277,48 @@ func (w *Watcher) Dismiss() { return } w.dismissedAt = w.current - // Cancel the armed final-warning timer eagerly. fireFinal would also - // gate on dismissedAt, but stopping the timer avoids a wakeup with - // nothing to do and makes the intent visible. - if w.finalTimer != nil { - w.finalTimer.Stop() - w.finalTimer = nil - } log.Infof("auth session final-warning dismissed for deadline %s", w.current.Format(time.RFC3339)) } -// Close stops any pending timer. Update calls after Close are ignored. -// The recorder keeps its deadline: the watcher is engine-scoped and closes -// on every engine restart (network change, sleep/wake, stream errors) -// while the SSO deadline stays valid across those, so clearing here would -// blank the UI's "expires in" row on every transient reconnect. The -// client run loop clears the server-scoped recorder when it exits for -// real (Down, profile switch, permanent login failure). +// Close stops the evaluation loop and waits for it to exit. Update calls +// after Close are ignored. The recorder keeps its deadline: the watcher is +// engine-scoped and closes on every engine restart (network change, +// sleep/wake, stream errors) while the SSO deadline stays valid across +// those, so clearing here would blank the UI's "expires in" row on every +// transient reconnect. The client run loop clears the server-scoped +// recorder when it exits for real (Down, profile switch, permanent login +// failure). func (w *Watcher) Close() { w.mu.Lock() - defer w.mu.Unlock() if w.closed { + // A concurrent Close is already tearing down. w.done outlives it, + // so this caller waits for the same loop rather than returning + // while a warning is still on its way to the recorder. + done := w.done + w.mu.Unlock() + if done != nil { + <-done + } return } w.closed = true - w.stopTimerLocked() w.current = time.Time{} w.firedAt = time.Time{} w.finalFiredAt = time.Time{} w.dismissedAt = time.Time{} + w.announcedAt = time.Time{} + // Copy the channels out before releasing the lock: the loop takes w.mu + // on every tick, so waiting for it while holding the lock would + // deadlock. w.done stays on the receiver for the branch above. + stop, done := w.stop, w.done + w.stop, w.wake = nil, nil + w.mu.Unlock() + + if stop == nil { + return + } + close(stop) + <-done } // clearLocked drops the tracked deadline and notifies the recorder so @@ -259,11 +330,11 @@ func (w *Watcher) clearLocked() { w.mu.Unlock() return } - w.stopTimerLocked() w.current = time.Time{} w.firedAt = time.Time{} w.finalFiredAt = time.Time{} w.dismissedAt = time.Time{} + w.announcedAt = time.Time{} recorder := w.recorder w.mu.Unlock() if recorder != nil { @@ -272,87 +343,121 @@ func (w *Watcher) clearLocked() { log.Infof("auth session deadline cleared") } -func (w *Watcher) stopTimerLocked() { - if w.timer != nil { - w.timer.Stop() - w.timer = nil +// startPollLocked starts the evaluation loop unless it is already +// running. The loop starts lazily on the first accepted deadline, so a +// client whose server never publishes a session expiry never pays for a +// ticker, and it runs until Close: the watcher is engine-scoped, and a +// cleared deadline is normally followed by a fresh one on the next sync. +// Caller must hold w.mu. +func (w *Watcher) startPollLocked() { + if w.stop != nil { + return } - if w.finalTimer != nil { - w.finalTimer.Stop() - w.finalTimer = nil + stop := make(chan struct{}) + done := make(chan struct{}) + wake := make(chan struct{}, 1) + w.stop, w.done, w.wake = stop, done, wake + go w.poll(stop, done, wake, w.interval) +} + +// poll re-evaluates the tracked deadline every interval, and as soon as a +// new deadline is announced. It is the only caller of evaluate, so Close +// waiting for it to exit is enough to know no warning is still on its way +// to the recorder. Its channels and interval are passed in rather than +// read off the receiver, so Close can clear them without racing it. +func (w *Watcher) poll(stop <-chan struct{}, done chan<- struct{}, wake <-chan struct{}, interval time.Duration) { + defer close(done) + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-stop: + return + case <-wake: + w.evaluate() + case <-ticker.C: + w.evaluate() + } } } -func (w *Watcher) armTimerLocked(deadline time.Time) { - w.timer = armOneShotLocked(deadline.Add(-w.lead), func() { w.fire(deadline) }) - // finalLead <= 0 disables the final-warning timer entirely. Used by - // tests that predate the final-warning fallback so a millisecond-scale - // deadline does not flush both timers at once. - if w.finalLead > 0 { - w.finalTimer = armOneShotLocked(deadline.Add(-w.finalLead), func() { w.fireFinal(deadline) }) - } -} - -func (w *Watcher) fire(armedFor time.Time) { +// evaluate compares the tracked deadline against the wall clock and +// publishes whichever warning the remaining time calls for. Inside the +// final-warning window the interactive warning is stale, so the final one +// is published in its place: a device that resumes there never saw the +// T-WarningLead notification. +func (w *Watcher) evaluate() { w.mu.Lock() - if w.closed || !w.current.Equal(armedFor) { - // Deadline moved while we were waiting (e.g. a successful extend). - // The reschedule path armed a fresh timer; this one is stale. + if w.closed || w.deadlineOnly || w.current.IsZero() { w.mu.Unlock() return } - if !w.firedAt.IsZero() && w.firedAt.Equal(armedFor) { + + deadline := w.current + if !w.announcedAt.Equal(deadline) { + // Update is still on its way to the recorder with this deadline. w.mu.Unlock() return } - w.firedAt = armedFor + // Round(0) strips the monotonic reading so the comparison is wall + // clock on both sides, whether the deadline came off the wire or from + // a caller that derived it from time.Now. + remaining := deadline.Round(0).Sub(w.nowFn().Round(0)) + + switch { + case remaining <= 0: + // Already expired: the post-mortem SessionExpired flow owns it. + w.mu.Unlock() + case w.finalLead > 0 && remaining <= w.finalLead: + w.publishFinalLocked(deadline, remaining) + case remaining <= w.lead: + w.publishWarningLocked(deadline, remaining) + default: + w.mu.Unlock() + } +} + +// publishWarningLocked emits the interactive T-WarningLead warning, at +// most once per deadline value. Caller must hold w.mu; this helper +// releases it. +func (w *Watcher) publishWarningLocked(deadline time.Time, remaining time.Duration) { + if w.firedAt.Equal(deadline) { + w.mu.Unlock() + return + } + w.firedAt = deadline recorder := w.recorder w.mu.Unlock() if recorder == nil { return } - log.Infof("auth session expiry soon warning fired") - publishWarning(recorder, armedFor, false) + log.Infof("auth session expiry soon warning fired for deadline %s (in %s)", + deadline.Format(time.RFC3339), remaining.Round(time.Second)) + publishWarning(recorder, deadline, false) } -// fireFinal mirrors fire for the T-FinalWarningLead timer with an extra -// dismiss-gate: if the user dismissed the T-WarningLead notification for -// this deadline, the final warning is suppressed entirely. -func (w *Watcher) fireFinal(armedFor time.Time) { - w.mu.Lock() - if w.closed || !w.current.Equal(armedFor) { +// publishFinalLocked emits the final warning, at most once per deadline +// value and never once the user dismissed that deadline. It marks the +// interactive warning as handled too: the final window is open, so a +// "expires in WarningLead minutes" notification would be wrong. Caller +// must hold w.mu; this helper releases it. +func (w *Watcher) publishFinalLocked(deadline time.Time, remaining time.Duration) { + if w.finalFiredAt.Equal(deadline) || w.dismissedAt.Equal(deadline) { w.mu.Unlock() return } - if !w.finalFiredAt.IsZero() && w.finalFiredAt.Equal(armedFor) { - w.mu.Unlock() - return - } - if w.dismissedAt.Equal(armedFor) { - w.mu.Unlock() - log.Infof("auth session final-warning skipped (dismissed by user)") - return - } - w.finalFiredAt = armedFor + w.firedAt = deadline + w.finalFiredAt = deadline recorder := w.recorder w.mu.Unlock() if recorder == nil { return } - log.Infof("auth session final-warning fired") - publishWarning(recorder, armedFor, true) -} - -// armOneShotLocked schedules cb at fireAt. When fireAt is already in the -// past it dispatches on the next scheduler tick so a state-change recorder -// notification (invoked after w.mu is released) lands first. Caller must -// hold w.mu. -func armOneShotLocked(fireAt time.Time, cb func()) *time.Timer { - delay := time.Until(fireAt) - if delay <= 0 { - return time.AfterFunc(0, cb) - } - return time.AfterFunc(delay, cb) + log.Infof("auth session final-warning fired for deadline %s (in %s)", + deadline.Format(time.RFC3339), remaining.Round(time.Second)) + publishWarning(recorder, deadline, true) } // publishWarning composes the SystemEvent for a watcher-fired warning and diff --git a/client/internal/auth/sessionwatch/watcher_test.go b/client/internal/auth/sessionwatch/watcher_test.go index 0445736c6..fe6216d73 100644 --- a/client/internal/auth/sessionwatch/watcher_test.go +++ b/client/internal/auth/sessionwatch/watcher_test.go @@ -19,6 +19,9 @@ type fakeRecorder struct { mu sync.Mutex events []event lastDeadline time.Time + // setDelay stalls SetSessionExpiresAt, widening the window in which a + // concurrent evaluation could publish a warning out of order. + setDelay time.Duration } type eventKind int @@ -43,6 +46,12 @@ type event struct { // is the zero time, so an initial clear before any deadline is set emits // nothing — matching the real recorder. func (r *fakeRecorder) SetSessionExpiresAt(deadline time.Time) { + r.mu.Lock() + delay := r.setDelay + r.mu.Unlock() + // Stall without the lock held, so a concurrent publish can still record. + time.Sleep(delay) + r.mu.Lock() defer r.mu.Unlock() if r.lastDeadline.Equal(deadline) { @@ -117,10 +126,54 @@ func waitForEvents(t *testing.T, r *fakeRecorder, want int) []event { return nil } -// newWatcher builds a watcher with the final timer disabled (finalLead=0), +// testInterval keeps the ticker-driven tests fast; the leads they use are +// in the same millisecond scale. +const testInterval = 2 * time.Millisecond + +// newWatcher builds a watcher with the final warning disabled (finalLead=0), // matching the lead-only behaviour the pre-final-warning tests assume. func newWatcher(lead time.Duration, r *fakeRecorder) *Watcher { - return NewWithLeads(lead, 0, r) + return newWatcherWithLeads(lead, 0, r) +} + +// newWatcherWithLeads builds a watcher that evaluates on testInterval. The +// interval is set before Update, so the evaluation loop does not exist yet +// and the write cannot race it. +func newWatcherWithLeads(lead, final time.Duration, r StatusRecorder) *Watcher { + w := NewWithLeads(lead, final, r) + w.interval = testInterval + return w +} + +// fakeClock is the watcher's wall clock under test. Reads come from the +// evaluation goroutine while the test writes, so both go through the mutex. +type fakeClock struct { + mu sync.Mutex + t time.Time +} + +func newFakeClock(t time.Time) *fakeClock { + return &fakeClock{t: t} +} + +func (c *fakeClock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +// set jumps the clock, standing in for a resume from suspension or for an +// NTP correction. +func (c *fakeClock) set(t time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = t +} + +// settle waits out a handful of evaluation ticks so a test can assert that +// nothing was published. +func settle() { + time.Sleep(20 * testInterval) } func TestUpdateZeroBeforeAnythingIsNoop(t *testing.T) { @@ -411,7 +464,7 @@ func TestCloseWithoutDeadlineLeavesRecorderUntouched(t *testing.T) { func TestFinalWarningFiresAfterRegularWarning(t *testing.T) { r := &fakeRecorder{} // Warning fires at deadline-80ms, final at deadline-30ms. - w := NewWithLeads(80*time.Millisecond, 30*time.Millisecond, r) + w := newWatcherWithLeads(80*time.Millisecond, 30*time.Millisecond, r) defer w.Close() d := time.Now().Add(100 * time.Millisecond) @@ -447,7 +500,7 @@ func TestFinalWarningFiresAfterRegularWarning(t *testing.T) { func TestDismissSuppressesFinalWarning(t *testing.T) { r := &fakeRecorder{} - w := NewWithLeads(80*time.Millisecond, 30*time.Millisecond, r) + w := newWatcherWithLeads(80*time.Millisecond, 30*time.Millisecond, r) defer w.Close() d := time.Now().Add(100 * time.Millisecond) @@ -478,7 +531,7 @@ func TestDismissSuppressesFinalWarning(t *testing.T) { func TestDismissResetByNewDeadline(t *testing.T) { r := &fakeRecorder{} - w := NewWithLeads(80*time.Millisecond, 30*time.Millisecond, r) + w := newWatcherWithLeads(80*time.Millisecond, 30*time.Millisecond, r) defer w.Close() first := time.Now().Add(100 * time.Millisecond) @@ -508,7 +561,7 @@ func TestDismissResetByNewDeadline(t *testing.T) { func TestDismissBeforeUpdateIsNoop(t *testing.T) { r := &fakeRecorder{} - w := NewWithLeads(80*time.Millisecond, 30*time.Millisecond, r) + w := newWatcherWithLeads(80*time.Millisecond, 30*time.Millisecond, r) defer w.Close() // No deadline tracked yet; Dismiss must be a no-op (no panic, no state). @@ -528,3 +581,340 @@ func TestDismissBeforeUpdateIsNoop(t *testing.T) { } t.Fatalf("final-warning did not publish after no-op pre-Update Dismiss, events=%+v", r.snapshot()) } + +// The tests below drive the watcher's wall clock directly. The deadline sits +// an hour out in real time and the fake clock jumps, standing in for a device +// that was suspended and resumed somewhere inside — or past — the warning +// windows. The evaluation loop is what reacts to the jump, so these exercise +// the same path production takes on a resume. +func newResumeWatcher(t *testing.T, r *fakeRecorder) (*Watcher, *fakeClock, time.Time) { + t.Helper() + + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + start := time.Now() + clock := newFakeClock(start) + // Set before Update: the evaluation loop does not exist yet. + w.nowFn = clock.now + t.Cleanup(w.Close) + + deadline := start.Add(time.Hour).Round(0) + if err := w.Update(deadline); err != nil { + t.Fatalf("Update: %v", err) + } + return w, clock, deadline +} + +func TestResumeInsideWarningWindowWarns(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(-5 * time.Minute)) + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected the interactive warning after the resume, got %+v", events[1]) + } + if n := countWhere(events, event.isFinalWarning); n != 0 { + t.Fatalf("final-warning must wait for its own window, got %d: %+v", n, events) + } +} + +func TestResumeInsideFinalWindowSendsFinalWarningOnly(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(-time.Minute)) + + events := waitForEvents(t, r, 2) + if !events[1].isFinalWarning() { + t.Fatalf("expected the final warning after the resume, got %+v", events[1]) + } + settle() + if n := countWhere(r.snapshot(), event.isWarning); n != 0 { + t.Fatalf("the interactive warning is stale inside the final window, got %d: %+v", n, r.snapshot()) + } +} + +func TestResumePastDeadlinePublishesNothing(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(time.Minute)) + + settle() + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("an expired session must not warn, got %d publishes: %+v", n, r.snapshot()) + } +} + +func TestWarningPublishesOncePerDeadline(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(-5 * time.Minute)) + waitForEvents(t, r, 2) + + // Many ticks pass inside the same window. + settle() + if n := countWhere(r.snapshot(), event.isWarning); n != 1 { + t.Fatalf("expected exactly 1 warning publish across ticks, got %d: %+v", n, r.snapshot()) + } +} + +// TestWarningRecoversFromAClockRunningAhead covers the device that boots +// before NTP has corrected it: the deadline looks long gone, nothing is +// published, and the warning still arrives once the clock is fixed. +func TestWarningRecoversFromAClockRunningAhead(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(2 * time.Hour)) + settle() + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("a deadline that looks expired must not warn, got %d publishes: %+v", n, r.snapshot()) + } + + clock.set(deadline.Add(-5 * time.Minute)) + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected the warning once the clock was corrected, got %+v", events[1]) + } +} + +func TestResumeInsideFinalWindowRespectsDismiss(t *testing.T) { + r := &fakeRecorder{} + w, clock, deadline := newResumeWatcher(t, r) + + // The user dismissed the warning for this deadline before the device + // was suspended; resuming inside the final window must not reopen it. + w.Dismiss() + clock.set(deadline.Add(-time.Minute)) + + settle() + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("a dismissed deadline must not warn on resume, got %d: %+v", n, r.snapshot()) + } +} + +func TestCloseStopsTheEvaluationLoop(t *testing.T) { + r := &fakeRecorder{} + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + start := time.Now() + clock := newFakeClock(start) + w.nowFn = clock.now + + deadline := start.Add(time.Hour).Round(0) + if err := w.Update(deadline); err != nil { + t.Fatalf("Update: %v", err) + } + w.Close() + + clock.set(deadline.Add(-5 * time.Minute)) + settle() + + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("a closed watcher must not publish, got %d: %+v", n, r.snapshot()) + } +} + +func TestDeadlineOnlyRecordsDeadlineWithoutWarnings(t *testing.T) { + r := &fakeRecorder{} + w := NewDeadlineOnly(r) + defer w.Close() + + // With the default leads this deadline sits inside the final-warning + // window, so a watcher that warns at all would publish on the spot. + d := time.Now().Add(50 * time.Millisecond).Round(0) + if err := w.Update(d); err != nil { + t.Fatalf("Update: %v", err) + } + if got := r.deadline(); !got.Equal(d) { + t.Fatalf("expected recorder deadline %v, got %v", d, got) + } + + time.Sleep(100 * time.Millisecond) + + events := r.snapshot() + if got := countWhere(events, func(e event) bool { return e.kind == publish }); got != 0 { + t.Fatalf("expected no publish in deadline-only mode, got %d: %+v", got, events) + } + w.mu.Lock() + polling := w.stop != nil + w.mu.Unlock() + if polling { + t.Fatal("expected no evaluation loop in deadline-only mode") + } +} + +func TestDeadlineOnlyStillRejectsOutOfRangeDeadlines(t *testing.T) { + r := &fakeRecorder{} + w := NewDeadlineOnly(r) + defer w.Close() + + if err := w.Update(time.Now().Add(time.Hour)); err != nil { + t.Fatalf("Update: %v", err) + } + + err := w.Update(time.Now().Add(-maxPastHorizon - time.Hour)) + if !errors.Is(err, ErrDeadlineInPast) { + t.Fatalf("expected ErrDeadlineInPast, got %v", err) + } + if got := r.deadline(); !got.IsZero() { + t.Fatalf("expected recorder cleared after rejection, got %v", got) + } +} + +// TestClockAheadAtUpdateStillWarnsOnceCorrected covers a client that connects +// while its clock runs ahead of real time: the deadline reads as already +// expired when it arrives, so nothing publishes, and the warning must still +// come once NTP pulls the clock back. +func TestClockAheadAtUpdateStillWarnsOnceCorrected(t *testing.T) { + r := &fakeRecorder{} + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + deadline := time.Now().Add(time.Hour).Round(0) + // Two hours past the deadline from the device's point of view, well + // inside maxPastHorizon, so Update accepts and records it. + clock := newFakeClock(deadline.Add(2 * time.Hour)) + w.nowFn = clock.now + t.Cleanup(w.Close) + + if err := w.Update(deadline); err != nil { + t.Fatalf("Update: %v", err) + } + + settle() + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("a deadline that reads as expired must not warn, got %d: %+v", n, r.snapshot()) + } + + clock.set(deadline.Add(-5 * time.Minute)) + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected the warning once the clock was corrected, got %+v", events[1]) + } +} + +// TestWarningNeverPrecedesTheDeadlineStateChange pins the ordering Update +// documents: consumers learn the new deadline before they see a warning that +// refers to it. The deadline here already sits inside the warning window, and +// the recorder stalls, so the evaluation loop gets many chances to publish +// while Update is still announcing. +func TestWarningNeverPrecedesTheDeadlineStateChange(t *testing.T) { + r := &fakeRecorder{setDelay: 50 * time.Millisecond} + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + deadline := time.Now().Add(time.Hour).Round(0) + clock := newFakeClock(deadline.Add(-5 * time.Minute)) + w.nowFn = clock.now + t.Cleanup(w.Close) + + if err := w.Update(deadline); err != nil { + t.Fatalf("Update: %v", err) + } + + events := waitForEvents(t, r, 2) + if events[0].kind != stateChange { + t.Fatalf("event[0] should be the deadline state change, got %+v", events) + } + if !events[1].isWarning() { + t.Fatalf("event[1] should be the warning, got %+v", events) + } +} + +// TestUpdateWakesTheLoopWithoutWaitingForATick pins the hand-off: Update +// publishes nothing itself, it nudges the evaluation loop, so a deadline that +// already sits inside a warning window is warned about at once even though the +// ticker here would not fire for an hour. +func TestUpdateWakesTheLoopWithoutWaitingForATick(t *testing.T) { + r := &fakeRecorder{} + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + w.interval = time.Hour + t.Cleanup(w.Close) + + d := time.Now().Add(5 * time.Minute).Round(0) + if err := w.Update(d); err != nil { + t.Fatalf("Update: %v", err) + } + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected the warning on the wake-up, got %+v", events[1]) + } +} + +// blockingRecorder holds a publish open until the test releases it, so the +// evaluation loop can be parked mid-publish while Close is called. +type blockingRecorder struct { + fakeRecorder + entered chan struct{} + release chan struct{} +} + +func (r *blockingRecorder) PublishEvent( + severity cProto.SystemEvent_Severity, + category cProto.SystemEvent_Category, + message string, + userMessage string, + metadata map[string]string, +) bool { + select { + case r.entered <- struct{}{}: + default: + } + <-r.release + return r.fakeRecorder.PublishEvent(severity, category, message, userMessage, metadata) +} + +// TestConcurrentCloseWaitsForTheLoop pins the contract for the caller that +// loses the race: Close returns only once the loop is done, whichever of the +// two calls got there first. +func TestConcurrentCloseWaitsForTheLoop(t *testing.T) { + r := &blockingRecorder{entered: make(chan struct{}, 1), release: make(chan struct{})} + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(r.release) }) } + + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + w.interval = time.Hour + // Order matters: Close waits for the loop, which stays parked in the + // publish until the release, so a t.Fatal below would hang the cleanup + // instead of reporting the failure. + t.Cleanup(func() { + release() + w.Close() + }) + + d := time.Now().Add(5 * time.Minute).Round(0) + if err := w.Update(d); err != nil { + t.Fatalf("Update: %v", err) + } + + // The loop is now parked inside the warning publish. + select { + case <-r.entered: + case <-time.After(2 * time.Second): + t.Fatal("the loop never reached the publish") + } + + first := make(chan struct{}) + second := make(chan struct{}) + go func() { w.Close(); close(first) }() + go func() { w.Close(); close(second) }() + + select { + case <-first: + t.Fatal("Close returned while the loop was still publishing") + case <-second: + t.Fatal("Close returned while the loop was still publishing") + case <-time.After(100 * time.Millisecond): + } + + release() + for _, done := range []chan struct{}{first, second} { + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("Close did not return after the publish completed") + } + } +} diff --git a/client/internal/certproof/collect.go b/client/internal/certproof/collect.go new file mode 100644 index 000000000..310fd2b19 --- /dev/null +++ b/client/internal/certproof/collect.go @@ -0,0 +1,168 @@ +package certproof + +import ( + "context" + "crypto" + "crypto/sha256" + "crypto/x509" + "time" + + log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// Collect answers the certificate challenges in checks: for each challenge it picks a +// stored certificate that chains to the challenge's CAs and signs the nonce with its +// key. The same certificate is proven once even if several checks accept it. +func Collect(ctx context.Context, store Store, checks []*proto.Checks, peerKey []byte) []certposture.Proof { + challenges := certificateChallenges(checks) + if len(challenges) == 0 { + logNoChallenges(checks) + return nil + } + return CollectChallenges(ctx, store, challenges, peerKey) +} + +func logNoChallenges(checks []*proto.Checks) { + if len(checks) > 0 { + log.Debugf("certificate posture: %d posture checks received, none carries a certificate challenge", len(checks)) + } +} + +// CollectChallenges answers challenges already extracted from the posture checks, so a +// caller that ships them across a process boundary reuses the same matching and signing. +// Only nonces of the size management issues are signed, for a peer key of the size of +// ours, so the keys behind the store never sign arbitrary caller-chosen data. +func CollectChallenges(ctx context.Context, store Store, challenges []*proto.CertificateChallenge, peerKey []byte) []certposture.Proof { + if len(peerKey) != wgtypes.KeyLen { + log.Warnf("certificate posture: refusing to sign for a %d byte peer key", len(peerKey)) + return nil + } + challenges = wellFormed(challenges) + if len(challenges) == 0 { + return nil + } + log.Debugf("certificate posture: answering %d certificate challenges from store %T", len(challenges), store) + + candidates, err := store.Candidates(ctx) + if err != nil { + log.Warnf("failed loading certificates for posture checks: %v", err) + return nil + } + if len(candidates) == 0 { + log.Debug("certificate posture: certificate store holds no candidates, no proof will be sent") + return nil + } + log.Debugf("certificate posture: store holds %d candidate certificates", len(candidates)) + + now := time.Now() + proven := make(map[[sha256.Size]byte]struct{}) + var proofs []certposture.Proof + for i, challenge := range challenges { + roots, err := certposture.ParseCAs(challenge.GetCaCertificates()) + if err != nil { + log.Warnf("skipping certificate challenge with invalid CA certificates: %v", err) + continue + } + log.Debugf("certificate posture: challenge %d accepts %d CA certificates, nonce is %d bytes", i, len(challenge.GetCaCertificates()), len(challenge.GetNonce())) + + matched := false + for _, candidate := range candidates { + if len(candidate.Chain) == 0 { + continue + } + leaf := candidate.Chain[0] + chain, err := certposture.VerifiedChain(leaf, candidate.issuers(), roots, now) + if err != nil { + log.Debugf("certificate posture: challenge %d rejected %q issued by %q: %v", i, leaf.Subject, leaf.Issuer, err) + continue + } + matched = true + + // The same leaf can chain to different CAs for different challenges, and + // management checks each chain against each check's CAs, so a proof is + // deduplicated by its whole chain rather than by its leaf. + fingerprint := chainFingerprint(chain) + if _, done := proven[fingerprint]; done { + log.Debugf("certificate posture: challenge %d matched %q, already proven for an earlier challenge", i, leaf.Subject) + break + } + proof, err := prove(candidate.Signer, chain, challenge.GetNonce(), peerKey) + if err != nil { + log.Warnf("failed signing certificate proof for %s: %v", leaf.Subject, err) + continue + } + log.Debugf("certificate posture: challenge %d proven by %q with %s, signature %d bytes, chain of %d", i, leaf.Subject, proof.SigAlg, len(proof.Signature), len(proof.Chain)) + proven[fingerprint] = struct{}{} + proofs = append(proofs, proof) + break + } + if !matched { + log.Debugf("certificate posture: challenge %d matched none of the %d candidates", i, len(candidates)) + } + } + log.Debugf("certificate posture: %d challenges produced %d proofs", len(challenges), len(proofs)) + return proofs +} + +// HasChallenges reports whether any of checks asks for a certificate proof. +func HasChallenges(checks []*proto.Checks) bool { + return len(certificateChallenges(checks)) > 0 +} + +func certificateChallenges(checks []*proto.Checks) []*proto.CertificateChallenge { + var challenges []*proto.CertificateChallenge + for _, check := range checks { + if challenge := check.GetCertificateChallenge(); challenge != nil { + challenges = append(challenges, challenge) + } + } + return wellFormed(challenges) +} + +// wellFormed drops challenges whose nonce is not one management could have issued. +func wellFormed(challenges []*proto.CertificateChallenge) []*proto.CertificateChallenge { + var kept []*proto.CertificateChallenge + for _, challenge := range challenges { + if len(challenge.GetNonce()) != certposture.NonceSize { + log.Debugf("certificate posture: skipping challenge with a %d byte nonce", len(challenge.GetNonce())) + continue + } + kept = append(kept, challenge) + } + return kept +} + +func prove(signer crypto.Signer, chain []*x509.Certificate, nonce, peerKey []byte) (certposture.Proof, error) { + sigAlg, sig, err := certposture.Sign(signer, nonce, peerKey) + if err != nil { + return certposture.Proof{}, err + } + der := make([][]byte, 0, len(chain)) + for _, cert := range chain { + der = append(der, cert.Raw) + } + return certposture.Proof{Nonce: nonce, Chain: der, SigAlg: sigAlg, Signature: sig}, nil +} + +func chainFingerprint(chain []*x509.Certificate) [sha256.Size]byte { + ders := make([][]byte, 0, len(chain)) + for _, cert := range chain { + ders = append(ders, cert.Raw) + } + return chainDERFingerprint(ders) +} + +// chainDERFingerprint identifies a chain of DER certificates by the hashes of all of them, +// in order. +func chainDERFingerprint(chain [][]byte) [sha256.Size]byte { + buf := make([]byte, 0, len(chain)*sha256.Size) + for _, der := range chain { + certHash := sha256.Sum256(der) + buf = append(buf, certHash[:]...) + } + return sha256.Sum256(buf) +} diff --git a/client/internal/certproof/collect_darwin.go b/client/internal/certproof/collect_darwin.go new file mode 100644 index 000000000..cc21563c1 --- /dev/null +++ b/client/internal/certproof/collect_darwin.go @@ -0,0 +1,137 @@ +//go:build !ios + +package certproof + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +const helperTimeout = 30 * time.Second + +// userHelperBackoff holds off asking a user's keychain again after it proved nothing. +var userHelperBackoff = newHelperBackoff() + +// CollectProofs answers the certificate challenges in checks from every store this Mac +// can reach. The root daemon reads the System keychain itself, which is where MDM +// installs device identities, and reaches the console user's login keychain only by +// launching a helper into that user's session. A Mac sitting at the login window +// therefore yields device proofs alone. +func CollectProofs(ctx context.Context, checks []*proto.Checks, peerKey []byte, cfg Config) []certposture.Proof { + challenges := certificateChallenges(checks) + if len(challenges) == 0 { + logNoChallenges(checks) + return nil + } + + // A helper already runs inside the user's session, so it reads its own keychain + // directly and must never launch another one. + if os.Geteuid() != 0 { + return CollectChallenges(ctx, DefaultStore(), challenges, peerKey) + } + + proofs := CollectChallenges(ctx, DefaultStore(), challenges, peerKey) + if cfg.OwnerUnknown { + return proofs + } + + userProofs, err := collectAsConsoleUser(ctx, cfg.ProfileOwner, challenges, peerKey) + if err != nil { + log.Debugf("certificate posture: console user keychain unavailable: %v", err) + } + return mergeProofs(proofs, userProofs) +} + +// UserContext identifies the user whose keychain a collection would include: the console +// user when it owns the active profile, or empty when no user keychain would be asked. A +// change means a collection made earlier no longer reflects what this Mac can prove. +func UserContext(cfg Config) string { + if os.Geteuid() != 0 || cfg.OwnerUnknown { + return "" + } + user, ok := currentConsoleUser() + if !ok || !user.isOwner(cfg.ProfileOwner) { + return "" + } + return strconv.FormatUint(uint64(user.UID), 10) + ":" + user.Name +} + +// collectAsConsoleUser runs the helper inside the desktop session of the logged-in +// user. Dropping to their uid is not enough: keychain access is an XPC call to a +// per-session securityd, so the helper has to enter their Mach bootstrap namespace, +// which is what launchctl asuser does. +func collectAsConsoleUser(ctx context.Context, owner string, challenges []*proto.CertificateChallenge, peerKey []byte) ([]certposture.Proof, error) { + user, ok := currentConsoleUser() + if !ok { + return nil, nil + } + if !user.isOwner(owner) { + log.Debugf("certificate posture: console user %s does not own the active profile, no user keychain is asked", user.Name) + return nil, nil + } + + uid := strconv.FormatUint(uint64(user.UID), 10) + backoffKey := helperBackoffKey(uid, challenges) + if !userHelperBackoff.allow(backoffKey, time.Now()) { + log.Debugf("certificate posture: the keychain of uid %s proved nothing recently, not asking again yet", uid) + return nil, nil + } + + binary, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("resolve own binary: %w", err) + } + + parent := ctx + ctx, cancel := context.WithTimeout(ctx, helperTimeout) + defer cancel() + + // Absolute paths, because the daemon's PATH is configurable through the service + // environment, and sudo selects the user by uid so the name never has to round-trip. + cmd := exec.CommandContext(ctx, "/bin/launchctl", "asuser", uid, "/usr/bin/sudo", "-u", "#"+uid, "-H", "--", binary, "posture", "cert-proof") + killHelperGroupOnCancel(cmd) + + log.Debugf("certificate posture: asking the desktop session of uid %s to answer %d challenges", uid, len(challenges)) + proofs, err := runHelperCmd(cmd, helperRequest(challenges, peerKey)) + // A run that completed, or ran into the timeout waiting on a prompt nobody answered, + // tells whether the keychain proves anything. A launch or output failure, or a run the + // caller cut short, says nothing about it and must not hold off the next one. + timedOut := errors.Is(ctx.Err(), context.DeadlineExceeded) && parent.Err() == nil + if err == nil || timedOut { + userHelperBackoff.record(backoffKey, err == nil && len(proofs) > 0, time.Now()) + } + if err != nil { + return nil, fmt.Errorf("run helper as uid %s: %w", uid, err) + } + log.Debugf("certificate posture: desktop session of uid %s returned %d proofs", uid, len(proofs)) + return proofs, nil +} + +// helperStore is the store the helper reads: the user's login keychain alone. The +// session's search list also holds the System keychain, which the daemon reads itself, +// and using a System keychain key from the user's session would ask for an +// administrator's approval. +func helperStore() Store { + home, err := os.UserHomeDir() + if err != nil { + log.Debugf("certificate posture: no home directory, searching the default keychain list: %v", err) + return newKeychainStore() + } + login := filepath.Join(home, "Library", "Keychains", "login.keychain-db") + if _, err := os.Stat(login); err != nil { + // Keychains created before macOS 10.12 keep the old file name. + login = filepath.Join(home, "Library", "Keychains", "login.keychain") + } + return newKeychainStore(login) +} diff --git a/client/internal/certproof/collect_js.go b/client/internal/certproof/collect_js.go new file mode 100644 index 000000000..fd7c4c690 --- /dev/null +++ b/client/internal/certproof/collect_js.go @@ -0,0 +1,31 @@ +//go:build js + +package certproof + +import ( + "context" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// CollectProofs proves nothing in a browser: it has no TPM, certificate store or file +// store to sign with, so the stores stay out of the WebAssembly build. +func CollectProofs(context.Context, []*proto.Checks, []byte, Config) []certposture.Proof { + return nil +} + +// UserContext identifies the user whose certificates a collection would include. A +// browser has no per-user store, so it never changes. +func UserContext(Config) string { + return "" +} + +// DefaultStore is empty in a browser. +func DefaultStore() Store { + return Stores{} +} + +func helperStore() Store { + return DefaultStore() +} diff --git a/client/internal/certproof/collect_other.go b/client/internal/certproof/collect_other.go new file mode 100644 index 000000000..785d3db65 --- /dev/null +++ b/client/internal/certproof/collect_other.go @@ -0,0 +1,31 @@ +//go:build ((!darwin && !windows) || ios) && !js + +package certproof + +import ( + "context" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// CollectProofs answers the certificate challenges in checks from the PEM directory cfg +// names, joined by its PKCS#11 token when it names one. Only macOS and Windows keep +// per-user certificates out of reach of a privileged daemon, so every other platform +// reads its store in the daemon itself. +func CollectProofs(ctx context.Context, checks []*proto.Checks, peerKey []byte, cfg Config) []certposture.Proof { + return Collect(ctx, storeWithToken(cfg), checks, peerKey) +} + +// UserContext identifies the user whose certificates a collection would include. These +// platforms have no per-user store, so it never changes. +func UserContext(Config) string { + return "" +} + +// helperStore is the store the helper reads. Nothing launches a helper on these +// platforms, so it is the store the daemon reads, configured from the same environment, +// which lets an administrator check a setup by running the helper by hand. +func helperStore() Store { + return storeWithToken(Config{PKCS11: PKCS11FromEnv()}) +} diff --git a/client/internal/certproof/collect_test.go b/client/internal/certproof/collect_test.go new file mode 100644 index 000000000..aa033dea5 --- /dev/null +++ b/client/internal/certproof/collect_test.go @@ -0,0 +1,258 @@ +package certproof + +import ( + "context" + "crypto/rand" + "crypto/x509" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/certposture/certtest" + "github.com/netbirdio/netbird/shared/management/proto" +) + +var peerKey = []byte("peer-public-key-aaaaaaaaaaaaaaaa") + +func TestCollect_ProvesOneMatchingCertificatePerChallenge(t *testing.T) { + corpCA := certtest.NewCA(t, "corp-root") + otherCA := certtest.NewCA(t, "other-root") + unrelatedCA := certtest.NewCA(t, "unrelated-root") + + dir := storeDir(t) + deviceKey := certtest.ECDSAKey(t) + device := corpCA.Issue(t, deviceKey, "device") + writeFile(t, dir, "device.pem", certtest.CertPEM(device)+certtest.KeyPEM(t, deviceKey)) + + otherKey := certtest.RSAKey(t) + writeFile(t, dir, "other.crt", certtest.CertPEM(otherCA.Issue(t, otherKey, "other"))) + writeFile(t, dir, "other.key", certtest.KeyPEM(t, otherKey)) + + writeFile(t, dir, "keyless.crt", certtest.CertPEM(corpCA.Issue(t, certtest.ECDSAKey(t), "keyless"))) + writeFile(t, dir, "notes.txt", "ignored") + + challenger := certposture.NewChallenger([]byte("secret")) + nonce := challenger.Nonce(peerKey, time.Now()) + challenge := func(cas ...string) *proto.Checks { + return &proto.Checks{CertificateChallenge: &proto.CertificateChallenge{Nonce: nonce, CaCertificates: cas}} + } + checks := []*proto.Checks{ + {Files: []string{"/usr/bin/agent"}}, + challenge(corpCA.PEM), + challenge(corpCA.PEM), + challenge(otherCA.PEM), + challenge(unrelatedCA.PEM), + challenge("not a pem"), + } + + proofs := Collect(context.Background(), newFileStore(dir), checks, peerKey) + + require.Len(t, proofs, 2) + var subjects []string + for _, p := range proofs { + chain, err := challenger.Verify(p, peerKey, time.Now()) + require.NoError(t, err) + subjects = append(subjects, chain[0].Subject.CommonName) + } + assert.ElementsMatch(t, []string{"device", "other"}, subjects) +} + +func TestCollect_NothingToProve(t *testing.T) { + dir := storeDir(t) + key := certtest.ECDSAKey(t) + ca := certtest.NewCA(t, "root") + writeFile(t, dir, "device.pem", certtest.CertPEM(ca.Issue(t, key, "device"))+certtest.KeyPEM(t, key)) + + tests := []struct { + name string + store Store + checks []*proto.Checks + }{ + {"no checks", newFileStore(dir), nil}, + {"files only", newFileStore(dir), []*proto.Checks{{Files: []string{"/bin/x"}}}}, + {"challenge without nonce", newFileStore(dir), []*proto.Checks{{CertificateChallenge: &proto.CertificateChallenge{CaCertificates: []string{ca.PEM}}}}}, + {"missing store dir", newFileStore(filepath.Join(dir, "missing")), []*proto.Checks{{CertificateChallenge: &proto.CertificateChallenge{Nonce: make([]byte, certposture.NonceSize), CaCertificates: []string{ca.PEM}}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Nil(t, Collect(context.Background(), tt.store, tt.checks, peerKey)) + }) + } +} + +func TestFileStore_ChainWithIntermediate(t *testing.T) { + root := certtest.NewCA(t, "root") + intermediate := certtest.NewIntermediate(t, root, "intermediate") + key := certtest.ECDSAKey(t) + leaf := intermediate.Issue(t, key, "device") + + dir := storeDir(t) + writeFile(t, dir, "device.pem", certtest.CertPEM(leaf)+certtest.CertPEM(intermediate.Cert)+certtest.KeyPEM(t, key)) + + candidates, err := newFileStore(dir).Candidates(context.Background()) + require.NoError(t, err) + require.Len(t, candidates, 1) + require.Len(t, candidates[0].Chain, 2) + + roots, err := certposture.ParseCAs([]string{root.PEM}) + require.NoError(t, err) + assert.NoError(t, certposture.VerifyChain(candidates[0].Chain, roots, time.Now())) +} + +// storeDir is a PEM directory the store accepts: t.TempDir follows the umask, which +// leaves the directory group-writable on systems with a user-private group umask. +func storeDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.Chmod(dir, 0o700)) + return dir +} + +func writeFile(t *testing.T, dir, name, content string) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600)) +} + +func TestCollectChallenges_RefusesMalformedInput(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + dir := storeDir(t) + key := certtest.ECDSAKey(t) + writeFile(t, dir, "device.pem", certtest.CertPEM(ca.Issue(t, key, "device"))+certtest.KeyPEM(t, key)) + store := newFileStore(dir) + nonce := certposture.NewChallenger([]byte("secret")).Nonce(peerKey, time.Now()) + + tests := []struct { + name string + nonce []byte + peerKey []byte + want int + }{ + {"issued nonce and peer key are signed", nonce, peerKey, 1}, + {"short nonce is not signed", nonce[:8], peerKey, 0}, + {"oversized nonce is not signed", append(append([]byte{}, nonce...), 0), peerKey, 0}, + {"short peer key is not signed", nonce, peerKey[:16], 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + challenges := []*proto.CertificateChallenge{{Nonce: tt.nonce, CaCertificates: []string{ca.PEM}}} + assert.Len(t, CollectChallenges(context.Background(), store, challenges, tt.peerKey), tt.want, + "the device key signs only what management could have issued") + }) + } +} + +func TestFileStore_SkipsKeyOfAnotherCertificate(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + dir := storeDir(t) + + // A stale key next to a renewed certificate, sorted before the good pair, must not + // produce a proof that management rejects and stop the search there. + writeFile(t, dir, "a-renewed.crt", certtest.CertPEM(ca.Issue(t, certtest.ECDSAKey(t), "renewed"))) + writeFile(t, dir, "a-renewed.key", certtest.KeyPEM(t, certtest.ECDSAKey(t))) + goodKey := certtest.ECDSAKey(t) + good := ca.Issue(t, goodKey, "good") + writeFile(t, dir, "b-good.pem", certtest.CertPEM(good)+certtest.KeyPEM(t, goodKey)) + + candidates, err := newFileStore(dir).Candidates(context.Background()) + require.NoError(t, err) + require.Len(t, candidates, 1, "only the certificate whose key matches is a candidate") + assert.True(t, good.Equal(candidates[0].Chain[0]), "the matching pair is kept") + + challenger := certposture.NewChallenger([]byte("secret")) + now := time.Now() + nonce := challenger.Nonce(peerKey, now) + proofs := CollectChallenges(context.Background(), newFileStore(dir), []*proto.CertificateChallenge{{Nonce: nonce, CaCertificates: []string{ca.PEM}}}, peerKey) + require.Len(t, proofs, 1) + _, err = challenger.Verify(proofs[0], peerKey, now) + assert.NoError(t, err, "the proof sent is one management accepts") +} + +// staticStore hands out fixed candidates, for scenarios no on-disk layout can express. +type staticStore []Candidate + +func (s staticStore) Candidates(context.Context) ([]Candidate, error) { return s, nil } + +func TestCollectChallenges_RoutesAroundExpiredCopyOfRenewedIntermediate(t *testing.T) { + root := certtest.NewCA(t, "root") + intermediate := certtest.NewIntermediate(t, root, "issuing-ca") + + // Renewing a CA with the same key pair leaves two certificates with the same + // subject and key in the store. The expired one sorts first here. + expiredTmpl := *intermediate.Cert + expiredTmpl.SerialNumber = big.NewInt(1) + expiredTmpl.NotBefore = time.Now().Add(-72 * time.Hour) + expiredTmpl.NotAfter = time.Now().Add(-48 * time.Hour) + der, err := x509.CreateCertificate(rand.Reader, &expiredTmpl, root.Cert, intermediate.Key.Public(), root.Key) + require.NoError(t, err) + expired, err := x509.ParseCertificate(der) + require.NoError(t, err) + + key := certtest.ECDSAKey(t) + leaf := intermediate.Issue(t, key, "device") + pool := []*x509.Certificate{expired, intermediate.Cert} + + chain := buildChain(leaf, pool) + require.Len(t, chain, 2) + require.True(t, expired.Equal(chain[1]), "precondition: the first-match chain runs through the expired copy") + + challenger := certposture.NewChallenger([]byte("secret")) + now := time.Now() + nonce := challenger.Nonce(peerKey, now) + store := staticStore{{Chain: chain, Signer: key, Intermediates: pool}} + + proofs := CollectChallenges(context.Background(), store, []*proto.CertificateChallenge{{Nonce: nonce, CaCertificates: []string{root.PEM}}}, peerKey) + + require.Len(t, proofs, 1, "a valid path through the renewed intermediate exists, so the challenge is answered") + verified, err := challenger.Verify(proofs[0], peerKey, now) + require.NoError(t, err) + assert.True(t, intermediate.Cert.Equal(verified[1]), "the proof carries the valid intermediate, not the expired copy") + assert.True(t, certposture.ChainMatchesCAs(certposture.EncodeChainPEM(verified), []string{root.PEM}, now), + "management's own check accepts the chain the proof carries") +} + +func TestCollectChallenges_ProvesALeafOncePerDistinctChain(t *testing.T) { + rootA := certtest.NewCA(t, "root-a") + rootB := certtest.NewCA(t, "root-b") + issuer := certtest.NewIntermediate(t, rootA, "issuing-ca") + + // The same issuing CA cross-signed by a second root: one leaf, two valid paths. + crossTmpl := *issuer.Cert + crossTmpl.SerialNumber = big.NewInt(2) + der, err := x509.CreateCertificate(rand.Reader, &crossTmpl, rootB.Cert, issuer.Key.Public(), rootB.Key) + require.NoError(t, err) + cross, err := x509.ParseCertificate(der) + require.NoError(t, err) + + key := certtest.ECDSAKey(t) + leaf := issuer.Issue(t, key, "device") + pool := []*x509.Certificate{issuer.Cert, cross} + store := staticStore{{Chain: buildChain(leaf, pool), Signer: key, Intermediates: pool}} + + challenger := certposture.NewChallenger([]byte("secret")) + now := time.Now() + nonce := challenger.Nonce(peerKey, now) + challenges := []*proto.CertificateChallenge{ + {Nonce: nonce, CaCertificates: []string{rootA.PEM}}, + {Nonce: nonce, CaCertificates: []string{rootB.PEM}}, + {Nonce: nonce, CaCertificates: []string{rootA.PEM}}, + } + + proofs := CollectChallenges(context.Background(), store, challenges, peerKey) + + require.Len(t, proofs, 2, "one proof per distinct chain, the repeated root-a challenge reuses the first") + for _, root := range []*certtest.CA{rootA, rootB} { + matched := false + for _, p := range proofs { + chain, err := challenger.Verify(p, peerKey, now) + require.NoError(t, err) + matched = matched || certposture.ChainMatchesCAs(certposture.EncodeChainPEM(chain), []string{root.PEM}, now) + } + assert.True(t, matched, "management can match a chain for the check that trusts %s", root.Cert.Subject.CommonName) + } +} diff --git a/client/internal/certproof/collect_windows.go b/client/internal/certproof/collect_windows.go new file mode 100644 index 000000000..3b21ec147 --- /dev/null +++ b/client/internal/certproof/collect_windows.go @@ -0,0 +1,108 @@ +package certproof + +import ( + "context" + "fmt" + "os" + "os/exec" + "syscall" + "time" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +const helperTimeout = 30 * time.Second + +// CollectProofs answers the certificate challenges in checks from every store this +// machine can reach. The service reads the local machine store itself, where AD and +// Intune enrol device certificates, and reaches the signed-in user's store by launching +// a helper with that session's token. A machine at the sign-in screen therefore proves +// device certificates alone. +func CollectProofs(ctx context.Context, checks []*proto.Checks, peerKey []byte, cfg Config) []certposture.Proof { + challenges := certificateChallenges(checks) + if len(challenges) == 0 { + logNoChallenges(checks) + return nil + } + + proofs := CollectChallenges(ctx, DefaultStore(), challenges, peerKey) + + // The helper already runs as the signed-in user, and an ordinary process has no + // right to a session token, so only the service goes looking for one. + if !runningAsLocalSystem() || cfg.OwnerUnknown { + return proofs + } + + userProofs, err := collectAsDesktopUser(ctx, cfg.ProfileOwner, challenges, peerKey) + if err != nil { + log.Debugf("certificate posture: user certificate store unavailable: %v", err) + } + return mergeProofs(proofs, userProofs) +} + +// UserContext identifies the session whose store a collection would include: a session +// of the profile owner, or empty when no user store would be asked. A change means a +// collection made earlier no longer reflects what this machine can prove. +func UserContext(cfg Config) string { + if !runningAsLocalSystem() || cfg.OwnerUnknown { + return "" + } + user, ok := currentDesktopUser(cfg.ProfileOwner) + if !ok { + return "" + } + defer user.Close() + return fmt.Sprintf("%d:%s", user.Session, user.Name) +} + +// helperStore is the store the helper reads. It runs as the signed-in user, so it wants +// that user's store rather than the machine store the service already read. +func helperStore() Store { + return newUserStore() +} + +// collectAsDesktopUser runs the helper inside the interactive session of the signed-in +// user. Unlike a keychain on macOS, a Windows service can assume a user identity +// directly, so the session token goes straight into the child process. +func collectAsDesktopUser(ctx context.Context, owner string, challenges []*proto.CertificateChallenge, peerKey []byte) ([]certposture.Proof, error) { + user, ok := currentDesktopUser(owner) + if !ok { + return nil, nil + } + defer user.Close() + + binary, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("resolve own binary: %w", err) + } + + // The user's own environment, not the service's: the service environment may carry + // secrets such as a setup key that the signed-in user must not be able to read. + env, err := user.Token.Environ(false) + if err != nil { + return nil, fmt.Errorf("build environment of %s: %w", user.Name, err) + } + + ctx, cancel := context.WithTimeout(ctx, helperTimeout) + defer cancel() + + cmd := exec.CommandContext(ctx, binary, "posture", "cert-proof") + cmd.Env = env + cmd.SysProcAttr = &syscall.SysProcAttr{ + Token: syscall.Token(user.Token), + HideWindow: true, + CreationFlags: windows.CREATE_NO_WINDOW, + } + + log.Debugf("certificate posture: asking session %d to answer %d challenges", user.Session, len(challenges)) + proofs, err := runHelperCmd(cmd, helperRequest(challenges, peerKey)) + if err != nil { + return nil, fmt.Errorf("run helper in session %d: %w", user.Session, err) + } + log.Debugf("certificate posture: session %d returned %d proofs", user.Session, len(proofs)) + return proofs, nil +} diff --git a/client/internal/certproof/collector.go b/client/internal/certproof/collector.go new file mode 100644 index 000000000..2931f4b00 --- /dev/null +++ b/client/internal/certproof/collector.go @@ -0,0 +1,189 @@ +package certproof + +import ( + "context" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +const ( + collectTimeout = 45 * time.Second + + // lostAfter is how many deadlines an abandoned collection is waited for before a new + // one may start beside it. A store that is merely slow finishes well within that and + // releases its slot; only one wedged for good is worked around. + lostAfter = 10 + + // maxInFlight bounds how many abandoned collections may pile up. A store that never + // answers would otherwise leak a goroutine on every collection for the life of the + // daemon. + maxInFlight = 2 +) + +// collectSlots tracks the collections running in the process. Collectors share one by +// default: a collection abandoned in a token, TPM or keychain call keeps running after +// the engine that started it has stopped, and the next engine must account for it. +type collectSlots struct { + mu sync.Mutex + next uint64 + // running holds the start time of each collection still running, by slot id. + running map[uint64]time.Time +} + +var processSlots collectSlots + +// Collector runs CollectProofs with a deadline and, in the normal case, one collection +// at a time in the process. Token, TPM and keychain calls cannot be interrupted, and on +// Linux there is not even a process to kill, so a collection that overruns is abandoned +// rather than awaited. A single wedged call would then refuse every later collection for +// the life of the daemon, so once an abandoned collection has been running for lostAfter +// deadlines another may start beside it, up to maxInFlight. The zero value is ready to +// use. +type Collector struct { + // slots overrides the process-wide slots when set. + slots *collectSlots + // timeout overrides collectTimeout when set. + timeout time.Duration + // now overrides the clock when set. + now func() time.Time +} + +// Collect answers the certificate challenges in checks, returning no proofs when there +// are no challenges, when previous collections still hold the slots, or when this one +// does not finish in time. Missing proofs fail the certificate check on management. +func (c *Collector) Collect(ctx context.Context, checks []*proto.Checks, peerKey []byte, cfg Config) []certposture.Proof { + return c.collect(ctx, checks, func(ctx context.Context) []certposture.Proof { + return CollectProofs(ctx, checks, peerKey, cfg) + }) +} + +func (c *Collector) collect(ctx context.Context, checks []*proto.Checks, run func(context.Context) []certposture.Proof) []certposture.Proof { + if len(certificateChallenges(checks)) == 0 { + return nil + } + slots := c.slotsInUse() + slot, ok := slots.start(c.clock(), time.Duration(lostAfter)*c.deadline()) + if !ok { + return nil + } + + ctx, cancel := context.WithTimeout(ctx, c.deadline()) + defer cancel() + + done := make(chan []certposture.Proof, 1) + go func() { + // The slot is freed before the result is delivered, so a caller that starts the + // next collection right after this one returned is not turned away. + proofs := run(ctx) + slots.finish(slot) + done <- proofs + }() + + select { + case proofs := <-done: + return proofs + case <-ctx.Done(): + log.Warnf("certificate posture: proof collection did not finish within %s, sending no proofs", c.deadline()) + return nil + } +} + +// Stuck reports whether lost collections hold every slot. No collection starts again +// until one of them returns, which for a store that never answers means a restart. +func (c *Collector) Stuck() bool { + return c.slotsInUse().allLost(c.clock(), time.Duration(lostAfter)*c.deadline()) +} + +func (c *Collector) slotsInUse() *collectSlots { + if c.slots != nil { + return c.slots + } + return &processSlots +} + +func (c *Collector) clock() time.Time { + if c.now != nil { + return c.now() + } + return time.Now() +} + +func (c *Collector) deadline() time.Duration { + if c.timeout > 0 { + return c.timeout + } + return collectTimeout +} + +// start claims a slot at now, reporting its id and whether the caller may collect. A +// collection that has held a slot for lost is taken to be wedged. +func (s *collectSlots) start(now time.Time, lost time.Duration) (uint64, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.running) > 0 { + oldest := s.oldest() + switch { + case len(s.running) >= maxInFlight: + log.Warnf("certificate posture: %d proof collections are wedged, sending no proofs", len(s.running)) + return 0, false + case now.Sub(oldest) < lost: + log.Warnf("certificate posture: previous proof collection is still running, sending no proofs") + return 0, false + default: + log.Warnf("certificate posture: a proof collection has been stuck since %s, starting another", oldest.Format(time.RFC3339)) + } + } + if s.running == nil { + s.running = make(map[uint64]time.Time, maxInFlight) + } + s.next++ + s.running[s.next] = now + return s.next, true +} + +// finish releases the slot with the given id. +func (s *collectSlots) finish(slot uint64) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.running, slot) +} + +// allLost reports whether every slot is held by a collection that has run for lost. +func (s *collectSlots) allLost(now time.Time, lost time.Duration) bool { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.running) < maxInFlight { + return false + } + for _, started := range s.running { + if now.Sub(started) < lost { + return false + } + } + return true +} + +// idle reports whether no collection is running. +func (s *collectSlots) idle() bool { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.running) == 0 +} + +// oldest returns the start time of the longest-running collection. The caller holds mu +// and has checked that one is running. +func (s *collectSlots) oldest() time.Time { + var oldest time.Time + for _, started := range s.running { + if oldest.IsZero() || started.Before(oldest) { + oldest = started + } + } + return oldest +} diff --git a/client/internal/certproof/collector_test.go b/client/internal/certproof/collector_test.go new file mode 100644 index 000000000..a28c47da2 --- /dev/null +++ b/client/internal/certproof/collector_test.go @@ -0,0 +1,207 @@ +package certproof + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +var challengeChecks = []*proto.Checks{{CertificateChallenge: &proto.CertificateChallenge{ + Nonce: certposture.NewChallenger([]byte("secret")).Nonce(peerKey, time.Now()), +}}} + +func TestCollector_SkipsChecksWithoutChallenges(t *testing.T) { + var c Collector + called := false + + proofs := c.collect(context.Background(), []*proto.Checks{{Files: []string{"/bin/agent"}}}, func(context.Context) []certposture.Proof { + called = true + return nil + }) + + assert.Nil(t, proofs) + assert.False(t, called, "no store is touched when no check carries a challenge") +} + +func TestCollector_ReturnsProofs(t *testing.T) { + var c Collector + want := []certposture.Proof{{Nonce: []byte("nonce")}} + + proofs := c.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { return want }) + + assert.Equal(t, want, proofs, "a collection that finishes in time is returned as is") +} + +func TestCollector_AbandonsStuckCollection(t *testing.T) { + c := Collector{timeout: 50 * time.Millisecond, slots: &collectSlots{}} + release := make(chan struct{}) + finished := make(chan struct{}) + + // A token or keychain call that ignores its context and blocks well past the deadline. + stuck := func(context.Context) []certposture.Proof { + defer close(finished) + <-release + return []certposture.Proof{{Nonce: []byte("late")}} + } + + start := time.Now() + proofs := c.collect(context.Background(), challengeChecks, stuck) + assert.Nil(t, proofs, "an overrunning collection yields no proofs") + assert.Less(t, time.Since(start), time.Second, "the caller is released at the deadline, not when the call returns") + + called := false + proofs = c.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { + called = true + return nil + }) + assert.Nil(t, proofs) + assert.False(t, called, "no second collection starts while the first is still running") + + close(release) + <-finished + require.Eventually(t, func() bool { return c.slots.idle() }, time.Second, 5*time.Millisecond, "the collector frees up once the stuck call returns") + + want := []certposture.Proof{{Nonce: []byte("nonce")}} + assert.Equal(t, want, c.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { return want }), + "collection works again after the stuck call returned") +} + +func TestCollector_CancelsContextAtDeadline(t *testing.T) { + c := Collector{timeout: 20 * time.Millisecond, slots: &collectSlots{}} + cancelled := make(chan struct{}) + + c.collect(context.Background(), challengeChecks, func(ctx context.Context) []certposture.Proof { + <-ctx.Done() + close(cancelled) + return nil + }) + + select { + case <-cancelled: + case <-time.After(time.Second): + t.Fatal("a collection that honours its context, like the helper process, must see it cancelled") + } +} + +// TestCollector_SingleFlightAcrossCollectors: each engine has its own Collector, and a +// collection stuck in a token call outlives the engine that started it, so the next +// engine's Collector must not start another one until it has finished. +func TestCollector_SingleFlightAcrossCollectors(t *testing.T) { + shared := &collectSlots{} + first := Collector{timeout: 20 * time.Millisecond, slots: shared} + second := Collector{timeout: time.Second, slots: shared} + + release := make(chan struct{}) + stuck := func(context.Context) []certposture.Proof { + <-release + return nil + } + assert.Nil(t, first.collect(context.Background(), challengeChecks, stuck), "the first collection is abandoned at its deadline") + + called := false + proofs := second.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { + called = true + return []certposture.Proof{{}} + }) + assert.Nil(t, proofs, "no proofs while the abandoned collection still runs") + assert.False(t, called, "a second collection does not start on top of the abandoned one") + + close(release) + require.Eventually(t, func() bool { return shared.idle() }, time.Second, 5*time.Millisecond) + assert.Len(t, second.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { return []certposture.Proof{{}} }), 1, + "collections resume once the abandoned one finished") +} + +// TestCollector_ZeroValuesShareTheProcessFlag: the zero value uses the process-wide flag. +func TestCollector_ZeroValuesShareTheProcessFlag(t *testing.T) { + var a, b Collector + assert.Same(t, a.slotsInUse(), b.slotsInUse(), "separate Collectors share the process slots") +} + +// TestCollector_LostCollectionDoesNotBlockForever: a store call that never answers, such +// as a wedged token on Linux where there is no process to kill, must not refuse every +// later collection. Once it has run for lostAfter deadlines another starts beside it, +// and no more than maxInFlight ever run. +func TestCollector_LostCollectionDoesNotBlockForever(t *testing.T) { + now := time.Now() + slots := &collectSlots{} + c := Collector{timeout: 10 * time.Millisecond, slots: slots, now: func() time.Time { return now }} + + release := make(chan struct{}) + defer close(release) + var started atomic.Int32 + wedged := func(context.Context) []certposture.Proof { + started.Add(1) + <-release + return nil + } + + c.collect(context.Background(), challengeChecks, wedged) + waitStarted(t, &started, 1) + c.collect(context.Background(), challengeChecks, wedged) + assert.Equal(t, int32(1), started.Load(), "a second collection is refused while the first may still finish") + assert.False(t, c.Stuck(), "one slow collection is not stuck yet") + + now = now.Add(time.Duration(lostAfter) * c.deadline()) + c.collect(context.Background(), challengeChecks, wedged) + waitStarted(t, &started, 2) + assert.False(t, c.Stuck(), "the second collection may still return, so the collector is not stuck yet") + + now = now.Add(time.Duration(lostAfter) * c.deadline()) + c.collect(context.Background(), challengeChecks, wedged) + assert.Equal(t, int32(2), started.Load(), "no more than maxInFlight collections run") + assert.True(t, c.Stuck(), "with every slot held by a lost collection the collector reports itself stuck") +} + +// TestCollector_LostWindowFollowsTheOldestRunningCollection: A wedges, B starts beside it +// once A counts as lost, then A returns. B is recent, so a third collection must wait for +// B to be lost in its own right rather than inherit A's start time. +func TestCollector_LostWindowFollowsTheOldestRunningCollection(t *testing.T) { + now := time.Now() + slots := &collectSlots{} + c := Collector{timeout: 10 * time.Millisecond, slots: slots, now: func() time.Time { return now }} + lost := time.Duration(lostAfter) * c.deadline() + + releaseA := make(chan struct{}) + releaseB := make(chan struct{}) + defer close(releaseB) + var started atomic.Int32 + blockOn := func(release chan struct{}) func(context.Context) []certposture.Proof { + return func(context.Context) []certposture.Proof { + started.Add(1) + <-release + return nil + } + } + + c.collect(context.Background(), challengeChecks, blockOn(releaseA)) + waitStarted(t, &started, 1) + now = now.Add(lost) + c.collect(context.Background(), challengeChecks, blockOn(releaseB)) + waitStarted(t, &started, 2) + + close(releaseA) + require.Eventually(t, func() bool { + slots.mu.Lock() + defer slots.mu.Unlock() + return len(slots.running) == 1 + }, time.Second, 5*time.Millisecond, "A frees its slot once it returns") + + c.collect(context.Background(), challengeChecks, blockOn(make(chan struct{}))) + assert.Equal(t, int32(2), started.Load(), "a third collection waits while B is recent") +} + +// waitStarted waits for the collections the test launched to have started. collect may +// return at its deadline before the goroutine running the store call was scheduled. +func waitStarted(t *testing.T, started *atomic.Int32, n int32) { + t.Helper() + require.Eventually(t, func() bool { return started.Load() == n }, time.Second, time.Millisecond, + "%d collections should have started", n) +} diff --git a/client/internal/certproof/consoleuser_darwin.go b/client/internal/certproof/consoleuser_darwin.go new file mode 100644 index 000000000..1045bc014 --- /dev/null +++ b/client/internal/certproof/consoleuser_darwin.go @@ -0,0 +1,123 @@ +//go:build !ios + +package certproof + +import ( + "bytes" + "fmt" + "strconv" + "sync" + + "github.com/ebitengine/purego" + log "github.com/sirupsen/logrus" +) + +const ( + systemConfigurationFramework = "/System/Library/Frameworks/SystemConfiguration.framework/SystemConfiguration" + + encodingUTF8 = 0x08000100 + consoleNameBufSize = 256 +) + +var ( + consoleOnce sync.Once + consoleErr error + + scDynamicStoreCopyConsoleUser func(store uintptr, uid, gid *uint32) uintptr + cfStringGetCString func(str uintptr, buffer *byte, size int, encoding uint32) bool +) + +// ConsoleUser is the account whose desktop session owns the display. Its login keychain +// is the only user keychain a NetBird daemon can reach, and only while it is logged in. +type ConsoleUser struct { + Name string + UID uint32 + GID uint32 +} + +// currentConsoleUser reports the user sitting at the desktop. The second return value is +// false when nobody is: at the login window macOS either reports no console user at all +// or attributes the session to root, and neither has a login keychain to offer. +func currentConsoleUser() (ConsoleUser, bool) { + if err := loadConsoleUser(); err != nil { + log.Debugf("console user lookup unavailable: %v", err) + return ConsoleUser{}, false + } + + var uid, gid uint32 + name := scDynamicStoreCopyConsoleUser(0, &uid, &gid) + if name == 0 { + log.Debug("no console user is logged in, no login keychain is reachable") + return ConsoleUser{}, false + } + defer release(name) + + user := ConsoleUser{Name: cfString(name), UID: uid, GID: gid} + if !user.hasDesktop() { + log.Debugf("console session belongs to %q uid=%d, which is not a desktop login, no login keychain is reachable", user.Name, user.UID) + return ConsoleUser{}, false + } + return user, true +} + +// hasDesktop reports whether the console session is a real user desktop. The login +// window runs as root and some macOS releases name it "loginwindow" instead. +func (u ConsoleUser) hasDesktop() bool { + switch u.Name { + case "", "root", "loginwindow": + return false + } + return u.UID != 0 +} + +// isOwner reports whether the console user is owner, the account of the active profile, +// which is recorded as a short user name or, for an account without one, a numeric uid. +// With no owner the console user counts, as macOS has a single console user. +func (u ConsoleUser) isOwner(owner string) bool { + return owner == "" || owner == u.Name || owner == strconv.FormatUint(uint64(u.UID), 10) +} + +func cfString(str uintptr) string { + buf := make([]byte, consoleNameBufSize) + if !cfStringGetCString(str, &buf[0], len(buf), encodingUTF8) { + return "" + } + if end := bytes.IndexByte(buf, 0); end >= 0 { + return string(buf[:end]) + } + return string(buf) +} + +// loadConsoleUser resolves the console user symbols. It loads the keychain bindings +// first because CFRelease is resolved there and released strings depend on it. +func loadConsoleUser() error { + if err := loadKeychain(); err != nil { + return err + } + consoleOnce.Do(func() { consoleErr = resolveConsoleUser() }) + return consoleErr +} + +func resolveConsoleUser() error { + systemConfiguration, err := purego.Dlopen(systemConfigurationFramework, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return fmt.Errorf("open %s: %w", systemConfigurationFramework, err) + } + coreFoundation, err := purego.Dlopen(coreFoundationFramework, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return fmt.Errorf("open %s: %w", coreFoundationFramework, err) + } + + symbol, err := purego.Dlsym(systemConfiguration, "SCDynamicStoreCopyConsoleUser") + if err != nil { + return fmt.Errorf("resolve SCDynamicStoreCopyConsoleUser: %w", err) + } + purego.RegisterFunc(&scDynamicStoreCopyConsoleUser, symbol) + + symbol, err = purego.Dlsym(coreFoundation, "CFStringGetCString") + if err != nil { + return fmt.Errorf("resolve CFStringGetCString: %w", err) + } + purego.RegisterFunc(&cfStringGetCString, symbol) + return nil +} diff --git a/client/internal/certproof/consoleuser_darwin_test.go b/client/internal/certproof/consoleuser_darwin_test.go new file mode 100644 index 000000000..051cc4ab0 --- /dev/null +++ b/client/internal/certproof/consoleuser_darwin_test.go @@ -0,0 +1,51 @@ +//go:build !ios + +package certproof + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConsoleUser_OnlyADesktopSessionCanBeValidated(t *testing.T) { + tests := []struct { + name string + user ConsoleUser + desktop bool + }{ + {"logged in user", ConsoleUser{Name: "maycon", UID: 501, GID: 20}, true}, + {"login window as root", ConsoleUser{Name: "root", UID: 0}, false}, + {"login window by name", ConsoleUser{Name: "loginwindow", UID: 0}, false}, + {"named user still at uid 0", ConsoleUser{Name: "admin", UID: 0}, false}, + {"no console user", ConsoleUser{}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.desktop, tt.user.hasDesktop(), "only a real desktop session offers a login keychain") + }) + } +} + +// currentConsoleUser runs against the real SystemConfiguration framework. A machine with +// a desktop open must report a non-root user; a headless runner must report none. +func TestCurrentConsoleUser_AgreesWithItself(t *testing.T) { + user, ok := currentConsoleUser() + if !ok { + t.Log("no console user, running headless") + return + } + assert.NotEmpty(t, user.Name, "a console user must have a name") + assert.NotZero(t, user.UID, "a desktop session never belongs to uid 0") + assert.True(t, user.hasDesktop(), "a reported console user must be a desktop session") +} + +func TestConsoleUser_IsOwner(t *testing.T) { + user := ConsoleUser{Name: "maycon", UID: 501, GID: 20} + + assert.True(t, user.isOwner(""), "a profile without owner accepts the single console user") + assert.True(t, user.isOwner("maycon"), "the owner by short name") + assert.True(t, user.isOwner("501"), "the owner recorded as a numeric uid") + assert.False(t, user.isOwner("viktor"), "another account's profile must not read this user's keychain") + assert.False(t, user.isOwner("502"), "another uid is another account") +} diff --git a/client/internal/certproof/desktopuser_windows.go b/client/internal/certproof/desktopuser_windows.go new file mode 100644 index 000000000..0a9d63b36 --- /dev/null +++ b/client/internal/certproof/desktopuser_windows.go @@ -0,0 +1,182 @@ +package certproof + +import ( + "fmt" + "strings" + "unsafe" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" +) + +const ( + noActiveSession = 0xFFFFFFFF + servicesSession = 0 + + // wtsCurrentServer is WTS_CURRENT_SERVER_HANDLE, which x/sys/windows does not export. + wtsCurrentServer = windows.Handle(0) +) + +// DesktopUser is an interactive session and the account signed into it. The user's +// certificate store is readable only from a process running as that account, because +// its private keys are protected against the user profile rather than the machine. +type DesktopUser struct { + Session uint32 + Name string + Token windows.Token +} + +// Close releases the session token. +func (u DesktopUser) Close() { + if err := u.Token.Close(); err != nil { + log.Debugf("failed closing desktop session token: %v", err) + } +} + +// currentDesktopUser returns a token for the session whose certificate store should be +// asked: a session of owner, the account the active profile belongs to, with the +// physical console preferred over remote sessions. With no owner only the console user +// counts. Picking any signed-in user instead would let whoever else is logged in to a +// terminal server or VDI host decide the result. The second return is false when no +// such session exists, and only machine certificates can then be proven. +// +// Obtaining the token needs SE_TCB_NAME, which the LocalSystem service has and an +// ordinary process does not. +func currentDesktopUser(owner string) (DesktopUser, bool) { + console := windows.WTSGetActiveConsoleSessionId() + if owner == "" { + return consoleUser(console) + } + + sessions, err := userSessions(console) + if err != nil { + log.Debugf("cannot enumerate terminal sessions: %v", err) + return DesktopUser{}, false + } + var found DesktopUser + var ok bool + for _, session := range sessions { + user, signedIn := desktopUser(session) + if !signedIn { + continue + } + switch { + case !sameAccountName(user.Name, owner): + user.Close() + case !ok: + found, ok = user, true + case strings.EqualFold(user.Name, found.Name): + // Another session of the same account; the first one in preference order wins. + user.Close() + default: + // An owner recorded without a domain matches accounts of several domains here. + // Picking one would let another domain's user answer for the owner. + log.Debugf("certificate posture: profile owner %s matches both %s and %s, no user certificate store is used", owner, found.Name, user.Name) + user.Close() + found.Close() + return DesktopUser{}, false + } + } + if !ok { + log.Debugf("certificate posture: profile owner %s has no signed-in session, no user certificate store is reachable", owner) + } + return found, ok +} + +func consoleUser(console uint32) (DesktopUser, bool) { + if console == noActiveSession || console == servicesSession { + log.Debug("no console session, no user certificate store is reachable") + return DesktopUser{}, false + } + user, ok := desktopUser(console) + if !ok { + log.Debugf("console session %d has nobody signed in, no user certificate store is reachable", console) + } + return user, ok +} + +// sameAccountName compares DOMAIN\account names case-insensitively, as Windows does, and an +// owner given without a domain against the account part alone. The session side comes from +// the session token's own SID, which Windows resolves from its cache of signed-in users. +// Resolving the owner name to a SID instead would ask the domain controller, which on a +// laptop that cannot reach it yet blocks for tens of seconds, past the collection deadline. +func sameAccountName(sessionName, owner string) bool { + if strings.EqualFold(sessionName, owner) { + return true + } + if strings.Contains(owner, `\`) { + return false + } + _, account, found := strings.Cut(sessionName, `\`) + return found && strings.EqualFold(account, owner) +} + +func desktopUser(session uint32) (DesktopUser, bool) { + var token windows.Token + if err := windows.WTSQueryUserToken(session, &token); err != nil { + log.Debugf("no user token for session %d: %v", session, err) + return DesktopUser{}, false + } + + name, err := tokenAccount(token) + if err != nil { + log.Debugf("session %d token has no readable account: %v", session, err) + if closeErr := token.Close(); closeErr != nil { + log.Debugf("failed closing session token: %v", closeErr) + } + return DesktopUser{}, false + } + return DesktopUser{Session: session, Name: name, Token: token}, true +} + +func tokenAccount(token windows.Token) (string, error) { + user, err := token.GetTokenUser() + if err != nil { + return "", fmt.Errorf("read token user: %w", err) + } + account, domain, _, err := user.User.Sid.LookupAccount("") + if err != nil { + return "", fmt.Errorf("look up account: %w", err) + } + if domain == "" { + return account, nil + } + return domain + `\` + account, nil +} + +// userSessions lists the sessions a user can be signed in to, console first, then +// active remote sessions, then disconnected ones, whose user is still signed in. Session +// 0 is skipped: it hosts services and never belongs to an interactive user. +func userSessions(console uint32) ([]uint32, error) { + var info *windows.WTS_SESSION_INFO + var count uint32 + if err := windows.WTSEnumerateSessions(wtsCurrentServer, 0, 1, &info, &count); err != nil { + return nil, fmt.Errorf("enumerate sessions: %w", err) + } + defer windows.WTSFreeMemory(uintptr(unsafe.Pointer(info))) + + var sessions []uint32 + if console != noActiveSession && console != servicesSession { + sessions = append(sessions, console) + } + for _, state := range []uint32{windows.WTSActive, windows.WTSDisconnected} { + for _, session := range unsafe.Slice(info, count) { + if session.SessionID == servicesSession || session.SessionID == console || session.State != state { + continue + } + sessions = append(sessions, session.SessionID) + } + } + return sessions, nil +} + +// runningAsLocalSystem reports whether this process is the service. The helper runs as +// the signed-in user and must read its own store rather than launching another helper. +func runningAsLocalSystem() bool { + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + log.Debugf("failed reading own token user: %v", err) + return false + } + return user.User.Sid.IsWellKnown(windows.WinLocalSystemSid) +} diff --git a/client/internal/certproof/desktopuser_windows_test.go b/client/internal/certproof/desktopuser_windows_test.go new file mode 100644 index 000000000..c3c4022e5 --- /dev/null +++ b/client/internal/certproof/desktopuser_windows_test.go @@ -0,0 +1,61 @@ +package certproof + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSameAccountName(t *testing.T) { + tests := []struct { + session, owner string + want bool + }{ + {`CORP\alice`, `CORP\alice`, true}, + {`CORP\alice`, `corp\ALICE`, true}, + {`CORP\alice`, `alice`, true}, + {`CORP\alice`, `OTHER\alice`, false}, + {`CORP\alice`, `bob`, false}, + {`alice`, `alice`, true}, + {`CORP\alice`, `CORP\alic`, false}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, sameAccountName(tt.session, tt.owner), "session %q, owner %q", tt.session, tt.owner) + } +} + +// currentDesktopUser against the real session manager: an owner no session belongs to +// must never yield a session, whoever else is signed in. +func TestCurrentDesktopUser_UnknownOwnerHasNoSession(t *testing.T) { + user, ok := currentDesktopUser(`NO-SUCH-DOMAIN\no-such-user-netbird`) + if ok { + user.Close() + } + assert.False(t, ok, "a profile owner without a session must not borrow another user's store") +} + +// TestCurrentDesktopUser_FindsTheOwnersSession runs against a real machine as LocalSystem, +// with NB_TEST_DESKTOP_OWNER naming an account that is signed in. That account's session +// must be found, and only that account's. +func TestCurrentDesktopUser_FindsTheOwnersSession(t *testing.T) { + owner := os.Getenv("NB_TEST_DESKTOP_OWNER") + if owner == "" { + t.Skip("set NB_TEST_DESKTOP_OWNER to a signed-in account and run as LocalSystem") + } + user, ok := currentDesktopUser(owner) + require.True(t, ok, "the signed-in owner %s must have a session", owner) + defer user.Close() + t.Logf("owner %s resolved to session %d as %s", owner, user.Session, user.Name) + assert.True(t, sameAccountName(user.Name, owner), "the session found belongs to %s, not %s", owner, user.Name) + assert.NotZero(t, user.Session, "session 0 hosts services and never belongs to the owner") + + console, consoleOK := currentDesktopUser("") + if consoleOK { + defer console.Close() + t.Logf("without an owner the console user counts: session %d as %s", console.Session, console.Name) + } else { + t.Log("without an owner nobody counts: no user at the console") + } +} diff --git a/client/internal/certproof/doc.go b/client/internal/certproof/doc.go new file mode 100644 index 000000000..c8dc62f0c --- /dev/null +++ b/client/internal/certproof/doc.go @@ -0,0 +1,26 @@ +// Package certproof answers certificate posture challenges: it finds the certificates a +// peer holds a private key for and signs the challenge nonce, bound to the peer's +// WireGuard key, with that key. Management verifies the signature and the chain against +// the CAs of the check, so only a holder of the key passes and a proof made for one peer +// cannot be replayed by another. Keys in the OS stores, the TPM and a PKCS#11 token are +// used through the platform, which signs; a plain PEM key file is the exception, parsed +// and used in the daemon's memory. +// +// Machine stores are read by the daemon itself: the Windows LocalMachine store, the macOS +// System keychain, and on Linux a directory of PEM files, TSS2 key files the TPM signs +// with, and a PKCS#11 token. On Linux the directory and the token URI come from the +// daemon's environment (NB_CERT_STORE_DIR, NB_CERT_PKCS11_URI), as does the token PIN +// (NB_TPM_PIN); none of them is read from the profile config. +// +// User stores cannot be read by a privileged daemon, so it starts this binary as +// "netbird posture cert-proof" inside the user's session and receives only signatures +// and chains on its stdout. On macOS uid is not what unlocks a login keychain, the +// session's bootstrap namespace is, which is why the helper enters it through launchctl +// asuser. On Windows a service opening CURRENT_USER silently reads its own empty hive, so +// the helper runs with the session's token instead. Its output is untrusted: sizes are +// capped and only proofs for requested nonces are kept. +// +// A user proof shows that some process in that user's session could use a key whose +// certificate chains to the CA. It does not show which binary signed, or that the key is +// hardware-bound; device stores are the stronger signal. +package certproof diff --git a/client/internal/certproof/helper.go b/client/internal/certproof/helper.go new file mode 100644 index 000000000..81a72ac70 --- /dev/null +++ b/client/internal/certproof/helper.go @@ -0,0 +1,66 @@ +package certproof + +import ( + "context" + "encoding/json" + "fmt" + "io" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// HelperRequest is the work the daemon hands to a helper running in a user session. The +// peer key binds every signature to this machine, so a proof cannot be replayed onto +// another peer. +type HelperRequest struct { + PeerKey []byte `json:"peerKey"` + Challenges []HelperChallenge `json:"challenges"` +} + +// HelperChallenge is one certificate challenge in the form the helper needs. +type HelperChallenge struct { + Nonce []byte `json:"nonce"` + CACertificates []string `json:"caCertificates"` +} + +// HelperResponse carries the proofs the helper produced from its own keychain. +type HelperResponse struct { + Proofs []certposture.Proof `json:"proofs"` +} + +// RunHelper answers the challenges on in from the store of the user running this +// process and writes the proofs to out. It is the child half of the console user +// lookup: the daemon cannot read a login keychain, so it launches this in the user's +// session instead. Only the signature crosses back, never the private key. +func RunHelper(ctx context.Context, in io.Reader, out io.Writer) error { + return runHelper(ctx, helperStore(), in, out) +} + +func runHelper(ctx context.Context, store Store, in io.Reader, out io.Writer) error { + var req HelperRequest + if err := json.NewDecoder(in).Decode(&req); err != nil { + return fmt.Errorf("decode helper request: %w", err) + } + + challenges := make([]*proto.CertificateChallenge, 0, len(req.Challenges)) + for _, challenge := range req.Challenges { + challenges = append(challenges, &proto.CertificateChallenge{ + Nonce: challenge.Nonce, + CaCertificates: challenge.CACertificates, + }) + } + + var proofs []certposture.Proof + if len(challenges) > 0 { + proofs = CollectChallenges(ctx, store, challenges, req.PeerKey) + } + log.Debugf("certificate posture helper: answering %d challenges with %d proofs", len(challenges), len(proofs)) + + if err := json.NewEncoder(out).Encode(HelperResponse{Proofs: proofs}); err != nil { + return fmt.Errorf("encode helper response: %w", err) + } + return nil +} diff --git a/client/internal/certproof/helper_backoff.go b/client/internal/certproof/helper_backoff.go new file mode 100644 index 000000000..3372b026b --- /dev/null +++ b/client/internal/certproof/helper_backoff.go @@ -0,0 +1,78 @@ +package certproof + +import ( + "crypto/sha256" + "encoding/hex" + "slices" + "strings" + "sync" + "time" + + "github.com/netbirdio/netbird/shared/management/proto" +) + +// helperQuietPeriod is how long a user whose helper proved nothing is not asked again +// for the same CAs. On macOS each helper run may show a keychain prompt, so retrying on +// every collection would put up a new one each time the user ignored or denied the last. +const helperQuietPeriod = time.Hour + +// helperBackoff remembers, per user and set of CAs asked about, until when the helper +// is not launched again because its last run proved nothing. +type helperBackoff struct { + mu sync.Mutex + until map[string]time.Time +} + +func newHelperBackoff() *helperBackoff { + return &helperBackoff{until: map[string]time.Time{}} +} + +// allow reports whether the helper may be launched for key now. +func (b *helperBackoff) allow(key string, now time.Time) bool { + b.mu.Lock() + defer b.mu.Unlock() + return !now.Before(b.until[key]) +} + +// record stores the outcome of a helper run for key: one that proved something clears +// the back-off, one that proved nothing starts it. Expired entries are dropped, so the +// map holds only users currently held off. +func (b *helperBackoff) record(key string, proven bool, now time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + for other, until := range b.until { + if !now.Before(until) { + delete(b.until, other) + } + } + if proven { + delete(b.until, key) + return + } + b.until[key] = now.Add(helperQuietPeriod) +} + +// helperBackoffKey identifies a user and the set of CAs the challenges accept, in any +// order, leaving out the nonces, which rotate without changing what the user is asked to +// prove. +func helperBackoffKey(user string, challenges []*proto.CertificateChallenge) string { + sets := make([]string, 0, len(challenges)) + for _, challenge := range challenges { + cas := make([]string, 0, len(challenge.GetCaCertificates())) + for _, ca := range challenge.GetCaCertificates() { + sum := sha256.Sum256([]byte(ca)) + cas = append(cas, hex.EncodeToString(sum[:])) + } + slices.Sort(cas) + sets = append(sets, strings.Join(cas, ",")) + } + slices.Sort(sets) + + h := sha256.New() + h.Write([]byte(user)) + for _, set := range sets { + h.Write([]byte{0}) + h.Write([]byte(set)) + } + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/client/internal/certproof/helper_backoff_test.go b/client/internal/certproof/helper_backoff_test.go new file mode 100644 index 000000000..0d3cafea1 --- /dev/null +++ b/client/internal/certproof/helper_backoff_test.go @@ -0,0 +1,69 @@ +package certproof + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/netbirdio/netbird/shared/management/proto" +) + +// TestHelperBackoff: a user whose helper proved nothing, for one ignoring or denying a +// keychain prompt, is not asked again for the same CAs within the quiet period, while a +// success or other CAs are asked at once. +func TestHelperBackoff(t *testing.T) { + now := time.Now() + b := newHelperBackoff() + key := helperBackoffKey("501", []*proto.CertificateChallenge{{CaCertificates: []string{"ca-a"}}}) + + assert.True(t, b.allow(key, now), "a user never asked is asked") + + b.record(key, false, now) + assert.False(t, b.allow(key, now.Add(helperQuietPeriod-time.Second)), "no new prompt within the quiet period") + assert.True(t, b.allow(key, now.Add(helperQuietPeriod)), "asked again once the quiet period passed") + + other := helperBackoffKey("501", []*proto.CertificateChallenge{{CaCertificates: []string{"ca-b"}}}) + assert.True(t, b.allow(other, now), "a check for other CAs is asked at once") + assert.True(t, b.allow(helperBackoffKey("502", []*proto.CertificateChallenge{{CaCertificates: []string{"ca-a"}}}), now), "another user is asked at once") + + b.record(key, true, now) + assert.True(t, b.allow(key, now), "a run that proved something clears the quiet period") +} + +// TestHelperBackoffKey_IgnoresNonces: nonces rotate every window without changing what +// the user is asked to prove, so they do not reset the quiet period. +func TestHelperBackoffKey_IgnoresNonces(t *testing.T) { + a := helperBackoffKey("501", []*proto.CertificateChallenge{{Nonce: []byte("one"), CaCertificates: []string{"ca"}}}) + b := helperBackoffKey("501", []*proto.CertificateChallenge{{Nonce: []byte("two"), CaCertificates: []string{"ca"}}}) + assert.Equal(t, a, b, "the key does not depend on the nonce") +} + +// TestHelperBackoffKey_IgnoresOrder: the same CAs, listed in another order or with the +// challenges reordered, are the same question to the user. +func TestHelperBackoffKey_IgnoresOrder(t *testing.T) { + a := helperBackoffKey("501", []*proto.CertificateChallenge{ + {CaCertificates: []string{"ca-1", "ca-2"}}, + {CaCertificates: []string{"ca-3"}}, + }) + b := helperBackoffKey("501", []*proto.CertificateChallenge{ + {CaCertificates: []string{"ca-3"}}, + {CaCertificates: []string{"ca-2", "ca-1"}}, + }) + assert.Equal(t, a, b, "the key does not depend on the order of CAs or challenges") + + c := helperBackoffKey("501", []*proto.CertificateChallenge{{CaCertificates: []string{"ca-1", "ca-2", "ca-3"}}}) + assert.NotEqual(t, a, c, "the same CAs grouped into other challenges are another question") +} + +// TestHelperBackoff_PrunesExpired: entries whose quiet period passed are dropped when the +// next outcome is recorded, so the map does not grow with every user and CA set seen. +func TestHelperBackoff_PrunesExpired(t *testing.T) { + now := time.Now() + b := newHelperBackoff() + b.record("old", false, now) + b.record("new", false, now.Add(helperQuietPeriod)) + + assert.NotContains(t, b.until, "old", "an expired entry is pruned") + assert.Contains(t, b.until, "new", "a current entry is kept") +} diff --git a/client/internal/certproof/helper_kill_other.go b/client/internal/certproof/helper_kill_other.go new file mode 100644 index 000000000..2ececb27e --- /dev/null +++ b/client/internal/certproof/helper_kill_other.go @@ -0,0 +1,10 @@ +//go:build !unix && !windows + +package certproof + +import "os/exec" + +// startHelper starts cmd. +func startHelper(cmd *exec.Cmd) (func(), error) { + return func() {}, cmd.Start() +} diff --git a/client/internal/certproof/helper_kill_unix.go b/client/internal/certproof/helper_kill_unix.go new file mode 100644 index 000000000..e434d4962 --- /dev/null +++ b/client/internal/certproof/helper_kill_unix.go @@ -0,0 +1,35 @@ +//go:build unix + +package certproof + +import ( + "errors" + "os/exec" + "syscall" +) + +// killHelperGroupOnCancel puts cmd in a process group of its own and kills the whole +// group when cmd's context ends. The helper may run below launchers such as launchctl +// and sudo, which do not pass a kill on, and it keeps the output pipes open, so killing +// only the direct child would leave the helper running and Wait blocked until the +// helper exits on its own, for one waiting on a keychain prompt nobody answers. +func killHelperGroupOnCancel(cmd *exec.Cmd) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.Setpgid = true + cmd.Cancel = func() error { + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if errors.Is(err, syscall.ESRCH) { + return nil + } + return err + } + cmd.WaitDelay = helperWaitDelay +} + +// startHelper starts cmd. Killing the helper with everything below it is arranged by +// killHelperGroupOnCancel where the helper runs under a launcher. +func startHelper(cmd *exec.Cmd) (func(), error) { + return func() {}, cmd.Start() +} diff --git a/client/internal/certproof/helper_kill_windows.go b/client/internal/certproof/helper_kill_windows.go new file mode 100644 index 000000000..1e143c132 --- /dev/null +++ b/client/internal/certproof/helper_kill_windows.go @@ -0,0 +1,77 @@ +//go:build windows + +package certproof + +import ( + "fmt" + "os/exec" + "unsafe" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" +) + +// startHelper starts cmd inside a job object that is terminated when cmd's context ends +// and closed, killing whatever is left in it, once the helper has been awaited. Killing +// only the helper on Windows leaves its descendants running, and one that holds the +// output pipe would block Wait until it exits, so WaitDelay bounds that wait as well. +func startHelper(cmd *exec.Cmd) (func(), error) { + job, err := newKillOnCloseJob() + if err != nil { + return nil, err + } + closeJob := func() { + if err := windows.CloseHandle(job); err != nil { + log.Debugf("failed to close certificate proof helper job: %v", err) + } + } + + cmd.Cancel = func() error { + return windows.TerminateJobObject(job, 1) + } + cmd.WaitDelay = helperWaitDelay + if err := cmd.Start(); err != nil { + closeJob() + return nil, err + } + + // A helper outside the job is still killed directly and bounded by WaitDelay; only + // its descendants would outlive it. + if err := assignToJob(job, cmd.Process.Pid); err != nil { + log.Debugf("failed to put certificate proof helper %d in its job: %v", cmd.Process.Pid, err) + cmd.Cancel = func() error { return cmd.Process.Kill() } + } + return closeJob, nil +} + +// newKillOnCloseJob creates a job object whose processes are killed when its last handle +// is closed. +func newKillOnCloseJob() (windows.Handle, error) { + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return 0, fmt.Errorf("create job object: %w", err) + } + info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{ + BasicLimitInformation: windows.JOBOBJECT_BASIC_LIMIT_INFORMATION{ + LimitFlags: windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + }, + } + if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(info))); err != nil { + _ = windows.CloseHandle(job) + return 0, fmt.Errorf("set job object limits: %w", err) + } + return job, nil +} + +// assignToJob puts the process pid into job. +func assignToJob(job windows.Handle, pid int) error { + process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(pid)) + if err != nil { + return fmt.Errorf("open process: %w", err) + } + defer func() { + _ = windows.CloseHandle(process) + }() + return windows.AssignProcessToJobObject(job, process) +} diff --git a/client/internal/certproof/helper_kill_windows_test.go b/client/internal/certproof/helper_kill_windows_test.go new file mode 100644 index 000000000..4778d253b --- /dev/null +++ b/client/internal/certproof/helper_kill_windows_test.go @@ -0,0 +1,29 @@ +//go:build windows + +package certproof + +import ( + "context" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRunHelperCmd_KillsHelperTree: the helper's child holds stdout and runs for a minute, +// like a helper stuck below a launcher. Killing only the helper would leave Wait blocked +// on the child's pipe until WaitDelay gives up; terminating the job ends both at once. +func TestRunHelperCmd_KillsHelperTree(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + + cmd := exec.CommandContext(ctx, "cmd.exe", "/c", "ping -n 60 127.0.0.1") + + start := time.Now() + _, err := runHelperCmd(cmd, HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}}) + + require.Error(t, err) + assert.Less(t, time.Since(start), helperWaitDelay, "the whole tree is killed at the timeout, not left for WaitDelay") +} diff --git a/client/internal/certproof/helper_run.go b/client/internal/certproof/helper_run.go new file mode 100644 index 000000000..52c9db775 --- /dev/null +++ b/client/internal/certproof/helper_run.go @@ -0,0 +1,126 @@ +package certproof + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os/exec" + "strings" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" +) + +const ( + maxHelperStdout = 1 << 20 + maxHelperStderr = 4 << 10 + + // helperWaitDelay bounds how long a helper's output is awaited after it was killed. + helperWaitDelay = 2 * time.Second +) + +var errHelperOutputTooLarge = errors.New("helper output exceeds the size limit") + +// runHelperCmd feeds req to the helper process cmd and returns the proofs it answered. +// The helper runs as an unprivileged user who can control its output, so both streams +// are capped and only proofs for a nonce req asked about are kept, one per challenge. +func runHelperCmd(cmd *exec.Cmd, req HelperRequest) ([]certposture.Proof, error) { + payload, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("encode helper request: %w", err) + } + + stdout := &cappedBuffer{limit: maxHelperStdout} + stderr := &cappedBuffer{limit: maxHelperStderr} + cmd.Stdin = bytes.NewReader(payload) + cmd.Stdout = stdout + cmd.Stderr = stderr + + release, err := startHelper(cmd) + if err != nil { + return nil, fmt.Errorf("start helper: %w", err) + } + defer release() + if err := cmd.Wait(); err != nil { + return nil, fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String())) + } + logHelperStderr(stderr.String()) + if stdout.truncated { + return nil, errHelperOutputTooLarge + } + + var resp HelperResponse + if err := json.Unmarshal(stdout.Bytes(), &resp); err != nil { + return nil, fmt.Errorf("decode helper response: %w", err) + } + return requestedProofs(req, resp.Proofs), nil +} + +// logHelperStderr records what a helper that succeeded wrote to stderr, such as a user +// certificate it could not sign with, which is otherwise only visible in the user's +// session. Each line is quoted because the helper's user controls its content. +func logHelperStderr(stderr string) { + for _, line := range strings.Split(strings.TrimSpace(stderr), "\n") { + if line = strings.TrimSpace(line); line != "" { + log.Debugf("certificate posture helper: %q", line) + } + } +} + +// requestedProofs keeps the proofs whose nonce belongs to one of req's challenges, at +// most as many as req has challenges. +func requestedProofs(req HelperRequest, proofs []certposture.Proof) []certposture.Proof { + var kept []certposture.Proof + for _, proof := range proofs { + if len(kept) == len(req.Challenges) { + break + } + if !req.asked(proof.Nonce) { + log.Debugf("certificate posture: dropping helper proof for a nonce that was not requested") + continue + } + kept = append(kept, proof) + } + return kept +} + +func (r HelperRequest) asked(nonce []byte) bool { + for _, challenge := range r.Challenges { + if len(challenge.Nonce) > 0 && bytes.Equal(challenge.Nonce, nonce) { + return true + } + } + return false +} + +// cappedBuffer keeps the first limit bytes written to it and discards the rest, so a +// misbehaving child cannot grow the parent's memory without bound. The buffer is a +// named field rather than embedded: an embedded bytes.Buffer would promote ReadFrom, +// which io.Copy prefers over Write, bypassing the cap. +type cappedBuffer struct { + buf bytes.Buffer + limit int + truncated bool +} + +func (b *cappedBuffer) Write(p []byte) (int, error) { + if room := b.limit - b.buf.Len(); room < len(p) { + b.truncated = true + if room > 0 { + b.buf.Write(p[:room]) + } + return len(p), nil + } + return b.buf.Write(p) +} + +func (b *cappedBuffer) Bytes() []byte { + return b.buf.Bytes() +} + +func (b *cappedBuffer) String() string { + return b.buf.String() +} diff --git a/client/internal/certproof/helper_run_test.go b/client/internal/certproof/helper_run_test.go new file mode 100644 index 000000000..ea1246e93 --- /dev/null +++ b/client/internal/certproof/helper_run_test.go @@ -0,0 +1,119 @@ +//go:build !windows && !js + +package certproof + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" +) + +// fakeHelper is a child process that drains its stdin and then prints script's output, +// standing in for a helper running in a user session the daemon cannot trust. +func fakeHelper(t *testing.T, script string) *exec.Cmd { + t.Helper() + return exec.Command("/bin/sh", "-c", "cat >/dev/null; "+script) +} + +func printJSON(t *testing.T, v any) string { + t.Helper() + out, err := json.Marshal(v) + require.NoError(t, err) + return fmt.Sprintf("printf '%%s' '%s'", out) +} + +func TestRunHelperCmd_KeepsOnlyRequestedProofs(t *testing.T) { + req := HelperRequest{PeerKey: peerKey, Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + resp := HelperResponse{Proofs: []certposture.Proof{ + {Nonce: []byte("injected"), Signature: []byte("x")}, + {Nonce: []byte("asked"), Signature: []byte("first")}, + {Nonce: []byte("asked"), Signature: []byte("second")}, + }} + + proofs, err := runHelperCmd(fakeHelper(t, printJSON(t, resp)), req) + + require.NoError(t, err) + require.Len(t, proofs, 1, "a helper may not return more proofs than challenges or proofs for nonces it was not asked about") + assert.Equal(t, []byte("first"), proofs[0].Signature, "the first proof for the requested nonce is kept") +} + +func TestRunHelperCmd_RejectsOversizedOutput(t *testing.T) { + req := HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + script := fmt.Sprintf("head -c %d /dev/zero", maxHelperStdout+1) + + _, err := runHelperCmd(fakeHelper(t, script), req) + + assert.ErrorIs(t, err, errHelperOutputTooLarge) +} + +func TestRunHelperCmd_CapsStderrInError(t *testing.T) { + req := HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + script := fmt.Sprintf("head -c %d /dev/zero | tr '\\0' 'a' >&2; exit 3", 10*maxHelperStderr) + + _, err := runHelperCmd(fakeHelper(t, script), req) + + require.Error(t, err) + assert.LessOrEqual(t, len(err.Error()), maxHelperStderr+100, "a chatty helper must not blow up the daemon's error or log line") + assert.True(t, strings.Contains(err.Error(), "exit status 3"), "the exit status is kept: %v", err) +} + +// TestRunHelperCmd_LogsStderrOfSuccessfulHelper covers a helper that answers but warns, +// for one about a certificate it could not sign with: the warning reaches the daemon's +// log, quoted so a helper cannot forge log lines with embedded newlines. +func TestRunHelperCmd_LogsStderrOfSuccessfulHelper(t *testing.T) { + var logged bytes.Buffer + log.SetOutput(&logged) + level := log.GetLevel() + log.SetLevel(log.DebugLevel) + t.Cleanup(func() { + log.SetOutput(os.Stderr) + log.SetLevel(level) + }) + + req := HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + script := "printf 'failed signing certificate proof for CN=user\\nforged line\\n' >&2; " + printJSON(t, HelperResponse{}) + + _, err := runHelperCmd(fakeHelper(t, script), req) + + require.NoError(t, err) + assert.Contains(t, logged.String(), "failed signing certificate proof for CN=user", "the helper's warning reaches the daemon log") + assert.NotContains(t, logged.String(), "\nforged line", "helper output cannot start a log line of its own") +} + +// TestRunHelperCmd_KillsHelperBelowLauncher stands in for launchctl and sudo starting the +// helper: the direct child spawns a grandchild that holds stdout and never exits, like a +// helper waiting on a keychain prompt. The timeout must end both, not wait for the +// grandchild. +func TestRunHelperCmd_KillsHelperBelowLauncher(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", "sleep 30 & cat >/dev/null; wait") + killHelperGroupOnCancel(cmd) + + start := time.Now() + _, err := runHelperCmd(cmd, HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}}) + + require.Error(t, err) + assert.Less(t, time.Since(start), 5*time.Second, "the helper is killed at the timeout, not awaited until it exits") +} + +func TestRunHelperCmd_RejectsGarbage(t *testing.T) { + req := HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + + _, err := runHelperCmd(fakeHelper(t, "printf 'not json'"), req) + + assert.ErrorContains(t, err, "decode helper response") +} diff --git a/client/internal/certproof/helper_spawn.go b/client/internal/certproof/helper_spawn.go new file mode 100644 index 000000000..54ed265ec --- /dev/null +++ b/client/internal/certproof/helper_spawn.go @@ -0,0 +1,18 @@ +//go:build (darwin && !ios) || windows + +package certproof + +import ( + "github.com/netbirdio/netbird/shared/management/proto" +) + +func helperRequest(challenges []*proto.CertificateChallenge, peerKey []byte) HelperRequest { + req := HelperRequest{PeerKey: peerKey, Challenges: make([]HelperChallenge, 0, len(challenges))} + for _, challenge := range challenges { + req.Challenges = append(req.Challenges, HelperChallenge{ + Nonce: challenge.GetNonce(), + CACertificates: challenge.GetCaCertificates(), + }) + } + return req +} diff --git a/client/internal/certproof/helper_spawn_test.go b/client/internal/certproof/helper_spawn_test.go new file mode 100644 index 000000000..74a792a49 --- /dev/null +++ b/client/internal/certproof/helper_spawn_test.go @@ -0,0 +1,29 @@ +//go:build (darwin && !ios) || windows + +package certproof + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture/certtest" + "github.com/netbirdio/netbird/shared/management/proto" +) + +func TestHelperRequest_CarriesEveryChallenge(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + challenges := []*proto.CertificateChallenge{ + {Nonce: []byte("first"), CaCertificates: []string{ca.PEM}}, + {Nonce: []byte("second")}, + } + + req := helperRequest(challenges, peerKey) + + require.Len(t, req.Challenges, 2, "every challenge must reach the helper") + assert.Equal(t, peerKey, req.PeerKey, "the peer key binds the signature to this machine") + assert.Equal(t, []byte("first"), req.Challenges[0].Nonce, "the nonce must survive unchanged") + assert.Equal(t, []string{ca.PEM}, req.Challenges[0].CACertificates, "the accepted CAs must survive unchanged") + assert.Empty(t, req.Challenges[1].CACertificates, "a challenge without CAs stays without CAs") +} diff --git a/client/internal/certproof/helper_test.go b/client/internal/certproof/helper_test.go new file mode 100644 index 000000000..61fcc8532 --- /dev/null +++ b/client/internal/certproof/helper_test.go @@ -0,0 +1,66 @@ +package certproof + +import ( + "bytes" + "context" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/certposture/certtest" +) + +// TestRunHelper_AnswersAChallengeFromItsStore runs the helper's request and response +// handling in-process: the proof it writes verifies as the daemon will verify it. The +// process boundary itself is covered by the runHelperCmd tests. +func TestRunHelper_AnswersAChallengeFromItsStore(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + dir := storeDir(t) + key := certtest.ECDSAKey(t) + writeFile(t, dir, "device.pem", certtest.CertPEM(ca.Issue(t, key, "device"))+certtest.KeyPEM(t, key)) + + challenger := certposture.NewChallenger([]byte("secret")) + nonce := challenger.Nonce(peerKey, time.Now()) + request, err := json.Marshal(HelperRequest{ + PeerKey: peerKey, + Challenges: []HelperChallenge{{Nonce: nonce, CACertificates: []string{ca.PEM}}}, + }) + require.NoError(t, err, "request must encode") + + var stdout bytes.Buffer + require.NoError(t, runHelper(context.Background(), newFileStore(dir), bytes.NewReader(request), &stdout)) + + var resp HelperResponse + require.NoError(t, json.Unmarshal(stdout.Bytes(), &resp), "helper must emit decodable JSON") + require.Len(t, resp.Proofs, 1, "the matching certificate should produce one proof") + + // Verify exactly as management does, so the proof is proven to survive the encode, + // the process boundary and the decode intact. + chain, err := challenger.Verify(resp.Proofs[0], peerKey, time.Now()) + require.NoError(t, err, "the decoded proof must verify against the issued nonce") + assert.Equal(t, "device", chain[0].Subject.CommonName, "the proven leaf should be the device certificate") +} + +func TestRunHelper_NoChallengesYieldsEmptyResponse(t *testing.T) { + request, err := json.Marshal(HelperRequest{PeerKey: peerKey}) + require.NoError(t, err) + + var stdout bytes.Buffer + require.NoError(t, runHelper(context.Background(), newFileStore(storeDir(t)), bytes.NewReader(request), &stdout)) + + var resp HelperResponse + require.NoError(t, json.Unmarshal(stdout.Bytes(), &resp), "an empty request must still emit valid JSON") + assert.Empty(t, resp.Proofs, "no challenges should produce no proofs") +} + +func TestRunHelper_RejectsMalformedRequest(t *testing.T) { + var stdout bytes.Buffer + err := runHelper(context.Background(), newFileStore(storeDir(t)), bytes.NewReader([]byte("not json")), &stdout) + + require.Error(t, err, "a malformed request must fail rather than emit an empty proof set") + assert.Empty(t, stdout.String(), "nothing should be written to stdout on a decode failure") +} diff --git a/client/internal/certproof/keychain_darwin.go b/client/internal/certproof/keychain_darwin.go new file mode 100644 index 000000000..5b1426997 --- /dev/null +++ b/client/internal/certproof/keychain_darwin.go @@ -0,0 +1,490 @@ +//go:build !ios + +package certproof + +import ( + "bytes" + "context" + "crypto" + "crypto/x509" + "errors" + "fmt" + "io" + "os" + "sync" + "unsafe" + + "github.com/ebitengine/purego" + log "github.com/sirupsen/logrus" +) + +const ( + securityFramework = "/System/Library/Frameworks/Security.framework/Security" + coreFoundationFramework = "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation" + + errSecItemNotFound = -25300 + errSecInteractionNotAllowed = -25308 +) + +// errKeyNeedsApproval reports a key whose access list does not include netbird, so using +// it needs the user's approval, which a daemon has no UI to ask for. +var errKeyNeedsApproval = errors.New("the key's access control requires user approval for netbird; " + + "import the identity with netbird allowed (security import -k " + + "-T /Applications/NetBird.app/Contents/MacOS/netbird), " + + "or set AllowAllAppsAccess in the MDM certificate payload, which allows every application") + +var ( + keychainOnce sync.Once + keychainErr error + + secItemCopyMatching func(query uintptr, result *uintptr) int32 + secIdentityCopyCertificate func(identity uintptr, cert *uintptr) int32 + secIdentityCopyPrivateKey func(identity uintptr, key *uintptr) int32 + secCertificateCopyData func(cert uintptr) uintptr + secKeyCreateSignature func(key, algorithm, data uintptr, err *uintptr) uintptr + + secKeychainOpen func(path *byte, keychain *uintptr) int32 + secKeychainCopySearchList func(searchList *uintptr) int32 + secKeychainGetPath func(keychain uintptr, pathLength *uint32, path *byte) int32 + cfArrayCreate func(alloc uintptr, values *uintptr, count int, callBacks uintptr) uintptr + + cfDictionaryCreate func(alloc uintptr, keys, values *uintptr, count int, keyCallBacks, valueCallBacks uintptr) uintptr + cfArrayGetCount func(array uintptr) int + cfArrayGetValueAtIndex func(array uintptr, index int) uintptr + cfDataCreate func(alloc uintptr, data *byte, length int) uintptr + cfDataGetLength func(data uintptr) int + cfDataGetBytePtr func(data uintptr) unsafe.Pointer + cfErrorGetCode func(err uintptr) int + cfRelease func(ref uintptr) + + kSecClass, kSecClassIdentity, kSecClassCertificate, kSecMatchLimit, kSecMatchLimitAll, kSecReturnRef uintptr + kSecMatchSearchList uintptr + kSecKeyAlgorithmECDSASHA256, kSecKeyAlgorithmECDSASHA384, kSecKeyAlgorithmRSAPSSSHA256 uintptr + kCFBooleanTrue, kCFTypeDictionaryKeyCallBacks, kCFTypeDictionaryValueCallBacks, kCFTypeArrayCallBacks uintptr +) + +// systemKeychain is where MDM installs device identities. +const systemKeychain = "/Library/Keychains/System.keychain" + +// DefaultStore is the System keychain for the root daemon, so it never touches a user's +// keychain, whose keys would ask for approval in a session the daemon has no UI in. Run +// by an ordinary user, it is that user's keychain search list. +func DefaultStore() Store { + if os.Geteuid() == 0 { + return newKeychainStore(systemKeychain) + } + return newKeychainStore() +} + +// keychainStore yields the identities of a set of keychains, or of the process's keychain +// search list when none is named, reached through purego so the client keeps building +// with CGO_ENABLED=0. +type keychainStore struct { + keychains []string +} + +// newKeychainStore returns a store that searches the keychain files at paths, or the +// process's keychain search list when no path is given. +func newKeychainStore(paths ...string) *keychainStore { + return &keychainStore{keychains: paths} +} + +func (s *keychainStore) Candidates(_ context.Context) ([]Candidate, error) { + if err := loadKeychain(); err != nil { + return nil, err + } + searchList, done, err := searchListOf(s.keychains) + if err != nil { + return nil, err + } + defer done() + + var leaves []*x509.Certificate + err = eachIdentity(searchList, func(_ uintptr, der []byte) (bool, error) { + cert, err := x509.ParseCertificate(der) + if err != nil { + log.Warnf("skipping keychain identity: %v", err) + return false, nil + } + log.Debugf("keychain identity: subject=%q issuer=%q serial=%s expires=%s", cert.Subject, cert.Issuer, cert.SerialNumber, cert.NotAfter) + leaves = append(leaves, cert) + return false, nil + }) + if err != nil { + return nil, err + } + // The certificate query runs even without identities: it separates a keychain that is + // readable but holds no identity from one the process cannot read at all. + pool, err := keychainCertificates(searchList) + if err != nil { + return nil, err + } + if len(leaves) == 0 { + log.Debugf("keychain search list holds no identities usable for certificate posture, but %d readable certificates: an identity needs its private key in the same keychain", len(pool)) + return nil, nil + } + log.Debugf("keychain search list holds %d identities and %d certificates for chain building", len(leaves), len(pool)) + + candidates := make([]Candidate, 0, len(leaves)) + for _, leaf := range leaves { + chain := buildChain(leaf, pool) + log.Debugf("keychain candidate %q issued by %q built a chain of %d certificates", leaf.Subject, leaf.Issuer, len(chain)) + if len(chain) == 1 && leaf.CheckSignatureFrom(leaf) != nil { + log.Debugf("keychain candidate %q has no issuer in the keychain, its proof carries the leaf alone and only verifies if the challenge supplies %q", leaf.Subject, leaf.Issuer) + } + candidates = append(candidates, Candidate{Chain: chain, Signer: &keychainSigner{leaf: leaf, keychains: s.keychains}, Intermediates: pool}) + } + return candidates, nil +} + +// keychainSigner holds only the certificate; the identity is looked up again at signing +// time so no keychain references outlive a call. +type keychainSigner struct { + leaf *x509.Certificate + keychains []string +} + +func (s *keychainSigner) Public() crypto.PublicKey { + return s.leaf.PublicKey +} + +func (s *keychainSigner) Sign(_ io.Reader, digest []byte, opts crypto.SignerOpts) ([]byte, error) { + scheme, err := schemeFor(s.leaf.PublicKey, opts) + if err != nil { + return nil, err + } + log.Debugf("signing certificate posture challenge with keychain key of %q", s.leaf.Subject) + + searchList, done, err := searchListOf(s.keychains) + if err != nil { + return nil, err + } + defer done() + + algorithm := keychainAlgorithm(scheme) + var signature []byte + var attempts signAttempts + err = eachIdentity(searchList, func(identity uintptr, der []byte) (bool, error) { + if !bytes.Equal(der, s.leaf.Raw) { + return false, nil + } + // The same certificate can sit in the keychain with more than one key, of which + // only one may be usable, so a failure moves on to the next match. + var ok bool + signature, ok = attempts.try(func() ([]byte, error) { return signWithIdentity(identity, algorithm, digest) }) + return ok, nil + }) + if err != nil { + return nil, err + } + if signature == nil { + if err := attempts.err(); err != nil { + return nil, err + } + return nil, errors.New("certificate is no longer in the keychain") + } + log.Debugf("keychain signed certificate posture challenge for %q, %d bytes", s.leaf.Subject, len(signature)) + return signature, nil +} + +func keychainAlgorithm(scheme sigScheme) uintptr { + switch scheme { + case schemeECDSASHA384: + return kSecKeyAlgorithmECDSASHA384 + case schemeRSAPSSSHA256: + return kSecKeyAlgorithmRSAPSSSHA256 + default: + return kSecKeyAlgorithmECDSASHA256 + } +} + +func signWithIdentity(identity, algorithm uintptr, digest []byte) ([]byte, error) { + var key uintptr + if status := secIdentityCopyPrivateKey(identity, &key); status != 0 { + return nil, fmt.Errorf("SecIdentityCopyPrivateKey: %d", status) + } + defer release(key) + + data := cfDataCreate(0, &digest[0], len(digest)) + if data == 0 { + return nil, errors.New("CFDataCreate returned NULL") + } + defer release(data) + + var cfErr uintptr + signature := secKeyCreateSignature(key, algorithm, data, &cfErr) + if signature == 0 { + if cfErr == 0 { + return nil, errors.New("SecKeyCreateSignature failed without a CFError") + } + defer release(cfErr) + code := cfErrorGetCode(cfErr) + if code == errSecInteractionNotAllowed { + return nil, fmt.Errorf("SecKeyCreateSignature: CFError %d: %w", code, errKeyNeedsApproval) + } + return nil, fmt.Errorf("SecKeyCreateSignature: CFError %d", code) + } + defer release(signature) + return dataBytes(signature), nil +} + +// searchListOf opens the keychain files at paths as a CFArray for kSecMatchSearchList, +// or yields 0, the process's own search list, when paths is empty. The caller calls done +// once it no longer uses the list. +func searchListOf(paths []string) (searchList uintptr, done func(), err error) { + if len(paths) == 0 { + return 0, func() {}, nil + } + list, err := openSearchList(paths) + if err != nil { + return 0, nil, err + } + return list, func() { release(list) }, nil +} + +// openSearchList opens the keychain files at paths, at least one, as a CFArray. The +// caller releases the array. +func openSearchList(paths []string) (uintptr, error) { + refs := make([]uintptr, 0, len(paths)) + defer func() { + for _, ref := range refs { + release(ref) + } + }() + for _, path := range paths { + cpath := append([]byte(path), 0) + var keychain uintptr + if status := secKeychainOpen(&cpath[0], &keychain); status != 0 { + return 0, fmt.Errorf("SecKeychainOpen %s: %d", path, status) + } + refs = append(refs, keychain) + } + // The array retains the keychains, so the references opened here are released. + list := cfArrayCreate(0, &refs[0], len(refs), kCFTypeArrayCallBacks) + if list == 0 { + return 0, errors.New("CFArrayCreate returned NULL") + } + return list, nil +} + +// eachIdentity calls fn with every identity in searchList, or in the process's search +// list when it is 0, and its certificate. An identity whose certificate cannot be read is +// skipped rather than ending the walk. +func eachIdentity(searchList uintptr, fn func(identity uintptr, der []byte) (bool, error)) error { + return eachMatching(searchList, kSecClassIdentity, "identity", func(identity uintptr) (bool, error) { + var cert uintptr + if status := secIdentityCopyCertificate(identity, &cert); status != 0 { + log.Debugf("skipping keychain identity: SecIdentityCopyCertificate: %d", status) + return false, nil + } + der := certificateDER(cert) + release(cert) + if der == nil { + log.Debug("skipping keychain identity whose certificate has no DER data") + return false, nil + } + return fn(identity, der) + }) +} + +func keychainCertificates(searchList uintptr) ([]*x509.Certificate, error) { + var certs []*x509.Certificate + var unparsable int + err := eachMatching(searchList, kSecClassCertificate, "certificate", func(item uintptr) (bool, error) { + if cert, err := x509.ParseCertificate(certificateDER(item)); err == nil { + certs = append(certs, cert) + return false, nil + } + unparsable++ + return false, nil + }) + log.Debugf("keychain holds %d parsable certificates, %d unparsable", len(certs), unparsable) + return certs, err +} + +func eachMatching(searchList, class uintptr, name string, fn func(item uintptr) (bool, error)) error { + keys := []uintptr{kSecClass, kSecMatchLimit, kSecReturnRef} + values := []uintptr{class, kSecMatchLimitAll, kCFBooleanTrue} + if searchList != 0 { + keys = append(keys, kSecMatchSearchList) + values = append(values, searchList) + } + query := cfDictionaryCreate(0, &keys[0], &values[0], len(keys), kCFTypeDictionaryKeyCallBacks, kCFTypeDictionaryValueCallBacks) + defer release(query) + + var items uintptr + switch status := secItemCopyMatching(query, &items); status { + case 0: + case errSecItemNotFound: + log.Debugf("keychain %s query returned errSecItemNotFound (%d): the search list holds no item of this class", name, errSecItemNotFound) + return nil + default: + log.Debugf("keychain %s query returned OSStatus %d", name, status) + return fmt.Errorf("SecItemCopyMatching: %d", status) + } + defer release(items) + + n := cfArrayGetCount(items) + log.Debugf("keychain %s query returned %d items", name, n) + for i := 0; i < n; i++ { + if stop, err := fn(cfArrayGetValueAtIndex(items, i)); stop || err != nil { + return err + } + } + return nil +} + +// certificateDER returns the DER form of cert, or nil when SecCertificateCopyData +// returns NULL, which it does for an object that is not a valid certificate. +func certificateDER(cert uintptr) []byte { + data := secCertificateCopyData(cert) + if data == 0 { + return nil + } + defer release(data) + return dataBytes(data) +} + +func dataBytes(data uintptr) []byte { + n := cfDataGetLength(data) + if n <= 0 { + return nil + } + return bytes.Clone(unsafe.Slice((*byte)(cfDataGetBytePtr(data)), n)) +} + +// release drops a CoreFoundation reference. CFRelease crashes the process on NULL, and +// several Security calls return NULL on failure, so every release goes through here. +func release(ref uintptr) { + if ref != 0 { + cfRelease(ref) + } +} + +func loadKeychain() error { + keychainOnce.Do(func() { + if keychainErr = resolveKeychain(); keychainErr != nil { + log.Debugf("macOS keychain unavailable for certificate posture: %v", keychainErr) + return + } + log.Debugf("macOS Security framework loaded for certificate posture, running as uid=%d euid=%d", os.Getuid(), os.Geteuid()) + logSearchList() + }) + return keychainErr +} + +// logSearchList reports the keychains the process searches. The root daemon sees the +// System keychain and System Roots, never a user's login keychain. +func logSearchList() { + if secKeychainCopySearchList == nil || secKeychainGetPath == nil { + log.Debug("keychain search list diagnostics unavailable on this macOS version") + return + } + + var list uintptr + if status := secKeychainCopySearchList(&list); status != 0 { + log.Debugf("SecKeychainCopySearchList returned OSStatus %d", status) + return + } + defer release(list) + + n := cfArrayGetCount(list) + log.Debugf("keychain search list contains %d keychains", n) + for i := 0; i < n; i++ { + log.Debugf("keychain search list[%d]: %s", i, keychainPath(cfArrayGetValueAtIndex(list, i))) + } +} + +func keychainPath(keychain uintptr) string { + path := make([]byte, 1024) + length := uint32(len(path)) + if status := secKeychainGetPath(keychain, &length, &path[0]); status != 0 { + return fmt.Sprintf("", status) + } + return string(path[:length]) +} + +func resolveKeychain() error { + security, err := purego.Dlopen(securityFramework, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return fmt.Errorf("open %s: %w", securityFramework, err) + } + coreFoundation, err := purego.Dlopen(coreFoundationFramework, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return fmt.Errorf("open %s: %w", coreFoundationFramework, err) + } + + for _, fn := range []struct { + ptr any + lib uintptr + name string + }{ + {&secItemCopyMatching, security, "SecItemCopyMatching"}, + {&secIdentityCopyCertificate, security, "SecIdentityCopyCertificate"}, + {&secIdentityCopyPrivateKey, security, "SecIdentityCopyPrivateKey"}, + {&secCertificateCopyData, security, "SecCertificateCopyData"}, + {&secKeyCreateSignature, security, "SecKeyCreateSignature"}, + {&secKeychainOpen, security, "SecKeychainOpen"}, + {&cfArrayCreate, coreFoundation, "CFArrayCreate"}, + {&cfDictionaryCreate, coreFoundation, "CFDictionaryCreate"}, + {&cfArrayGetCount, coreFoundation, "CFArrayGetCount"}, + {&cfArrayGetValueAtIndex, coreFoundation, "CFArrayGetValueAtIndex"}, + {&cfDataCreate, coreFoundation, "CFDataCreate"}, + {&cfDataGetLength, coreFoundation, "CFDataGetLength"}, + {&cfDataGetBytePtr, coreFoundation, "CFDataGetBytePtr"}, + {&cfErrorGetCode, coreFoundation, "CFErrorGetCode"}, + {&cfRelease, coreFoundation, "CFRelease"}, + } { + symbol, err := purego.Dlsym(fn.lib, fn.name) + if err != nil { + return fmt.Errorf("resolve %s: %w", fn.name, err) + } + purego.RegisterFunc(fn.ptr, symbol) + } + + for _, global := range []struct { + ptr *uintptr + lib uintptr + name string + deref bool + }{ + {&kSecClass, security, "kSecClass", true}, + {&kSecClassIdentity, security, "kSecClassIdentity", true}, + {&kSecClassCertificate, security, "kSecClassCertificate", true}, + {&kSecMatchLimit, security, "kSecMatchLimit", true}, + {&kSecMatchLimitAll, security, "kSecMatchLimitAll", true}, + {&kSecReturnRef, security, "kSecReturnRef", true}, + {&kSecMatchSearchList, security, "kSecMatchSearchList", true}, + {&kSecKeyAlgorithmECDSASHA256, security, "kSecKeyAlgorithmECDSASignatureDigestX962SHA256", true}, + {&kSecKeyAlgorithmECDSASHA384, security, "kSecKeyAlgorithmECDSASignatureDigestX962SHA384", true}, + {&kSecKeyAlgorithmRSAPSSSHA256, security, "kSecKeyAlgorithmRSASignatureDigestPSSSHA256", true}, + {&kCFBooleanTrue, coreFoundation, "kCFBooleanTrue", true}, + {&kCFTypeDictionaryKeyCallBacks, coreFoundation, "kCFTypeDictionaryKeyCallBacks", false}, + {&kCFTypeDictionaryValueCallBacks, coreFoundation, "kCFTypeDictionaryValueCallBacks", false}, + {&kCFTypeArrayCallBacks, coreFoundation, "kCFTypeArrayCallBacks", false}, + } { + addr, err := purego.Dlsym(global.lib, global.name) + if err != nil { + return fmt.Errorf("resolve %s: %w", global.name, err) + } + if global.deref { + addr = **(**uintptr)(unsafe.Pointer(&addr)) + } + *global.ptr = addr + } + + resolveOptional(security, "SecKeychainCopySearchList", &secKeychainCopySearchList) + resolveOptional(security, "SecKeychainGetPath", &secKeychainGetPath) + return nil +} + +// resolveOptional binds a diagnostic-only symbol, leaving it nil when the framework no +// longer exports it so keychain lookups keep working without it. +func resolveOptional(lib uintptr, name string, ptr any) { + symbol, err := purego.Dlsym(lib, name) + if err != nil { + log.Debugf("keychain diagnostics: %s unavailable: %v", name, err) + return + } + purego.RegisterFunc(ptr, symbol) +} diff --git a/client/internal/certproof/keystore.go b/client/internal/certproof/keystore.go new file mode 100644 index 000000000..7ce676b4c --- /dev/null +++ b/client/internal/certproof/keystore.go @@ -0,0 +1,77 @@ +package certproof + +import ( + "crypto" + "crypto/ecdsa" + "crypto/rsa" + "crypto/x509" + "encoding/asn1" + "errors" + "fmt" + "math/big" + "slices" +) + +var errUnsupportedScheme = errors.New("unsupported signature scheme for OS keystore") + +type sigScheme int + +const ( + schemeECDSASHA256 sigScheme = iota + 1 + schemeECDSASHA384 + schemeRSAPSSSHA256 +) + +// schemeFor maps a crypto.Signer request onto the schemes the OS keystores perform. +func schemeFor(pub crypto.PublicKey, opts crypto.SignerOpts) (sigScheme, error) { + switch pub.(type) { + case *ecdsa.PublicKey: + switch opts.HashFunc() { + case crypto.SHA256: + return schemeECDSASHA256, nil + case crypto.SHA384: + return schemeECDSASHA384, nil + } + case *rsa.PublicKey: + if pss, ok := opts.(*rsa.PSSOptions); ok && pss.Hash == crypto.SHA256 { + return schemeRSAPSSSHA256, nil + } + } + return 0, fmt.Errorf("%w: %T with %v", errUnsupportedScheme, pub, opts.HashFunc()) +} + +// buildChain extends leaf with the issuers found in pool up to a self-signed certificate. +func buildChain(leaf *x509.Certificate, pool []*x509.Certificate) []*x509.Certificate { + chain := []*x509.Certificate{leaf} + current := leaf + for current.CheckSignatureFrom(current) != nil { + issuer := issuerIn(current, pool, chain) + if issuer == nil { + break + } + chain = append(chain, issuer) + current = issuer + } + return chain +} + +func issuerIn(cert *x509.Certificate, pool, seen []*x509.Certificate) *x509.Certificate { + for _, candidate := range pool { + if slices.ContainsFunc(seen, candidate.Equal) { + continue + } + if cert.CheckSignatureFrom(candidate) == nil { + return candidate + } + } + return nil +} + +// ecdsaSignatureASN1 converts the fixed-width r||s form emitted by CNG into the DER form Go verifies. +func ecdsaSignatureASN1(raw []byte) ([]byte, error) { + if len(raw) == 0 || len(raw)%2 != 0 { + return nil, errors.New("malformed raw ECDSA signature") + } + half := len(raw) / 2 + return asn1.Marshal(struct{ R, S *big.Int }{new(big.Int).SetBytes(raw[:half]), new(big.Int).SetBytes(raw[half:])}) +} diff --git a/client/internal/certproof/keystore_test.go b/client/internal/certproof/keystore_test.go new file mode 100644 index 000000000..7d028c4fb --- /dev/null +++ b/client/internal/certproof/keystore_test.go @@ -0,0 +1,91 @@ +package certproof + +import ( + "crypto" + "crypto/ecdsa" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/certposture/certtest" +) + +func TestBuildChain_FollowsIssuersThroughThePool(t *testing.T) { + root := certtest.NewCA(t, "root") + intermediate := certtest.NewIntermediate(t, root, "intermediate") + unrelated := certtest.NewCA(t, "unrelated") + leaf := intermediate.Issue(t, certtest.ECDSAKey(t), "device") + pool := []*x509.Certificate{unrelated.Cert, root.Cert, leaf, intermediate.Cert} + + chain := buildChain(leaf, pool) + + require.Equal(t, []*x509.Certificate{leaf, intermediate.Cert, root.Cert}, chain) + roots, err := certposture.ParseCAs([]string{root.PEM}) + require.NoError(t, err) + assert.NoError(t, certposture.VerifyChain(chain, roots, time.Now())) +} + +func TestBuildChain_StopsWhereThePoolEnds(t *testing.T) { + root := certtest.NewCA(t, "root") + intermediate := certtest.NewIntermediate(t, root, "intermediate") + leaf := intermediate.Issue(t, certtest.ECDSAKey(t), "device") + + assert.Equal(t, []*x509.Certificate{leaf}, buildChain(leaf, nil)) + assert.Equal(t, []*x509.Certificate{leaf, intermediate.Cert}, buildChain(leaf, []*x509.Certificate{intermediate.Cert})) +} + +func TestSchemeFor(t *testing.T) { + ecKey := certtest.ECDSAKey(t) + rsaKey := certtest.RSAKey(t) + pss := &rsa.PSSOptions{SaltLength: rsa.PSSSaltLengthEqualsHash, Hash: crypto.SHA256} + + tests := []struct { + name string + pub crypto.PublicKey + opts crypto.SignerOpts + want sigScheme + }{ + {"ecdsa sha256", ecKey.Public(), crypto.SHA256, schemeECDSASHA256}, + {"ecdsa sha384", ecKey.Public(), crypto.SHA384, schemeECDSASHA384}, + {"rsa pss sha256", rsaKey.Public(), pss, schemeRSAPSSSHA256}, + {"rsa pkcs1v15", rsaKey.Public(), crypto.SHA256, 0}, + {"ecdsa sha512", ecKey.Public(), crypto.SHA512, 0}, + {"rsa pss sha384", rsaKey.Public(), &rsa.PSSOptions{SaltLength: rsa.PSSSaltLengthEqualsHash, Hash: crypto.SHA384}, 0}, + {"ed25519", certtest.Ed25519Key(t).Public(), crypto.Hash(0), 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := schemeFor(tc.pub, tc.opts) + if tc.want == 0 { + assert.ErrorIs(t, err, errUnsupportedScheme) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestECDSASignatureASN1(t *testing.T) { + key := certtest.ECDSAKey(t).(*ecdsa.PrivateKey) + digest := sha256.Sum256([]byte("nonce")) + r, s, err := ecdsa.Sign(rand.Reader, key, digest[:]) + require.NoError(t, err) + raw := append(r.FillBytes(make([]byte, 32)), s.FillBytes(make([]byte, 32))...) + + der, err := ecdsaSignatureASN1(raw) + require.NoError(t, err) + assert.True(t, ecdsa.VerifyASN1(&key.PublicKey, digest[:], der)) + + _, err = ecdsaSignatureASN1(raw[:63]) + assert.Error(t, err, "an odd length cannot split into r and s") + _, err = ecdsaSignatureASN1(nil) + assert.Error(t, err, "an empty signature is malformed") +} diff --git a/client/internal/certproof/merge.go b/client/internal/certproof/merge.go new file mode 100644 index 000000000..d539ce927 --- /dev/null +++ b/client/internal/certproof/merge.go @@ -0,0 +1,50 @@ +package certproof + +import ( + "crypto/sha256" + "crypto/x509" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" +) + +// mergeProofs appends the user session proofs to the device proofs, dropping one whose +// chain the device already proved. Management checks each chain against each check's CAs, +// so the same leaf proven through another chain is kept. +func mergeProofs(device, user []certposture.Proof) []certposture.Proof { + if len(user) == 0 { + return device + } + + seen := make(map[[sha256.Size]byte]struct{}, len(device)+len(user)) + for _, proof := range device { + if len(proof.Chain) > 0 { + seen[chainDERFingerprint(proof.Chain)] = struct{}{} + } + } + + merged := device + for _, proof := range user { + if len(proof.Chain) == 0 { + continue + } + fingerprint := chainDERFingerprint(proof.Chain) + if _, done := seen[fingerprint]; done { + continue + } + seen[fingerprint] = struct{}{} + merged = append(merged, proof) + logUserProof(proof) + } + return merged +} + +func logUserProof(proof certposture.Proof) { + leaf, err := x509.ParseCertificate(proof.Chain[0]) + if err != nil { + log.Debugf("certificate posture: user proof carries an unparsable leaf: %v", err) + return + } + log.Debugf("certificate posture: signed-in user proved %q issued by %q", leaf.Subject, leaf.Issuer) +} diff --git a/client/internal/certproof/merge_test.go b/client/internal/certproof/merge_test.go new file mode 100644 index 000000000..7db2dd421 --- /dev/null +++ b/client/internal/certproof/merge_test.go @@ -0,0 +1,54 @@ +package certproof + +import ( + "crypto/sha256" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/certposture/certtest" +) + +func TestMergeProofs_ProvesACertificateHeldByBothStoresOnce(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + shared := ca.Issue(t, certtest.ECDSAKey(t), "shared") + userOnly := ca.Issue(t, certtest.ECDSAKey(t), "user-only") + + device := []certposture.Proof{{Chain: [][]byte{shared.Raw}}} + user := []certposture.Proof{{Chain: [][]byte{shared.Raw}}, {Chain: [][]byte{userOnly.Raw}}, {}} + + merged := mergeProofs(device, user) + + require.Len(t, merged, 2, "the shared leaf is proven once and the empty chain is dropped") + assert.Equal(t, shared.Raw, merged[0].Chain[0], "the device proof keeps its place") + assert.Equal(t, userOnly.Raw, merged[1].Chain[0], "the user-only certificate is appended") +} + +func TestMergeProofs_KeepsDeviceProofsWhenNoUserSession(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + device := []certposture.Proof{{Chain: [][]byte{ca.Issue(t, certtest.ECDSAKey(t), "device").Raw}}} + + merged := mergeProofs(device, nil) + + require.Len(t, merged, 1, "a machine with nobody signed in still sends its device proof") + assert.Equal(t, sha256.Sum256(device[0].Chain[0]), sha256.Sum256(merged[0].Chain[0]), "the device proof is unchanged") +} + +func TestMergeProofs_KeepsAnotherChainOfTheSameLeaf(t *testing.T) { + root := certtest.NewCA(t, "corp-root") + interA := root.Issue(t, certtest.ECDSAKey(t), "inter-a") + interB := root.Issue(t, certtest.ECDSAKey(t), "inter-b") + leaf := root.Issue(t, certtest.ECDSAKey(t), "leaf") + + // Management checks each chain against each check's CAs, so the same leaf proven + // through another chain is a different proof, not a duplicate. + device := []certposture.Proof{{Chain: [][]byte{leaf.Raw, interA.Raw}}} + user := []certposture.Proof{{Chain: [][]byte{leaf.Raw, interB.Raw}}, {Chain: [][]byte{leaf.Raw, interA.Raw}}} + + merged := mergeProofs(device, user) + + require.Len(t, merged, 2, "the chain only the user store holds is kept, the one both hold is not repeated") + assert.Equal(t, interB.Raw, merged[1].Chain[1], "the user's chain through the other intermediate is appended") +} diff --git a/client/internal/certproof/pinlatch.go b/client/internal/certproof/pinlatch.go new file mode 100644 index 000000000..79e396b11 --- /dev/null +++ b/client/internal/certproof/pinlatch.go @@ -0,0 +1,54 @@ +package certproof + +import ( + "crypto/sha256" + "encoding/binary" + "errors" + "sync" +) + +// errPINRejectedBefore is returned instead of logging in with a PIN the token already +// refused: every failed login counts towards the token's lockout, which for a TPM is +// shared with everything else on the machine, and proofs are collected on every sync. +var errPINRejectedBefore = errors.New("PKCS#11 token rejected this PIN before, not trying it again") + +// errPINNeedsToken refuses a PIN that names no token to log in to. +var errPINNeedsToken = errors.New("a PKCS#11 PIN needs the token named in NB_CERT_PKCS11_URI, as token=