diff --git a/.github/scripts/test-homebrew-cask.sh b/.github/scripts/test-homebrew-cask.sh new file mode 100755 index 000000000..9c1041bdd --- /dev/null +++ b/.github/scripts/test-homebrew-cask.sh @@ -0,0 +1,338 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { + echo "::error::$*" >&2 + exit 1 +} + +if [[ ${RUNNER_ENVIRONMENT:-} != github-hosted || ${RUNNER_OS:-} != macOS || $(uname -s) != Darwin ]]; then + fail "This test installs a system daemon and must run on a disposable GitHub macOS runner." +fi +if [[ $EUID == 0 ]]; then + fail "Run this script as the Homebrew user, not root." +fi + +readonly test_dir="${RUNNER_TEMP:?}/homebrew-cask" +readonly results_dir="$test_dir/results" +readonly app='/Applications/Netbird UI.app' +readonly plist='/Library/LaunchDaemons/netbird.plist' +readonly cask='netbirdio/tap/netbird-ui' +readonly formula='netbirdio/tap/netbird' +readonly published_cask="$test_dir/published-netbird-ui.rb" +readonly legacy_cask="$test_dir/legacy-netbird-ui.rb" +readonly rendered_cask="$test_dir/rendered-netbird-ui.rb" +readonly fixture_dir="$test_dir/fixture" +readonly serve_dir="$test_dir/serve" +readonly fixture_zip="$serve_dir/netbird-ui.zip" +readonly fixture_port=18080 +readonly fixture_url="http://127.0.0.1:$fixture_port/netbird-ui.zip" +readonly marker="$test_dir/installer.marker" + +mkdir -p "$results_dir" "$fixture_dir/netbird_ui_darwin" "$serve_dir" "$test_dir/downloads" +exec > >(tee "$results_dir/test.log") 2>&1 + +sudo -n true +if command -v netbird || [[ -e "$app" || -e "$plist" ]] || pgrep -x netbird-ui; then + fail "The runner already has NetBird installed or running." +fi +if sudo launchctl print system/netbird > "$results_dir/initial-service.log" 2>&1; then + fail "The runner already has a NetBird service loaded." +fi + +install_attempted=false +server_pid='' +daemon_pid='' +version='' + +stop_ui() { + local status=0 + sudo pkill -x netbird-ui || status=$? + # pkill returns 1 when the UI is already closed. + [[ $status == 0 || $status == 1 ]] +} + +cleanup() { + local status=$? + trap - EXIT + set +e + + if [[ $install_attempted == true ]]; then + stop_ui || status=1 + if [[ -S /var/run/netbird.sock ]]; then + sudo netbird down || status=1 + fi + if brew list --cask "$cask" >/dev/null 2>&1 || [[ -e "$app" ]]; then + brew uninstall --cask --force "$cask" || status=1 + fi + # A failed cask install can leave a daemon even after Homebrew rolls back the app. + if sudo launchctl print system/netbird > "$results_dir/cleanup-service.log" 2>&1; then + sudo netbird service stop || status=1 + fi + if [[ -e "$plist" ]]; then + sudo netbird service uninstall || status=1 + fi + fi + if [[ -f /var/log/netbird/client.log ]]; then + sudo cat /var/log/netbird/client.log > "$results_dir/client.log" || status=1 + fi + if command -v netbird >/dev/null; then + brew uninstall --formula "$formula" || status=1 + fi + if [[ -n $server_pid ]]; then + kill "$server_pid" 2>/dev/null || true + fi + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +run_logged() { + local name=$1 + shift + "$@" 2>&1 | tee "$results_dir/$name.log" +} + +cask_field() { + local stanza=$1 file=$2 + sed -nE "s/^[[:space:]]*$stanza \"([^\"]+)\".*/\\1/p" "$file" +} + +release_fields() { + local file=$1 + grep -E '^[[:space:]]*(version|url|sha256|app) ' "$file" +} + +use_cask() { + local file=$1 + cp "$file" "$tap_dir/Casks/netbird-ui.rb" +} + +# The released installer opens the UI as root, which never returns on a headless +# runner. The cask only needs two script paths and a version argument, so the test +# ships a stub bundle that records what it received and starts the daemon. +build_fixture() { + local bundle="$fixture_dir/netbird_ui_darwin" + printf '#!/bin/sh\nexit 0\n' > "$bundle/netbird-ui" + chmod 755 "$bundle/netbird-ui" + # After a bootout launchd keeps tearing the previous daemon down for a couple of + # seconds, and loading the same label again fails until that finishes. + cat > "$bundle/installer.sh" < '$marker' +netbird service install +attempt=0 +until netbird service start; do + attempt=\$((attempt + 1)) + [ "\$attempt" -lt 15 ] || exit 1 + sleep 1 +done +EOF + printf '#!/bin/sh\nexit 0\n' > "$bundle/uninstaller.sh" + # Shipped without the executable bit so the 0755 seen after install can only come from the cask. + chmod 644 "$bundle/installer.sh" "$bundle/uninstaller.sh" + rm -f "$fixture_zip" + (cd "$fixture_dir" && zip -qr "$fixture_zip" netbird_ui_darwin) +} + +start_fixture_server() { + python3 -m http.server "$fixture_port" --bind 127.0.0.1 --directory "$serve_dir" \ + > "$results_dir/fixture-server.log" 2>&1 & + server_pid=$! + local attempt + for attempt in {1..20}; do + if curl --silent --fail --output /dev/null "$fixture_url"; then + return + fi + sleep 0.5 + done + fail "The fixture HTTP server did not come up on port $fixture_port." +} + +assert_published_layout() { + local url archive script + while read -r url; do + archive="$test_dir/downloads/${url##*/}" + curl --fail --location --silent --retry 3 --output "$archive" "$url" + for script in installer.sh uninstaller.sh; do + unzip -l "$archive" | grep -q " netbird_ui_darwin/$script\$" || + fail "The published archive ${url##*/} has no netbird_ui_darwin/$script." + done + done < <(cask_field url "$published_cask") +} + +assert_no_deprecations() { + if grep -Ei '(postflight|uninstall_preflight).*deprecated|deprecated.*(postflight|uninstall_preflight)' "$@"; then + fail "Homebrew reported a deprecated cask lifecycle hook." + fi +} + +wait_for_daemon() { + local attempt + for attempt in {1..30}; do + if sudo launchctl print system/netbird > "$results_dir/service.log" 2>&1 && + grep -Eq '^[[:space:]]*state = running$' "$results_dir/service.log"; then + return + fi + sleep 1 + done + cat "$results_dir/service.log" + fail "The installed daemon did not reach the running state." +} + +wait_for_exit() { + local pid=$1 attempt + for attempt in {1..30}; do + if ! sudo kill -0 "$pid" 2>/dev/null; then + return + fi + sleep 1 + done + fail "Daemon process $pid is still running after removal." +} + +assert_service_absent() { + if sudo launchctl print system/netbird > "$results_dir/removed-service.log" 2>&1; then + fail "The NetBird service is still loaded after removal." + fi +} + +assert_installed() { + local script + [[ -f $marker ]] || fail "The cask did not run installer.sh." + grep -qx "version=$version" "$marker" || fail "installer.sh did not receive the cask version: $(cat "$marker")" + grep -qx 'uid=0' "$marker" || fail "installer.sh did not run as root: $(cat "$marker")" + [[ -d "$app" && -x "$app/netbird-ui" ]] || fail "The UI was not installed." + for script in installer.sh uninstaller.sh; do + [[ $(stat -f '%Lp' "$app/$script") == 755 ]] || fail "Incorrect permissions on $script." + done + [[ -f "$plist" ]] || fail "The installer did not create the daemon plist." + wait_for_daemon + daemon_pid=$(awk '/^[[:space:]]*pid = / { print $3; exit }' "$results_dir/service.log") + [[ $daemon_pid =~ ^[0-9]+$ ]] || fail "The running daemon has no PID." + sudo kill -0 "$daemon_pid" +} + +assert_uninstalled() { + local log=$1 + assert_no_deprecations "$log" + [[ ! -e "$app" ]] || fail "The UI app remains after uninstall." + [[ ! -e "$plist" ]] || fail "The daemon plist remains after uninstall." + assert_service_absent + wait_for_exit "$daemon_pid" + [[ $(netbird version) == "$version" ]] || fail "Cask uninstall removed the CLI dependency." +} + +installed_caskfiles() { + local extension=$1 + find "$(brew --caskroom)/netbird-ui/.metadata" -name "netbird-ui.$extension" 2>/dev/null +} + +assert_legacy_metadata() { + installed_caskfiles rb | grep -q . || fail "The legacy cask did not leave a Ruby caskfile behind." +} + +assert_steps_metadata() { + if installed_caskfiles rb | grep -q .; then + fail "Homebrew still keeps the legacy Ruby caskfile after reinstall." + fi + installed_caskfiles json | grep -q . || fail "Homebrew did not save the reinstalled cask as JSON." +} + +brew --version +sw_vers +brew tap netbirdio/tap "${GITHUB_WORKSPACE:?}/.homebrew-cask-tap" +tap_dir=$(brew --repository netbirdio/tap) +readonly tap_dir + +[[ -f "$tap_dir/Casks/netbird-ui.rb" ]] || fail "The tap has no Casks/netbird-ui.rb." +cp "$tap_dir/Casks/netbird-ui.rb" "$published_cask" +cp "$published_cask" "$results_dir/published-netbird-ui.rb" + +version=$(brew info --json=v2 --formula "$formula" | jq -r '.formulae[0].versions.stable') +readonly version +[[ -n $version && $version != null ]] || fail "Could not read the formula version from the tap." + +assert_published_layout + +build_fixture +fixture_sha=$(shasum -a 256 "$fixture_zip" | cut -d' ' -f1) +readonly fixture_sha +start_fixture_server + +export PROJECT=netbird-ui VERSION="$version" +export AMD="$fixture_zip" ARM="$fixture_zip" AMD_URL="$fixture_url" ARM_URL="$fixture_url" +gomplate -f "$GITHUB_WORKSPACE/client/ui/netbird-ui.rb.tmpl" -o "$rendered_cask" +cp "$rendered_cask" "$results_dir/rendered-netbird-ui.rb" + +sed -E "s|^([[:space:]]*version) \"[^\"]+\"|\\1 \"$version\"|; s|^([[:space:]]*url) \"[^\"]+\"|\\1 \"$fixture_url\"|; s|^([[:space:]]*sha256) \"[^\"]+\"|\\1 \"$fixture_sha\"|" \ + "$published_cask" > "$legacy_cask" +cp "$legacy_cask" "$results_dir/legacy-netbird-ui.rb" +if ! diff <(release_fields "$legacy_cask") <(release_fields "$rendered_cask"); then + fail "The rendered cask changes release data, not only lifecycle stanzas." +fi + +use_cask "$rendered_cask" +brew info --json=v2 --cask "$cask" > "$results_dir/cask.json" 2> "$results_dir/load.log" +cat "$results_dir/load.log" +assert_no_deprecations "$results_dir/load.log" +run_logged style brew style --cask --only-cops=Cask/InstallSteps "$cask" + +run_logged install-cli brew install --formula "$formula" +[[ $(netbird version) == "$version" ]] || fail "The installed CLI does not report the formula version." + +for scenario in running stopped missing; do + echo "::group::Uninstall with $scenario service" + install_attempted=true + sudo rm -f "$marker" + run_logged "install-$scenario" brew install --cask "$cask" + assert_no_deprecations "$results_dir/install-$scenario.log" + assert_installed + stop_ui + + case "$scenario" in + running) ;; + stopped) + run_logged stop-daemon sudo netbird service stop + wait_for_exit "$daemon_pid" + [[ -f "$plist" ]] || fail "Stopping the daemon unexpectedly removed its plist." + ;; + missing) + run_logged stop-missing-daemon sudo netbird service stop + run_logged remove-daemon sudo netbird service uninstall + wait_for_exit "$daemon_pid" + [[ ! -e "$plist" ]] || fail "The missing-service scenario still has a plist." + assert_service_absent + ;; + *) fail "Unknown uninstall scenario: $scenario" ;; + esac + + run_logged "uninstall-$scenario" brew uninstall --cask "$cask" + assert_uninstalled "$results_dir/uninstall-$scenario.log" + echo "::endgroup::" +done + +# Every existing user first meets the new cask through an upgrade of the published +# one, whose legacy flight blocks Homebrew replays from the saved Ruby caskfile. +echo "::group::Reinstall over the published legacy cask" +install_attempted=true +use_cask "$legacy_cask" +sudo rm -f "$marker" +run_logged install-legacy brew install --cask "$cask" +assert_installed +assert_legacy_metadata +stop_ui + +use_cask "$rendered_cask" +sudo rm -f "$marker" +run_logged reinstall-legacy brew reinstall --cask "$cask" +assert_installed +assert_steps_metadata +stop_ui + +run_logged uninstall-legacy brew uninstall --cask "$cask" +assert_uninstalled "$results_dir/uninstall-legacy.log" +echo "::endgroup::" diff --git a/.github/workflows/agent-network-e2e.yml b/.github/workflows/agent-network-e2e.yml index 9501c5fba..41bfe153d 100644 --- a/.github/workflows/agent-network-e2e.yml +++ b/.github/workflows/agent-network-e2e.yml @@ -52,7 +52,7 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-anet-e2e-buildx-${{ hashFiles('go.sum', 'combined/Dockerfile.multistage', 'proxy/Dockerfile.multistage', 'e2e/harness/Dockerfile.client') }} + key: ${{ runner.os }}-anet-e2e-buildx-${{ hashFiles('go.sum', 'combined/Dockerfile.multistage', 'proxy/Dockerfile.multistage', 'e2e/harness/Dockerfile.client', 'e2e/harness/Dockerfile.nblink') }} restore-keys: | ${{ runner.os }}-anet-e2e-buildx- diff --git a/.github/workflows/check-license-dependencies.yml b/.github/workflows/check-license-dependencies.yml index 81d293e4f..27c59f8d6 100644 --- a/.github/workflows/check-license-dependencies.yml +++ b/.github/workflows/check-license-dependencies.yml @@ -34,7 +34,7 @@ jobs: while IFS= read -r dir; do echo "=== Checking $dir ===" # Search for problematic imports, excluding test files - RESULTS=$(grep -r "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\)" "$dir" --include="*.go" 2>/dev/null | grep -v "_test.go" | grep -v "test_" | grep -v "/test/" | grep -v "tools/idp-migrate/" || true) + RESULTS=$(grep -r "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\)" "$dir" --include="*.go" 2>/dev/null | grep -v "_test.go" | grep -v "test_" | grep -v "/test/" | grep -v "tools/idp-migrate/" | grep -v "tools/mysql-migrate/" || true) if [ -n "$RESULTS" ]; then echo "❌ Found problematic dependencies:" echo "$RESULTS" @@ -93,7 +93,7 @@ jobs: IMPORTERS=$(go list -json -deps ./... 2>/dev/null | jq -r "select(.Imports[]? == \"$package\") | .ImportPath") # Check if any importer is NOT in management/signal/relay - BSD_IMPORTER=$(echo "$IMPORTERS" | grep -v "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\|combined\|tools/idp-migrate\)" | head -1) + BSD_IMPORTER=$(echo "$IMPORTERS" | grep -v "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\|combined\|tools/idp-migrate\|tools/mysql-migrate\)" | head -1) if [ -n "$BSD_IMPORTER" ]; then echo "❌ $package ($license) is imported by BSD-licensed code: $BSD_IMPORTER" diff --git a/.github/workflows/docs-ack.yml b/.github/workflows/docs-ack.yml index 7e34e2f8a..edd1eebff 100644 --- a/.github/workflows/docs-ack.yml +++ b/.github/workflows/docs-ack.yml @@ -12,6 +12,8 @@ jobs: docs-ack: name: Require docs PR URL or explicit "not needed" runs-on: ubuntu-latest + # Crowdin's translation-sync service PRs are auto-generated without the PR template. + if: github.event.pull_request.user.login != 'netbirddev' steps: - name: Read PR body diff --git a/.github/workflows/golang-test-freebsd.yml b/.github/workflows/golang-test-freebsd.yml index 65c39147a..7bd48e3d0 100644 --- a/.github/workflows/golang-test-freebsd.yml +++ b/.github/workflows/golang-test-freebsd.yml @@ -33,7 +33,7 @@ jobs: with: usesh: true copyback: false - release: "15.0" + release: "15.1" envs: "GO_VERSION" prepare: | pkg install -y curl pkgconf xorg diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml index 449eb14fa..880370ede 100644 --- a/.github/workflows/golang-test-linux.yml +++ b/.github/workflows/golang-test-linux.yml @@ -207,6 +207,43 @@ jobs: # regression test, and need no frontend bundle. run: CGO_ENABLED=1 go test -timeout 5m ./client/ui/authsession/... ./client/ui/i18n/... ./client/ui/preferences/... ./client/ui/services/... + test_client_pkcs11: + name: "Client PKCS#11 / Unit" + # The deb and rpm packages ship the client built with the pkcs11 tag, which loads + # PKCS#11 modules through purego. The other jobs build without the tag, so this one + # compiles that driver and signs with a real SoftHSM token, the build release ships. + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Install Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: "go.mod" + cache: false + + - name: Install SoftHSM + run: sudo apt update && sudo apt install -y -q softhsm2 + + - name: Initialise token + run: | + mkdir -p "$RUNNER_TEMP/softhsm/tokens" + printf 'directories.tokendir = %s\nobjectstore.backend = file\n' "$RUNNER_TEMP/softhsm/tokens" > "$RUNNER_TEMP/softhsm/softhsm2.conf" + echo "SOFTHSM2_CONF=$RUNNER_TEMP/softhsm/softhsm2.conf" >> "$GITHUB_ENV" + SOFTHSM2_CONF="$RUNNER_TEMP/softhsm/softhsm2.conf" softhsm2-util --init-token --free --label netbird --pin 1234 --so-pin 1234 + + - name: Vet + run: CGO_ENABLED=0 go vet -tags pkcs11 ./client/internal/pkcs11/... ./client/internal/certproof/... + + - name: Test + env: + NB_TEST_PKCS11_URI: "pkcs11:token=netbird?module-path=/usr/lib/softhsm/libsofthsm2.so&pin-value=1234" + NB_TEST_PKCS11_DISPOSABLE: "1" + run: CGO_ENABLED=0 go test -tags pkcs11 -timeout 5m ./client/internal/pkcs11/... ./client/internal/certproof/... + test_client_on_docker: name: "Client (Docker) / Unit" needs: [build-cache] @@ -270,7 +307,7 @@ jobs: sh -c ' \ apk update; apk add --no-cache \ ca-certificates iptables ip6tables dbus dbus-dev libpcap-dev build-base; \ - go test -buildvcs=false -tags "devcert privileged" -v -timeout 10m -p 1 $(go list -e -buildvcs=false ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui -e /upload-server -e /client/testutil/privileged) + go test -buildvcs=false -tags "devcert privileged" -v -timeout 10m -p 1 $(go list -e -buildvcs=false ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui -e /upload-server -e /client/testutil/privileged -e tools/mysql-migrate) ' test_relay: diff --git a/.github/workflows/golang-test-windows.yml b/.github/workflows/golang-test-windows.yml index ca300f7df..49be3af58 100644 --- a/.github/workflows/golang-test-windows.yml +++ b/.github/workflows/golang-test-windows.yml @@ -76,7 +76,7 @@ jobs: # resolve; the Where-Object pipeline then drops the broken package by # path. Without -e, go list aborts with empty stdout. run: | - $packages = go list -e ./... | Where-Object { $_ -notmatch '/management' } | Where-Object { $_ -notmatch '/relay' } | Where-Object { $_ -notmatch '/signal' } | Where-Object { $_ -notmatch '/proxy' } | Where-Object { $_ -notmatch '/combined' } | Where-Object { $_ -notmatch '/client/ui$' } + $packages = go list -e ./... | Where-Object { $_ -notmatch '/management' } | Where-Object { $_ -notmatch '/relay' } | Where-Object { $_ -notmatch '/signal' } | Where-Object { $_ -notmatch '/proxy' } | Where-Object { $_ -notmatch '/combined' } | Where-Object { $_ -notmatch '/client/ui$' } | Where-Object { $_ -notmatch '/tools/mysql-migrate' } $goExe = "C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe" $cmd = "$goExe test -tags `"devcert privileged`" -timeout 10m -p 1 $($packages -join ' ') > test-out.txt 2>&1" Set-Content -Path "${{ github.workspace }}\run-tests.cmd" -Value $cmd diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index ff36a0854..843177b88 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -30,7 +30,7 @@ jobs: # segment by codespell and behave the same across versions; the # recursive "**" form did not take effect with the codespell shipped # by this action. - skip: go.mod,go.sum,*/proxy/web/*,*pnpm-lock.yaml,*package-lock.json,*/locales/de/*,*/locales/es/*,*/locales/fr/*,*/locales/hu/*,*/locales/it/*,*/locales/pt/*,*/locales/ru/*,*/locales/zh-CN/*,*/i18n/TRANSLATING.md + skip: go.mod,go.sum,*/proxy/web/*,*pnpm-lock.yaml,*package-lock.json,*/locales/de/*,*/locales/es/*,*/locales/fr/*,*/locales/gl/*,*/locales/hu/*,*/locales/it/*,*/locales/pt/*,*/locales/ru/*,*/locales/zh-CN/*,*/i18n/TRANSLATING.md golangci: strategy: fail-fast: false diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml index 24d81b50f..5b769e2f7 100644 --- a/.github/workflows/pr-title-check.yml +++ b/.github/workflows/pr-title-check.yml @@ -7,6 +7,8 @@ on: jobs: check-title: runs-on: ubuntu-latest + # Crowdin's translation-sync service PRs are auto-generated with a fixed title. + if: github.event.pull_request.user.login != 'netbirddev' steps: - name: Validate PR title prefix uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 diff --git a/.github/workflows/redhat-certify.yml b/.github/workflows/redhat-certify.yml new file mode 100644 index 000000000..96d50de9f --- /dev/null +++ b/.github/workflows/redhat-certify.yml @@ -0,0 +1,207 @@ +name: Red Hat Certification + +# Certify published UBI images in the Red Hat Ecosystem Catalog. Called by +# release.yml on stable tags, or run by hand to (re)certify any released +# version. preflight submits every architecture of an image's manifest list +# to Pyxis; auto-publish on the component makes it public once certified. +# +# Each component's Partner Connect ID comes from the REDHAT_CERT_ID_ +# repository variable, e.g. REDHAT_CERT_ID_CLIENT_ROOTLESS. The run fails +# before certifying anything if a selected component's variable is not set. + +on: + workflow_call: + inputs: + component: + type: string + required: true + version: + type: string + required: true + secrets: + PYXIS_API_TOKEN: + required: true + workflow_dispatch: + inputs: + component: + description: "Component to certify" + type: choice + required: true + default: all + options: + - all + - client-rootless + - reverse-proxy + - netbird-server + - management + - signal + - relay + version: + description: "Released version, e.g. v0.80.0" + type: string + required: true + +permissions: + contents: read + +jobs: + resolve: + name: Resolve components + runs-on: ubuntu-24.04 + outputs: + version: ${{ steps.resolve.outputs.version }} + matrix: ${{ steps.resolve.outputs.matrix }} + steps: + - name: Resolve components and images + id: resolve + env: + COMPONENT: ${{ inputs.component }} + INPUT_VERSION: ${{ inputs.version }} + REPO_VARS: ${{ toJSON(vars) }} + run: | + set -euo pipefail + version="${INPUT_VERSION#v}" + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Only stable x.y.z versions are certified, got '${INPUT_VERSION}'" + exit 1 + fi + # name, image repository, tag suffix (must match .goreleaser.yaml). + # Keep the names in sync with the workflow_dispatch options above. + components=( + "client-rootless ghcr.io/netbirdio/netbird -rootless-ubi" + "reverse-proxy ghcr.io/netbirdio/reverse-proxy -ubi" + "netbird-server ghcr.io/netbirdio/netbird-server -ubi" + "management ghcr.io/netbirdio/management -ubi" + "signal ghcr.io/netbirdio/signal -ubi" + "relay ghcr.io/netbirdio/relay -ubi" + ) + matrix="[]" + missing=() + for c in "${components[@]}"; do + read -r name repo suffix <<< "$c" + [[ "$COMPONENT" == all || "$COMPONENT" == "$name" ]] || continue + var="REDHAT_CERT_ID_${name^^}"; var="${var//-/_}" + id="$(jq -r --arg v "$var" '.[$v] // empty' <<< "$REPO_VARS")" + if [[ -z "$id" ]]; then + missing+=("$var") + continue + fi + matrix="$(jq -c --arg n "$name" --arg t "${version}${suffix}" --arg r "${repo}:${version}${suffix}" --arg i "$id" \ + '. + [{component: $n, tag: $t, ref: $r, component_id: $i}]' <<< "$matrix")" + done + if (( ${#missing[@]} )); then + echo "::error::Set these repository variables to the Partner Connect component IDs: ${missing[*]}" + exit 1 + fi + if [[ "$matrix" == "[]" ]]; then + echo "::error::No component to certify for '${COMPONENT}'" + exit 1 + fi + echo "Components to certify: ${matrix}" + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" + + certify: + name: "Certify ${{ matrix.component }} UBI image" + needs: resolve + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.resolve.outputs.matrix) }} + env: + PREFLIGHT_VERSION: "1.21.0" + # sha256 of preflight-linux-amd64 from the 1.21.0 GitHub release. + # Red Hat publishes no checksum file, so the value is pinned here. + PREFLIGHT_SHA256: "5e653135503c72f8702bbe31d7643197d12937c68086879133dd6b9650a9a449" + steps: + - name: Verify the multi-arch image is on ghcr.io + env: + IMAGE_REF: ${{ matrix.ref }} + run: | + set -euo pipefail + docker buildx imagetools inspect "$IMAGE_REF" --raw > manifest.json + for arch in amd64 arm64; do + if ! jq -e --arg a "$arch" '.manifests[] | select(.platform.architecture == $a)' manifest.json > /dev/null; then + echo "::error::${IMAGE_REF} has no ${arch} manifest" + exit 1 + fi + done + echo "Manifest list for ${IMAGE_REF}:" + jq -r '.manifests[] | "\(.platform.os)/\(.platform.architecture) \(.digest)"' manifest.json + + - name: Install preflight + run: | + set -euo pipefail + curl -fsSL --proto '=https' --proto-redir '=https' -o preflight \ + "https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/download/${PREFLIGHT_VERSION}/preflight-linux-amd64" + echo "${PREFLIGHT_SHA256} preflight" | sha256sum -c - + chmod +x preflight + ./preflight --version + + - name: Run preflight checks and submit to Red Hat + env: + IMAGE_REF: ${{ matrix.ref }} + PFLT_PYXIS_API_TOKEN: ${{ secrets.PYXIS_API_TOKEN }} + PFLT_CERTIFICATION_COMPONENT_ID: ${{ matrix.component_id }} + PFLT_ARTIFACTS: artifacts + PFLT_LOGFILE: artifacts/preflight.log + PFLT_LOGLEVEL: info + PFLT_JUNIT: "true" + run: | + set -euo pipefail + # No --platform: preflight walks the manifest list and submits every + # architecture in one run, grouped under one manifest-list digest. + # preflight does not create the PFLT_LOGFILE directory, and --submit + # fails if the log file is missing. + mkdir -p artifacts + ./preflight check container "$IMAGE_REF" --submit + + - name: Fail if any check did not pass + run: | + set -euo pipefail + shopt -s nullglob globstar + results=(artifacts/**/results.json) + if [[ ${#results[@]} -eq 0 ]]; then + echo "::error::preflight produced no results.json" + exit 1 + fi + status=0 + for f in "${results[@]}"; do + arch="$(basename "$(dirname "$f")")" + passed="$(jq -r '.passed' "$f")" + failed="$(jq -r '[.results.failed[]?.name] | join(", ")' "$f")" + echo "${arch}: passed=${passed} ${failed:+failed checks: ${failed}}" + [[ "$passed" == "true" ]] || status=1 + done + exit $status + + - name: Upload preflight artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: redhat-preflight-${{ matrix.component }}-${{ needs.resolve.outputs.version }} + path: artifacts/ + retention-days: 30 + + - name: Wait for Pyxis to mark both architectures certified + env: + TAG: ${{ matrix.tag }} + COMPONENT_ID: ${{ matrix.component_id }} + PFLT_PYXIS_API_TOKEN: ${{ secrets.PYXIS_API_TOKEN }} + run: | + set -euo pipefail + # Filter on the tag server-side so older versions are found past the first page. + url="https://catalog.redhat.com/api/containers/v1/projects/certification/id/${COMPONENT_ID}/images?filter=repositories.tags.name==${TAG}&page_size=100" + for attempt in $(seq 1 20); do + certified="$(curl -fsS --proto '=https' --proto-redir '=https' -H "X-API-KEY: ${PFLT_PYXIS_API_TOKEN}" "$url" \ + | jq -r --arg t "$TAG" '[.data[] | select(.repositories[]?.tags[]?.name == $t) | select(.certified == true) | .architecture] | unique | join(",")')" + echo "attempt ${attempt}: certified architectures for ${TAG}: ${certified:-none}" + if [[ "$certified" == "amd64,arm64" ]]; then + echo "Both architectures certified. Auto-publish is enabled on the component, so the catalog updates on its own." + exit 0 + fi + sleep 30 + done + echo "::error::Pyxis has not marked both architectures certified after 10 minutes. Check https://connect.redhat.com/component/view/${COMPONENT_ID}/images" + exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 673fcc281..d133d87f1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,7 +69,7 @@ jobs: with: usesh: true copyback: false - release: "15.0" + release: "15.1" envs: "GO_VERSION" prepare: | # Install required packages @@ -199,7 +199,7 @@ jobs: with: node-version: '22' - name: Install proxy web dependencies for license collection - # proxy/collect-licenses.sh reads the UI's license terms from node_modules. + # release_files/collect-licenses.sh -w reads the proxy UI's license terms from node_modules. working-directory: proxy/web run: npm ci --ignore-scripts - name: Set up QEMU @@ -363,7 +363,9 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 with: name: linux-packages - path: dist/netbird_linux** + path: | + dist/netbird_linux** + dist/netbird-pkcs11_linux** retention-days: 7 - name: upload windows packages id: upload_windows_packages @@ -380,131 +382,23 @@ jobs: path: dist/netbird_darwin** retention-days: 7 - # Certify and publish the rootless UBI client image in the Red Hat Ecosystem - # Catalog. Stable tags only: goreleaser pushes -rootless-ubi to - # ghcr.io in the release job above, and preflight submits every architecture - # of that manifest list to Pyxis. Auto-publish on the component makes the new - # version public once certification passes. + # Certify the UBI images in the Red Hat Ecosystem Catalog on stable tags. + # See redhat-certify.yml, which can also be run by hand for any released version. redhat_certification: - name: "Red Hat / Certify rootless UBI image" + name: "Red Hat" needs: release if: | github.repository == 'netbirdio/netbird' && startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-') - runs-on: ubuntu-24.04 permissions: contents: read - env: - PREFLIGHT_VERSION: "1.21.0" - # sha256 of preflight-linux-amd64 from the 1.21.0 GitHub release. - # Red Hat publishes no checksum file, so the value is pinned here. - PREFLIGHT_SHA256: "5e653135503c72f8702bbe31d7643197d12937c68086879133dd6b9650a9a449" - IMAGE_REPOSITORY: "ghcr.io/netbirdio/netbird" - # Component "NetBird Client Container Image (rootless)" in Partner Connect. - # Override with the REDHAT_CERT_COMPONENT_ID repository variable if it changes. - DEFAULT_COMPONENT_ID: "6aa3ca4b4676aefdf07aaa97" - steps: - - name: Resolve image reference - id: image - env: - INPUT_VERSION: ${{ github.ref_name }} - run: | - set -euo pipefail - version="${INPUT_VERSION#v}" - if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "::error::Only stable x.y.z versions are certified, got '${INPUT_VERSION}'" - exit 1 - fi - echo "version=${version}" >> "$GITHUB_OUTPUT" - echo "ref=${IMAGE_REPOSITORY}:${version}-rootless-ubi" >> "$GITHUB_OUTPUT" - - - name: Verify the multi-arch image is on ghcr.io - env: - IMAGE_REF: ${{ steps.image.outputs.ref }} - run: | - set -euo pipefail - docker buildx imagetools inspect "$IMAGE_REF" --raw > manifest.json - for arch in amd64 arm64; do - if ! jq -e --arg a "$arch" '.manifests[] | select(.platform.architecture == $a)' manifest.json > /dev/null; then - echo "::error::${IMAGE_REF} has no ${arch} manifest" - exit 1 - fi - done - echo "Manifest list for ${IMAGE_REF}:" - jq -r '.manifests[] | "\(.platform.os)/\(.platform.architecture) \(.digest)"' manifest.json - - - name: Install preflight - run: | - set -euo pipefail - curl -fsSL --proto '=https' --proto-redir '=https' -o preflight \ - "https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/download/${PREFLIGHT_VERSION}/preflight-linux-amd64" - echo "${PREFLIGHT_SHA256} preflight" | sha256sum -c - - chmod +x preflight - ./preflight --version - - - name: Run preflight checks and submit to Red Hat - env: - IMAGE_REF: ${{ steps.image.outputs.ref }} - PFLT_PYXIS_API_TOKEN: ${{ secrets.PYXIS_API_TOKEN }} - PFLT_CERTIFICATION_COMPONENT_ID: ${{ vars.REDHAT_CERT_COMPONENT_ID || env.DEFAULT_COMPONENT_ID }} - PFLT_ARTIFACTS: artifacts - PFLT_LOGFILE: artifacts/preflight.log - PFLT_LOGLEVEL: info - PFLT_JUNIT: "true" - run: | - set -euo pipefail - # No --platform: preflight walks the manifest list and submits every - # architecture in one run, grouped under one manifest-list digest. - ./preflight check container "$IMAGE_REF" --submit - - - name: Fail if any check did not pass - run: | - set -euo pipefail - shopt -s nullglob - results=(artifacts/results.json artifacts/*/results.json) - if [[ ${#results[@]} -eq 0 ]]; then - echo "::error::preflight produced no results.json" - exit 1 - fi - status=0 - for f in "${results[@]}"; do - arch="$(basename "$(dirname "$f")")" - passed="$(jq -r '.passed' "$f")" - failed="$(jq -r '[.results.failed[]?.name] | join(", ")' "$f")" - echo "${arch}: passed=${passed} ${failed:+failed checks: ${failed}}" - [[ "$passed" == "true" ]] || status=1 - done - exit $status - - - name: Upload preflight artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: redhat-preflight-${{ steps.image.outputs.version }} - path: artifacts/ - retention-days: 30 - - - name: Wait for Pyxis to mark both architectures certified - env: - VERSION: ${{ steps.image.outputs.version }} - PFLT_PYXIS_API_TOKEN: ${{ secrets.PYXIS_API_TOKEN }} - COMPONENT_ID: ${{ vars.REDHAT_CERT_COMPONENT_ID || env.DEFAULT_COMPONENT_ID }} - run: | - set -euo pipefail - tag="${VERSION}-rootless-ubi" - url="https://catalog.redhat.com/api/containers/v1/projects/certification/id/${COMPONENT_ID}/images?page_size=100" - for attempt in $(seq 1 20); do - certified="$(curl -fsS --proto '=https' --proto-redir '=https' -H "X-API-KEY: ${PFLT_PYXIS_API_TOKEN}" "$url" \ - | jq -r --arg t "$tag" '[.data[] | select(.repositories[]?.tags[]?.name == $t) | select(.certified == true) | .architecture] | unique | join(",")')" - echo "attempt ${attempt}: certified architectures for ${tag}: ${certified:-none}" - if [[ "$certified" == "amd64,arm64" ]]; then - echo "Both architectures certified. Auto-publish is enabled on the component, so the catalog updates on its own." - exit 0 - fi - sleep 30 - done - echo "::warning::Pyxis has not marked both architectures certified after 10 minutes. Check https://connect.redhat.com/component/view/${COMPONENT_ID}/images" + uses: ./.github/workflows/redhat-certify.yml + with: + component: all + version: ${{ github.ref_name }} + secrets: + PYXIS_API_TOKEN: ${{ secrets.PYXIS_API_TOKEN }} release_ui: runs-on: ubuntu-latest diff --git a/.github/workflows/reverse-proxy-e2e.yml b/.github/workflows/reverse-proxy-e2e.yml new file mode 100644 index 000000000..934a6d4c9 --- /dev/null +++ b/.github/workflows/reverse-proxy-e2e.yml @@ -0,0 +1,53 @@ +name: Reverse Proxy E2E + +on: + pull_request: + paths: + - '.github/workflows/reverse-proxy-e2e.yml' + - '**/*.go' + - 'e2e/**' + - 'shared/management/**' + - 'combined/Dockerfile*' + - 'proxy/Dockerfile*' + - '.dockerignore' + - 'go.mod' + - 'go.sum' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + reverse-proxy: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Install Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version-file: go.mod + + - name: Set up Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + - name: Cache Docker layers + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: /tmp/.reverse-proxy-buildx-cache + key: ${{ runner.os }}-reverse-proxy-e2e-${{ hashFiles('go.sum', 'combined/Dockerfile.multistage', 'proxy/Dockerfile.multistage') }} + restore-keys: | + ${{ runner.os }}-reverse-proxy-e2e- + + - name: Run reverse-proxy E2E + env: + NB_E2E_BUILDX_CACHE: /tmp/.reverse-proxy-buildx-cache + run: go test -tags e2e -timeout 25m -v ./e2e/reverseproxy/... diff --git a/.github/workflows/test-homebrew-cask.yml b/.github/workflows/test-homebrew-cask.yml new file mode 100644 index 000000000..a75951425 --- /dev/null +++ b/.github/workflows/test-homebrew-cask.yml @@ -0,0 +1,46 @@ +name: Test Homebrew cask + +on: + pull_request: + paths: + - "client/ui/netbird-ui.rb.tmpl" + - ".github/scripts/test-homebrew-cask.sh" + - ".github/workflows/test-homebrew-cask.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }} + cancel-in-progress: true + +jobs: + install-uninstall: + runs-on: macos-latest + timeout-minutes: 20 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Clone the Homebrew tap + run: git clone https://github.com/netbirdio/homebrew-tap.git .homebrew-cask-tap + + - name: Update Homebrew and install gomplate + # The runner image disables auto-update; the cask steps DSL needs Homebrew 6.0.20 or newer. + run: | + brew update + brew install gomplate + + - name: Install and uninstall the cask + run: .github/scripts/test-homebrew-cask.sh + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: homebrew-cask-results + path: ${{ runner.temp }}/homebrew-cask/results + if-no-files-found: ignore diff --git a/.github/workflows/test-infrastructure-files.yml b/.github/workflows/test-infrastructure-files.yml index 1313379ee..4875adc31 100644 --- a/.github/workflows/test-infrastructure-files.yml +++ b/.github/workflows/test-infrastructure-files.yml @@ -258,6 +258,18 @@ jobs: with: persist-credentials: false + - name: Verify enterprise Traefik tag validation + run: bash infrastructure_files/tests/test-getting-started-enterprise.sh + + - name: Verify community Traefik image validation + run: bash infrastructure_files/tests/test-getting-started.sh + + - name: Verify community Compose network scoping + run: bash infrastructure_files/tests/test-getting-started-networks.sh + + - name: Verify migration Compose network scoping + run: bash infrastructure_files/tests/test-getting-started-networks.sh --migration + - name: Verify fresh-install session cookie key hardening run: | grep -Fxq ' SESSION_COOKIE_ENCRYPTION_KEY=$(openssl rand -base64 32)' infrastructure_files/getting-started.sh diff --git a/.github/workflows/ui-translations.yml b/.github/workflows/ui-translations.yml index 24b7c9de2..9ac524495 100644 --- a/.github/workflows/ui-translations.yml +++ b/.github/workflows/ui-translations.yml @@ -36,7 +36,8 @@ jobs: with: node-version: "22" - # English (en) is the source of truth for translation keys; every other - # locale declared in _index.json must carry the exact same key set. + # English (en) is the source of truth for translation keys. Locales declared + # in _index.json fail on orphaned keys or placeholder mismatches; missing + # keys only warn, since they fall back to English at runtime. - name: Check translation key parity run: node client/ui/i18n/check-translations.mjs diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 275c1cd7b..de5f7419c 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -40,7 +40,28 @@ builds: tags: - load_wgnt_from_rsrc - # Single-arch builds: nfpm provides is not templated, so the RPM splits per arch. + # deb and rpm packages target glibc distributions, so they carry the PKCS#11 store, which + # links libc. Tarballs and the Alpine-based images keep the static build above. + - id: netbird-pkcs11 + dir: client + binary: netbird + env: [CGO_ENABLED=0] + goos: + - linux + goarch: + - arm + - amd64 + - arm64 + - 386 + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + tags: + - load_wgnt_from_rsrc + - pkcs11 + + # Single-arch builds: nfpm provides is not templated, so the RPM splits per arch. They + # carry the PKCS#11 store like the deb build above. - &netbird_rpm_build id: netbird-rpm-amd64 dir: client @@ -53,6 +74,7 @@ builds: mod_timestamp: "{{ .CommitTimestamp }}" tags: - load_wgnt_from_rsrc + - pkcs11 - <<: *netbird_rpm_build id: netbird-rpm-arm64 @@ -194,6 +216,29 @@ builds: - -s -w -X main.Version={{.Version}} -X main.Commit={{.Commit}} -X main.BuildDate={{.CommitDate}} mod_timestamp: "{{ .CommitTimestamp }}" + - id: nblink + dir: client/link/cmd/nblink + env: [CGO_ENABLED=0] + binary: nblink + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + - arm + goarm: + - 7 + ignore: + - goos: windows + goarch: arm + - goos: darwin + goarch: arm + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} + mod_timestamp: "{{ .CommitTimestamp }}" + - id: netbird-idp-migrate dir: tools/idp-migrate env: @@ -216,6 +261,28 @@ builds: - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser mod_timestamp: "{{ .CommitTimestamp }}" + - id: netbird-mysql-migrate + dir: tools/mysql-migrate + env: + - CGO_ENABLED=1 + - >- + {{- if eq .Runtime.Goos "linux" }} + {{- if eq .Arch "arm64"}}CC=aarch64-linux-gnu-gcc{{- end }} + {{- if eq .Arch "arm"}}CC=arm-linux-gnueabihf-gcc{{- end }} + {{- end }} + binary: netbird-mysql-migrate + goos: + - linux + goarch: + - amd64 + - arm64 + - arm + goarm: + - 7 + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + universal_binaries: - id: netbird @@ -232,6 +299,19 @@ archives: builds: - netbird-idp-migrate name_template: "netbird-idp-migrate_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + - id: netbird-mysql-migrate + builds: + - netbird-mysql-migrate + name_template: "netbird-mysql-migrate_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + - id: nblink + builds: + - nblink + name_template: "nblink_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + # goreleaser archives as tar.gz everywhere unless told otherwise, and + # nblink is downloaded and extracted by hand on Windows too. + format_overrides: + - goos: windows + format: zip nfpms: - maintainer: Netbird @@ -242,7 +322,7 @@ nfpms: id: netbird_deb bindir: /usr/bin builds: - - netbird + - netbird-pkcs11 formats: - deb scripts: @@ -285,6 +365,9 @@ nfpms: - src: release_files/netbird.sysconfig dst: /etc/sysconfig/netbird type: config|noreplace + # May hold the PKCS#11 token PIN (NB_TPM_PIN), so readable by root alone. + file_info: + mode: 0600 scripts: postinstall: "release_files/post_install.sh" preremove: "release_files/pre_remove.sh" @@ -385,7 +468,7 @@ dockers_v2: RELEASE: "{{ .Timestamp }}" hooks: pre: - - cmd: 'sh client/collect-licenses.sh "{{ .ContextDir }}/licenses" amd64 arm64' + - cmd: 'sh release_files/collect-licenses.sh -t load_wgnt_from_rsrc "{{ .ContextDir }}/licenses" ./client amd64 arm64' env: - GOOS=linux - CGO_ENABLED=0 @@ -423,6 +506,41 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: relay-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-relay + images: + - netbirdio/relay + - ghcr.io/netbirdio/relay + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: relay/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l relay/LICENSE "{{ .ContextDir }}/licenses" ./relay amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=0 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: signal disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -445,6 +563,41 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: signal-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-signal + images: + - netbirdio/signal + - ghcr.io/netbirdio/signal + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: signal/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l signal/LICENSE "{{ .ContextDir }}/licenses" ./signal amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=0 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: management disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -467,6 +620,41 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: management-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-mgmt + images: + - netbirdio/management + - ghcr.io/netbirdio/management + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: management/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l management/LICENSE "{{ .ContextDir }}/licenses" ./management amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=1 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: upload disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -511,6 +699,41 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: netbird-server-ubi + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - netbird-server + images: + - netbirdio/netbird-server + - ghcr.io/netbirdio/netbird-server + tags: + - "{{ .Version }}-ubi" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}ubi-latest{{ end }}" + dockerfile: combined/Dockerfile.ubi + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{ .Version }}" + RELEASE: "{{ .Timestamp }}" + hooks: + pre: + - cmd: 'sh release_files/collect-licenses.sh -l combined/LICENSE "{{ .ContextDir }}/licenses" ./combined amd64 arm64' + env: + - GOOS=linux + - CGO_ENABLED=1 + labels: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "{{.ProjectName}}" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: netbird-proxy disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -533,6 +756,28 @@ dockers_v2: "org.opencontainers.image.revision": "{{.FullCommit}}" "org.opencontainers.image.source": "{{.GitURL}}" "maintainer": "dev@netbird.io" + - id: nblink + disable: "{{ .Env.SKIP_DOCKER_PUSH }}" + ids: + - nblink + images: + - netbirdio/nblink + - ghcr.io/netbirdio/nblink + tags: + - "{{ .Version }}" + - "{{ if eq .Env.SKIP_PUBLISH \"false\" }}latest{{ end }}" + dockerfile: client/Dockerfile-nblink + platforms: + - linux/amd64 + - linux/arm64 + - linux/arm + annotations: + "org.opencontainers.image.created": "{{.Date}}" + "org.opencontainers.image.title": "nblink" + "org.opencontainers.image.version": "{{.Version}}" + "org.opencontainers.image.revision": "{{.FullCommit}}" + "org.opencontainers.image.source": "{{.GitURL}}" + "maintainer": "dev@netbird.io" - id: proxy-ubi disable: "{{ .Env.SKIP_DOCKER_PUSH }}" ids: @@ -552,7 +797,7 @@ dockers_v2: RELEASE: "{{ .Timestamp }}" hooks: pre: - - cmd: 'sh proxy/collect-licenses.sh "{{ .ContextDir }}/licenses" amd64 arm64' + - cmd: 'sh release_files/collect-licenses.sh -l proxy/LICENSE -w "{{ .ContextDir }}/licenses" ./proxy/cmd/proxy amd64 arm64' env: - GOOS=linux - CGO_ENABLED=0 diff --git a/LICENSE b/LICENSE index d922f155a..4d0551de8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,9 @@ -This BSD‑3‑Clause license applies to all parts of the repository except for the directories management/, signal/, relay/ and combined/. +This BSD-3-Clause license applies to all parts of the repository except for the directories management/, signal/, relay/ and combined/. Those directories are licensed under the GNU Affero General Public License version 3.0 (AGPLv3). See the respective LICENSE files inside each directory. BSD 3-Clause License -Copyright (c) 2022 NetBird GmbH & AUTHORS +Copyright (c) 2022-2026 NetBird GmbH & AUTHORS Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/LICENSES/BSD-3-Clause.txt b/LICENSES/BSD-3-Clause.txt index 7cba76dfd..5e9a2539c 100644 --- a/LICENSES/BSD-3-Clause.txt +++ b/LICENSES/BSD-3-Clause.txt @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2022 NetBird GmbH & AUTHORS +Copyright (c) 2022-2026 NetBird GmbH & AUTHORS Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/base62/base62.go b/base62/base62.go index efafbc768..1a02e98e2 100644 --- a/base62/base62.go +++ b/base62/base62.go @@ -3,56 +3,75 @@ package base62 import ( "fmt" "math" - "strings" ) const ( - alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - base = uint32(len(alphabet)) + alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + base = uint32(len(alphabet)) + maxBase62Digits = 6 // max number of digits required to encode MaxUint32 + ) +var ( + ErrEmptyString = fmt.Errorf("empty string") + ErrInvalidChar = fmt.Errorf("invalid character") + ErrOverflow = fmt.Errorf("integer overflow") +) + +// Fixed-size arrays have better performance and lower memory overhead compared to maps for small static sets of data +var charToIndex [123]int8 // Assuming ASCII from '\0' - 'z' + +func init() { + for i := range charToIndex { + charToIndex[i] = -1 + } + for i, c := range alphabet { + charToIndex[c] = int8(i) + } +} + // Encode encodes a uint32 value to a base62 string. -func Encode(num uint32) string { - if num == 0 { - return string(alphabet[0]) +// The returned string will be between 1-6 characters long. +func Encode(n uint32) string { + if n < base { + return string(alphabet[n]) + } + // avoid dynamic memory usage for small, fixed size data + buf := [maxBase62Digits]byte{} + idx := len(buf) + + for n > 0 { + idx-- + buf[idx] = alphabet[n%base] + n /= base } - var encoded strings.Builder - - for num > 0 { - remainder := num % base - encoded.WriteByte(alphabet[remainder]) - num /= base - } - - // Reverse the encoded string - encodedString := encoded.String() - reversed := reverse(encodedString) - return reversed + return string(buf[idx:]) } // Decode decodes a base62 string to a uint32 value. +// Returns an error if the input string is empty, contains invalid characters, +// or would result in integer overflow. func Decode(encoded string) (uint32, error) { + if len(encoded) == 0 { + return 0, ErrEmptyString + } var decoded uint32 - strLen := len(encoded) - - for i, char := range encoded { - index := strings.IndexRune(alphabet, char) + for _, char := range encoded { + index := int8(-1) + if int(char) < len(charToIndex) { + index = charToIndex[char] + } if index < 0 { - return 0, fmt.Errorf("invalid character: %c", char) + return 0, fmt.Errorf("%w: %c", ErrInvalidChar, char) + } + // Add overflow check when calculating the decoded value to prevent silent overflow of uint32 + if decoded > (math.MaxUint32-uint32(index))/base { + return 0, fmt.Errorf("%w: %s", ErrOverflow, encoded) } - decoded += uint32(index) * uint32(math.Pow(float64(base), float64(strLen-i-1))) + decoded = decoded*base + uint32(index) } return decoded, nil } - -// Reverse a string. -func reverse(s string) string { - runes := []rune(s) - for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { - runes[i], runes[j] = runes[j], runes[i] - } - return string(runes) -} diff --git a/base62/base62_test.go b/base62/base62_test.go index 00da2124a..f2ad06d6f 100644 --- a/base62/base62_test.go +++ b/base62/base62_test.go @@ -1,31 +1,67 @@ package base62 import ( + "errors" + "math" "testing" ) func TestEncodeDecode(t *testing.T) { - tests := []struct { - num uint32 + testCases := []struct { + input uint32 + expected string }{ - {0}, - {1}, - {42}, - {12345}, - {99999}, - {123456789}, + {0, "0"}, + {1, "1"}, + {5, "5"}, + {9, "9"}, + {10, "A"}, + {42, "g"}, + {61, "z"}, + {62, "10"}, + {'0', "m"}, + {'9', "v"}, + {'A', "13"}, + {'Z', "1S"}, + {'a', "1Z"}, + {'z', "1y"}, + {99999, "Q0t"}, + {12345, "3D7"}, + {123456789, "8M0kX"}, + {math.MaxUint32, "4gfFC3"}, } - for _, tt := range tests { - encoded := Encode(tt.num) + for _, tc := range testCases { + encoded := Encode(tc.input) + if encoded != tc.expected { + t.Errorf("Encode(%d) = %s; want %s", tc.input, encoded, tc.expected) + } decoded, err := Decode(encoded) - if err != nil { - t.Errorf("Decode error: %v", err) + t.Errorf("Expected error nil, got %v", err) } - if decoded != tt.num { - t.Errorf("Decode(%v) = %v, want %v", encoded, decoded, tt.num) + if decoded != tc.input { + t.Errorf("Decode(%v) = %v, want %v", encoded, decoded, tc.input) } } } + +// Decode handles empty string input with appropriate error +func TestDecodeEmptyString(t *testing.T) { + if _, err := Decode(""); !errors.Is(err, ErrEmptyString) { + t.Errorf("Expected error %v, got %v", ErrEmptyString, err) + } +} + +func TestDecodeOverflow(t *testing.T) { + if _, err := Decode("4gfFC4"); !errors.Is(err, ErrOverflow) { + t.Errorf("Expected error %v, got %v", ErrOverflow, err) + } +} + +func TestDecodeInvalid(t *testing.T) { + if _, err := Decode("/"); !errors.Is(err, ErrInvalidChar) { + t.Errorf("Expected error %v, got %v", ErrInvalidChar, err) + } +} diff --git a/client/Dockerfile-nblink b/client/Dockerfile-nblink new file mode 100644 index 000000000..63177a6c0 --- /dev/null +++ b/client/Dockerfile-nblink @@ -0,0 +1,34 @@ +# nblink runs unprivileged: no TUN device, no capabilities, no host networking. +# +# The image is built for platforms that assign an arbitrary UID at run time, so +# it must not depend on a passwd entry and its state directory has to be +# writable by the root group rather than by one named user. +FROM alpine:3.22 AS build + +# Group 0 owns the state directory so a container started as any UID in that +# group can still persist its identity. Mode 770 rather than g=u, because the +# directory holds the peer's keys and nothing outside owner and group should +# read them, including when the path is bind-mounted from a host. +RUN mkdir -p /tmp/state/nblink \ + && chgrp -R 0 /tmp/state/nblink \ + && chmod -R 770 /tmp/state/nblink + +FROM gcr.io/distroless/static-debian13:nonroot + +ARG TARGETPLATFORM +COPY ${TARGETPLATFORM}/nblink /usr/local/bin/nblink +# COPY of a directory copies its contents and creates the target with mode +# 755, which group 0 cannot write. Copying the parent makes nblink/ itself the +# copied entry, so its mode survives. +COPY --from=build --chown=65532:0 /tmp/state/ /var/lib/ + +# Numeric so the runtime can substitute its own UID without a passwd lookup. +USER 65532:0 + +# No NB_STATE_DIR by default: state stays in memory, so the image runs on a +# read-only root filesystem without a volume. Set NB_STATE_DIR=/var/lib/nblink +# and mount a volume there to keep the peer identity; a volume mounted over +# the directory starts out with its group-0 ownership. +ENV NB_LOG_LEVEL=info + +ENTRYPOINT ["/usr/local/bin/nblink"] diff --git a/client/android/client.go b/client/android/client.go index e47a1c13d..9705db8e0 100644 --- a/client/android/client.go +++ b/client/android/client.go @@ -104,8 +104,7 @@ type Client struct { stateChangeMu sync.Mutex stateChangeSubID string - eventSub *peer.EventSubscription - // Closed to stop the watch goroutines from delivering buffered items to a + // Closed to stop the watch goroutine from delivering buffered ticks to a // listener that has been removed or replaced. See stopStateChangeWatchLocked. stateChangeDone chan struct{} @@ -213,6 +212,7 @@ func (c *Client) Run(platformFiles PlatformFiles, urlOpener URLOpener, isAndroid connectClient := internal.NewConnectClient(ctx, cfg, c.recorder, internal.WithNetEvents(c.netMgr)) c.setState(cfg, cacheDir, cfgFile, connectClient) + connectClient.SetSyncResponsePersistence(true) // This path runs the interactive SSO flow, so reaching here means the peer // is authenticated again — release the latch Status() reports from. Clear // only once the fresh connect client is installed: until then Status() @@ -256,6 +256,7 @@ func (c *Client) RunWithoutLogin(platformFiles PlatformFiles, dns *DNSList, dnsR connectClient := internal.NewConnectClient(ctx, cfg, c.recorder, internal.WithNetEvents(c.netMgr)) c.setState(cfg, cacheDir, cfgFile, connectClient) + connectClient.SetSyncResponsePersistence(true) return connectClient.RunOnAndroid(c.tunAdapter, c.iFaceDiscover, c.networkChangeListener, slices.Clone(dns.items), dnsReadyListener, stateFile, cacheDir) } @@ -327,6 +328,19 @@ func (c *Client) NotifyNetworkChange() { // or "strict"; strict also anonymizes internal IP ranges, peer names, and // WireGuard public keys, and implies anonymize. func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool, anonymizeLevel string) (string, error) { + return c.debugBundle(platformFiles, anonymize, anonymizeLevel, true) +} + +// DebugBundleFile generates a debug bundle and returns the path of the zip in +// the cache directory instead of uploading it, so the app can hand the file to +// the user for inspection. The caller owns the file and removes it once done; +// the stale-bundle cleanup of later runs removes it only after a day. +// anonymize and anonymizeLevel behave as in DebugBundle. +func (c *Client) DebugBundleFile(platformFiles PlatformFiles, anonymize bool, anonymizeLevel string) (string, error) { + return c.debugBundle(platformFiles, anonymize, anonymizeLevel, false) +} + +func (c *Client) debugBundle(platformFiles PlatformFiles, anonymize bool, anonymizeLevel string, upload bool) (string, error) { cfg, cacheDir, cc := c.stateSnapshot() // If the engine hasn't been started, load config from disk @@ -342,6 +356,11 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool, anonym cacheDir = platformFiles.CacheDir() } + // Clear what an interrupted earlier run may have left in the cache before + // adding to it. Remote debug jobs write to the same directory, so anything + // younger than an hour is treated as possibly still in use. + debug.RemoveStaleBundles(cacheDir, time.Hour) + deps := debug.GeneratorDependencies{ InternalConfig: cfg, StatusRecorder: c.recorder, @@ -379,6 +398,9 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool, anonym if err != nil { return "", fmt.Errorf("generate debug bundle: %w", err) } + if !upload { + return debug.ExportBundle(path) + } defer func() { if err := os.Remove(path); err != nil { log.Errorf("failed to remove debug bundle file: %v", err) @@ -475,6 +497,7 @@ func (c *Client) Networks() *NetworkArray { routesMap := routeManager.GetClientRoutesWithNetID() v6Merged := route.V6ExitMergeSet(routesMap) resolvedDomains := c.recorder.GetResolvedDomainsStates() + activeRoutePeers := c.recorder.GetActiveRoutePeers() networkArray := &NetworkArray{ items: make([]Network, 0), @@ -488,7 +511,7 @@ func (c *Client) Networks() *NetworkArray { continue } - network := c.buildNetwork(id, routes, routeSelector.IsSelected(id), resolvedDomains, v6Merged) + network := c.buildNetwork(id, routes, routeSelector.IsSelected(id), resolvedDomains, v6Merged, activeRoutePeers) if network == nil { continue } @@ -497,14 +520,14 @@ func (c *Client) Networks() *NetworkArray { return networkArray } -func (c *Client) buildNetwork(id route.NetID, routes []*route.Route, selected bool, resolvedDomains map[domain.Domain]peer.ResolvedDomainInfo, v6Merged map[route.NetID]struct{}) *Network { +func (c *Client) buildNetwork(id route.NetID, routes []*route.Route, selected bool, resolvedDomains map[domain.Domain]peer.ResolvedDomainInfo, v6Merged map[route.NetID]struct{}, activeRoutePeers map[route.HAUniqueID]string) *Network { r := routes[0] netStr := r.Network.String() if r.IsDynamic() { netStr = r.Domains.SafeString() } - routePeer, err := c.findBestRoutePeer(routes) + routePeer, err := c.findBestRoutePeer(routes, activeRoutePeers) if err != nil { log.Errorf("could not get peer info for route %s: %v", id, err) return nil @@ -528,12 +551,9 @@ func (c *Client) buildNetwork(id route.NetID, routes []*route.Route, selected bo // findBestRoutePeer returns the peer actively routing traffic for the given // HA route group. Falls back to the first connected peer, then the first peer. -func (c *Client) findBestRoutePeer(routes []*route.Route) (peer.State, error) { - netStr := routes[0].Network.String() - - fullStatus := c.recorder.GetFullStatus() - for _, p := range fullStatus.Peers { - if _, ok := p.GetRoutes()[netStr]; ok { +func (c *Client) findBestRoutePeer(routes []*route.Route, activeRoutePeers map[route.HAUniqueID]string) (peer.State, error) { + if peerKey, ok := activeRoutePeers[routes[0].GetHAUniqueID()]; ok { + if p, err := c.recorder.GetPeer(peerKey); err == nil { return p, nil } } diff --git a/client/android/login.go b/client/android/login.go index 155c6eadd..b9ec21b39 100644 --- a/client/android/login.go +++ b/client/android/login.go @@ -194,12 +194,49 @@ func (a *Auth) login(urlOpener URLOpener, isAndroidTV bool) error { } func (a *Auth) foregroundGetTokenInfo(authClient *auth.Auth, urlOpener URLOpener, isAndroidTV bool) (*auth.TokenInfo, error) { - oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, isAndroidTV, profileLoginHint(a.cfgPath)) + return a.foregroundGetTokenInfoFlow(authClient, urlOpener, isAndroidTV, false) +} + +// foregroundGetTokenInfoFlow runs the interactive flow. sessionExtend tells the +// server the token will renew this peer's session rather than log a peer in, so +// it can rule out a silent authorization the IdP could answer from an unrelated +// account. See PKCEAuthorizationFlowRequest. +func (a *Auth) foregroundGetTokenInfoFlow(authClient *auth.Auth, urlOpener URLOpener, isAndroidTV bool, sessionExtend bool) (*auth.TokenInfo, error) { + hint := profileLoginHint(a.cfgPath) + + oAuthFlow, err := authClient.GetOAuthFlow(a.ctx, isAndroidTV, sessionExtend, hint) if err != nil { return nil, fmt.Errorf("failed to get OAuth flow: %v", err) } - return runOAuthFlow(a.ctx, oAuthFlow, urlOpener, nil) + tokenInfo, err := runOAuthFlow(a.ctx, oAuthFlow, urlOpener, nil) + if err != nil { + return nil, err + } + + if tokenInfo.MatchesAccount(hint) { + return tokenInfo, nil + } + + // The IdP answered from a session belonging to another account. Retrying is + // what makes this recoverable: on a peer already registered the server would + // reject the token, and on a fresh one it would silently register the peer + // under the wrong account and bind the profile to it. + log.Infof("login returned an account other than the one this profile is bound to, retrying with an account prompt") + retryFlow := auth.RetryFlowForAccount(oAuthFlow) + if retryFlow == nil { + return tokenInfo, nil + } + + retryToken, err := runOAuthFlow(a.ctx, retryFlow, urlOpener, nil) + if err != nil { + return nil, err + } + if !retryToken.MatchesAccount(hint) { + log.Warnf("login still returned a different account after the prompt, continuing with it") + } + + return retryToken, nil } // profileLoginHint returns the stored account email for the profile at cfgPath. diff --git a/client/android/preferences.go b/client/android/preferences.go index 5ce31026c..3623de23f 100644 --- a/client/android/preferences.go +++ b/client/android/preferences.go @@ -46,7 +46,7 @@ func (p *Preferences) GetManagementURL() (string, error) { return p.configInput.ManagementURL, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return "", err } @@ -64,7 +64,7 @@ func (p *Preferences) GetAdminURL() (string, error) { return p.configInput.AdminURL, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return "", err } @@ -86,7 +86,7 @@ func (p *Preferences) HasPreSharedKey() (bool, error) { return *p.configInput.PreSharedKey != "", nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -112,7 +112,7 @@ func (p *Preferences) GetRosenpassEnabled() (bool, error) { return *p.configInput.RosenpassEnabled, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -133,7 +133,7 @@ func (p *Preferences) GetRosenpassPermissive() (bool, error) { return *p.configInput.RosenpassPermissive, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -149,7 +149,7 @@ func (p *Preferences) GetDisableClientRoutes() (bool, error) { return *p.configInput.DisableClientRoutes, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -170,7 +170,7 @@ func (p *Preferences) GetDisableServerRoutes() (bool, error) { return *p.configInput.DisableServerRoutes, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -188,7 +188,7 @@ func (p *Preferences) GetDisableDNS() (bool, error) { return *p.configInput.DisableDNS, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -206,7 +206,7 @@ func (p *Preferences) GetDisableFirewall() (bool, error) { return *p.configInput.DisableFirewall, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -227,7 +227,7 @@ func (p *Preferences) GetServerSSHAllowed() (bool, error) { return *p.configInput.ServerSSHAllowed, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -249,7 +249,7 @@ func (p *Preferences) GetEnableSSHRoot() (bool, error) { return *p.configInput.EnableSSHRoot, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -271,7 +271,7 @@ func (p *Preferences) GetEnableSSHSFTP() (bool, error) { return *p.configInput.EnableSSHSFTP, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -293,7 +293,7 @@ func (p *Preferences) GetEnableSSHLocalPortForwarding() (bool, error) { return *p.configInput.EnableSSHLocalPortForwarding, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -315,7 +315,7 @@ func (p *Preferences) GetEnableSSHRemotePortForwarding() (bool, error) { return *p.configInput.EnableSSHRemotePortForwarding, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -340,7 +340,7 @@ func (p *Preferences) GetBlockInbound() (bool, error) { return *p.configInput.BlockInbound, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -358,7 +358,7 @@ func (p *Preferences) GetDisableIPv6() (bool, error) { return *p.configInput.DisableIPv6, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } @@ -377,7 +377,7 @@ func (p *Preferences) GetRemoteJobsAllowed() (bool, error) { return *p.configInput.RemoteJobsAllowed, nil } - cfg, err := profilemanager.ReadConfig(p.configInput.ConfigPath) + cfg, err := profilemanager.ReadConfigOrDefault(p.configInput.ConfigPath) if err != nil { return false, err } diff --git a/client/android/session.go b/client/android/session.go index d5da09c93..b2de8dadb 100644 --- a/client/android/session.go +++ b/client/android/session.go @@ -6,13 +6,8 @@ import ( "context" "fmt" - log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/auth" - "github.com/netbirdio/netbird/client/internal/auth/sessionwatch" - "github.com/netbirdio/netbird/client/internal/peer" - cProto "github.com/netbirdio/netbird/client/proto" ) // StateChangeListener receives client state notifications. @@ -21,16 +16,11 @@ import ( // changed: connection state, the run-loop status label (e.g. NeedsLogin) or // the session deadline. It mirrors the daemon's SubscribeStatus stream // trigger — on each signal the consumer pulls the fresh values via -// Status() / SessionExpiresAtUnix(). -// -// OnSessionExpiring forwards the engine's session-expiry warnings, fired at -// sessionwatch.WarningLead before the deadline and again at FinalWarningLead -// (finalWarning true). The second one is suppressed when the user dismissed -// the first via DismissSessionWarning. The daemon turns the same events into -// its tray notification. +// Status() / SessionExpiresAtUnix(). The engine arms no expiry-warning +// timers on Android; the app schedules the warnings from the deadline it +// reads here. type StateChangeListener interface { OnStateChanged() - OnSessionExpiring(expiresAtUnix int64, leadMinutes int64, finalWarning bool) } // Status returns the connect run-loop's status label — the same value the @@ -110,11 +100,11 @@ func (c *Client) SetStateChangeListener(listener StateChangeListener) { return } - // Both subscriptions are buffered (one pending tick, ten pending events), - // so unsubscribing is not enough to stop callbacks: the loops would drain - // what is already queued and deliver it to a listener the caller has - // already removed or replaced. Gate every callback on this registration's - // own signal, which is closed before unsubscribing. + // The subscription is buffered (one pending tick), so unsubscribing is + // not enough to stop callbacks: the loop would drain what is already + // queued and deliver it to a listener the caller has already removed or + // replaced. Gate every callback on this registration's own signal, which + // is closed before unsubscribing. done := make(chan struct{}) c.stateChangeDone = done @@ -133,9 +123,6 @@ func (c *Client) SetStateChangeListener(listener StateChangeListener) { listener.OnStateChanged() } }() - - c.eventSub = c.recorder.SubscribeToEvents() - go watchSessionWarnings(c.eventSub, listener, done) } // RemoveStateChangeListener unregisters the state notification listener. @@ -145,21 +132,6 @@ func (c *Client) RemoveStateChangeListener() { c.stopStateChangeWatchLocked() } -// DismissSessionWarning records the user's "Dismiss" on the first expiry -// warning and suppresses the final one for the current deadline. A refreshed -// deadline re-arms both. No-op while the engine is not running. -func (c *Client) DismissSessionWarning() { - cc := c.getConnectClient() - if cc == nil { - return - } - engine := cc.Engine() - if engine == nil { - return - } - engine.DismissSessionWarning() -} - // ExtendAuthSession runs the interactive SSO flow to obtain a fresh JWT and // asks the management server to extend the session deadline. The tunnel is // untouched: no resync, no reconnect. Async; the result arrives on the @@ -201,8 +173,8 @@ func (c *Client) CancelExtendAuthSession() { } func (c *Client) stopStateChangeWatchLocked() { - // Signal first, unsubscribe second: closing the channels only stops new - // items, and the loops would still hand whatever is buffered to a listener + // Signal first, unsubscribe second: closing the channel only stops new + // items, and the loop would still hand whatever is buffered to a listener // that is no longer registered. if c.stateChangeDone != nil { close(c.stateChangeDone) @@ -212,49 +184,6 @@ func (c *Client) stopStateChangeWatchLocked() { c.recorder.UnsubscribeFromStateChanges(c.stateChangeSubID) c.stateChangeSubID = "" } - if c.eventSub != nil { - // Closes the channel, which ends watchSessionWarnings. - c.recorder.UnsubscribeFromEvents(c.eventSub) - c.eventSub = nil - } -} - -// watchSessionWarnings forwards the engine's session-expiry warnings to the -// listener. The event stream also carries unrelated traffic — network-map -// updates on every sync, DNS and route errors — so everything but an -// AUTHENTICATION event carrying the session-warning marker is dropped. Exits -// when the subscription is closed by UnsubscribeFromEvents, or earlier when -// done is closed — the stream buffers up to ten events, and a deregistered -// listener must not receive the ones already queued. -func watchSessionWarnings(sub *peer.EventSubscription, listener StateChangeListener, done <-chan struct{}) { - for ev := range sub.Events() { - select { - case <-done: - return - default: - } - if ev.GetCategory() != cProto.SystemEvent_AUTHENTICATION { - continue - } - meta := ev.GetMetadata() - if meta[sessionwatch.MetaSessionWarning] != "true" { - // Other AUTHENTICATION events exist (e.g. a deadline rejected as - // out of range); they carry no warning marker. - continue - } - deadline, err := sessionwatch.ParseExpiresAt(meta[sessionwatch.MetaSessionExpiresAt]) - if err != nil { - log.Warnf("session warning event with unparsable deadline: %v", err) - continue - } - lead, err := sessionwatch.ParseLeadMinutes(meta[sessionwatch.MetaSessionLeadMinutes]) - if err != nil { - // Informational only — the deadline above is what drives the UI. - lead = 0 - } - listener.OnSessionExpiring(deadline.Unix(), int64(lead), - meta[sessionwatch.MetaSessionFinal] == "true") - } } func (c *Client) beginExtend() (context.Context, error) { @@ -293,11 +222,13 @@ func (c *Client) extendAuthSession(ctx context.Context, urlOpener URLOpener, isA } defer authClient.Close() - // Passing the config path makes the flow pick up the login_hint: an extend - // renews the session of the account already signed in, so it must not stop to - // offer a choice. + // Passing the config path makes the flow pick up the login_hint. That alone + // cannot keep the IdP on this profile's account though — a hint is only a + // suggestion, and a silent authorization is answered from whatever session the + // IdP already has, which need not be this peer's when several accounts are + // signed in. Marking the flow as an extend lets the server rule that out. a := NewAuthWithConfig(ctx, cfg, cfgPath) - tokenInfo, err := a.foregroundGetTokenInfo(authClient, urlOpener, isAndroidTV) + tokenInfo, err := a.foregroundGetTokenInfoFlow(authClient, urlOpener, isAndroidTV, true) if err != nil { return fmt.Errorf("interactive sso login failed: %v", err) } diff --git a/client/android/ssh_client.go b/client/android/ssh_client.go index 9a11044ba..e05249e98 100644 --- a/client/android/ssh_client.go +++ b/client/android/ssh_client.go @@ -469,7 +469,7 @@ func (s *SSHClient) requestJWTToken(cfg *profilemanager.Config, cfgPath string) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - flow, err := auth.NewOAuthFlow(ctx, cfg, false, true, profileLoginHint(cfgPath)) + flow, err := auth.NewOAuthFlow(ctx, cfg, false, true, profileLoginHint(cfgPath), false) if err != nil { return "", fmt.Errorf("create oauth flow: %w", err) } diff --git a/client/cmd/debug.go b/client/cmd/debug.go index 98fe53626..c4d5ad6d7 100644 --- a/client/cmd/debug.go +++ b/client/cmd/debug.go @@ -23,7 +23,10 @@ import ( "github.com/netbirdio/netbird/version" ) -const errCloseConnection = "Failed to close connection: %v" +const ( + errCloseConnection = "Failed to close connection: %v" + noUpDownFlag = "no-updown" +) var ( logFileCount uint32 @@ -257,13 +260,14 @@ func runForDuration(cmd *cobra.Command, args []string) error { } stateWasDown := stat.Status != string(internal.StatusConnected) && stat.Status != string(internal.StatusConnecting) + noUpDown, _ := cmd.Flags().GetBool(noUpDownFlag) initialLogLevel, err := client.GetLogLevel(cmd.Context(), &proto.GetLogLevelRequest{}) if err != nil { return fmt.Errorf("failed to get log level: %v", status.Convert(err).Message()) } - if stateWasDown { + if stateWasDown && !noUpDown { if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil { cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message()) } else { @@ -284,34 +288,20 @@ func runForDuration(cmd *cobra.Command, args []string) error { } needsRestoreUp := false - if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil { - cmd.PrintErrf("Failed to bring service down: %v\n", status.Convert(err).Message()) + if noUpDown { + enableSyncResponsePersistence(cmd, client) } else { - needsRestoreUp = !stateWasDown - cmd.Println("netbird down") + needsRestoreUp = restartDaemon(cmd, client, stateWasDown) } - time.Sleep(1 * time.Second) - - // Enable sync response persistence before bringing the service up - if _, err := client.SetSyncResponsePersistence(cmd.Context(), &proto.SetSyncResponsePersistenceRequest{ - Enabled: true, - }); err != nil { - cmd.PrintErrf("Failed to enable sync response persistence: %v\n", status.Convert(err).Message()) - } - - if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil { - cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message()) - } else { - needsRestoreUp = false - cmd.Println("netbird up") - } - - time.Sleep(3 * time.Second) - cpuProfilingStarted := false if _, err := client.StartCPUProfile(cmd.Context(), &proto.StartCPUProfileRequest{}); err != nil { - cmd.PrintErrf("Failed to start CPU profiling: %v\n", err) + if msg := status.Convert(err).Message(); strings.Contains(msg, "already in progress") { + cmd.PrintErrln("CPU profiling is already running (started with `netbird debug cpu start`). " + + "It is left running and is included in a bundle created after `netbird debug cpu stop`.") + } else { + cmd.PrintErrf("Failed to start CPU profiling: %v\n", msg) + } } else { cpuProfilingStarted = true defer func() { @@ -401,7 +391,7 @@ func runForDuration(cmd *cobra.Command, args []string) error { } } - if stateWasDown { + if stateWasDown && !noUpDown { if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil { cmd.PrintErrf("Failed to restore service down state: %v\n", status.Convert(err).Message()) } else { @@ -458,6 +448,47 @@ func setSyncResponsePersistence(cmd *cobra.Command, args []string) error { return nil } +// enableSyncResponsePersistence asks the daemon to keep the latest sync +// response so the bundle carries the network map. With a running daemon only +// syncs received after the call are kept. +func enableSyncResponsePersistence(cmd *cobra.Command, client proto.DaemonServiceClient) { + if _, err := client.SetSyncResponsePersistence(cmd.Context(), &proto.SetSyncResponsePersistenceRequest{ + Enabled: true, + }); err != nil { + cmd.PrintErrf("Failed to enable sync response persistence: %v\n", status.Convert(err).Message()) + } +} + +// restartDaemon cycles the daemon down and up with sync response persistence +// enabled so the bundle carries the network map. It reports whether the +// daemon was left down although it was running before, so the caller can +// bring it back up. +func restartDaemon(cmd *cobra.Command, client proto.DaemonServiceClient, stateWasDown bool) bool { + needsRestoreUp := false + if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil { + cmd.PrintErrf("Failed to bring service down: %v\n", status.Convert(err).Message()) + } else { + needsRestoreUp = !stateWasDown + cmd.Println("netbird down") + } + + time.Sleep(1 * time.Second) + + // Enable sync response persistence before bringing the service up + enableSyncResponsePersistence(cmd, client) + + if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil { + cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message()) + } else { + needsRestoreUp = false + cmd.Println("netbird up") + } + + time.Sleep(3 * time.Second) + + return needsRestoreUp +} + func waitForDurationOrCancel(ctx context.Context, duration time.Duration, cmd *cobra.Command) error { ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() @@ -546,4 +577,5 @@ func init() { forCmd.Flags().StringVar(&uploadBundleURLFlag, "upload-bundle-url", types.DefaultBundleURL, "Service URL to get an URL to upload the debug bundle") forCmd.Flags().BoolVar(&uploadBundleInsecureFlag, "upload-bundle-insecure", false, "Allow uploading to an http or untrusted-TLS upload server (self-hosted); requires root") forCmd.Flags().Bool("capture", false, "Capture packets during the debug duration and include in bundle") + forCmd.Flags().Bool(noUpDownFlag, false, "Keep the daemon running instead of bringing it down and up before collecting. The bundle only includes the network map if a sync arrives during the run") } diff --git a/client/cmd/debug_cpu.go b/client/cmd/debug_cpu.go new file mode 100644 index 000000000..a01b845cf --- /dev/null +++ b/client/cmd/debug_cpu.go @@ -0,0 +1,83 @@ +package cmd + +import ( + "fmt" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "google.golang.org/grpc/status" + + "github.com/netbirdio/netbird/client/proto" +) + +var debugCPUCmd = &cobra.Command{ + Use: "cpu", + Short: "Profile the daemon's CPU usage", + Long: `Starts and stops CPU profiling in the running daemon without restarting it. +The profile is included in the next debug bundle as cpu.prof. + +Profiling is not time limited: it keeps running, and keeps costing CPU, until +"netbird debug cpu stop" is run.`, +} + +var debugCPUStartCmd = &cobra.Command{ + Use: "start", + Short: "Start CPU profiling in the daemon", + Example: " netbird debug cpu start", + Args: cobra.NoArgs, + RunE: debugCPUStart, +} + +var debugCPUStopCmd = &cobra.Command{ + Use: "stop", + Short: "Stop CPU profiling in the daemon", + Long: `Stops CPU profiling. The captured profile stays in the daemon until the next +debug bundle is created, which includes it as cpu.prof.`, + Example: " netbird debug cpu stop && netbird debug bundle", + Args: cobra.NoArgs, + RunE: debugCPUStop, +} + +func debugCPUStart(cmd *cobra.Command, _ []string) error { + conn, err := getClient(cmd) + if err != nil { + return err + } + defer func() { + if err := conn.Close(); err != nil { + log.Errorf(errCloseConnection, err) + } + }() + + if _, err := proto.NewDaemonServiceClient(conn).StartCPUProfile(cmd.Context(), &proto.StartCPUProfileRequest{}); err != nil { + return fmt.Errorf("start CPU profiling: %v", status.Convert(err).Message()) + } + + cmd.Println("CPU profiling started and runs until stopped. Run `netbird debug cpu stop` and then `netbird debug bundle` to collect it.") + return nil +} + +func debugCPUStop(cmd *cobra.Command, _ []string) error { + conn, err := getClient(cmd) + if err != nil { + return err + } + defer func() { + if err := conn.Close(); err != nil { + log.Errorf(errCloseConnection, err) + } + }() + + if _, err := proto.NewDaemonServiceClient(conn).StopCPUProfile(cmd.Context(), &proto.StopCPUProfileRequest{}); err != nil { + return fmt.Errorf("stop CPU profiling: %v", status.Convert(err).Message()) + } + + cmd.Println("CPU profiling stopped. Run `netbird debug bundle` to include cpu.prof.") + return nil +} + +func init() { + debugCPUCmd.AddCommand(debugCPUStartCmd) + debugCPUCmd.AddCommand(debugCPUStopCmd) + debugCmd.AddCommand(debugCPUCmd) +} diff --git a/client/cmd/debug_cpu_test.go b/client/cmd/debug_cpu_test.go new file mode 100644 index 000000000..29702bb53 --- /dev/null +++ b/client/cmd/debug_cpu_test.go @@ -0,0 +1,164 @@ +package cmd + +import ( + "bytes" + "context" + "os/user" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/internal" + "github.com/netbirdio/netbird/client/internal/profilemanager" +) + +// startDebugTestDaemon starts an in-process daemon with an isolated profile +// directory and returns the address the CLI should dial. +func startDebugTestDaemon(t *testing.T) string { + t.Helper() + + tempDir := t.TempDir() + origDefaultProfileDir := profilemanager.DefaultConfigPathDir + origActiveProfileStatePath := profilemanager.ActiveProfileStatePath + origConfigDirOverride := profilemanager.ConfigDirOverride + origDaemonAddr := daemonAddr + t.Cleanup(func() { + profilemanager.DefaultConfigPathDir = origDefaultProfileDir + profilemanager.ActiveProfileStatePath = origActiveProfileStatePath + profilemanager.ConfigDirOverride = origConfigDirOverride + daemonAddr = origDaemonAddr + }) + + profilemanager.DefaultConfigPathDir = tempDir + profilemanager.ActiveProfileStatePath = tempDir + "/active_profile.json" + profilemanager.ConfigDirOverride = tempDir + + currUser, err := user.Current() + require.NoError(t, err) + sm := profilemanager.ServiceManager{} + created, err := sm.AddProfile("test1", nil) + require.NoError(t, err) + require.NoError(t, sm.SetActiveProfileState(&profilemanager.ActiveProfileState{ + ID: created.ID, + Username: currUser.Username, + })) + + ctx, cancel := context.WithCancel(internal.CtxInitState(context.Background())) + srv, lis := startClientDaemon(t, ctx, "", tempDir+"/config.json") + t.Cleanup(func() { + cancel() + srv.Stop() + }) + + return "tcp://" + lis.Addr().String() +} + +// runDebugCmd runs `netbird debug ` against the daemon at addr and +// returns everything the command printed. +func runDebugCmd(addr string, args ...string) (string, error) { + daemonAddr = addr + var out bytes.Buffer + rootCmd.SetOut(&out) + rootCmd.SetErr(&out) + rootCmd.SetArgs(append(append([]string{"debug"}, args...), "--daemon-addr", addr, "--log-file", "")) + err := rootCmd.Execute() + rootCmd.SetOut(nil) + rootCmd.SetErr(nil) + rootCmd.SetArgs(nil) + resetFlags(rootCmd) + return out.String(), err +} + +// resetFlags puts every flag of the command and its subcommands back to its +// default so a value parsed in one run does not leak into the next in-process +// execution. +func resetFlags(cmd *cobra.Command) { + reset := func(f *pflag.Flag) { + // Set appends to a slice flag and would parse the "[a,b]" default + // text as elements, so slices are replaced instead. + if sv, ok := f.Value.(pflag.SliceValue); ok { + var def []string + if trimmed := strings.Trim(f.DefValue, "[]"); trimmed != "" { + def = strings.Split(trimmed, ",") + } + _ = sv.Replace(def) + } else { + _ = f.Value.Set(f.DefValue) + } + f.Changed = false + } + cmd.Flags().VisitAll(reset) + cmd.PersistentFlags().VisitAll(reset) + // Commands pin their writers to the buffer of the run that first used + // them, so a later run would print into the old buffer. + cmd.SetOut(nil) + cmd.SetErr(nil) + for _, sub := range cmd.Commands() { + resetFlags(sub) + } +} + +// TestResetFlagsSliceDefault guards against Set("[]") on slice flags, which +// stores a literal "[]" element instead of the empty default. +func TestResetFlagsSliceDefault(t *testing.T) { + cmd := &cobra.Command{Use: "x"} + var env, withDefault []string + cmd.Flags().StringSliceVar(&env, "env", nil, "") + cmd.Flags().StringSliceVar(&withDefault, "names", []string{"a", "b"}, "") + require.NoError(t, cmd.Flags().Parse([]string{"--env", "K=V", "--names", "c"})) + + resetFlags(cmd) + + assert.Empty(t, env, "slice flag with no default must reset to empty") + assert.Equal(t, []string{"a", "b"}, withDefault, "slice flag must reset to its default") +} + +func TestDebugCPUStartStop(t *testing.T) { + addr := startDebugTestDaemon(t) + + run := func(args ...string) error { + _, err := runDebugCmd(addr, append([]string{"cpu"}, args...)...) + return err + } + + require.Error(t, run("stop"), "stop without a running profile must fail") + require.NoError(t, run("start")) + assert.Error(t, run("start"), "second start must be rejected while profiling") + require.NoError(t, run("stop")) + assert.Error(t, run("stop"), "second stop must be rejected") + assert.NoError(t, run("start"), "profiling can be started again after a stop") + assert.NoError(t, run("stop")) +} + +// TestDebugForKeepsRunningCPUProfile covers `debug for` started while a +// profile from `debug cpu start` is running: it must say so, leave the +// profile alone, and still create the bundle. +func TestDebugForKeepsRunningCPUProfile(t *testing.T) { + addr := startDebugTestDaemon(t) + + _, err := runDebugCmd(addr, "cpu", "start") + require.NoError(t, err) + + out, err := runDebugCmd(addr, "for", "1s", "-S=false", "--no-updown") + require.NoError(t, err, "output: %s", out) + assert.Contains(t, out, "CPU profiling is already running", "the conflict must be explained") + assert.NotContains(t, out, "rpc error", "the raw RPC error must not reach the user") + assert.Contains(t, out, "Local file:", "the bundle must still be created") + + _, err = runDebugCmd(addr, "cpu", "stop") + assert.NoError(t, err, "the profile started by the user must still be running") +} + +func TestDebugForNoUpDown(t *testing.T) { + addr := startDebugTestDaemon(t) + + out, err := runDebugCmd(addr, "for", "1s", "-S=false", "--no-updown") + require.NoError(t, err, "output: %s", out) + assert.NotContains(t, out, "netbird down", "--no-updown must not bring the daemon down") + assert.NotContains(t, out, "netbird up", "--no-updown must not bring the daemon up") + assert.Contains(t, out, "Local file:", "the bundle must still be created") +} diff --git a/client/cmd/forwarding_rules.go b/client/cmd/forwarding_rules.go deleted file mode 100644 index b3052746a..000000000 --- a/client/cmd/forwarding_rules.go +++ /dev/null @@ -1,98 +0,0 @@ -package cmd - -import ( - "fmt" - "sort" - - "github.com/spf13/cobra" - "google.golang.org/grpc/status" - - "github.com/netbirdio/netbird/client/proto" -) - -var forwardingRulesCmd = &cobra.Command{ - Use: "forwarding", - Short: "List forwarding rules", - Long: `Commands to list forwarding rules.`, -} - -var forwardingRulesListCmd = &cobra.Command{ - Use: "list", - Aliases: []string{"ls"}, - Short: "List forwarding rules", - Example: " netbird forwarding list", - Long: "Commands to list forwarding rules.", - RunE: listForwardingRules, -} - -func listForwardingRules(cmd *cobra.Command, _ []string) error { - conn, err := getClient(cmd) - if err != nil { - return err - } - defer conn.Close() - - client := proto.NewDaemonServiceClient(conn) - resp, err := client.ForwardingRules(cmd.Context(), &proto.EmptyRequest{}) - if err != nil { - return fmt.Errorf("failed to list network: %v", status.Convert(err).Message()) - } - - if len(resp.GetRules()) == 0 { - cmd.Println("No forwarding rules available.") - return nil - } - - printForwardingRules(cmd, resp.GetRules()) - return nil -} - -func printForwardingRules(cmd *cobra.Command, rules []*proto.ForwardingRule) { - cmd.Println("Available forwarding rules:") - - // Sort rules by translated address - sort.Slice(rules, func(i, j int) bool { - if rules[i].GetTranslatedAddress() != rules[j].GetTranslatedAddress() { - return rules[i].GetTranslatedAddress() < rules[j].GetTranslatedAddress() - } - if rules[i].GetProtocol() != rules[j].GetProtocol() { - return rules[i].GetProtocol() < rules[j].GetProtocol() - } - - return getFirstPort(rules[i].GetDestinationPort()) < getFirstPort(rules[j].GetDestinationPort()) - }) - - var lastIP string - for _, rule := range rules { - dPort := portToString(rule.GetDestinationPort()) - tPort := portToString(rule.GetTranslatedPort()) - if lastIP != rule.GetTranslatedAddress() { - lastIP = rule.GetTranslatedAddress() - cmd.Printf("\nTranslated peer: %s\n", rule.GetTranslatedHostname()) - } - - cmd.Printf(" Local %s/%s to %s:%s\n", rule.GetProtocol(), dPort, rule.GetTranslatedAddress(), tPort) - } -} - -func getFirstPort(portInfo *proto.PortInfo) int { - switch v := portInfo.PortSelection.(type) { - case *proto.PortInfo_Port: - return int(v.Port) - case *proto.PortInfo_Range_: - return int(v.Range.GetStart()) - default: - return 0 - } -} - -func portToString(translatedPort *proto.PortInfo) string { - switch v := translatedPort.PortSelection.(type) { - case *proto.PortInfo_Port: - return fmt.Sprintf("%d", v.Port) - case *proto.PortInfo_Range_: - return fmt.Sprintf("%d-%d", v.Range.GetStart(), v.Range.GetEnd()) - default: - return "No port specified" - } -} diff --git a/client/cmd/login.go b/client/cmd/login.go index a5b5562b5..c715d4f1d 100644 --- a/client/cmd/login.go +++ b/client/cmd/login.go @@ -9,8 +9,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/term" - "google.golang.org/grpc/codes" - gstatus "google.golang.org/grpc/status" "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/auth" @@ -145,10 +143,7 @@ func doDaemonLogin(ctx context.Context, cmd *cobra.Command, providedSetupKey str err = WithBackOff(func() error { var backOffErr error loginResp, backOffErr = client.Login(ctx, &loginRequest) - if s, ok := gstatus.FromError(backOffErr); ok && (s.Code() == codes.InvalidArgument || - s.Code() == codes.PermissionDenied || - s.Code() == codes.NotFound || - s.Code() == codes.Unimplemented) { + if terminalLoginError(backOffErr) { loginErr = backOffErr return nil } @@ -327,10 +322,28 @@ func doForegroundLogin(ctx context.Context, cmd *cobra.Command, setupKey string, } - config, err := profilemanager.ReadConfig(configFilePath) + config, err := profilemanager.ReadConfigOrDefault(configFilePath) if err != nil { return fmt.Errorf("read config file %s: %v", configFilePath, err) } + // Reading a config does not provision one: this login is about to dial + // management with the profile's identity, so mint the keys if the profile + // has none yet and put them on disk — a key that stayed in memory would + // come back different on the next run and register a second peer. + // + // Before the MDM overlay below, on purpose: the file must keep the + // profile's own values. The overlay is runtime-only and re-derived on + // every load, so persisting it would turn an enforced management URL or + // pre-shared key into one the user appears to own once the policy is + // withdrawn. + if generated, err := config.EnsureIdentity(); err != nil { + return fmt.Errorf("ensure profile identity: %v", err) + } else if generated { + if err := profilemanager.WriteOutConfig(configFilePath, config); err != nil { + return fmt.Errorf("write out config file %s: %v", configFilePath, err) + } + } + // CLI standalone login: profilemanager no longer auto-applies MDM, // so layer in the OS-native policy here. Desktop builds construct // a Loader with no fetcher — the build-tagged loadPlatform reads @@ -412,11 +425,44 @@ func foregroundGetTokenInfo(ctx context.Context, cmd *cobra.Command, config *pro hint = profileState.Email } - oAuthFlow, err := auth.NewOAuthFlow(ctx, config, util.HasGraphicalSession(), false, hint) + oAuthFlow, err := auth.NewOAuthFlow(ctx, config, util.HasGraphicalSession(), false, hint, false) if err != nil { return nil, err } + tokenInfo, err := runInteractiveFlow(cmd, oAuthFlow) + if err != nil { + return nil, err + } + + if tokenInfo.MatchesAccount(hint) { + return tokenInfo, nil + } + + // The IdP answered from a session belonging to another account. Retrying is + // what makes this recoverable: on a peer already registered the server would + // reject the token, and on a fresh one it would silently register the peer + // under the wrong account and bind the profile to it. + cmd.Println("The login returned a different account than this profile uses. Asking to sign in again.") + retryFlow := auth.RetryFlowForAccount(oAuthFlow) + if retryFlow == nil { + return tokenInfo, nil + } + + retryToken, err := runInteractiveFlow(cmd, retryFlow) + if err != nil { + return nil, err + } + if !retryToken.MatchesAccount(hint) { + log.Warnf("login still returned a different account after the prompt, continuing with it") + } + + return retryToken, nil +} + +// runInteractiveFlow requests the authorization info, shows the URL to the user +// and blocks until the token comes back. +func runInteractiveFlow(cmd *cobra.Command, oAuthFlow auth.OAuthFlow) (*auth.TokenInfo, error) { flowInfo, err := oAuthFlow.RequestAuthInfo(context.TODO()) if err != nil { return nil, fmt.Errorf("getting a request OAuth flow info failed: %v", err) diff --git a/client/cmd/posture.go b/client/cmd/posture.go new file mode 100644 index 000000000..b0870233d --- /dev/null +++ b/client/cmd/posture.go @@ -0,0 +1,30 @@ +package cmd + +import ( + "github.com/spf13/cobra" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/certproof" +) + +var postureCmd = &cobra.Command{ + Use: "posture", + Short: "Posture helpers invoked by the NetBird daemon", + Hidden: true, +} + +var postureCertProofCmd = &cobra.Command{ + Use: "cert-proof", + Short: "Answer certificate posture challenges from the calling user's keychain", + Long: "Reads a certificate challenge set as JSON on stdin and writes the proofs as JSON on stdout.\n" + + "The daemon launches this in the console user's desktop session, because a login keychain\n" + + "cannot be reached from a root daemon. Not intended to be run by hand.", + Hidden: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + // Proofs travel on stdout, so every log line has to go elsewhere. + log.SetOutput(cmd.ErrOrStderr()) + return certproof.RunHelper(cmd.Context(), cmd.InOrStdin(), cmd.OutOrStdout()) + }, +} diff --git a/client/cmd/root.go b/client/cmd/root.go index 2ee9ef729..26b5f8f5d 100644 --- a/client/cmd/root.go +++ b/client/cmd/root.go @@ -20,6 +20,8 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + gstatus "google.golang.org/grpc/status" "github.com/netbirdio/netbird/client/anonymize" daddr "github.com/netbirdio/netbird/client/internal/daemonaddr" @@ -182,16 +184,16 @@ func init() { rootCmd.AddCommand(versionCmd) rootCmd.AddCommand(sshCmd) rootCmd.AddCommand(networksCMD) - rootCmd.AddCommand(forwardingRulesCmd) rootCmd.AddCommand(debugCmd) rootCmd.AddCommand(profileCmd) rootCmd.AddCommand(exposeCmd) + rootCmd.AddCommand(postureCmd) + + postureCmd.AddCommand(postureCertProofCmd) networksCMD.AddCommand(routesListCmd) networksCMD.AddCommand(routesSelectCmd, routesDeselectCmd) - forwardingRulesCmd.AddCommand(forwardingRulesListCmd) - debugCmd.AddCommand(debugBundleCmd) debugCmd.AddCommand(logCmd) logCmd.AddCommand(logLevelCmd) @@ -299,6 +301,43 @@ func DialClientGRPCServer(ctx context.Context, addr string) (*grpc.ClientConn, e return grpc.DialContext(ctx, target, opts...) } +// terminalLoginError reports whether a Login failure is final, so the backoff +// cycle stops and the caller is told what the daemon said instead of "login +// backoff cycle failed" thirty seconds later. Retrying cannot change any of +// these answers: the request is malformed, the caller is not allowed, the +// target does not exist, a precondition on the daemon refuses it (the +// update-settings kill switch, an MDM-managed field), or the method is not +// implemented. +// +// Both `netbird up` and `netbird login` run Login through the backoff, and +// they each carried their own copy of this list — which is how one of them +// ended up retrying a refusal the other treated as final. +func terminalLoginError(err error) bool { + // A successful Login reaches here with a nil error, and that is not a + // terminal failure. Handled explicitly rather than left to + // gstatus.FromError, which answers (nil, true) for a nil error and leans on + // Status.Code tolerating a nil receiver to come back as codes.OK. + if err == nil { + return false + } + + s, ok := gstatus.FromError(err) + if !ok { + return false + } + + switch s.Code() { + case codes.InvalidArgument, + codes.PermissionDenied, + codes.NotFound, + codes.FailedPrecondition, + codes.Unimplemented: + return true + default: + return false + } +} + // WithBackOff execute function in backoff cycle. func WithBackOff(bf func() error) error { return backoff.RetryNotify(bf, CLIBackOffSettings, func(err error, duration time.Duration) { diff --git a/client/cmd/testutil_test.go b/client/cmd/testutil_test.go index 328a15454..46bf31837 100644 --- a/client/cmd/testutil_test.go +++ b/client/cmd/testutil_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" - "go.uber.org/mock/gomock" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" + "go.uber.org/mock/gomock" "google.golang.org/grpc" "github.com/netbirdio/netbird/management/server/integrations/integrated_validator/validator" @@ -28,7 +28,6 @@ import ( mgmt "github.com/netbirdio/netbird/management/server" "github.com/netbirdio/netbird/management/server/activity" "github.com/netbirdio/netbird/management/server/groups" - "github.com/netbirdio/netbird/management/server/integrations/port_forwarding" "github.com/netbirdio/netbird/management/server/permissions" "github.com/netbirdio/netbird/management/server/settings" "github.com/netbirdio/netbird/management/server/store" @@ -124,9 +123,9 @@ func startManagement(t *testing.T, config *config.Config, testFile string) (*grp updateManager := update_channel.NewPeersUpdateManager(metrics) requestBuffer := mgmt.NewAccountRequestBuffer(ctx, store) - networkMapController := controller.NewController(ctx, store, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.cloud", port_forwarding.NewControllerMock(), manager.NewEphemeralManager(store, peersmanager), config, nil) + networkMapController := controller.NewController(ctx, store, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.cloud", manager.NewEphemeralManager(store, peersmanager), config, nil) - accountManager, err := mgmt.BuildManager(ctx, config, store, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManagerMock, false, cacheStore) + accountManager, err := mgmt.BuildManager(ctx, config, store, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, settingsMockManager, permissionsManagerMock, false, cacheStore) if err != nil { t.Fatal(err) } diff --git a/client/cmd/up.go b/client/cmd/up.go index 2c7a807a2..5a6a3bb26 100644 --- a/client/cmd/up.go +++ b/client/cmd/up.go @@ -367,9 +367,17 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command, pm *profilemanager // set the new config req := setupSetConfigReq(customDNSAddressConverted, cmd, activeProf.ID.String(), username.Username) if _, err := client.SetConfig(ctx, req); err != nil { - if st, ok := gstatus.FromError(err); ok && st.Code() == codes.Unavailable { - log.Warnf("setConfig method is not available in the daemon: %s", st.Message()) - } else { + switch reason, refused := refusedSettingsUpdate(err); { + case refused: + // Failing here is the point: carrying on would connect while + // silently dropping the settings the caller asked for, since + // nothing further down the line applies them. + return fmt.Errorf("the daemon refused the settings update: %s", reason) + case gstatus.Code(err) == codes.Unavailable: + // The daemon cannot serve the method at all, which is what this + // code means; an older daemon without it lands here. + log.Warnf("the daemon did not apply the settings update: %s", gstatus.Convert(err).Message()) + default: return daemonCallError("call service setConfig method", err) } } @@ -410,10 +418,7 @@ func doDaemonUp(ctx context.Context, cmd *cobra.Command, client proto.DaemonServ err = WithBackOff(func() error { var backOffErr error loginResp, backOffErr = client.Login(ctx, loginRequest) - if s, ok := gstatus.FromError(backOffErr); ok && (s.Code() == codes.InvalidArgument || - s.Code() == codes.PermissionDenied || - s.Code() == codes.NotFound || - s.Code() == codes.Unimplemented) { + if terminalLoginError(backOffErr) { loginErr = backOffErr return nil } @@ -482,6 +487,22 @@ func setSSHSetConfigFields(req *proto.SetConfigRequest, cmd *cobra.Command) { } } +// refusedSettingsUpdate reports whether err is the daemon refusing the settings +// a request carried — the update-settings kill switch, or a field an MDM policy +// manages — and returns the reason it gave. +// +// The distinction that matters is against codes.Unavailable, which means the +// daemon cannot serve the call: that one is worth a warning, because an older +// daemon without the method lands there and the rest of `netbird up` still +// works. A refusal is not, because the settings would be silently dropped. +func refusedSettingsUpdate(err error) (string, bool) { + st, ok := gstatus.FromError(err) + if !ok || st.Code() != codes.FailedPrecondition { + return "", false + } + return st.Message(), true +} + func setupSetConfigReq(customDNSAddressConverted []byte, cmd *cobra.Command, profileName, username string) *proto.SetConfigRequest { var req proto.SetConfigRequest req.ProfileName = profileName diff --git a/client/cmd/up_setconfig_refusal_test.go b/client/cmd/up_setconfig_refusal_test.go new file mode 100644 index 000000000..fdf580102 --- /dev/null +++ b/client/cmd/up_setconfig_refusal_test.go @@ -0,0 +1,85 @@ +package cmd + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + gstatus "google.golang.org/grpc/status" +) + +// A refused settings update has to fail `netbird up`, or a caller that asked +// for a setting the daemon will not apply connects as if it had been applied. +// The daemon being unable to serve the call is the case that stays a warning. +func TestRefusedSettingsUpdate(t *testing.T) { + tests := []struct { + name string + err error + wantRefused bool + }{ + { + name: "the kill switch refused the change", + err: gstatus.Errorf(codes.FailedPrecondition, "update settings are disabled, you cannot use this feature without update settings enabled"), + wantRefused: true, + }, + { + name: "an MDM policy manages the field", + err: gstatus.Errorf(codes.FailedPrecondition, "fields managed by MDM policy: managementURL"), + wantRefused: true, + }, + { + name: "the daemon cannot serve the call", + err: gstatus.Errorf(codes.Unavailable, "connection refused"), + wantRefused: false, + }, + { + name: "any other RPC failure", + err: gstatus.Errorf(codes.Internal, "boom"), + wantRefused: false, + }, + { + name: "not a status error at all", + err: errors.New("boom"), + wantRefused: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reason, refused := refusedSettingsUpdate(tt.err) + require.Equal(t, tt.wantRefused, refused) + if tt.wantRefused { + require.Equal(t, gstatus.Convert(tt.err).Message(), reason, "the daemon's reason must reach the caller") + } + }) + } +} + +// Both `netbird up` and `netbird login` drive Login through the backoff cycle, +// and a final answer has to stop it: retrying a refusal only replaces the +// daemon's reason with "login backoff cycle failed" thirty seconds later. +func TestTerminalLoginError(t *testing.T) { + tests := []struct { + name string + err error + wantTerminal bool + }{ + {name: "settings refused by the kill switch", err: gstatus.Errorf(codes.FailedPrecondition, "update settings are disabled"), wantTerminal: true}, + {name: "field managed by MDM", err: gstatus.Errorf(codes.FailedPrecondition, "fields managed by MDM policy: managementURL"), wantTerminal: true}, + {name: "caller not allowed", err: gstatus.Errorf(codes.PermissionDenied, "nope"), wantTerminal: true}, + {name: "malformed request", err: gstatus.Errorf(codes.InvalidArgument, "nope"), wantTerminal: true}, + {name: "profile not found", err: gstatus.Errorf(codes.NotFound, "nope"), wantTerminal: true}, + {name: "method missing on an older daemon", err: gstatus.Errorf(codes.Unimplemented, "nope"), wantTerminal: true}, + {name: "daemon unreachable, worth retrying", err: gstatus.Errorf(codes.Unavailable, "connection refused"), wantTerminal: false}, + {name: "transient internal failure", err: gstatus.Errorf(codes.Internal, "boom"), wantTerminal: false}, + {name: "not a status error", err: errors.New("boom"), wantTerminal: false}, + {name: "no error at all, the login succeeded", err: nil, wantTerminal: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.wantTerminal, terminalLoginError(tt.err)) + }) + } +} diff --git a/client/collect-licenses.sh b/client/collect-licenses.sh deleted file mode 100644 index 7dfabada9..000000000 --- a/client/collect-licenses.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/sh -set -eu - -if [ "$#" -lt 2 ]; then - printf '%s\n' "usage: $0 OUTPUT_DIRECTORY GOARCH..." >&2 - exit 2 -fi - -repo_root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -output_name=$(basename "$1") -if [ -z "$output_name" ] || [ "$output_name" = "." ] || - [ "$output_name" = ".." ] || [ "$output_name" = "/" ]; then - printf '%s\n' "OUTPUT_DIRECTORY must name a directory" >&2 - exit 2 -fi -output_parent=$(CDPATH= cd -- "$(dirname "$1")" && pwd) -output="$output_parent/$output_name" -shift -modules=$(mktemp "${TMPDIR:-/tmp}/netbird-client-licenses.modules.XXXXXX") -sorted_modules=$(mktemp "${TMPDIR:-/tmp}/netbird-client-licenses.sorted.XXXXXX") -trap 'rm -f "$modules" "$sorted_modules"' EXIT HUP INT TERM - -if [ -e "$output" ] || [ -L "$output" ]; then - printf 'output directory already exists: %s\n' "$output" >&2 - exit 1 -fi -mkdir "$output" -mkdir "$output/third_party" - -cp "$repo_root/LICENSE" "$output/BSD-3-Clause.txt" - -cd "$repo_root" -for arch in "$@"; do - GOOS=${GOOS:-linux} GOARCH="$arch" CGO_ENABLED=${CGO_ENABLED:-0} \ - go list -deps -f '{{with .Module}}{{if .Replace}}{{.Replace.Path}}{{"\t"}}{{.Replace.Version}}{{"\t"}}{{.Replace.Dir}}{{else}}{{.Path}}{{"\t"}}{{.Version}}{{"\t"}}{{.Dir}}{{end}}{{end}}' -tags load_wgnt_from_rsrc ./client >>"$modules" -done -LC_ALL=C sort -u "$modules" >"$sorted_modules" - -goroot=$(go env GOROOT) -for term in LICENSE PATENTS; do - if [ ! -f "$goroot/$term" ]; then - printf 'missing Go standard-library term: %s\n' "$goroot/$term" >&2 - exit 1 - fi - cp "$goroot/$term" "$output/Go-$term" -done - -while IFS=' ' read -r module version module_dir; do - [ -n "$module" ] || continue - [ "$module" = "github.com/netbirdio/netbird" ] && continue - - if [ -z "$version" ] || [ ! -d "$module_dir" ]; then - printf 'cannot collect terms for module %s at version %s\n' "$module" "$version" >&2 - exit 1 - fi - - destination="$output/third_party/$module/$version" - mkdir -p "$destination" - printf 'module: %s\nversion: %s\n' "$module" "$version" >"$destination/MODULE" - - found=false - for term in \ - "$module_dir"/LICENSE* "$module_dir"/License* "$module_dir"/license* \ - "$module_dir"/LICENCE* "$module_dir"/Licence* "$module_dir"/licence* \ - "$module_dir"/COPYING* "$module_dir"/Copying* "$module_dir"/copying* \ - "$module_dir"/NOTICE* "$module_dir"/Notice* "$module_dir"/notice* \ - "$module_dir"/PATENTS* "$module_dir"/Patents* "$module_dir"/patents*; do - [ -f "$term" ] || continue - cp "$term" "$destination/" - found=true - done - - if [ "$found" = false ]; then - printf 'no root license terms found for module %s at %s\n' "$module" "$module_dir" >&2 - exit 1 - fi -done <"$sorted_modules" diff --git a/client/embed/embed.go b/client/embed/embed.go index 5a3d540ec..6fdddab48 100644 --- a/client/embed/embed.go +++ b/client/embed/embed.go @@ -54,6 +54,9 @@ type Client struct { jwtToken string connect *internal.ConnectClient recorder *peer.Status + // inMemoryState is set when no StatePath was given in netstack mode, + // where the client changes nothing on the host that needs restoring. + inMemoryState bool } // Options configures a new Client. @@ -78,7 +81,8 @@ type Options struct { NoUserspace bool // ConfigPath is the path to the netbird config file. If empty, the config will be stored in memory and not persisted. ConfigPath string - // StatePath is the path to the netbird state file + // StatePath is the path to the netbird state file. When empty in netstack + // mode, state is kept in memory; with NoUserspace the default path is used. StatePath string // DisableClientRoutes disables the client routes DisableClientRoutes bool @@ -193,7 +197,6 @@ func New(opts Options) (*Client, error) { } if opts.StatePath != "" { - // TODO: Disable state if path not provided if err := os.Setenv("NB_DNS_STATE_FILE", opts.StatePath); err != nil { return nil, fmt.Errorf("setenv: %w", err) } @@ -261,6 +264,9 @@ func New(opts Options) (*Client, error) { jwtToken: opts.JWTToken, config: config, recorder: peer.NewRecorder(config.ManagementURL.String()), + // Without a path the default would be the installed agent's state + // file, which this client must neither read nor overwrite. + inMemoryState: opts.StatePath == "" && !opts.NoUserspace, }, nil } @@ -292,7 +298,11 @@ func (c *Client) Start(startCtx context.Context) error { if err, _ := authClient.Login(ctx, c.setupKey, c.jwtToken); err != nil { return fmt.Errorf("login: %w", err) } - client := internal.NewConnectClient(ctx, c.config, c.recorder) + var connectOpts []internal.ConnectClientOption + if c.inMemoryState { + connectOpts = append(connectOpts, internal.WithInMemoryState()) + } + client := internal.NewConnectClient(ctx, c.config, c.recorder, connectOpts...) client.SetSyncResponsePersistence(true) // either startup error (permanent backoff err) or nil err (successful engine up) diff --git a/client/embed/embed_test.go b/client/embed/embed_test.go index 4ff5c9978..a818af055 100644 --- a/client/embed/embed_test.go +++ b/client/embed/embed_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "go.uber.org/mock/gomock" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "google.golang.org/grpc" "github.com/netbirdio/netbird/management/internals/controllers/network_map/controller" @@ -21,7 +21,6 @@ import ( nbcache "github.com/netbirdio/netbird/management/server/cache" "github.com/netbirdio/netbird/management/server/groups" "github.com/netbirdio/netbird/management/server/integrations/integrated_validator/validator" - "github.com/netbirdio/netbird/management/server/integrations/port_forwarding" "github.com/netbirdio/netbird/management/server/job" "github.com/netbirdio/netbird/management/server/permissions" "github.com/netbirdio/netbird/management/server/settings" @@ -146,8 +145,8 @@ func startManagement(t *testing.T, signalAddr string) string { updateManager := update_channel.NewPeersUpdateManager(metrics) requestBuffer := mgmt.NewAccountRequestBuffer(context.Background(), testStore) - networkMapController := controller.NewController(context.Background(), testStore, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.selfhosted", port_forwarding.NewControllerMock(), manager.NewEphemeralManager(testStore, peersManager), cfg, nil) - accountManager, err := mgmt.BuildManager(context.Background(), cfg, testStore, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, port_forwarding.NewControllerMock(), settingsMockManager, permissionsManager, false, cacheStore) + networkMapController := controller.NewController(context.Background(), testStore, metrics, updateManager, requestBuffer, mgmt.MockIntegratedValidator{}, settingsMockManager, "netbird.selfhosted", manager.NewEphemeralManager(testStore, peersManager), cfg, nil) + accountManager, err := mgmt.BuildManager(context.Background(), cfg, testStore, networkMapController, jobManager, nil, "", eventStore, nil, false, iv, metrics, settingsMockManager, permissionsManager, false, cacheStore) require.NoError(t, err) secretsManager, err := nbgrpc.NewTimeBasedAuthSecretsManager(updateManager, cfg.TURNConfig, cfg.Relay, settingsMockManager, groupsManager) diff --git a/client/firewall/iptables/dnat_linux.go b/client/firewall/iptables/dnat_linux.go index eca8386c0..f118c9dfe 100644 --- a/client/firewall/iptables/dnat_linux.go +++ b/client/firewall/iptables/dnat_linux.go @@ -8,177 +8,11 @@ import ( "strconv" "strings" - "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" - nberrors "github.com/netbirdio/netbird/client/errors" firewall "github.com/netbirdio/netbird/client/firewall/manager" ) -func (r *family) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - ruleID := rule.ID() - if _, exists := r.rules[ruleID+dnatSuffix]; exists { - return rule, nil - } - - toDestination := rule.TranslatedAddress.String() - switch { - case len(rule.TranslatedPort.Values) == 0: - // no translated port, use original port - case len(rule.TranslatedPort.Values) == 1: - toDestination += fmt.Sprintf(":%d", rule.TranslatedPort.Values[0]) - case rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2: - // need the "/originalport" suffix to avoid dnat port randomization - toDestination += fmt.Sprintf(":%d-%d/%d", rule.TranslatedPort.Values[0], rule.TranslatedPort.Values[1], rule.DestinationPort.Values[0]) - default: - return nil, fmt.Errorf("invalid translated port: %v", rule.TranslatedPort) - } - - proto := strings.ToLower(string(rule.Protocol)) - - rules := make(map[firewall.RuleID]ruleInfo, 3) - - // DNAT rule - dnatRule := []string{ - "!", "-i", r.wgIface.Name(), - "-p", proto, - "-j", "DNAT", - "--to-destination", toDestination, - } - dnatRule = append(dnatRule, applyPort("--dport", &rule.DestinationPort)...) - rules[ruleID+dnatSuffix] = ruleInfo{ - table: tableNat, - chain: chainRTRdr, - rule: dnatRule, - } - - // SNAT rule - snatRule := []string{ - "-o", r.wgIface.Name(), - "-p", proto, - "-d", rule.TranslatedAddress.String(), - "-j", "MASQUERADE", - } - snatRule = append(snatRule, applyPort("--dport", &rule.TranslatedPort)...) - rules[ruleID+snatSuffix] = ruleInfo{ - table: tableNat, - chain: chainRTNAT, - rule: snatRule, - } - - // Forward filtering rule, if fwd policy is DROP - forwardRule := []string{ - "-o", r.wgIface.Name(), - "-p", proto, - "-d", rule.TranslatedAddress.String(), - "-j", "ACCEPT", - } - forwardRule = append(forwardRule, applyPort("--dport", &rule.TranslatedPort)...) - rules[ruleID+fwdSuffix] = ruleInfo{ - table: tableFilter, - chain: chainRTFwdOut, - rule: forwardRule, - } - - for key, ruleInfo := range rules { - if err := r.iptablesClient.Append(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { - r.cleanupFailedDNATAdd(rules) - return nil, fmt.Errorf("add rule %s: %w", key, err) - } - r.rules[key] = ruleInfo.rule - } - - if err := r.ipFwdState.RequestForwarding(r.v6); err != nil { - r.cleanupFailedDNATAdd(rules) - return nil, fmt.Errorf("enable forwarding: %w", err) - } - - r.updateState() - return rule, nil -} - -// cleanupFailedDNATAdd removes the bookkeeping written by a partially applied -// AddDNATRule before rolling back the kernel rules, so no entries remain that -// never got a forwarding refcount. rollbackRules re-adds entries it failed to -// remove from the kernel. -func (r *family) cleanupFailedDNATAdd(rules map[firewall.RuleID]ruleInfo) { - for key := range rules { - delete(r.rules, key) - } - if err := r.rollbackRules(rules); err != nil { - log.Errorf("rollback failed: %v", err) - } -} - -func (r *family) rollbackRules(rules map[firewall.RuleID]ruleInfo) error { - var merr *multierror.Error - for key, ruleInfo := range rules { - if err := r.iptablesClient.DeleteIfExists(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("rollback rule %s: %w", key, err)) - // On rollback error, add to rules map for next cleanup - r.rules[key] = ruleInfo.rule - } - } - if merr != nil { - r.updateState() - } - return nberrors.FormatErrorOrNil(merr) -} - -func (r *family) DeleteDNATRule(rule firewall.Rule) error { - ruleID := rule.ID() - - _, hadDNAT := r.rules[ruleID+dnatSuffix] - _, hadSNAT := r.rules[ruleID+snatSuffix] - _, hadFWD := r.rules[ruleID+fwdSuffix] - if !hadDNAT && !hadSNAT && !hadFWD { - return nil - } - - var merr *multierror.Error - if dnatRule, exists := r.rules[ruleID+dnatSuffix]; exists { - if err := r.iptablesClient.Delete(tableNat, chainRTRdr, dnatRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete DNAT rule: %w", err)) - } else { - delete(r.rules, ruleID+dnatSuffix) - } - } - - if snatRule, exists := r.rules[ruleID+snatSuffix]; exists { - if err := r.iptablesClient.Delete(tableNat, chainRTNAT, snatRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete SNAT rule: %w", err)) - } else { - delete(r.rules, ruleID+snatSuffix) - } - } - - if fwdRule, exists := r.rules[ruleID+fwdSuffix]; exists { - if err := r.iptablesClient.Delete(tableFilter, chainRTFwdOut, fwdRule...); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete forward rule: %w", err)) - } else { - delete(r.rules, ruleID+fwdSuffix) - } - } - - // Release the refcount only once all rules are gone from the kernel. On - // partial failure the failed entries stay in r.rules so a retry can remove - // them and release then. - if merr == nil { - r.releaseForwarding() - } - - r.updateState() - - return nberrors.FormatErrorOrNil(merr) -} - -// releaseForwarding drops one IP forwarding reference, logging any error. -func (r *family) releaseForwarding() { - if err := r.ipFwdState.ReleaseForwarding(r.v6); err != nil { - log.Errorf("release IP forwarding: %v", err) - } -} - func (r *family) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) diff --git a/client/firewall/iptables/dnat_refcount_linux_test.go b/client/firewall/iptables/dnat_refcount_linux_test.go deleted file mode 100644 index 40ebc6cc3..000000000 --- a/client/firewall/iptables/dnat_refcount_linux_test.go +++ /dev/null @@ -1,240 +0,0 @@ -//go:build privileged - -package iptables - -import ( - "net/netip" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - fw "github.com/netbirdio/netbird/client/firewall/manager" - "github.com/netbirdio/netbird/client/iface" - "github.com/netbirdio/netbird/client/iface/wgaddr" -) - -func iptRefcountIfaceV4() *iFaceMock { - return &iFaceMock{ - NameFunc: func() string { return "wt-refcount" }, - AddressFunc: func() wgaddr.Address { - return wgaddr.Address{ - IP: netip.MustParseAddr("10.20.0.1"), - Network: netip.MustParsePrefix("10.20.0.0/24"), - } - }, - } -} - -func iptRefcountIfaceDual() *iFaceMock { - return &iFaceMock{ - NameFunc: func() string { return "wt-refcount" }, - AddressFunc: func() wgaddr.Address { - return wgaddr.Address{ - IP: netip.MustParseAddr("10.20.0.1"), - Network: netip.MustParsePrefix("10.20.0.0/24"), - IPv6: netip.MustParseAddr("fd00::1"), - IPv6Net: netip.MustParsePrefix("fd00::/64"), - } - }, - } -} - -func newIptRefcountManager(t *testing.T, dual bool) *Manager { - t.Helper() - var ifMock *iFaceMock - if dual { - ifMock = iptRefcountIfaceDual() - } else { - ifMock = iptRefcountIfaceV4() - } - m, err := Create(ifMock, iface.DefaultMTU) - require.NoError(t, err, "create manager") - require.NoError(t, m.Init(nil), "init manager") - t.Cleanup(func() { - require.NoError(t, m.Close(nil), "close manager") - }) - return m -} - -func iptDnatV4(port uint16) fw.ForwardRule { - return fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{port}}, - TranslatedAddress: netip.MustParseAddr("10.20.0.2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - } -} - -func iptDnatV6(port uint16) fw.ForwardRule { - return fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{port}}, - TranslatedAddress: netip.MustParseAddr("fd00::2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - } -} - -// TestIptablesRouting_RepeatedEnableSingleReference verifies that EnableRouting -// (called on every network-map update) holds at most one reference per family -// and a single DisableRouting drops both back to zero. -func TestIptablesRouting_RepeatedEnableSingleReference(t *testing.T) { - m := newIptRefcountManager(t, true) - state := m.family4.ipFwdState - - require.NoError(t, m.EnableRouting(), "first enable") - require.NoError(t, m.EnableRouting(), "second enable") - require.NoError(t, m.EnableRouting(), "third enable") - v4, v6 := state.Counts() - assert.Equal(t, 1, v4, "repeated enable holds a single v4 reference") - assert.Equal(t, 1, v6, "repeated enable holds a single v6 reference") - - require.NoError(t, m.DisableRouting(), "disable") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "single disable releases the v4 reference") - assert.Equal(t, 0, v6, "single disable releases the v6 reference") -} - -// TestIptablesRouting_DisableKeepsDNATReference verifies that an unpaired -// DisableRouting does not release references held by active DNAT rules. -func TestIptablesRouting_DisableKeepsDNATReference(t *testing.T) { - m := newIptRefcountManager(t, true) - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(iptDnatV6(9095)) - require.NoError(t, err, "add v6 dnat") - - require.NoError(t, m.DisableRouting(), "unpaired disable") - _, v6 := state.Counts() - assert.Equal(t, 1, v6, "DNAT-held reference survives unpaired DisableRouting") - - require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat") - _, v6 = state.Counts() - assert.Equal(t, 0, v6, "delete releases the DNAT reference") -} - -// TestIptablesDNAT_RefcountBalancedV4 covers a Balanced Add/Delete pair on v4. -func TestIptablesDNAT_RefcountBalancedV4(t *testing.T) { - m := newIptRefcountManager(t, false) - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(iptDnatV4(7081)) - require.NoError(t, err, "add v4 dnat 1") - v4, v6 := state.Counts() - assert.Equal(t, 1, v4, "v4 refcount after first add") - assert.Equal(t, 0, v6, "v6 refcount unchanged") - - r2, err := m.AddDNATRule(iptDnatV4(7082)) - require.NoError(t, err, "add v4 dnat 2") - v4, v6 = state.Counts() - assert.Equal(t, 2, v4, "v4 refcount after second add") - assert.Equal(t, 0, v6, "v6 refcount unchanged") - - require.NoError(t, m.DeleteDNATRule(r1)) - v4, v6 = state.Counts() - assert.Equal(t, 1, v4, "v4 refcount after first delete") - assert.Equal(t, 0, v6, "v6 refcount unchanged") - - require.NoError(t, m.DeleteDNATRule(r2)) - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "v4 refcount after second delete") - assert.Equal(t, 0, v6, "v6 refcount unchanged") -} - -// TestIptablesDNAT_RefcountBalancedV6 checks the v6 path increments v6 only and -// decrements back to zero. -func TestIptablesDNAT_RefcountBalancedV6(t *testing.T) { - m := newIptRefcountManager(t, true) - require.NotNil(t, m.family6, "v6 family") - require.Same(t, m.family4.ipFwdState, m.family6.ipFwdState, "shared state") - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(iptDnatV6(9081)) - require.NoError(t, err, "add v6 dnat 1") - v4, v6 := state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 1, v6, "v6 refcount after first add") - - r2, err := m.AddDNATRule(iptDnatV6(9082)) - require.NoError(t, err, "add v6 dnat 2") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "v4 refcount unchanged") - assert.Equal(t, 2, v6, "v6 refcount after second add") - - require.NoError(t, m.DeleteDNATRule(r1)) - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "v4 refcount unchanged") - assert.Equal(t, 1, v6, "v6 refcount after first delete") - - require.NoError(t, m.DeleteDNATRule(r2)) - v4, v6 = state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 0, v6, "v6 refcount after second delete") -} - -// TestIptablesDNAT_DuplicateAddNoLeak verifies the duplicate-rule path returns -// without bumping the refcount. -func TestIptablesDNAT_DuplicateAddNoLeak(t *testing.T) { - m := newIptRefcountManager(t, true) - state := m.family4.ipFwdState - - rule := iptDnatV4(7083) - r1, err := m.AddDNATRule(rule) - require.NoError(t, err) - v4, _ := state.Counts() - assert.Equal(t, 1, v4) - - _, err = m.AddDNATRule(rule) - require.NoError(t, err, "duplicate add") - v4, _ = state.Counts() - assert.Equal(t, 1, v4, "duplicate add must not increment") - - require.NoError(t, m.DeleteDNATRule(r1)) - v4, _ = state.Counts() - assert.Equal(t, 0, v4, "single delete must drop to zero") -} - -// TestIptablesDNAT_DeleteMissingNoUnderflow verifies Delete on an unknown rule -// neither errors nor releases the refcount. -func TestIptablesDNAT_DeleteMissingNoUnderflow(t *testing.T) { - m := newIptRefcountManager(t, true) - state := m.family4.ipFwdState - - phantom := iptDnatV4(7099) - require.NoError(t, m.DeleteDNATRule(&phantom), "delete missing v4") - v4, v6 := state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 0, v6) - - phantom6 := iptDnatV6(9099) - require.NoError(t, m.DeleteDNATRule(&phantom6), "delete missing v6") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 0, v6) - - r1, err := m.AddDNATRule(iptDnatV4(7100)) - require.NoError(t, err) - v4, _ = state.Counts() - assert.Equal(t, 1, v4, "real add still increments after phantom delete") - require.NoError(t, m.DeleteDNATRule(r1)) -} - -// TestIptablesDNAT_DoubleDeleteNoUnderflow verifies a second Delete on the same -// rule is a no-op. -func TestIptablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) { - m := newIptRefcountManager(t, true) - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(iptDnatV6(9083)) - require.NoError(t, err) - _, v6 := state.Counts() - assert.Equal(t, 1, v6) - - require.NoError(t, m.DeleteDNATRule(r1), "first delete") - _, v6 = state.Counts() - assert.Equal(t, 0, v6) - - require.NoError(t, m.DeleteDNATRule(r1), "second delete must be no-op") - _, v6 = state.Counts() - assert.Equal(t, 0, v6, "double delete must not underflow") -} diff --git a/client/firewall/iptables/family_linux.go b/client/firewall/iptables/family_linux.go index c5ed8cc20..2ac860a0a 100644 --- a/client/firewall/iptables/family_linux.go +++ b/client/firewall/iptables/family_linux.go @@ -24,6 +24,7 @@ const ( tableFilter = "filter" tableNat = "nat" tableMangle = "mangle" + tableRaw = "raw" // chainACLInput is the peer ACL chain that holds installed // peer-filtering rules. @@ -34,6 +35,7 @@ const ( mangleForwardKey chainKey = "MANGLE-FORWARD" chainInput = "INPUT" + chainOutput = "OUTPUT" chainPostrouting = "POSTROUTING" chainPrerouting = "PREROUTING" chainForward = "FORWARD" @@ -54,10 +56,6 @@ const ( markManglePost = "mark-mangle-post" matchSet = "--match-set" - dnatSuffix firewall.RuleID = "_dnat" - snatSuffix firewall.RuleID = "_snat" - fwdSuffix firewall.RuleID = "_fwd" - // ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation. ipv4TCPHeaderSize = 40 // ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation. diff --git a/client/firewall/iptables/filter_linux.go b/client/firewall/iptables/filter_linux.go index dc606da2d..30cd81018 100644 --- a/client/firewall/iptables/filter_linux.go +++ b/client/firewall/iptables/filter_linux.go @@ -81,15 +81,6 @@ func (r *family) hasRule(id nbid.RuleID) bool { return ok } -// hasDNATRule reports whether this family owns the DNAT rule set for -// the given user id. DNAT rules live in r.rules under the well-known -// "_dnat" key; the lookup here is used by Manager.DeleteDNATRule -// to pick the right family. -func (r *family) hasDNATRule(id firewall.RuleID) bool { - _, ok := r.rules[id+dnatSuffix] - return ok -} - // DeleteFilterRule removes a previously installed filter rule. The // rule's stored chain/table identify where to delete from; source set // references are recovered from the spec via findSets and dropped diff --git a/client/firewall/iptables/manager_linux.go b/client/firewall/iptables/manager_linux.go index 49b88f1ea..a566909c8 100644 --- a/client/firewall/iptables/manager_linux.go +++ b/client/firewall/iptables/manager_linux.go @@ -25,9 +25,8 @@ type Manager struct { wgIface iFaceMapper - ipv4Client *iptables.IPTables - family4 *family - rawSupported bool + ipv4Client *iptables.IPTables + family4 *family // IPv6 counterparts, nil when no v6 overlay ipv6Client *iptables.IPTables @@ -108,10 +107,6 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error { return err } - if err := m.initNoTrackChain(); err != nil { - log.Warnf("raw table not available, notrack rules will be disabled: %v", err) - } - // Trust after all fatal init steps so a later failure doesn't leave the // interface in firewalld's trusted zone without a corresponding Close. if err := firewalld.TrustInterface(m.wgIface.Name()); err != nil { @@ -285,10 +280,6 @@ func (m *Manager) Close(stateManager *statemanager.Manager) error { var merr *multierror.Error - if err := m.cleanupNoTrackChain(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("cleanup notrack chain: %w", err)) - } - if m.hasIPv6() { if err := m.family6.Reset(); err != nil { merr = multierror.Append(merr, fmt.Errorf("reset v6 family: %w", err)) @@ -332,31 +323,6 @@ func (m *Manager) DisableRouting() error { return m.family4.ipFwdState.ReleaseRouting() } -// AddDNATRule adds a DNAT rule -func (m *Manager) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - m.mutex.Lock() - defer m.mutex.Unlock() - - if rule.TranslatedAddress.Is6() { - if !m.hasIPv6() { - return nil, fmt.Errorf("add DNAT rule: %w", firewall.ErrIPv6NotInitialized) - } - return m.family6.AddDNATRule(rule) - } - return m.family4.AddDNATRule(rule) -} - -// DeleteDNATRule deletes a DNAT rule -func (m *Manager) DeleteDNATRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if m.hasIPv6() && !m.family4.hasDNATRule(rule.ID()) { - return m.family6.DeleteDNATRule(rule) - } - return m.family4.DeleteDNATRule(rule) -} - // UpdateSet updates the set with the given prefixes func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { m.mutex.Lock() @@ -440,134 +406,6 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } -const ( - chainNameRaw = "NETBIRD-RAW" - chainOutput = "OUTPUT" - tableRaw = "raw" -) - -// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic. -// This prevents conntrack from tracking WireGuard proxy traffic on loopback, which -// can interfere with MASQUERADE rules (e.g., from container runtimes like Podman/netavark). -// -// Traffic flows that need NOTRACK: -// -// 1. Egress: WireGuard -> fake endpoint (before eBPF rewrite) -// src=127.0.0.1:wgPort -> dst=127.0.0.1:fakePort -// Matched by: sport=wgPort -// -// 2. Egress: Proxy -> WireGuard (via raw socket) -// src=127.0.0.1:fakePort -> dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 3. Ingress: Packets to WireGuard -// dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 4. Ingress: Packets to proxy (after eBPF rewrite) -// dst=127.0.0.1:proxyPort -// Matched by: dport=proxyPort -// -// Rules are cleaned up when the firewall manager is closed. -func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if !m.rawSupported { - return fmt.Errorf("raw table not available") - } - - wgPortStr := fmt.Sprintf("%d", wgPort) - proxyPortStr := fmt.Sprintf("%d", proxyPort) - - // Egress rules: match outgoing loopback UDP packets - outputRuleSport := []string{"-o", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--sport", wgPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, outputRuleSport...); err != nil { - return fmt.Errorf("add output sport notrack rule: %w", err) - } - - outputRuleDport := []string{"-o", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, outputRuleDport...); err != nil { - return fmt.Errorf("add output dport notrack rule: %w", err) - } - - // Ingress rules: match incoming loopback UDP packets - preroutingRuleWg := []string{"-i", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, preroutingRuleWg...); err != nil { - return fmt.Errorf("add prerouting wg notrack rule: %w", err) - } - - preroutingRuleProxy := []string{"-i", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", proxyPortStr, "-j", "NOTRACK"} - if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, preroutingRuleProxy...); err != nil { - return fmt.Errorf("add prerouting proxy notrack rule: %w", err) - } - - log.Debugf("set up ebpf proxy notrack rules for ports %d,%d", proxyPort, wgPort) - return nil -} - -func (m *Manager) initNoTrackChain() error { - if err := m.cleanupNoTrackChain(); err != nil { - log.Debugf("cleanup notrack chain: %v", err) - } - - if err := m.ipv4Client.NewChain(tableRaw, chainNameRaw); err != nil { - return fmt.Errorf("create chain: %w", err) - } - - jumpRule := []string{"-j", chainNameRaw} - - if err := m.ipv4Client.InsertUnique(tableRaw, chainOutput, 1, jumpRule...); err != nil { - if delErr := m.ipv4Client.DeleteChain(tableRaw, chainNameRaw); delErr != nil { - log.Debugf("delete orphan chain: %v", delErr) - } - return fmt.Errorf("add output jump rule: %w", err) - } - - if err := m.ipv4Client.InsertUnique(tableRaw, chainPrerouting, 1, jumpRule...); err != nil { - if delErr := m.ipv4Client.DeleteIfExists(tableRaw, chainOutput, jumpRule...); delErr != nil { - log.Debugf("delete output jump rule: %v", delErr) - } - if delErr := m.ipv4Client.DeleteChain(tableRaw, chainNameRaw); delErr != nil { - log.Debugf("delete orphan chain: %v", delErr) - } - return fmt.Errorf("add prerouting jump rule: %w", err) - } - - m.rawSupported = true - return nil -} - -func (m *Manager) cleanupNoTrackChain() error { - exists, err := m.ipv4Client.ChainExists(tableRaw, chainNameRaw) - if err != nil { - if !m.rawSupported { - return nil - } - return fmt.Errorf("check chain exists: %w", err) - } - if !exists { - return nil - } - - jumpRule := []string{"-j", chainNameRaw} - - if err := m.ipv4Client.DeleteIfExists(tableRaw, chainOutput, jumpRule...); err != nil { - return fmt.Errorf("remove output jump rule: %w", err) - } - - if err := m.ipv4Client.DeleteIfExists(tableRaw, chainPrerouting, jumpRule...); err != nil { - return fmt.Errorf("remove prerouting jump rule: %w", err) - } - - if err := m.ipv4Client.ClearAndDeleteChain(tableRaw, chainNameRaw); err != nil { - return fmt.Errorf("clear and delete chain: %w", err) - } - - m.rawSupported = false - return nil -} - func getConntrackEstablished() []string { return []string{"-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"} } diff --git a/client/firewall/iptables/manager_linux_test.go b/client/firewall/iptables/manager_linux_test.go index 9f53352e1..8435bf6a5 100644 --- a/client/firewall/iptables/manager_linux_test.go +++ b/client/firewall/iptables/manager_linux_test.go @@ -497,16 +497,6 @@ func TestIptablesCloseRemovesAllState(t *testing.T) { require.NoError(t, manager.AddNatRule(pair), "add nat rule") require.NoError(t, manager.EnableRouting(), "enable routing") - // A DNAT redirect, which also holds a forwarding reference. - dnat := fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{8080}}, - TranslatedAddress: netip.MustParseAddr("10.20.0.44"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - } - _, err = manager.AddDNATRule(dnat) - require.NoError(t, err, "add dnat rule") - require.NotEqual(t, before, snapshotIptables(t, ipv4Client), "the manager must have installed state") // Everything above stays in place, so Close is what has to remove it. diff --git a/client/firewall/manager/firewall.go b/client/firewall/manager/firewall.go index 97a94d0f5..f8de1e2b5 100644 --- a/client/firewall/manager/firewall.go +++ b/client/firewall/manager/firewall.go @@ -172,12 +172,6 @@ type Manager interface { DisableRouting() error - // AddDNATRule adds outbound DNAT rule for forwarding external traffic to the NetBird network. - AddDNATRule(ForwardRule) (Rule, error) - - // DeleteDNATRule deletes the outbound DNAT rule. - DeleteDNATRule(Rule) error - // UpdateSet updates the set with the given prefixes UpdateSet(hash Set, prefixes []netip.Prefix) error @@ -192,10 +186,6 @@ type Manager interface { // RemoveOutputDNAT removes an OUTPUT chain DNAT rule. RemoveOutputDNAT(localAddr netip.Addr, protocol Protocol, originalPort, translatedPort uint16) error - - // SetupEBPFProxyNoTrack creates static notrack rules for eBPF proxy loopback traffic. - // This prevents conntrack from interfering with WireGuard proxy communication. - SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error } // GenKey builds the rule id for this pair from the given format. diff --git a/client/firewall/manager/forward_rule.go b/client/firewall/manager/forward_rule.go deleted file mode 100644 index c2e9e5c60..000000000 --- a/client/firewall/manager/forward_rule.go +++ /dev/null @@ -1,27 +0,0 @@ -package manager - -import ( - "fmt" - "net/netip" -) - -// ForwardRule todo figure out better place to this to avoid circular imports -type ForwardRule struct { - Protocol Protocol - DestinationPort Port - TranslatedAddress netip.Addr - TranslatedPort Port -} - -func (r ForwardRule) ID() RuleID { - id := fmt.Sprintf("%s;%s;%s;%s", - r.Protocol, - r.DestinationPort.String(), - r.TranslatedAddress.String(), - r.TranslatedPort.String()) - return RuleID(id) -} - -func (r ForwardRule) String() string { - return fmt.Sprintf("protocol: %s, destinationPort: %s, translatedAddress: %s, translatedPort: %s", r.Protocol, r.DestinationPort.String(), r.TranslatedAddress.String(), r.TranslatedPort.String()) -} diff --git a/client/firewall/nftables/dnat_linux.go b/client/firewall/nftables/dnat_linux.go index 8eae694a2..c179d60cc 100644 --- a/client/firewall/nftables/dnat_linux.go +++ b/client/firewall/nftables/dnat_linux.go @@ -9,332 +9,11 @@ import ( "github.com/google/nftables" "github.com/google/nftables/binaryutil" "github.com/google/nftables/expr" - "github.com/google/nftables/xt" - "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" - nberrors "github.com/netbirdio/netbird/client/errors" firewall "github.com/netbirdio/netbird/client/firewall/manager" ) -func (r *family) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - ruleID := rule.ID() - if _, exists := r.rules[ruleID+dnatSuffix]; exists { - return rule, nil - } - - protoNum, err := r.af.protoNum(rule.Protocol) - if err != nil { - return nil, fmt.Errorf("convert protocol to number: %w", err) - } - - // Request forwarding before queueing rules: addDnatRedirect/addDnatMasq - // buffer netlink messages on r.conn that the next caller's Flush would - // commit if we returned without flushing them ourselves. - if err := r.ipFwdState.RequestForwarding(r.isV6()); err != nil { - return nil, fmt.Errorf("enable forwarding: %w", err) - } - - if err := r.addDnatRedirect(rule, protoNum, ruleID); err != nil { - r.releaseForwarding() - return nil, err - } - - if err := r.addDnatMasq(rule, protoNum, ruleID); err != nil { - r.releaseForwarding() - delete(r.rules, ruleID+dnatSuffix) - return nil, err - } - - // Unlike iptables, there's no point in adding "out" rules in the forward chain here as our policy is ACCEPT. - // To overcome DROP policies in other chains, we'd have to add rules to the chains there. - // We also cannot just add "oif accept" there and filter in our own table as we don't know what is supposed to be allowed. - // TODO: find chains with drop policies and add rules there - - if err := r.conn.Flush(); err != nil { - r.releaseForwarding() - delete(r.rules, ruleID+dnatSuffix) - delete(r.rules, ruleID+snatSuffix) - return nil, fmt.Errorf("flush rules: %w", err) - } - - return &rule, nil -} - -func (r *family) addDnatRedirect(rule firewall.ForwardRule, protoNum uint8, ruleID firewall.RuleID) error { - dnatExprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpNeq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{protoNum}, - }, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseTransportHeader, - Offset: 2, - Len: 2, - }, - } - portExprs, err := r.applyPort(&rule.DestinationPort, false) - if err != nil { - return fmt.Errorf("apply destination port: %w", err) - } - dnatExprs = append(dnatExprs, portExprs...) - - // shifted translated port is not supported in nftables, so we hand this over to xtables - if rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2 { - if rule.TranslatedPort.Values[0] != rule.DestinationPort.Values[0] || - rule.TranslatedPort.Values[1] != rule.DestinationPort.Values[1] { - return r.addXTablesRedirect(dnatExprs, ruleID, rule) - } - } - - additionalExprs, regProtoMin, regProtoMax, err := r.handleTranslatedPort(rule) - if err != nil { - return err - } - dnatExprs = append(dnatExprs, additionalExprs...) - - dnatExprs = append(dnatExprs, - &expr.NAT{ - Type: expr.NATTypeDestNAT, - Family: uint32(r.af.tableFamily), - RegAddrMin: 1, - RegProtoMin: regProtoMin, - RegProtoMax: regProtoMax, - }, - ) - - dnatRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingRdr], - Exprs: dnatExprs, - UserData: []byte(ruleID + dnatSuffix), - } - r.conn.AddRule(dnatRule) - r.rules[ruleID+dnatSuffix] = dnatRule - - return nil -} - -func (r *family) handleTranslatedPort(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - switch { - case rule.TranslatedPort.IsRange && len(rule.TranslatedPort.Values) == 2: - return r.handlePortRange(rule) - case len(rule.TranslatedPort.Values) == 0: - return r.handleAddressOnly(rule) - case len(rule.TranslatedPort.Values) == 1: - return r.handleSinglePort(rule) - default: - return nil, 0, 0, fmt.Errorf("invalid translated port: %v", rule.TranslatedPort) - } -} - -func (r *family) handlePortRange(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - exprs := []expr.Any{ - &expr.Immediate{ - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - &expr.Immediate{ - Register: 2, - Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[0]), - }, - &expr.Immediate{ - Register: 3, - Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[1]), - }, - } - return exprs, 2, 3, nil -} - -func (r *family) handleAddressOnly(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - exprs := []expr.Any{ - &expr.Immediate{ - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - } - return exprs, 0, 0, nil -} - -func (r *family) handleSinglePort(rule firewall.ForwardRule) ([]expr.Any, uint32, uint32, error) { - exprs := []expr.Any{ - &expr.Immediate{ - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - &expr.Immediate{ - Register: 2, - Data: binaryutil.BigEndian.PutUint16(rule.TranslatedPort.Values[0]), - }, - } - return exprs, 2, 0, nil -} - -func (r *family) addXTablesRedirect(dnatExprs []expr.Any, ruleID firewall.RuleID, rule firewall.ForwardRule) error { - dnatExprs = append(dnatExprs, - &expr.Counter{}, - &expr.Target{ - Name: "DNAT", - Rev: 2, - Info: &xt.NatRange2{ - NatRange: xt.NatRange{ - Flags: uint(xt.NatRangeMapIPs | xt.NatRangeProtoSpecified | xt.NatRangeProtoOffset), - MinIP: rule.TranslatedAddress.AsSlice(), - MaxIP: rule.TranslatedAddress.AsSlice(), - MinPort: rule.TranslatedPort.Values[0], - MaxPort: rule.TranslatedPort.Values[1], - }, - BasePort: rule.DestinationPort.Values[0], - }, - }, - ) - - natTable := &nftables.Table{ - Name: tableNat, - Family: r.af.tableFamily, - } - dnatRule := &nftables.Rule{ - Table: natTable, - Chain: &nftables.Chain{ - Name: chainNameNatPrerouting, - Table: natTable, - Type: nftables.ChainTypeNAT, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityNATDest, - }, - Exprs: dnatExprs, - UserData: []byte(ruleID + dnatSuffix), - } - r.conn.AddRule(dnatRule) - r.rules[ruleID+dnatSuffix] = dnatRule - - return nil -} - -func (r *family) addDnatMasq(rule firewall.ForwardRule, protoNum uint8, ruleID firewall.RuleID) error { - portExprs, err := r.applyPort(&rule.TranslatedPort, false) - if err != nil { - return fmt.Errorf("apply translated port: %w", err) - } - - masqExprs := []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: ifname(r.wgIface.Name()), - }, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: []byte{protoNum}, - }, - &expr.Payload{ - DestRegister: 1, - Base: expr.PayloadBaseNetworkHeader, - Offset: r.af.dstAddrOffset, - Len: r.af.addrLen, - }, - &expr.Cmp{ - Op: expr.CmpOpEq, - Register: 1, - Data: rule.TranslatedAddress.AsSlice(), - }, - } - - masqExprs = append(masqExprs, portExprs...) - masqExprs = append(masqExprs, &expr.Masq{}) - - masqRule := &nftables.Rule{ - Table: r.workTable, - Chain: r.chains[chainNameRoutingNat], - Exprs: masqExprs, - UserData: []byte(ruleID + snatSuffix), - } - r.conn.AddRule(masqRule) - r.rules[ruleID+snatSuffix] = masqRule - - return nil -} - -func (r *family) DeleteDNATRule(rule firewall.Rule) error { - ruleID := rule.ID() - - if err := r.refreshRulesMap(); err != nil { - return fmt.Errorf(refreshRulesMapError, err) - } - - var merr *multierror.Error - var needsFlush bool - var found bool - - if dnatRule, exists := r.rules[ruleID+dnatSuffix]; exists { - found = true - if dnatRule.Handle == 0 { - log.Warnf("dnat rule %s has no handle, removing stale entry", ruleID+dnatSuffix) - delete(r.rules, ruleID+dnatSuffix) - } else if err := r.conn.DelRule(dnatRule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete dnat rule: %w", err)) - } else { - needsFlush = true - } - } - - if masqRule, exists := r.rules[ruleID+snatSuffix]; exists { - found = true - if masqRule.Handle == 0 { - log.Warnf("snat rule %s has no handle, removing stale entry", ruleID+snatSuffix) - delete(r.rules, ruleID+snatSuffix) - } else if err := r.conn.DelRule(masqRule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete snat rule: %w", err)) - } else { - needsFlush = true - } - } - - if needsFlush { - if err := r.conn.Flush(); err != nil { - merr = multierror.Append(merr, fmt.Errorf(flushError, err)) - } - } - - if merr != nil { - return nberrors.FormatErrorOrNil(merr) - } - - delete(r.rules, ruleID+dnatSuffix) - delete(r.rules, ruleID+snatSuffix) - - // Release once, only if the rule was present and removed. - if found { - r.releaseForwarding() - } - - return nil -} - -// releaseForwarding drops one IP forwarding reference, logging any error. -func (r *family) releaseForwarding() { - if err := r.ipFwdState.ReleaseForwarding(r.isV6()); err != nil { - log.Errorf("release IP forwarding: %v", err) - } -} - -// isV6 reports whether this family handles the IPv6 table. -func (r *family) isV6() bool { - return r.af.tableFamily == nftables.TableFamilyIPv6 -} - func (r *family) AddInboundDNAT(localAddr netip.Addr, protocol firewall.Protocol, originalPort, translatedPort uint16) error { ruleID := firewall.RuleID(fmt.Sprintf("inbound-dnat-%s-%s-%d-%d", localAddr.String(), protocol, originalPort, translatedPort)) diff --git a/client/firewall/nftables/dnat_refcount_linux_test.go b/client/firewall/nftables/dnat_refcount_linux_test.go deleted file mode 100644 index cdc24e77f..000000000 --- a/client/firewall/nftables/dnat_refcount_linux_test.go +++ /dev/null @@ -1,249 +0,0 @@ -//go:build privileged - -package nftables - -import ( - "net/netip" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - fw "github.com/netbirdio/netbird/client/firewall/manager" - "github.com/netbirdio/netbird/client/iface" - "github.com/netbirdio/netbird/client/iface/wgaddr" -) - -func nftRefcountIfaceV4() *iFaceMock { - return &iFaceMock{ - NameFunc: func() string { return "wt-refcount" }, - AddressFunc: func() wgaddr.Address { - return wgaddr.Address{ - IP: netip.MustParseAddr("100.96.0.1"), - Network: netip.MustParsePrefix("100.96.0.0/16"), - } - }, - } -} - -func nftRefcountIfaceDual() *iFaceMock { - return &iFaceMock{ - NameFunc: func() string { return "wt-refcount" }, - AddressFunc: func() wgaddr.Address { - return wgaddr.Address{ - IP: netip.MustParseAddr("100.96.0.1"), - Network: netip.MustParsePrefix("100.96.0.0/16"), - IPv6: netip.MustParseAddr("fd00::1"), - IPv6Net: netip.MustParsePrefix("fd00::/64"), - } - }, - } -} - -func newNftRefcountManager(t *testing.T, dual bool) *Manager { - t.Helper() - if check() != NFTABLES { - t.Skip("nftables not supported on this system") - } - var ifMock *iFaceMock - if dual { - ifMock = nftRefcountIfaceDual() - } else { - ifMock = nftRefcountIfaceV4() - } - m, err := Create(ifMock, iface.DefaultMTU) - require.NoError(t, err, "create manager") - require.NoError(t, m.Init(nil), "init manager") - t.Cleanup(func() { - require.NoError(t, m.Close(nil), "close manager") - }) - return m -} - -func dnatV4(port uint16) fw.ForwardRule { - return fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{port}}, - TranslatedAddress: netip.MustParseAddr("100.96.0.2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - } -} - -func dnatV6(port uint16) fw.ForwardRule { - return fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{port}}, - TranslatedAddress: netip.MustParseAddr("fd00::2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - } -} - -// TestNftablesDNAT_RefcountBalancedV4 verifies that Add/Delete pairs leave the -// v4 refcount at zero. -func TestNftablesDNAT_RefcountBalancedV4(t *testing.T) { - m := newNftRefcountManager(t, false) - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(dnatV4(8081)) - require.NoError(t, err, "add v4 dnat 1") - v4, v6 := state.Counts() - assert.Equal(t, 1, v4, "v4 refcount after first add") - assert.Equal(t, 0, v6, "v6 refcount unchanged") - - r2, err := m.AddDNATRule(dnatV4(8082)) - require.NoError(t, err, "add v4 dnat 2") - v4, v6 = state.Counts() - assert.Equal(t, 2, v4, "v4 refcount after second add") - assert.Equal(t, 0, v6, "v6 refcount unchanged") - - require.NoError(t, m.DeleteDNATRule(r1), "delete v4 dnat 1") - v4, v6 = state.Counts() - assert.Equal(t, 1, v4, "v4 refcount after first delete") - assert.Equal(t, 0, v6, "v6 refcount unchanged") - - require.NoError(t, m.DeleteDNATRule(r2), "delete v4 dnat 2") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "v4 refcount after second delete") - assert.Equal(t, 0, v6, "v6 refcount unchanged") -} - -// TestNftablesDNAT_RefcountBalancedV6 verifies the v6 path increments v6 only -// and decrements back to zero on Delete. -func TestNftablesDNAT_RefcountBalancedV6(t *testing.T) { - m := newNftRefcountManager(t, true) - require.NotNil(t, m.family6, "v6 family") - require.Same(t, m.family4.ipFwdState, m.family6.ipFwdState, "shared state") - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(dnatV6(9091)) - require.NoError(t, err, "add v6 dnat 1") - v4, v6 := state.Counts() - assert.Equal(t, 0, v4, "v4 refcount unchanged") - assert.Equal(t, 1, v6, "v6 refcount after first add") - - r2, err := m.AddDNATRule(dnatV6(9092)) - require.NoError(t, err, "add v6 dnat 2") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 2, v6, "v6 refcount after second add") - - require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat 1") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "v4 refcount unchanged") - assert.Equal(t, 1, v6, "v6 refcount after first delete") - - require.NoError(t, m.DeleteDNATRule(r2), "delete v6 dnat 2") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 0, v6, "v6 refcount after second delete") -} - -// TestNftablesDNAT_DuplicateAddNoLeak verifies that a duplicate Add (same -// ForwardRule) does not double-increment the refcount. -func TestNftablesDNAT_DuplicateAddNoLeak(t *testing.T) { - m := newNftRefcountManager(t, true) - state := m.family4.ipFwdState - - rule := dnatV4(8083) - r1, err := m.AddDNATRule(rule) - require.NoError(t, err, "add v4 dnat") - v4, _ := state.Counts() - assert.Equal(t, 1, v4) - - // duplicate add: same rule ID, must be a no-op for the refcount. - _, err = m.AddDNATRule(rule) - require.NoError(t, err, "duplicate add") - v4, _ = state.Counts() - assert.Equal(t, 1, v4, "duplicate add must not increment") - - require.NoError(t, m.DeleteDNATRule(r1), "delete v4 dnat") - v4, _ = state.Counts() - assert.Equal(t, 0, v4, "single delete must drop to zero") -} - -// TestNftablesDNAT_DeleteMissingNoUnderflow verifies deleting a rule that was -// never added does not underflow the refcount. -func TestNftablesDNAT_DeleteMissingNoUnderflow(t *testing.T) { - m := newNftRefcountManager(t, true) - state := m.family4.ipFwdState - - // Construct a Rule reference for something never added. The router stores - // rules by ID(), and DeleteDNATRule looks them up in r.rules; a missing - // entry must be a no-op rather than calling Release. - phantom := dnatV4(8099) - require.NoError(t, m.DeleteDNATRule(&phantom), "delete missing v4 dnat") - v4, v6 := state.Counts() - assert.Equal(t, 0, v4, "v4 refcount unaffected by missing delete") - assert.Equal(t, 0, v6, "v6 refcount unaffected") - - phantom6 := dnatV6(9099) - require.NoError(t, m.DeleteDNATRule(&phantom6), "delete missing v6 dnat") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4) - assert.Equal(t, 0, v6, "v6 refcount unaffected by missing delete") - - // And after a phantom delete, a real add still results in count=1. - r1, err := m.AddDNATRule(dnatV4(8100)) - require.NoError(t, err, "add v4 dnat after phantom delete") - v4, _ = state.Counts() - assert.Equal(t, 1, v4, "real add still increments after phantom delete") - require.NoError(t, m.DeleteDNATRule(r1)) -} - -// TestNftablesRouting_RepeatedEnableSingleReference verifies that EnableRouting -// (called on every network-map update) holds at most one reference per family -// and a single DisableRouting drops both back to zero. -func TestNftablesRouting_RepeatedEnableSingleReference(t *testing.T) { - m := newNftRefcountManager(t, true) - state := m.family4.ipFwdState - - require.NoError(t, m.EnableRouting(), "first enable") - require.NoError(t, m.EnableRouting(), "second enable") - require.NoError(t, m.EnableRouting(), "third enable") - v4, v6 := state.Counts() - assert.Equal(t, 1, v4, "repeated enable holds a single v4 reference") - assert.Equal(t, 1, v6, "repeated enable holds a single v6 reference") - - require.NoError(t, m.DisableRouting(), "disable") - v4, v6 = state.Counts() - assert.Equal(t, 0, v4, "single disable releases the v4 reference") - assert.Equal(t, 0, v6, "single disable releases the v6 reference") -} - -// TestNftablesRouting_DisableKeepsDNATReference verifies that an unpaired -// DisableRouting does not release references held by active DNAT rules. -func TestNftablesRouting_DisableKeepsDNATReference(t *testing.T) { - m := newNftRefcountManager(t, true) - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(dnatV6(9095)) - require.NoError(t, err, "add v6 dnat") - - require.NoError(t, m.DisableRouting(), "unpaired disable") - _, v6 := state.Counts() - assert.Equal(t, 1, v6, "DNAT-held reference survives unpaired DisableRouting") - - require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat") - _, v6 = state.Counts() - assert.Equal(t, 0, v6, "delete releases the DNAT reference") -} - -// TestNftablesDNAT_DoubleDeleteNoUnderflow verifies that deleting the same rule -// twice does not underflow the refcount (the second delete is a no-op). -func TestNftablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) { - m := newNftRefcountManager(t, true) - state := m.family4.ipFwdState - - r1, err := m.AddDNATRule(dnatV6(9093)) - require.NoError(t, err) - _, v6 := state.Counts() - assert.Equal(t, 1, v6) - - require.NoError(t, m.DeleteDNATRule(r1), "first delete") - _, v6 = state.Counts() - assert.Equal(t, 0, v6) - - require.NoError(t, m.DeleteDNATRule(r1), "second delete must be no-op") - _, v6 = state.Counts() - assert.Equal(t, 0, v6, "double delete must not underflow") -} diff --git a/client/firewall/nftables/family_linux.go b/client/firewall/nftables/family_linux.go index 7a5df3ed7..4169c9d2d 100644 --- a/client/firewall/nftables/family_linux.go +++ b/client/firewall/nftables/family_linux.go @@ -24,7 +24,6 @@ const ( tableRaw = "raw" tableSecurity = "security" - chainNameNatPrerouting = "PREROUTING" chainNameRoutingFw = "netbird-rt-fwd" chainNameRoutingNat = "netbird-rt-postrouting" chainNameRoutingRdr = "netbird-rt-redirect" @@ -47,9 +46,6 @@ const ( userDataAcceptForwardRuleOif = "frwacceptoif" userDataAcceptInputRule = "inputaccept" - dnatSuffix firewall.RuleID = "_dnat" - snatSuffix firewall.RuleID = "_snat" - // ipv4TCPHeaderSize is the minimum IPv4 (20) + TCP (20) header size for MSS calculation. ipv4TCPHeaderSize = 40 // ipv6TCPHeaderSize is the minimum IPv6 (40) + TCP (20) header size for MSS calculation. @@ -167,10 +163,6 @@ func (r *family) Reset() error { merr = multierror.Append(merr, err) } - if err := r.removeNatPreroutingRules(); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove filter prerouting rules: %w", err)) - } - return nberrors.FormatErrorOrNil(merr) } diff --git a/client/firewall/nftables/filter_linux.go b/client/firewall/nftables/filter_linux.go index ebd238063..bb3ac1dfe 100644 --- a/client/firewall/nftables/filter_linux.go +++ b/client/firewall/nftables/filter_linux.go @@ -197,11 +197,6 @@ func (r *family) hasRule(id firewall.RuleID) bool { return ok } -func (r *family) hasDNATRule(id firewall.RuleID) bool { - _, ok := r.rules[id+dnatSuffix] - return ok -} - // DeleteFilterRule removes a previously installed filter rule. Source // set references are recovered from the stored rule's expressions via // findSets and dropped from the shared refcounter. diff --git a/client/firewall/nftables/manager_linux.go b/client/firewall/nftables/manager_linux.go index dbd5e4fa2..75405e213 100644 --- a/client/firewall/nftables/manager_linux.go +++ b/client/firewall/nftables/manager_linux.go @@ -12,7 +12,6 @@ import ( "github.com/google/nftables/expr" "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" nberrors "github.com/netbirdio/netbird/client/errors" firewall "github.com/netbirdio/netbird/client/firewall/manager" @@ -55,9 +54,6 @@ type Manager struct { // IPv6 counterpart, nil when no v6 overlay. family6 *family - notrackOutputChain *nftables.Chain - notrackPreroutingChain *nftables.Chain - extMonitor *externalChainMonitor } @@ -170,10 +166,6 @@ func (m *Manager) initFirewall() (err error) { } } - if err := m.initNoTrackChains(workTable); err != nil { - log.Warnf("raw priority chains not available, notrack rules will be disabled: %v", err) - } - return nil } @@ -260,7 +252,7 @@ func (m *Manager) DeleteFilterRule(rule firewall.Rule) error { m.mutex.Lock() defer m.mutex.Unlock() - fam, err := m.familyForRuleID(rule.ID(), (*family).hasRule, false) + fam, err := m.familyForRuleID(rule.ID(), (*family).hasRule) if err != nil { return err } @@ -268,11 +260,8 @@ func (m *Manager) DeleteFilterRule(rule firewall.Rule) error { } // familyForRuleID picks the family holding the rule with the given id, using -// the supplied lookup. With refresh set, a miss in both cached maps reloads -// the NAT/DNAT rule maps from the kernel once and re-checks before falling -// back to the v4 family. Filter rules are tracked only in memory and have no -// kernel-backed reload, so their callers pass refresh as false. -func (m *Manager) familyForRuleID(id firewall.RuleID, has func(*family, firewall.RuleID) bool, refresh bool) (*family, error) { +// the supplied lookup, and falls back to the v4 family on a miss. +func (m *Manager) familyForRuleID(id firewall.RuleID, has func(*family, firewall.RuleID) bool) (*family, error) { if has(m.family4, id) { return m.family4, nil } @@ -282,18 +271,6 @@ func (m *Manager) familyForRuleID(id firewall.RuleID, has func(*family, firewall if has(m.family6, id) { return m.family6, nil } - if !refresh { - return m.family4, nil - } - if err := m.family4.refreshRulesMap(); err != nil { - return nil, fmt.Errorf("refresh v4 rules: %w", err) - } - if err := m.family6.refreshRulesMap(); err != nil { - return nil, fmt.Errorf("refresh v6 rules: %w", err) - } - if has(m.family6, id) && !has(m.family4, id) { - return m.family6, nil - } return m.family4, nil } @@ -455,39 +432,9 @@ func (m *Manager) Flush() error { } } - if err := m.refreshNoTrackChains(); err != nil { - log.Errorf("failed to refresh notrack chains: %v", err) - } - return nil } -// AddDNATRule adds a DNAT rule -func (m *Manager) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - m.mutex.Lock() - defer m.mutex.Unlock() - - if rule.TranslatedAddress.Is6() { - if !m.hasIPv6() { - return nil, fmt.Errorf("add DNAT rule: %w", firewall.ErrIPv6NotInitialized) - } - return m.family6.AddDNATRule(rule) - } - return m.family4.AddDNATRule(rule) -} - -// DeleteDNATRule deletes a DNAT rule -func (m *Manager) DeleteDNATRule(rule firewall.Rule) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - r, err := m.familyForRuleID(rule.ID(), (*family).hasDNATRule, true) - if err != nil { - return err - } - return r.DeleteDNATRule(rule) -} - // UpdateSet updates the set with the given prefixes func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { m.mutex.Lock() @@ -571,176 +518,6 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort) } -const ( - chainNameRawOutput = "netbird-raw-out" - chainNameRawPrerouting = "netbird-raw-pre" -) - -// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic. -// This prevents conntrack from tracking WireGuard proxy traffic on loopback, which -// can interfere with MASQUERADE rules (e.g., from container runtimes like Podman/netavark). -// -// Traffic flows that need NOTRACK: -// -// 1. Egress: WireGuard -> fake endpoint (before eBPF rewrite) -// src=127.0.0.1:wgPort -> dst=127.0.0.1:fakePort -// Matched by: sport=wgPort -// -// 2. Egress: Proxy -> WireGuard (via raw socket) -// src=127.0.0.1:fakePort -> dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 3. Ingress: Packets to WireGuard -// dst=127.0.0.1:wgPort -// Matched by: dport=wgPort -// -// 4. Ingress: Packets to proxy (after eBPF rewrite) -// dst=127.0.0.1:proxyPort -// Matched by: dport=proxyPort -// -// Rules are cleaned up when the firewall manager is closed. -func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if m.notrackOutputChain == nil || m.notrackPreroutingChain == nil { - return fmt.Errorf("notrack chains not initialized") - } - - proxyPortBytes := binaryutil.BigEndian.PutUint16(proxyPort) - wgPortBytes := binaryutil.BigEndian.PutUint16(wgPort) - loopback := []byte{127, 0, 0, 1} - - // Egress rules: match outgoing loopback UDP packets - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackOutputChain.Table, - Chain: m.notrackOutputChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // sport=wgPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackOutputChain.Table, - Chain: m.notrackOutputChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // dport=wgPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - - // Ingress rules: match incoming loopback UDP packets - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackPreroutingChain.Table, - Chain: m.notrackPreroutingChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // dport=wgPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - m.rConn.AddRule(&nftables.Rule{ - Table: m.notrackPreroutingChain.Table, - Chain: m.notrackPreroutingChain, - Exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}}, - &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: proxyPortBytes}, // dport=proxyPort - &expr.Counter{}, - &expr.Notrack{}, - }, - }) - - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf("flush notrack rules: %w", err) - } - - log.Debugf("set up ebpf proxy notrack rules for ports %d,%d", proxyPort, wgPort) - return nil -} - -func (m *Manager) initNoTrackChains(table *nftables.Table) error { - m.notrackOutputChain = m.rConn.AddChain(&nftables.Chain{ - Name: chainNameRawOutput, - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookOutput, - Priority: nftables.ChainPriorityRaw, - }) - - m.notrackPreroutingChain = m.rConn.AddChain(&nftables.Chain{ - Name: chainNameRawPrerouting, - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityRaw, - }) - - if err := m.rConn.Flush(); err != nil { - return fmt.Errorf("flush chain creation: %w", err) - } - - return nil -} - -func (m *Manager) refreshNoTrackChains() error { - chains, err := m.rConn.ListChainsOfTableFamily(nftables.TableFamilyIPv4) - if err != nil { - return fmt.Errorf("list chains: %w", err) - } - - tableName := getTableName() - for _, c := range chains { - if c.Table.Name != tableName { - continue - } - switch c.Name { - case chainNameRawOutput: - m.notrackOutputChain = c - case chainNameRawPrerouting: - m.notrackPreroutingChain = c - } - } - - return nil -} - func (m *Manager) createWorkTable() (*nftables.Table, error) { return m.createWorkTableFamily(nftables.TableFamilyIPv4) } diff --git a/client/firewall/nftables/manager_linux_test.go b/client/firewall/nftables/manager_linux_test.go index 0ca56409e..4d6eec3c1 100644 --- a/client/firewall/nftables/manager_linux_test.go +++ b/client/firewall/nftables/manager_linux_test.go @@ -378,18 +378,6 @@ func TestNftablesManagerCompatibilityWithIptables(t *testing.T) { err = manager.AddNatRule(pair) require.NoError(t, err, "failed to add NAT rule") - dnatRule, err := manager.AddDNATRule(fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{8080}}, - TranslatedAddress: netip.MustParseAddr("100.96.0.2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - }) - require.NoError(t, err, "failed to add DNAT rule") - - t.Cleanup(func() { - require.NoError(t, manager.DeleteDNATRule(dnatRule), "failed to delete DNAT rule") - }) - stdout, stderr = runIptablesSave(t) verifyIptablesOutput(t, stdout, stderr) } @@ -453,18 +441,6 @@ func TestNftablesManagerIPv6CompatibilityWithIp6tables(t *testing.T) { }) require.NoError(t, err, "add v6 NAT rule") - dnatRule, err := manager.AddDNATRule(fw.ForwardRule{ - Protocol: fw.ProtocolTCP, - DestinationPort: fw.Port{Values: []uint16{8080}}, - TranslatedAddress: netip.MustParseAddr("fd00::2"), - TranslatedPort: fw.Port{Values: []uint16{80}}, - }) - require.NoError(t, err, "add v6 DNAT rule") - - t.Cleanup(func() { - require.NoError(t, manager.DeleteDNATRule(dnatRule), "delete v6 DNAT rule") - }) - stdout, stderr := runIptablesSave(t) verifyIptablesOutput(t, stdout, stderr) diff --git a/client/firewall/nftables/routing_linux.go b/client/firewall/nftables/routing_linux.go index 4115c94bd..e98471e8f 100644 --- a/client/firewall/nftables/routing_linux.go +++ b/client/firewall/nftables/routing_linux.go @@ -192,7 +192,7 @@ func (r *family) addPostroutingRules() { Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade), }, - // We need to exclude the loopback interface as this changes the ebpf proxy port + // We need to exclude the loopback interface as this changes the wg proxy port &expr.Meta{ Key: expr.MetaKeyOIFNAME, Register: 1, @@ -459,41 +459,6 @@ func (r *family) RemoveAllLegacyRouteRules() error { return nberrors.FormatErrorOrNil(merr) } -func (r *family) removeNatPreroutingRules() error { - table := &nftables.Table{ - Name: tableNat, - Family: r.af.tableFamily, - } - chain := &nftables.Chain{ - Name: chainNameNatPrerouting, - Table: table, - Hooknum: nftables.ChainHookPrerouting, - Priority: nftables.ChainPriorityNATDest, - Type: nftables.ChainTypeNAT, - } - rules, err := r.conn.GetRules(table, chain) - if err != nil { - return fmt.Errorf("get rules from nat table: %w", err) - } - - var merr *multierror.Error - - // Delete rules that have our UserData suffix - for _, rule := range rules { - if len(rule.UserData) == 0 || !strings.HasSuffix(string(rule.UserData), string(dnatSuffix)) { - continue - } - if err := r.conn.DelRule(rule); err != nil { - merr = multierror.Append(merr, fmt.Errorf("delete rule %s: %w", rule.UserData, err)) - } - } - - if err := r.conn.Flush(); err != nil { - merr = multierror.Append(merr, fmt.Errorf(flushError, err)) - } - return nberrors.FormatErrorOrNil(merr) -} - func (r *family) RemoveNatRule(pair firewall.RouterPair) error { if err := r.refreshRulesMap(); err != nil { return fmt.Errorf(refreshRulesMapError, err) diff --git a/client/firewall/uspfilter/filter.go b/client/firewall/uspfilter/filter.go index 5e1366c1f..0c73400f1 100644 --- a/client/firewall/uspfilter/filter.go +++ b/client/firewall/uspfilter/filter.go @@ -879,12 +879,6 @@ func (m *Manager) resetState() { } } -// SetupEBPFProxyNoTrack is not supported by the userspace firewall: eBPF isn't -// used in userspace mode, so this should never be called. -func (m *Manager) SetupEBPFProxyNoTrack(uint16, uint16) error { - return errNotSupported -} - // UpdateSet updates the rule destinations associated with the given set // by merging the existing prefixes with the new ones, then deduplicating. func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error { diff --git a/client/firewall/uspfilter/nat.go b/client/firewall/uspfilter/nat.go index 06312aabf..49c26766a 100644 --- a/client/firewall/uspfilter/nat.go +++ b/client/firewall/uspfilter/nat.go @@ -486,16 +486,6 @@ func incrementalUpdate(oldChecksum uint16, oldBytes, newBytes []byte) uint16 { return ^uint16(sum) } -// AddDNATRule adds outbound DNAT rule for forwarding external traffic to NetBird network. -func (m *Manager) AddDNATRule(firewall.ForwardRule) (firewall.Rule, error) { - return nil, errNotSupported -} - -// DeleteDNATRule deletes outbound DNAT rule. -func (m *Manager) DeleteDNATRule(firewall.Rule) error { - return errNotSupported -} - // addPortRedirection adds a port redirection rule. func (m *Manager) addPortRedirection(targetIP netip.Addr, protocol gopacket.LayerType, originalPort, translatedPort uint16) error { m.portDNATMutex.Lock() diff --git a/client/grpc/dialer_generic.go b/client/grpc/dialer_generic.go index 737787223..72b3538ba 100644 --- a/client/grpc/dialer_generic.go +++ b/client/grpc/dialer_generic.go @@ -6,12 +6,9 @@ import ( "context" "fmt" "net" - "os/user" + "os" "runtime" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - log "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -45,18 +42,13 @@ func WithSweeper(sweeper Sweeper) grpc.DialOption { } func dialContext(ctx context.Context, addr string) (net.Conn, error) { - if runtime.GOOS == "linux" { - currentUser, err := user.Current() - if err != nil { - return nil, status.Errorf(codes.FailedPrecondition, "failed to get current user: %v", err) - } - - // the custom dialer requires root permissions which are not required for use cases run as non-root - if currentUser.Uid != "0" { - log.Debug("Not running as root, using standard dialer") - dialer := &net.Dialer{} - return dialer.DialContext(ctx, "tcp", addr) - } + // The custom dialer requires root permissions which are not required for + // use cases run as non-root. The effective UID is read directly: a passwd + // lookup fails without cgo for a UID that has no entry, as on OpenShift. + if runtime.GOOS == "linux" && os.Geteuid() != 0 { + log.Debug("Not running as root, using standard dialer") + dialer := &net.Dialer{} + return dialer.DialContext(ctx, "tcp", addr) } conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr) diff --git a/client/iface/configurer/allowedips.go b/client/iface/configurer/allowedips.go index 193197d4a..08a766ee0 100644 --- a/client/iface/configurer/allowedips.go +++ b/client/iface/configurer/allowedips.go @@ -1,3 +1,5 @@ +//go:build (linux && !android) || freebsd + package configurer import ( @@ -165,33 +167,6 @@ func (s *allowedIPStore) releaseLocked(k wgtypes.Key) { } } -// normalizePrefix puts a prefix into the form the store recognises it by. It clears the -// host bits, which a device does on its own, so a caller passing 10.20.0.1/16 still matches -// the 10.20.0.0/16 read back from the device; and it unmaps a v4-mapped prefix so that it -// compares equal to, and marshals like, the plain v4 prefix for the same network. -// -// Masking comes first because it also decides the address family: only a prefix at least 96 -// bits long keeps the mapped marker through the mask, so a shorter prefix inside the mapped -// range is a genuine v6 prefix and unmapping it would yield an invalid v4 prefix. -func normalizePrefix(prefix netip.Prefix) netip.Prefix { - masked := prefix.Masked() - - addr := masked.Addr() - if !addr.Is4In6() { - return masked - } - return netip.PrefixFrom(addr.Unmap(), masked.Bits()-96) -} - -// normalizePrefixes returns a normalized copy without changing the caller's slice. -func normalizePrefixes(prefixes []netip.Prefix) []netip.Prefix { - normalized := make([]netip.Prefix, len(prefixes)) - for i, prefix := range prefixes { - normalized[i] = normalizePrefix(prefix) - } - return normalized -} - // ipNetsToPrefixes converts addresses read back from a device. Unmap keeps a v4-mapped v6 // address comparable to the plain v4 prefix the configurer was given. func ipNetsToPrefixes(ipNets []net.IPNet) []netip.Prefix { diff --git a/client/iface/configurer/allowedips_test.go b/client/iface/configurer/allowedips_test.go index 1d272d8c4..405609e29 100644 --- a/client/iface/configurer/allowedips_test.go +++ b/client/iface/configurer/allowedips_test.go @@ -1,3 +1,5 @@ +//go:build (linux && !android) || freebsd + package configurer import ( @@ -125,24 +127,6 @@ func TestAllowedIPStoreNormalizesMappedPrefixes(t *testing.T) { assert.Equal(t, []netip.Prefix{v4}, prefixes, "a mapped prefix must be stored unmapped and not duplicated") } -func TestNormalizePrefix(t *testing.T) { - v4 := netip.MustParsePrefix("10.20.0.0/16") - v6 := netip.MustParsePrefix("fd00::/64") - - assert.Equal(t, v4, normalizePrefix(v4), "a plain v4 prefix is unchanged") - assert.Equal(t, v6, normalizePrefix(v6), "a real v6 prefix is unchanged") - assert.Equal(t, v4, normalizePrefix(netip.PrefixFrom(netip.AddrFrom16(v4.Addr().As16()), 112)), - "a mapped prefix under a 128 bit mask becomes plain v4") - // A prefix shorter than /96 inside the mapped range is a genuine v6 prefix. Unmapping it - // would pair a v4 address with a v6 sized mask, which is invalid, and the store would then - // record a zero prefix that can never recreate the allowed IP. - for _, tc := range []string{"::ffff:0:0/64", "::ffff:1.2.3.4/80", "::ffff:1.2.3.4/95"} { - got := normalizePrefix(netip.MustParsePrefix(tc)) - assert.True(t, got.IsValid(), "%s must normalize to a valid prefix", tc) - assert.False(t, got.Addr().Is4(), "%s must stay v6", tc) - } -} - func TestAllowedIPStoreAddExistingDoesNotCreate(t *testing.T) { s := newAllowedIPStore() routed := netip.MustParsePrefix("10.20.0.0/16") @@ -194,15 +178,6 @@ func TestAllowedIPStoreForgetReleasesOwnership(t *testing.T) { assert.Equal(t, []netip.Prefix{routed}, current, "the new owner must hold the prefix") } -func TestNormalizePrefixClearsHostBits(t *testing.T) { - // A device stores a prefix masked, so a caller passing host bits must still match what a - // device fallback seeded, otherwise that prefix could never be removed by value. - assert.Equal(t, netip.MustParsePrefix("10.20.0.0/16"), - normalizePrefix(netip.MustParsePrefix("10.20.0.1/16")), "host bits must be cleared") - assert.Equal(t, netip.MustParsePrefix("fd00::/64"), - normalizePrefix(netip.MustParsePrefix("fd00::1/64")), "host bits must be cleared for v6") -} - func TestIPNetsToPrefixesKeepsV6InTheMappedRange(t *testing.T) { // ::ffff:0:0/64 reads as v4-mapped but is a genuine v6 prefix: unmapping it would leave a // v4 address under a 64 bit mask, which is invalid, and the allowed IP would be dropped. @@ -215,49 +190,3 @@ func TestIPNetsToPrefixesKeepsV6InTheMappedRange(t *testing.T) { assert.False(t, got[0].Addr().Is4(), "a v6 prefix in the mapped range must not become v4") assert.Equal(t, 64, got[0].Bits(), "the prefix length must survive the conversion") } - -func TestPrefixesToIPNetsNormalizes(t *testing.T) { - // net.IPNet prints a v4-mapped address as v4 but takes the length from its 16 byte - // mask, so an unnormalized ::ffff:10.1.2.3/64 reaches a userspace device as 10.1.2.3/0, - // an allowed IP that matches every v4 address. - tests := []struct { - name string - given string - want string - }{ - {name: "mapped below /96", given: "::ffff:10.1.2.3/64", want: "::/64"}, - {name: "mapped at /112", given: "::ffff:10.1.2.3/112", want: "10.1.0.0/16"}, - {name: "host bits are cleared", given: "10.20.0.1/16", want: "10.20.0.0/16"}, - {name: "v6 is untouched", given: "fd00::1/64", want: "fd00::/64"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got := prefixesToIPNets([]netip.Prefix{netip.MustParsePrefix(tc.given)}) - require.Len(t, got, 1, "the prefix must be converted, not dropped") - assert.Equal(t, tc.want, got[0].String(), "what the device is given") - assert.NotEqual(t, 0, mustOnes(t, got[0]), "a device must never be given a zero length allowed IP") - }) - } -} - -func mustOnes(t *testing.T, ipNet net.IPNet) int { - t.Helper() - - ones, _ := ipNet.Mask.Size() - return ones -} - -// TestPrefixesToIPNetsAgreesWithTheStore pins the property the store depends on: what a -// device is given and what is recorded for it are the same prefix. -func TestPrefixesToIPNetsAgreesWithTheStore(t *testing.T) { - for _, given := range []string{"::ffff:10.1.2.3/64", "::ffff:10.1.2.3/112", "10.20.0.1/16", "fd00::1/64"} { - prefix := netip.MustParsePrefix(given) - - toDevice := prefixesToIPNets([]netip.Prefix{prefix}) - recorded := normalizePrefix(prefix) - - assert.Equal(t, recorded.String(), toDevice[0].String(), - "%s must reach the device in the form the store records", given) - } -} diff --git a/client/iface/configurer/common.go b/client/iface/configurer/common.go index 40f8209e9..50adf00d1 100644 --- a/client/iface/configurer/common.go +++ b/client/iface/configurer/common.go @@ -3,22 +3,7 @@ package configurer import ( "net" "net/netip" - - "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) - -// buildPresharedKeyConfig creates a wgtypes.Config for setting a preshared key on a peer. -// This is a shared helper used by both kernel and userspace configurers. -func buildPresharedKeyConfig(peerKey wgtypes.Key, psk wgtypes.Key, updateOnly bool) wgtypes.Config { - return wgtypes.Config{ - Peers: []wgtypes.PeerConfig{{ - PublicKey: peerKey, - PresharedKey: &psk, - UpdateOnly: updateOnly, - }}, - } -} - // prefixesToIPNets converts prefixes on their way to a device. It is the only place that // conversion happens, so it also normalizes: the device is then given the same form the // store records, and a v4-mapped prefix cannot reach net.IPNet, which prints such an @@ -35,3 +20,30 @@ func prefixesToIPNets(prefixes []netip.Prefix) []net.IPNet { } return ipNets } + +// normalizePrefix puts a prefix into the form the store recognises it by. It clears the +// host bits, which a device does on its own, so a caller passing 10.20.0.1/16 still matches +// the 10.20.0.0/16 read back from the device; and it unmaps a v4-mapped prefix so that it +// compares equal to, and marshals like, the plain v4 prefix for the same network. +// +// Masking comes first because it also decides the address family: only a prefix at least 96 +// bits long keeps the mapped marker through the mask, so a shorter prefix inside the mapped +// range is a genuine v6 prefix and unmapping it would yield an invalid v4 prefix. +func normalizePrefix(prefix netip.Prefix) netip.Prefix { + masked := prefix.Masked() + + addr := masked.Addr() + if !addr.Is4In6() { + return masked + } + return netip.PrefixFrom(addr.Unmap(), masked.Bits()-96) +} + +// normalizePrefixes returns a normalized copy without changing the caller's slice. +func normalizePrefixes(prefixes []netip.Prefix) []netip.Prefix { + normalized := make([]netip.Prefix, len(prefixes)) + for i, prefix := range prefixes { + normalized[i] = normalizePrefix(prefix) + } + return normalized +} diff --git a/client/iface/configurer/common_test.go b/client/iface/configurer/common_test.go new file mode 100644 index 000000000..487c1a734 --- /dev/null +++ b/client/iface/configurer/common_test.go @@ -0,0 +1,81 @@ +package configurer + +import ( + "net" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizePrefix(t *testing.T) { + v4 := netip.MustParsePrefix("10.20.0.0/16") + v6 := netip.MustParsePrefix("fd00::/64") + + assert.Equal(t, v4, normalizePrefix(v4), "a plain v4 prefix is unchanged") + assert.Equal(t, v6, normalizePrefix(v6), "a real v6 prefix is unchanged") + assert.Equal(t, v4, normalizePrefix(netip.PrefixFrom(netip.AddrFrom16(v4.Addr().As16()), 112)), + "a mapped prefix under a 128 bit mask becomes plain v4") + // A prefix shorter than /96 inside the mapped range is a genuine v6 prefix. Unmapping it + // would pair a v4 address with a v6 sized mask, which is invalid, and the store would then + // record a zero prefix that can never recreate the allowed IP. + for _, tc := range []string{"::ffff:0:0/64", "::ffff:1.2.3.4/80", "::ffff:1.2.3.4/95"} { + got := normalizePrefix(netip.MustParsePrefix(tc)) + assert.True(t, got.IsValid(), "%s must normalize to a valid prefix", tc) + assert.False(t, got.Addr().Is4(), "%s must stay v6", tc) + } +} + +func TestNormalizePrefixClearsHostBits(t *testing.T) { + // A device stores a prefix masked, so a caller passing host bits must still match what a + // device fallback seeded, otherwise that prefix could never be removed by value. + assert.Equal(t, netip.MustParsePrefix("10.20.0.0/16"), + normalizePrefix(netip.MustParsePrefix("10.20.0.1/16")), "host bits must be cleared") + assert.Equal(t, netip.MustParsePrefix("fd00::/64"), + normalizePrefix(netip.MustParsePrefix("fd00::1/64")), "host bits must be cleared for v6") +} + +func TestPrefixesToIPNetsNormalizes(t *testing.T) { + // net.IPNet prints a v4-mapped address as v4 but takes the length from its 16 byte + // mask, so an unnormalized ::ffff:10.1.2.3/64 reaches a userspace device as 10.1.2.3/0, + // an allowed IP that matches every v4 address. + tests := []struct { + name string + given string + want string + }{ + {name: "mapped below /96", given: "::ffff:10.1.2.3/64", want: "::/64"}, + {name: "mapped at /112", given: "::ffff:10.1.2.3/112", want: "10.1.0.0/16"}, + {name: "host bits are cleared", given: "10.20.0.1/16", want: "10.20.0.0/16"}, + {name: "v6 is untouched", given: "fd00::1/64", want: "fd00::/64"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := prefixesToIPNets([]netip.Prefix{netip.MustParsePrefix(tc.given)}) + require.Len(t, got, 1, "the prefix must be converted, not dropped") + assert.Equal(t, tc.want, got[0].String(), "what the device is given") + assert.NotEqual(t, 0, mustOnes(t, got[0]), "a device must never be given a zero length allowed IP") + }) + } +} + +func TestPrefixesToIPNetsAgreesWithTheStore(t *testing.T) { + for _, given := range []string{"::ffff:10.1.2.3/64", "::ffff:10.1.2.3/112", "10.20.0.1/16", "fd00::1/64"} { + prefix := netip.MustParsePrefix(given) + + toDevice := prefixesToIPNets([]netip.Prefix{prefix}) + recorded := normalizePrefix(prefix) + + assert.Equal(t, recorded.String(), toDevice[0].String(), + "%s must reach the device in the form the store records", given) + } +} + +func mustOnes(t *testing.T, ipNet net.IPNet) int { + t.Helper() + + ones, _ := ipNet.Mask.Size() + return ones +} diff --git a/client/iface/configurer/kernel_unix.go b/client/iface/configurer/kernel_unix.go index 3a95249c1..afe1300d5 100644 --- a/client/iface/configurer/kernel_unix.go +++ b/client/iface/configurer/kernel_unix.go @@ -3,10 +3,12 @@ package configurer import ( + "errors" "fmt" "net" "net/netip" "slices" + "sync" "time" log "github.com/sirupsen/logrus" @@ -16,18 +18,38 @@ import ( "github.com/netbirdio/netbird/monotime" ) +var errConfigurerClosed = errors.New("configurer closed") + +type wgClient interface { + Device(name string) (*wgtypes.Device, error) + ConfigureDevice(name string, cfg wgtypes.Config) error + Close() error +} + type KernelConfigurer struct { - deviceName string - statsCache *statsCache - allowedIPs *allowedIPStore + deviceName string + statsCache *statsCache + allowedIPs *allowedIPStore + newWgctrlFn func() (wgClient, error) + + clientMu sync.Mutex + client wgClient + closed bool } // NewKernelConfigurer creates a configurer with an empty allowed IP mirror // and a statistics cache for the named kernel device. func NewKernelConfigurer(deviceName string) *KernelConfigurer { + return newKernelConfigurer(deviceName, func() (wgClient, error) { + return wgctrl.New() + }) +} + +func newKernelConfigurer(deviceName string, newWgctrlFn func() (wgClient, error)) *KernelConfigurer { c := &KernelConfigurer{ - deviceName: deviceName, - allowedIPs: newAllowedIPStore(), + deviceName: deviceName, + allowedIPs: newAllowedIPStore(), + newWgctrlFn: newWgctrlFn, } c.statsCache = newStatsCache(statsCacheTTL, c.fetchStats) return c @@ -242,7 +264,7 @@ func (c *KernelConfigurer) peerAllowedIPs(peerKey wgtypes.Key) ([]netip.Prefix, return prefixes, nil } - existingPeer, err := c.getPeer(c.deviceName, peerKey) + existingPeer, err := c.getPeer(peerKey) if err != nil { return nil, fmt.Errorf("get peer: %w", err) } @@ -254,21 +276,10 @@ func (c *KernelConfigurer) peerAllowedIPs(peerKey wgtypes.Key) ([]netip.Prefix, // getPeer scans the device for one peer. wgtypes.Key is an array, so the comparison is a // plain equality: Key.String would base64 encode into a fresh allocation for every peer. -func (c *KernelConfigurer) getPeer(ifaceName string, peerPubKey wgtypes.Key) (wgtypes.Peer, error) { - wg, err := wgctrl.New() +func (c *KernelConfigurer) getPeer(peerPubKey wgtypes.Key) (wgtypes.Peer, error) { + wgDevice, err := c.device() if err != nil { - return wgtypes.Peer{}, fmt.Errorf("wgctl: %w", err) - } - defer func() { - err = wg.Close() - if err != nil { - log.Errorf("Got error while closing wgctl: %v", err) - } - }() - - wgDevice, err := wg.Device(ifaceName) - if err != nil { - return wgtypes.Peer{}, fmt.Errorf("get device %s: %w", ifaceName, err) + return wgtypes.Peer{}, err } for _, peer := range wgDevice.Peers { if peer.PublicKey == peerPubKey { @@ -279,37 +290,73 @@ func (c *KernelConfigurer) getPeer(ifaceName string, peerPubKey wgtypes.Key) (wg } func (c *KernelConfigurer) configure(config wgtypes.Config) error { - wg, err := wgctrl.New() + c.clientMu.Lock() + defer c.clientMu.Unlock() + + client, err := c.clientLocked() if err != nil { return err } - defer func() { - if err := wg.Close(); err != nil { - log.Errorf("Failed to close wgctrl client: %v", err) - } - }() - - return wg.ConfigureDevice(c.deviceName, config) + if err := client.ConfigureDevice(c.deviceName, config); err != nil { + c.discardLocked() + return err + } + return nil } -func (c *KernelConfigurer) Close() { +func (c *KernelConfigurer) device() (*wgtypes.Device, error) { + c.clientMu.Lock() + defer c.clientMu.Unlock() + + client, err := c.clientLocked() + if err != nil { + return nil, err + } + wgDevice, err := client.Device(c.deviceName) + if err != nil { + c.discardLocked() + return nil, fmt.Errorf("get device %s: %w", c.deviceName, err) + } + return wgDevice, nil } -func (c *KernelConfigurer) FullStats() (*Stats, error) { - wg, err := wgctrl.New() +func (c *KernelConfigurer) clientLocked() (wgClient, error) { + if c.closed { + return nil, errConfigurerClosed + } + if c.client != nil { + return c.client, nil + } + client, err := c.newWgctrlFn() if err != nil { return nil, fmt.Errorf("wgctl: %w", err) } - defer func() { - err = wg.Close() - if err != nil { - log.Errorf("Got error while closing wgctl: %v", err) - } - }() + c.client = client + return client, nil +} - wgDevice, err := wg.Device(c.deviceName) +func (c *KernelConfigurer) discardLocked() { + if c.client == nil { + return + } + if err := c.client.Close(); err != nil { + log.Warnf("failed to close wgctrl client: %v", err) + } + c.client = nil +} + +func (c *KernelConfigurer) Close() { + c.clientMu.Lock() + defer c.clientMu.Unlock() + + c.closed = true + c.discardLocked() +} + +func (c *KernelConfigurer) FullStats() (*Stats, error) { + wgDevice, err := c.device() if err != nil { - return nil, fmt.Errorf("get device %s: %w", c.deviceName, err) + return nil, err } fullStats := &Stats{ DeviceName: wgDevice.Name, @@ -345,23 +392,12 @@ func (c *KernelConfigurer) LastActivities() map[string]monotime.Time { } func (c *KernelConfigurer) fetchStats() (map[string]WGStats, error) { + wgDevice, err := c.device() + if err != nil { + return nil, err + } + stats := make(map[string]WGStats) - wg, err := wgctrl.New() - if err != nil { - return nil, fmt.Errorf("wgctl: %w", err) - } - defer func() { - err = wg.Close() - if err != nil { - log.Errorf("Got error while closing wgctl: %v", err) - } - }() - - wgDevice, err := wg.Device(c.deviceName) - if err != nil { - return nil, fmt.Errorf("get device %s: %w", c.deviceName, err) - } - for _, peer := range wgDevice.Peers { stats[peer.PublicKey.String()] = WGStats{ LastHandshake: peer.LastHandshakeTime, @@ -371,3 +407,13 @@ func (c *KernelConfigurer) fetchStats() (map[string]WGStats, error) { } return stats, nil } + +func buildPresharedKeyConfig(peerKey wgtypes.Key, psk wgtypes.Key, updateOnly bool) wgtypes.Config { + return wgtypes.Config{ + Peers: []wgtypes.PeerConfig{{ + PublicKey: peerKey, + PresharedKey: &psk, + UpdateOnly: updateOnly, + }}, + } +} diff --git a/client/iface/configurer/kernel_unix_test.go b/client/iface/configurer/kernel_unix_test.go new file mode 100644 index 000000000..8daa15a25 --- /dev/null +++ b/client/iface/configurer/kernel_unix_test.go @@ -0,0 +1,167 @@ +//go:build (linux && !android) || freebsd + +package configurer + +import ( + "errors" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +type fakeWGClient struct { + configureErr error + deviceErr error + calls int + closed bool +} + +type fakeWGClientFactory struct { + openErr error + clients []*fakeWGClient +} + +func (f *fakeWGClient) Device(string) (*wgtypes.Device, error) { + f.calls++ + if f.deviceErr != nil { + return nil, f.deviceErr + } + return &wgtypes.Device{}, nil +} + +func (f *fakeWGClient) ConfigureDevice(string, wgtypes.Config) error { + f.calls++ + return f.configureErr +} + +func (f *fakeWGClient) Close() error { + f.closed = true + return nil +} + +func (f *fakeWGClientFactory) open() (wgClient, error) { + if f.openErr != nil { + return nil, f.openErr + } + client := &fakeWGClient{} + f.clients = append(f.clients, client) + return client, nil +} + +func TestKernelConfigurer_ReusesClientAcrossRequests(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + require.NoError(t, c.RemovePeer(peerKey)) + require.NoError(t, c.RemovePeer(peerKey)) + _, err := c.FullStats() + require.NoError(t, err) + + require.Len(t, factory.clients, 1, "all requests should share one client") + assert.Equal(t, 3, factory.clients[0].calls, "every request should go through the shared client") + assert.False(t, factory.clients[0].closed, "a healthy client should stay open") +} + +func TestKernelConfigurer_ReopensClientAfterFailedConfigure(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + require.NoError(t, c.RemovePeer(peerKey)) + require.Len(t, factory.clients, 1) + + factory.clients[0].configureErr = errors.New("netlink down") + require.Error(t, c.RemovePeer(peerKey)) + assert.True(t, factory.clients[0].closed, "the failed client should be closed") + assert.Len(t, factory.clients, 1, "a failure must not open a client on its own") + + require.NoError(t, c.RemovePeer(peerKey)) + require.Len(t, factory.clients, 2, "the next request should open a fresh client") + assert.Equal(t, 1, factory.clients[1].calls, "the request should run on the fresh client") + assert.False(t, factory.clients[1].closed, "the fresh client should stay open") +} + +func TestKernelConfigurer_ReopensClientAfterFailedDeviceQuery(t *testing.T) { + c, factory := newFakeKernelConfigurer() + + _, err := c.FullStats() + require.NoError(t, err) + require.Len(t, factory.clients, 1) + + factory.clients[0].deviceErr = errors.New("netlink down") + _, err = c.FullStats() + require.Error(t, err) + assert.True(t, factory.clients[0].closed, "the failed client should be closed") + + _, err = c.FullStats() + require.NoError(t, err) + assert.Len(t, factory.clients, 2, "the next request should open a fresh client") +} + +func TestKernelConfigurer_RetriesOpenOnNextRequest(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + factory.openErr = errors.New("no netlink") + require.Error(t, c.RemovePeer(peerKey)) + assert.Empty(t, factory.clients, "a failed open leaves no client behind") + + factory.openErr = nil + require.NoError(t, c.RemovePeer(peerKey)) + assert.Len(t, factory.clients, 1, "the next request should open the client") +} + +func TestKernelConfigurer_CloseRejectsFurtherRequests(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + require.NoError(t, c.RemovePeer(peerKey)) + require.Len(t, factory.clients, 1) + + c.Close() + c.Close() + assert.True(t, factory.clients[0].closed, "Close should release the client") + + require.ErrorIs(t, c.RemovePeer(peerKey), errConfigurerClosed) + _, err := c.FullStats() + require.ErrorIs(t, err, errConfigurerClosed) + assert.Len(t, factory.clients, 1, "a closed configurer must not open a client") +} + +func TestKernelConfigurer_ConcurrentRequestsShareOneClient(t *testing.T) { + c, factory := newFakeKernelConfigurer() + peerKey := testPeerKey(t) + + const workers = 16 + errs := make(chan error, workers) + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + errs <- c.RemovePeer(peerKey) + }() + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + + require.Len(t, factory.clients, 1, "concurrent requests should share one client") + assert.Equal(t, workers, factory.clients[0].calls, "every request should reach the shared client") +} + +func newFakeKernelConfigurer() (*KernelConfigurer, *fakeWGClientFactory) { + factory := &fakeWGClientFactory{} + return newKernelConfigurer("wt0", factory.open), factory +} + +func testPeerKey(t *testing.T) string { + t.Helper() + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + return key.PublicKey().String() +} diff --git a/client/iface/configurer/usp.go b/client/iface/configurer/usp.go index 334d99369..f06c69a1e 100644 --- a/client/iface/configurer/usp.go +++ b/client/iface/configurer/usp.go @@ -1,19 +1,16 @@ package configurer import ( - "encoding/base64" - "encoding/hex" "fmt" + "math" "net" "net/netip" "os" "runtime" - "slices" - "strconv" - "strings" "time" log "github.com/sirupsen/logrus" + wgconn "golang.zx2c4.com/wireguard/conn" "golang.zx2c4.com/wireguard/device" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" @@ -22,19 +19,6 @@ import ( "github.com/netbirdio/netbird/monotime" ) -const ( - privateKey = "private_key" - ipcKeyLastHandshakeTimeSec = "last_handshake_time_sec" - ipcKeyTxBytes = "tx_bytes" - ipcKeyRxBytes = "rx_bytes" - allowedIP = "allowed_ip" - endpoint = "endpoint" - fwmark = "fwmark" - listenPort = "listen_port" - publicKey = "public_key" - presharedKey = "preshared_key" -) - var ErrAllowedIPNotFound = fmt.Errorf("allowed IP not found") type WGUSPConfigurer struct { @@ -49,13 +33,7 @@ type WGUSPConfigurer struct { // NewUSPConfigurer creates a userspace configurer and starts its UAPI listener. func NewUSPConfigurer(device *device.Device, deviceName string, activityRecorder *bind.ActivityRecorder) *WGUSPConfigurer { - wgCfg := &WGUSPConfigurer{ - device: device, - deviceName: deviceName, - activityRecorder: activityRecorder, - allowedIPs: newAllowedIPStore(), - } - wgCfg.statsCache = newStatsCache(statsCacheTTL, wgCfg.fetchStats) + wgCfg := NewUSPConfigurerNoUAPI(device, deviceName, activityRecorder) wgCfg.startUAPI() return wgCfg } @@ -73,26 +51,26 @@ func NewUSPConfigurerNoUAPI(device *device.Device, deviceName string, activityRe } // ConfigureInterface sets the device key, port and firewall mark, replacing all peers. -// The allowed IP mirror is reset only after the device accepts the configuration. func (c *WGUSPConfigurer) ConfigureInterface(privateKey string, port int) error { log.Debugf("adding Wireguard private key") key, err := wgtypes.ParseKey(privateKey) if err != nil { return err } - fwmark := getFwmark() - config := wgtypes.Config{ - PrivateKey: &key, - ReplacePeers: true, - FirewallMark: &fwmark, - ListenPort: &port, + if port < 0 || port > math.MaxUint16 { + return fmt.Errorf("invalid listen port %d", port) } - if err := c.device.IpcSet(toWgUserspaceString(config)); err != nil { - return err + if err := c.device.SetPrivateKey(device.NoisePrivateKey(key)); err != nil { + return fmt.Errorf("set private key: %w", err) + } + if err := c.device.SetListenPort(uint16(port)); err != nil { + return fmt.Errorf("set listen port %d: %w", port, err) + } + c.device.RemoveAllPeers() + if err := c.device.BindSetMark(uint32(getFwmark())); err != nil { + return fmt.Errorf("set fwmark: %w", err) } - - c.allowedIPs.reset() return nil } @@ -104,55 +82,48 @@ func (c *WGUSPConfigurer) SetPresharedKey(peerKey string, psk wgtypes.Key, updat return err } - cfg := buildPresharedKeyConfig(parsedPeerKey, psk, updateOnly) - if err := c.device.IpcSet(toWgUserspaceString(cfg)); err != nil { - return err - } - - // Without updateOnly this creates the peer when it is absent, so the store has to - // know about it even though no allowed IP was configured. - if !updateOnly { - c.allowedIPs.ensure(parsedPeerKey) - } - return nil + presharedKey := device.NoisePresharedKey(psk) + return c.device.ConfigurePeer(device.NoisePublicKey(parsedPeerKey), device.PeerConfig{ + PresharedKey: &presharedKey, + UpdateOnly: updateOnly, + }) } // UpdatePeer creates or updates a peer, merging allowed IPs with its existing set. -// It validates the endpoint before writing and records changes after a successful write. func (c *WGUSPConfigurer) UpdatePeer(peerKey string, allowedIps []netip.Prefix, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return err } - // Everything that can fail is done before the device is touched, so a failure here - // cannot leave the device holding a peer that the activity recorder and the allowed - // IP store never learned about. + prefixes, err := validPrefixes(allowedIps) + if err != nil { + return err + } + + cfg := device.PeerConfig{ + PersistentKeepalive: &keepAlive, + AllowedIPs: prefixes, + } + var addrPort netip.AddrPort if endpoint != nil { - addr, err := netip.ParseAddr(endpoint.IP.String()) - if err != nil { - return fmt.Errorf("parse endpoint address: %w", err) + addr, ok := netip.AddrFromSlice(endpoint.IP) + if !ok { + return fmt.Errorf("parse endpoint address %v", endpoint.IP) } - addrPort = netip.AddrPortFrom(addr.Unmap(), uint16(endpoint.Port)) + addr = addr.Unmap() + addrPort = netip.AddrPortFrom(addr, uint16(endpoint.Port)) + cfg.Endpoint = &bind.Endpoint{AddrPort: netip.AddrPortFrom(addr.WithZone(endpoint.Zone), addrPort.Port())} } - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - ReplaceAllowedIPs: false, - // don't replace allowed ips, wg will handle duplicated peer IP - AllowedIPs: prefixesToIPNets(allowedIps), - PersistentKeepaliveInterval: &keepAlive, - PresharedKey: preSharedKey, - Endpoint: endpoint, + if preSharedKey != nil { + psk := device.NoisePresharedKey(*preSharedKey) + cfg.PresharedKey = &psk } - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, - } - - if ipcErr := c.device.IpcSet(toWgUserspaceString(config)); ipcErr != nil { - return ipcErr + if err := c.device.ConfigurePeer(device.NoisePublicKey(peerKeyParsed), cfg); err != nil { + return fmt.Errorf("configure peer: %w", err) } if endpoint != nil { @@ -164,71 +135,40 @@ func (c *WGUSPConfigurer) UpdatePeer(peerKey string, allowedIps []netip.Prefix, } // RemoveEndpointAddress clears the endpoint of a peer while keeping it configured. -// The UAPI cannot clear an endpoint in place, so the peer is removed and re-added with the -// allowed IPs it already had. +// The session is reset and the keepalive is switched off until the next endpoint arrives. func (c *WGUSPConfigurer) RemoveEndpointAddress(peerKey string) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return fmt.Errorf("parse peer key: %w", err) } - allowedIPs, err := c.peerAllowedIPs(peerKeyParsed) - if err != nil { - return err + peer := c.device.LookupPeer(device.NoisePublicKey(peerKeyParsed)) + if peer == nil { + return ErrPeerNotFound } - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - Remove: true, - } + peer.ClearEndpoint() + peer.ZeroAndFlushAll() - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, + var noKeepAlive time.Duration + if err := c.device.ConfigurePeer(device.NoisePublicKey(peerKeyParsed), device.PeerConfig{ + PersistentKeepalive: &noKeepAlive, + UpdateOnly: true, + }); err != nil { + return fmt.Errorf("disable keepalive: %w", err) } - if ipcErr := c.device.IpcSet(toWgUserspaceString(config)); ipcErr != nil { - return fmt.Errorf("remove peer: %w", ipcErr) - } - - peer = wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - ReplaceAllowedIPs: true, - AllowedIPs: prefixesToIPNets(allowedIPs), - } - - config = wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, - } - - if err := c.device.IpcSet(toWgUserspaceString(config)); err != nil { - c.allowedIPs.forget(peerKeyParsed) - return fmt.Errorf("re-add peer without endpoint: %w", err) - } - return nil } -// RemovePeer removes a peer, then clears its activity and allowed IP records. -// A failed device write leaves both records intact. +// RemovePeer removes a peer and clears its activity record. func (c *WGUSPConfigurer) RemovePeer(peerKey string) error { peerKeyParsed, err := wgtypes.ParseKey(peerKey) if err != nil { return err } - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - Remove: true, - } - - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, - } - if ipcErr := c.device.IpcSet(toWgUserspaceString(config)); ipcErr != nil { - return ipcErr - } - + c.device.RemovePeer(device.NoisePublicKey(peerKeyParsed)) c.activityRecorder.Remove(peerKey) - c.allowedIPs.forget(peerKeyParsed) return nil } @@ -238,22 +178,16 @@ func (c *WGUSPConfigurer) AddAllowedIP(peerKey string, allowedIP netip.Prefix) e if err != nil { return err } - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - UpdateOnly: true, - ReplaceAllowedIPs: false, - AllowedIPs: prefixesToIPNets([]netip.Prefix{allowedIP}), + if !allowedIP.IsValid() { + return fmt.Errorf("invalid allowed IP %v", allowedIP) } - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, + peer := c.device.LookupPeer(device.NoisePublicKey(peerKeyParsed)) + if peer == nil { + return nil } - if err := c.device.IpcSet(toWgUserspaceString(config)); err != nil { - return err - } - - c.allowedIPs.addExisting(peerKeyParsed, []netip.Prefix{allowedIP}) + peer.AddAllowedIP(normalizePrefix(allowedIP)) return nil } @@ -264,86 +198,64 @@ func (c *WGUSPConfigurer) RemoveAllowedIP(peerKey string, allowedIP netip.Prefix if err != nil { return fmt.Errorf("parse peer key: %w", err) } - - currentAllowedIPs, err := c.peerAllowedIPs(peerKeyParsed) - if err != nil { - return err - } - - idx := slices.Index(currentAllowedIPs, normalizePrefix(allowedIP)) - if idx < 0 { + if !allowedIP.IsValid() { return ErrAllowedIPNotFound } - newAllowedIPs := slices.Delete(currentAllowedIPs, idx, idx+1) - peer := wgtypes.PeerConfig{ - PublicKey: peerKeyParsed, - UpdateOnly: true, - ReplaceAllowedIPs: true, - AllowedIPs: prefixesToIPNets(newAllowedIPs), + peer := c.device.LookupPeer(device.NoisePublicKey(peerKeyParsed)) + if peer == nil { + return ErrPeerNotFound } - config := wgtypes.Config{ - Peers: []wgtypes.PeerConfig{peer}, + if !peer.RemoveAllowedIP(normalizePrefix(allowedIP)) { + return ErrAllowedIPNotFound } - if err := c.device.IpcSet(toWgUserspaceString(config)); err != nil { - return fmt.Errorf("remove allowed IP %s: %w", allowedIP, err) - } - - c.allowedIPs.set(peerKeyParsed, newAllowedIPs) return nil } -// peerAllowedIPs returns the allowed IPs configured for a peer, reading them from the device -// only for a peer the store has not seen. Reading them back means dumping and parsing the -// whole device configuration, and this runs on every relay and ICE transition. -func (c *WGUSPConfigurer) peerAllowedIPs(peerKey wgtypes.Key) ([]netip.Prefix, error) { - if prefixes, ok := c.allowedIPs.get(peerKey); ok { - return prefixes, nil - } - - ipcStr, err := c.device.IpcGet() - if err != nil { - return nil, fmt.Errorf("get IPC config: %w", err) - } - - stats, err := parseStatus(c.deviceName, ipcStr) - if err != nil { - return nil, fmt.Errorf("parse IPC config: %w", err) - } - - // parseStatus reports keys in their textual form, so the comparison needs it once. - wanted := peerKey.String() - for _, peer := range stats.Peers { - if peer.PublicKey != wanted { - continue - } - - prefixes := ipNetsToPrefixes(peer.AllowedIPs) - c.allowedIPs.set(peerKey, prefixes) - return prefixes, nil - } - - return nil, ErrPeerNotFound -} - func (c *WGUSPConfigurer) FullStats() (*Stats, error) { - ipcStr, err := c.device.IpcGet() - if err != nil { - return nil, fmt.Errorf("IpcGet failed: %w", err) + peers := c.device.Peers() + stats := &Stats{ + DeviceName: c.deviceName, + PublicKey: wgtypes.Key(c.device.PublicKey()).String(), + ListenPort: int(c.device.ListenPort()), + FWMark: int(c.device.FirewallMark()), + Peers: make([]Peer, 0, len(peers)), } - - return parseStatus(c.deviceName, ipcStr) + for _, peer := range peers { + stats.Peers = append(stats.Peers, peerStats(peer)) + } + return stats, nil } func (c *WGUSPConfigurer) LastActivities() map[string]monotime.Time { return c.activityRecorder.GetLastActivities() } +func (c *WGUSPConfigurer) GetStats() (map[string]WGStats, error) { + return c.statsCache.get() +} + +func (c *WGUSPConfigurer) Close() { + if c.uapiListener != nil { + err := c.uapiListener.Close() + if err != nil { + log.Errorf("failed to close uapi listener: %v", err) + } + } + + if runtime.GOOS == "linux" { + sockPath := "/var/run/wireguard/" + c.deviceName + ".sock" + if _, statErr := os.Stat(sockPath); statErr == nil { + _ = os.Remove(sockPath) + } + } +} + // startUAPI starts the UAPI listener for managing the WireGuard interface via external tool -func (t *WGUSPConfigurer) startUAPI() { +func (c *WGUSPConfigurer) startUAPI() { var err error - t.uapiListener, err = openUAPI(t.deviceName) + c.uapiListener, err = openUAPI(c.deviceName) if err != nil { log.Errorf("failed to open uapi listener: %v", err) return @@ -357,175 +269,61 @@ func (t *WGUSPConfigurer) startUAPI() { return } go func() { - t.device.IpcHandle(uapiConn) + c.device.IpcHandle(uapiConn) }() } - }(t.uapiListener) + }(c.uapiListener) } -func (t *WGUSPConfigurer) Close() { - if t.uapiListener != nil { - err := t.uapiListener.Close() - if err != nil { - log.Errorf("failed to close uapi listener: %v", err) +func (c *WGUSPConfigurer) fetchStats() (map[string]WGStats, error) { + peers := c.device.Peers() + stats := make(map[string]WGStats, len(peers)) + for _, peer := range peers { + stats[wgtypes.Key(peer.PublicKey()).String()] = WGStats{ + LastHandshake: peer.LastHandshake(), + TxBytes: int64(peer.TxBytes()), + RxBytes: int64(peer.RxBytes()), } } - - if runtime.GOOS == "linux" { - sockPath := "/var/run/wireguard/" + t.deviceName + ".sock" - if _, statErr := os.Stat(sockPath); statErr == nil { - _ = os.Remove(sockPath) - } - } -} - -func (t *WGUSPConfigurer) GetStats() (map[string]WGStats, error) { - return t.statsCache.get() -} - -func (t *WGUSPConfigurer) fetchStats() (map[string]WGStats, error) { - ipc, err := t.device.IpcGet() - if err != nil { - return nil, fmt.Errorf("ipc get: %w", err) - } - - return parseTransfers(ipc) -} - -func parseTransfers(ipc string) (map[string]WGStats, error) { - stats := make(map[string]WGStats) - var ( - currentKey string - currentStats WGStats - hasPeer bool - ) - lines := strings.Split(ipc, "\n") - for _, line := range lines { - line = strings.TrimSpace(line) - - // If we're within the details of the found peer and encounter another public key, - // this means we're starting another peer's details. So, stop. - if strings.HasPrefix(line, "public_key=") { - peerID := strings.TrimPrefix(line, "public_key=") - h, err := hex.DecodeString(peerID) - if err != nil { - return nil, fmt.Errorf("decode peerID: %w", err) - } - currentKey = base64.StdEncoding.EncodeToString(h) - currentStats = WGStats{} // Reset stats for the new peer - hasPeer = true - stats[currentKey] = currentStats - continue - } - - if !hasPeer { - continue - } - - key := strings.SplitN(line, "=", 2) - if len(key) != 2 { - continue - } - switch key[0] { - case ipcKeyLastHandshakeTimeSec: - hs, err := toLastHandshake(key[1]) - if err != nil { - return nil, err - } - currentStats.LastHandshake = hs - stats[currentKey] = currentStats - case ipcKeyRxBytes: - rxBytes, err := toBytes(key[1]) - if err != nil { - return nil, fmt.Errorf("parse rx_bytes: %w", err) - } - currentStats.RxBytes = rxBytes - stats[currentKey] = currentStats - case ipcKeyTxBytes: - TxBytes, err := toBytes(key[1]) - if err != nil { - return nil, fmt.Errorf("parse tx_bytes: %w", err) - } - currentStats.TxBytes = TxBytes - stats[currentKey] = currentStats - } - } - return stats, nil } -func toWgUserspaceString(wgCfg wgtypes.Config) string { - var sb strings.Builder - if wgCfg.PrivateKey != nil { - hexKey := hex.EncodeToString(wgCfg.PrivateKey[:]) - sb.WriteString(fmt.Sprintf("private_key=%s\n", hexKey)) +func peerStats(peer *device.Peer) Peer { + stats := Peer{ + PublicKey: wgtypes.Key(peer.PublicKey()).String(), + AllowedIPs: prefixesToIPNets(peer.AllowedIPs()), + TxBytes: int64(peer.TxBytes()), + RxBytes: int64(peer.RxBytes()), + LastHandshake: peer.LastHandshake(), + PresharedKey: [32]byte(peer.PresharedKey()), } - - if wgCfg.ListenPort != nil { - sb.WriteString(fmt.Sprintf("listen_port=%d\n", *wgCfg.ListenPort)) + if addrPort, ok := endpointAddrPort(peer.Endpoint()); ok { + stats.Endpoint = net.UDPAddr{IP: addrPort.Addr().AsSlice(), Port: int(addrPort.Port())} } - - if wgCfg.ReplacePeers { - sb.WriteString("replace_peers=true\n") - } - - if wgCfg.FirewallMark != nil { - sb.WriteString(fmt.Sprintf("fwmark=%d\n", *wgCfg.FirewallMark)) - } - - for _, p := range wgCfg.Peers { - hexKey := hex.EncodeToString(p.PublicKey[:]) - sb.WriteString(fmt.Sprintf("public_key=%s\n", hexKey)) - - if p.Remove { - sb.WriteString("remove=true\n") - } - - if p.UpdateOnly { - sb.WriteString("update_only=true\n") - } - - if p.PresharedKey != nil { - preSharedHexKey := hex.EncodeToString(p.PresharedKey[:]) - sb.WriteString(fmt.Sprintf("preshared_key=%s\n", preSharedHexKey)) - } - - if p.Endpoint != nil { - sb.WriteString(fmt.Sprintf("endpoint=%s\n", p.Endpoint.String())) - } - - if p.PersistentKeepaliveInterval != nil { - sb.WriteString(fmt.Sprintf("persistent_keepalive_interval=%d\n", int(p.PersistentKeepaliveInterval.Seconds()))) - } - - if p.ReplaceAllowedIPs { - sb.WriteString("replace_allowed_ips=true\n") - } - - for _, aip := range p.AllowedIPs { - sb.WriteString(fmt.Sprintf("allowed_ip=%s\n", aip.String())) - } - } - return sb.String() + return stats } -func toLastHandshake(stringVar string) (time.Time, error) { - sec, err := strconv.ParseInt(stringVar, 10, 64) +func endpointAddrPort(endpoint wgconn.Endpoint) (netip.AddrPort, bool) { + if endpoint == nil { + return netip.AddrPort{}, false + } + if ep, ok := endpoint.(*bind.Endpoint); ok { + return ep.AddrPort, true + } + addrPort, err := netip.ParseAddrPort(endpoint.DstToString()) if err != nil { - return time.Time{}, fmt.Errorf("parse handshake sec: %w", err) + return netip.AddrPort{}, false } - - // If sec is 0 (Unix epoch), return zero time instead - // This indicates no handshake has occurred - if sec == 0 { - return time.Time{}, nil - } - - return time.Unix(sec, 0), nil + return addrPort, true } -func toBytes(s string) (int64, error) { - return strconv.ParseInt(s, 10, 64) +func validPrefixes(prefixes []netip.Prefix) ([]netip.Prefix, error) { + for _, prefix := range prefixes { + if !prefix.IsValid() { + return nil, fmt.Errorf("invalid allowed IP %v", prefix) + } + } + return normalizePrefixes(prefixes), nil } func getFwmark() int { @@ -534,138 +332,3 @@ func getFwmark() int { } return 0 } - -func hexToWireguardKey(hexKey string) (wgtypes.Key, error) { - // Decode hex string to bytes - keyBytes, err := hex.DecodeString(hexKey) - if err != nil { - return wgtypes.Key{}, fmt.Errorf("failed to decode hex key: %w", err) - } - - // Check if we have the right number of bytes (WireGuard keys are 32 bytes) - if len(keyBytes) != 32 { - return wgtypes.Key{}, fmt.Errorf("invalid key length: expected 32 bytes, got %d", len(keyBytes)) - } - - // Convert to wgtypes.Key - var key wgtypes.Key - copy(key[:], keyBytes) - - return key, nil -} - -func parseStatus(deviceName, ipcStr string) (*Stats, error) { - stats := &Stats{DeviceName: deviceName} - var currentPeer *Peer - for _, line := range strings.Split(strings.TrimSpace(ipcStr), "\n") { - if line == "" { - continue - } - parts := strings.SplitN(line, "=", 2) - if len(parts) != 2 { - continue - } - key := parts[0] - val := parts[1] - - switch key { - case privateKey: - key, err := hexToWireguardKey(val) - if err != nil { - log.Errorf("failed to parse private key: %v", err) - continue - } - stats.PublicKey = key.PublicKey().String() - case publicKey: - // Save previous peer - if currentPeer != nil { - stats.Peers = append(stats.Peers, *currentPeer) - } - key, err := hexToWireguardKey(val) - if err != nil { - log.Errorf("failed to parse public key: %v", err) - continue - } - currentPeer = &Peer{ - PublicKey: key.String(), - } - case listenPort: - if port, err := strconv.Atoi(val); err == nil { - stats.ListenPort = port - } - case fwmark: - if fwmark, err := strconv.Atoi(val); err == nil { - stats.FWMark = fwmark - } - case endpoint: - if currentPeer == nil { - continue - } - - host, portStr, err := net.SplitHostPort(val) - if err != nil { - log.Errorf("failed to parse endpoint: %v", err) - continue - } - port, err := strconv.Atoi(portStr) - if err != nil { - log.Errorf("failed to parse endpoint port: %v", err) - continue - } - currentPeer.Endpoint = net.UDPAddr{ - IP: net.ParseIP(host), - Port: port, - } - case allowedIP: - if currentPeer == nil { - continue - } - _, ipnet, err := net.ParseCIDR(val) - if err == nil { - currentPeer.AllowedIPs = append(currentPeer.AllowedIPs, *ipnet) - } - case ipcKeyTxBytes: - if currentPeer == nil { - continue - } - rxBytes, err := toBytes(val) - if err != nil { - continue - } - currentPeer.TxBytes = rxBytes - case ipcKeyRxBytes: - if currentPeer == nil { - continue - } - rxBytes, err := toBytes(val) - if err != nil { - continue - } - currentPeer.RxBytes = rxBytes - - case ipcKeyLastHandshakeTimeSec: - if currentPeer == nil { - continue - } - - ts, err := toLastHandshake(val) - if err != nil { - continue - } - currentPeer.LastHandshake = ts - case presharedKey: - if currentPeer == nil { - continue - } - if val != "" && val != "0000000000000000000000000000000000000000000000000000000000000000" { - if pskKey, err := hexToWireguardKey(val); err == nil { - currentPeer.PresharedKey = [32]byte(pskKey) - } - } - } - } - if currentPeer != nil { - stats.Peers = append(stats.Peers, *currentPeer) - } - return stats, nil -} diff --git a/client/iface/configurer/usp_allowedips_test.go b/client/iface/configurer/usp_allowedips_test.go index fba0ca546..effb360e9 100644 --- a/client/iface/configurer/usp_allowedips_test.go +++ b/client/iface/configurer/usp_allowedips_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/require" wgconn "golang.zx2c4.com/wireguard/conn" wgdevice "golang.zx2c4.com/wireguard/device" - "golang.zx2c4.com/wireguard/tun/tuntest" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/netbirdio/netbird/client/iface/bind" @@ -21,8 +20,7 @@ import ( func newTestUSPConfigurer(t *testing.T) *WGUSPConfigurer { t.Helper() - tun := tuntest.NewChannelTUN() - dev := wgdevice.NewDevice(tun.TUN(), wgconn.NewDefaultBind(), wgdevice.NewLogger(wgdevice.LogLevelSilent, "")) + dev := wgdevice.NewDevice(newIdleTUN(), wgconn.NewDefaultBind(), wgdevice.NewLogger(wgdevice.LogLevelSilent, "")) t.Cleanup(dev.Close) c := NewUSPConfigurerNoUAPI(dev, "wgtest0", bind.NewActivityRecorder()) @@ -118,27 +116,6 @@ func TestRemoveEndpointAddressDoesNotScaleWithPeerCount(t *testing.T) { large, small) } -// TestRemoveEndpointAddressFallsBackToDevice covers a peer the store never saw, which is what -// an out-of-band reconfiguration of the device leaves behind. The device stays the source of -// truth in that case, so the allowed IPs must still be preserved. -func TestRemoveEndpointAddressFallsBackToDevice(t *testing.T) { - c := newTestUSPConfigurer(t) - peerKey := seedPeers(t, c, 3)[1] - require.NoError(t, c.AddAllowedIP(peerKey, netip.MustParsePrefix("10.20.0.0/16")), "add routed prefix") - - before := peerAllowedIPs(t, c, peerKey) - c.allowedIPs.reset() - - require.NoError(t, c.RemoveEndpointAddress(peerKey), "remove endpoint address") - - assert.ElementsMatch(t, before, peerAllowedIPs(t, c, peerKey), - "allowed IPs recovered from the device must be preserved") - - recovered, ok := c.allowedIPs.get(mustParseKey(t, peerKey)) - assert.True(t, ok, "the fallback must seed the store so the next call skips the device dump") - assert.Len(t, recovered, 2, "seeded prefixes") -} - func TestRemoveAllowedIPKeepsTheOtherPrefixes(t *testing.T) { c := newTestUSPConfigurer(t) peerKey := seedPeers(t, c, 3)[0] @@ -156,10 +133,8 @@ func TestRemoveAllowedIPKeepsTheOtherPrefixes(t *testing.T) { } // TestAddAllowedIPOnAbsentPeerDoesNotResurrectIt covers the lazy connection window documented -// in #6863: AddAllowedIP is update-only, a silent no-op when the peer is absent, so it must not -// leave the store claiming prefixes the device never took. RemoveEndpointAddress re-adds a peer -// without update-only, so a phantom entry would create a peer the device had dropped, and a -// created peer would steal those allowed IPs from whichever peer legitimately holds them. +// in #6863: AddAllowedIP is update-only, a silent no-op when the peer is absent, and clearing +// the endpoint of an absent peer must not create one either. func TestAddAllowedIPOnAbsentPeerDoesNotResurrectIt(t *testing.T) { c := newTestUSPConfigurer(t) seedPeers(t, c, 2) @@ -253,16 +228,11 @@ func TestUpdatePeerDoesNotWidenAMappedPrefixOnTheDevice(t *testing.T) { onDevice := peerAllowedIPs(t, c, peerKey) assert.NotContains(t, onDevice, "0.0.0.0/0", "the device must not be given a catch-all allowed IP") assert.Equal(t, []string{"10.1.0.0/16"}, onDevice, "the device holds the normalized prefix") - - recorded, ok := c.allowedIPs.get(mustParseKey(t, peerKey)) - require.True(t, ok, "the peer must be recorded") - require.Len(t, recorded, 1, "one prefix recorded") - assert.Equal(t, onDevice[0], recorded[0].String(), "device and store must agree") } // TestUpdatePeerWithAnUnusableEndpointTouchesNothing pins the ordering: the endpoint is -// parsed before the device is configured, so a failure cannot leave the device holding a -// peer that the store never learned about, with the prefix handover skipped along with it. +// parsed before the device is configured, so a failure cannot leave a half-configured peer +// on the device. func TestUpdatePeerWithAnUnusableEndpointTouchesNothing(t *testing.T) { c := newTestUSPConfigurer(t) seedPeers(t, c, 2) @@ -279,40 +249,4 @@ func TestUpdatePeerWithAnUnusableEndpointTouchesNothing(t *testing.T) { stats, err := c.FullStats() require.NoError(t, err, "read device stats") assert.Len(t, stats.Peers, 2, "the peer must not have reached the device") - - _, ok := c.allowedIPs.get(mustParseKey(t, peerKey)) - assert.False(t, ok, "the peer must not have been recorded either") -} - -// TestRemovePeerKeepsTheRecordWhenTheDeviceRefuses covers a removal that never reached the -// device. A single peer removal is one write, so a failure leaves the peer on the device -// exactly as it was, and the record still describes it; dropping it would only force the -// next caller to read the whole device back for an answer it already had. -func TestRemovePeerKeepsTheRecordWhenTheDeviceRefuses(t *testing.T) { - c := newTestUSPConfigurer(t) - peerKey := seedPeers(t, c, 1)[0] - require.NoError(t, c.AddAllowedIP(peerKey, netip.MustParsePrefix("10.20.0.0/16")), "add routed prefix") - - before, ok := c.allowedIPs.get(mustParseKey(t, peerKey)) - require.True(t, ok, "the peer must be recorded before the removal") - require.Len(t, before, 2, "overlay address plus routed prefix") - - // A closed device refuses every write, which is the shape of any failed removal. - c.device.Close() - - require.Error(t, c.RemovePeer(peerKey), "the removal must report the failure") - - after, ok := c.allowedIPs.get(mustParseKey(t, peerKey)) - require.True(t, ok, "a peer still on the device must stay recorded") - assert.Equal(t, before, after, "the record must describe the peer the device kept") -} - -// mustParseKey turns the textual key the configurer API takes into the form the store -// keys on. -func mustParseKey(t *testing.T, key string) wgtypes.Key { - t.Helper() - - parsed, err := wgtypes.ParseKey(key) - require.NoError(t, err, "parse peer key") - return parsed } diff --git a/client/iface/configurer/usp_test.go b/client/iface/configurer/usp_test.go index e32491c54..48b5ad1f0 100644 --- a/client/iface/configurer/usp_test.go +++ b/client/iface/configurer/usp_test.go @@ -2,88 +2,222 @@ package configurer import ( "encoding/hex" + "net" + "net/netip" + "os" + "strings" "testing" + "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + wgdevice "golang.zx2c4.com/wireguard/device" + "golang.zx2c4.com/wireguard/tun" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) -var ipcFixture = ` -private_key=e84b5a6d2717c1003a13b431570353dbaca9146cf150c5f8575680feba52027a -listen_port=12912 -public_key=b85996fecc9c7f1fc6d2572a76eda11d59bcd20be8e543b15ce4bd85a8e75a33 -preshared_key=188515093e952f5f22e865cef3012e72f8b5f0b598ac0309d5dacce3b70fcf52 -allowed_ip=192.168.4.4/32 -endpoint=[abcd:23::33%2]:51820 -public_key=58402e695ba1772b1cc9309755f043251ea77fdcf10fbe63989ceb7e19321376 -tx_bytes=38333 -rx_bytes=2224 -allowed_ip=192.168.4.6/32 -persistent_keepalive_interval=111 -endpoint=182.122.22.19:3233 -public_key=662e14fd594556f522604703340351258903b64f35553763f19426ab2a515c58 -endpoint=5.152.198.39:51820 -allowed_ip=192.168.4.10/32 -allowed_ip=192.168.4.11/32 -tx_bytes=1212111 -rx_bytes=1929999999 -protocol_version=1 -errno=0 +// idleTUN never reports an up event, so the wireguard-go device stays down: no socket is +// opened, no peer goroutine starts, and the configuration state is all the tests observe. +type idleTUN struct { + events chan tun.Event + closed chan struct{} +} -` +func newIdleTUN() *idleTUN { + return &idleTUN{events: make(chan tun.Event), closed: make(chan struct{})} +} -func Test_parseTransfers(t *testing.T) { - tests := []struct { - name string - peerKey string - want WGStats - }{ - { - name: "single", - peerKey: "b85996fecc9c7f1fc6d2572a76eda11d59bcd20be8e543b15ce4bd85a8e75a33", - want: WGStats{ - TxBytes: 0, - RxBytes: 0, - }, - }, - { - name: "multiple", - peerKey: "58402e695ba1772b1cc9309755f043251ea77fdcf10fbe63989ceb7e19321376", - want: WGStats{ - TxBytes: 38333, - RxBytes: 2224, - }, - }, - { - name: "lastpeer", - peerKey: "662e14fd594556f522604703340351258903b64f35553763f19426ab2a515c58", - want: WGStats{ - TxBytes: 1212111, - RxBytes: 1929999999, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - res, err := hex.DecodeString(tt.peerKey) - require.NoError(t, err) +func (t *idleTUN) File() *os.File { return nil } - key, err := wgtypes.NewKey(res) - require.NoError(t, err) +func (t *idleTUN) Read([][]byte, []int, int) (int, error) { + <-t.closed + return 0, os.ErrClosed +} - stats, err := parseTransfers(ipcFixture) - if err != nil { - require.NoError(t, err) - return - } +func (t *idleTUN) Write(bufs [][]byte, _ int) (int, error) { return len(bufs), nil } - stat, ok := stats[key.String()] - if !ok { - require.True(t, ok) - return - } +func (t *idleTUN) MTU() (int, error) { return 1420, nil } - require.Equal(t, tt.want, stat) - }) +func (t *idleTUN) Name() (string, error) { return "idle", nil } + +func (t *idleTUN) Events() <-chan tun.Event { return t.events } + +func (t *idleTUN) Close() error { + close(t.closed) + close(t.events) + return nil +} + +func (t *idleTUN) BatchSize() int { return 1 } + +func TestConfigureInterfaceAppliesKeyAndPortAndReplacesPeers(t *testing.T) { + c := newTestUSPConfigurer(t) + seedPeers(t, c, 2) + + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate device private key") + require.NoError(t, c.ConfigureInterface(key.String(), 51820), "reconfigure the interface") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + assert.Equal(t, key.PublicKey().String(), stats.PublicKey, "device public key") + assert.Equal(t, 51820, stats.ListenPort, "listen port") + assert.Empty(t, stats.Peers, "configuring the interface replaces every peer") +} + +func TestConfigureInterfaceRejectsAnInvalidPort(t *testing.T) { + c := newTestUSPConfigurer(t) + + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate device private key") + assert.Error(t, c.ConfigureInterface(key.String(), 70000), "a port above 65535 must be rejected") +} + +func TestGetStatsIsKeyedByPeerKey(t *testing.T) { + c := newTestUSPConfigurer(t) + keys := seedPeers(t, c, 3) + + stats, err := c.GetStats() + require.NoError(t, err, "read stats") + require.Len(t, stats, 3, "one entry per peer") + for _, key := range keys { + entry, ok := stats[key] + require.True(t, ok, "peer %s must be reported", key) + assert.Zero(t, entry.TxBytes, "no traffic was sent") + assert.Zero(t, entry.RxBytes, "no traffic was received") + assert.True(t, entry.LastHandshake.IsZero(), "no handshake happened") } } + +func TestFullStatsMatchesTheDeviceDump(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + psk, err := wgtypes.GenerateKey() + require.NoError(t, err, "generate preshared key") + endpoint := &net.UDPAddr{IP: net.IPv4(192, 0, 2, 10), Port: 51820} + prefixes := []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32"), netip.MustParsePrefix("10.20.0.0/16")} + require.NoError(t, c.UpdatePeer(peerKey, prefixes, 25*time.Second, endpoint, &psk), "add peer") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + require.Len(t, stats.Peers, 1, "one peer on the device") + peer := stats.Peers[0] + assert.Equal(t, peerKey, peer.PublicKey, "peer key") + assert.Equal(t, "192.0.2.10:51820", peer.Endpoint.String(), "endpoint") + assert.Equal(t, [32]byte(psk), peer.PresharedKey, "preshared key") + assert.Equal(t, []string{"100.64.0.1/32", "10.20.0.0/16"}, ipNetStrings(peer.AllowedIPs), "allowed IPs") + + block := uapiPeerBlock(t, c, peerKey) + assert.Contains(t, block, "endpoint=192.0.2.10:51820", "the dump must show the endpoint") + assert.Contains(t, block, "preshared_key="+hex.EncodeToString(psk[:]), "the dump must show the preshared key") + assert.Contains(t, block, "persistent_keepalive_interval=25", "the dump must show the keepalive") + assert.Contains(t, block, "allowed_ip=100.64.0.1/32", "the dump must show the first prefix") + assert.Contains(t, block, "allowed_ip=10.20.0.0/16", "the dump must show the second prefix") +} + +func TestUpdatePeerKeepsTheEndpointZone(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + // The idle device opens no socket, so the zone name does not have to exist on the host. + endpoint := &net.UDPAddr{IP: net.ParseIP("fe80::1"), Port: 51820, Zone: "eth0"} + prefixes := []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")} + require.NoError(t, c.UpdatePeer(peerKey, prefixes, 25*time.Second, endpoint, nil), "add peer") + + assert.Contains(t, uapiPeerBlock(t, c, peerKey), "endpoint=[fe80::1%eth0]:51820", + "a link-local endpoint must keep its zone") +} + +func TestRemoveEndpointAddressKeepsThePeerAndStopsTheKeepalive(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + endpoint := &net.UDPAddr{IP: net.IPv4(192, 0, 2, 10), Port: 51820} + prefixes := []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")} + require.NoError(t, c.UpdatePeer(peerKey, prefixes, 25*time.Second, endpoint, nil), "add peer") + before := c.device.LookupPeer(wgdevice.NoisePublicKey(priv.PublicKey())) + require.NotNil(t, before, "the peer must be on the device") + + require.NoError(t, c.RemoveEndpointAddress(peerKey), "remove endpoint address") + + assert.Same(t, before, c.device.LookupPeer(wgdevice.NoisePublicKey(priv.PublicKey())), + "the peer object must survive the endpoint removal") + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + require.Len(t, stats.Peers, 1, "the peer stays configured") + assert.Nil(t, stats.Peers[0].Endpoint.IP, "the endpoint must be gone") + assert.Equal(t, []string{"100.64.0.1/32"}, ipNetStrings(stats.Peers[0].AllowedIPs), "allowed IPs survive") + assert.Contains(t, uapiPeerBlock(t, c, peerKey), "persistent_keepalive_interval=0", + "an endpoint-less peer must not keep sending keepalives") + + require.NoError(t, c.UpdatePeer(peerKey, prefixes, 25*time.Second, endpoint, nil), "restore the endpoint") + block := uapiPeerBlock(t, c, peerKey) + assert.Contains(t, block, "endpoint=192.0.2.10:51820", "the endpoint must be back") + assert.Contains(t, block, "persistent_keepalive_interval=25", "the keepalive must be back") +} + +func TestUpdatePeerRejectsAnInvalidPrefix(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + peerKey := priv.PublicKey().String() + + require.Error(t, c.UpdatePeer(peerKey, []netip.Prefix{{}}, 25*time.Second, nil, nil), "an invalid prefix must be rejected") + + stats, err := c.FullStats() + require.NoError(t, err, "read device stats") + assert.Empty(t, stats.Peers, "the peer must not have reached the device") +} + +func TestRemoveAllowedIPOnAbsentPeerReportsThePeer(t *testing.T) { + c := newTestUSPConfigurer(t) + + priv, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err, "generate peer private key") + + assert.ErrorIs(t, c.RemoveAllowedIP(priv.PublicKey().String(), netip.MustParsePrefix("10.20.0.0/16")), ErrPeerNotFound, + "removing a prefix from a peer the device does not have must report the peer") +} + +func ipNetStrings(ipNets []net.IPNet) []string { + out := make([]string, 0, len(ipNets)) + for _, ipNet := range ipNets { + out = append(out, ipNet.String()) + } + return out +} + +// uapiPeerBlock reads the device back through the UAPI dump, so a typed write is checked +// against an independent view of the device. +func uapiPeerBlock(t *testing.T, c *WGUSPConfigurer, peerKey string) []string { + t.Helper() + + parsed, err := wgtypes.ParseKey(peerKey) + require.NoError(t, err, "parse peer key") + wanted := "public_key=" + hex.EncodeToString(parsed[:]) + + dump, err := c.device.IpcGet() + require.NoError(t, err, "dump device") + + var block []string + inBlock := false + for _, line := range strings.Split(strings.TrimSpace(dump), "\n") { + if strings.HasPrefix(line, "public_key=") { + inBlock = line == wanted + continue + } + if inBlock { + block = append(block, line) + } + } + require.NotEmpty(t, block, "peer %s must appear in the dump", peerKey) + return block +} diff --git a/client/iface/device/device_kernel_unix.go b/client/iface/device/device_kernel_unix.go index 3c429fb96..ee965dd24 100644 --- a/client/iface/device/device_kernel_unix.go +++ b/client/iface/device/device_kernel_unix.go @@ -32,6 +32,7 @@ type TunKernelDevice struct { link *wgLink udpMuxConn net.PacketConn udpMux *udpmux.UniversalUDPMuxDefault + configurer WGConfigurer } func NewKernelDevice(name string, address wgaddr.Address, wgPort int, key string, mtu uint16, transportNet transport.Net) *TunKernelDevice { @@ -68,13 +69,18 @@ func (t *TunKernelDevice) Create() (WGConfigurer, error) { return nil, fmt.Errorf("set mtu: %w", err) } - configurer := configurer.NewKernelConfigurer(t.name) + cfg := configurer.NewKernelConfigurer(t.name) - if err := configurer.ConfigureInterface(t.key, t.wgPort); err != nil { + if err := cfg.ConfigureInterface(t.key, t.wgPort); err != nil { + cfg.Close() return nil, fmt.Errorf("error configuring interface: %s", err) } - return configurer, nil + if t.configurer != nil { + t.configurer.Close() + } + t.configurer = cfg + return cfg, nil } func (t *TunKernelDevice) Up() (*udpmux.UniversalUDPMuxDefault, error) { @@ -132,6 +138,10 @@ func (t *TunKernelDevice) Close() error { closErr = err } + if t.configurer != nil { + t.configurer.Close() + } + if t.udpMux != nil { if err := t.udpMux.Close(); err != nil { log.Debugf("failed to close udp mux: %s", err) diff --git a/client/iface/iface.go b/client/iface/iface.go index 247f421a2..f6006fa87 100644 --- a/client/iface/iface.go +++ b/client/iface/iface.go @@ -51,7 +51,6 @@ func ValidateMTU(mtu uint16) error { type wgProxyFactory interface { GetProxy() wgproxy.Proxy - GetProxyPort() uint16 Free() error } @@ -81,12 +80,6 @@ func (w *WGIface) GetProxy() wgproxy.Proxy { return w.wgProxyFactory.GetProxy() } -// GetProxyPort returns the proxy port used by the WireGuard proxy. -// Returns 0 if no proxy port is used (e.g., for userspace WireGuard). -func (w *WGIface) GetProxyPort() uint16 { - return w.wgProxyFactory.GetProxyPort() -} - // GetBind returns the EndpointManager userspace bind mode. func (w *WGIface) GetBind() device.EndpointManager { w.mu.Lock() diff --git a/client/iface/iface_close_test.go b/client/iface/iface_close_test.go index 171e15d0a..ea3115ec0 100644 --- a/client/iface/iface_close_test.go +++ b/client/iface/iface_close_test.go @@ -52,7 +52,6 @@ func (f *fakeTunDevice) Close() error { type fakeProxyFactory struct{} func (fakeProxyFactory) GetProxy() wgproxy.Proxy { return nil } -func (fakeProxyFactory) GetProxyPort() uint16 { return 0 } func (fakeProxyFactory) Free() error { return nil } // TestWGIface_CloseReleasesMutexBeforeTunClose guards against a deadlock diff --git a/client/iface/iface_test.go b/client/iface/iface_test.go index 89c8cd16e..690a0f66f 100644 --- a/client/iface/iface_test.go +++ b/client/iface/iface_test.go @@ -40,14 +40,18 @@ func init() { peerPubKey = peerPrivateKey.PublicKey().String() } +// testIFaceBlackList mirrors the prefixes profilemanager.DefaultInterfaceBlacklist +// carries for the overlay interface. These tests create their own utun device, and +// stdnet's filter probes with wgctrl every interface it is not told to skip, which +// on a userspace WireGuard platform reaches the UAPI socket of this same process. +// Declared here rather than imported because profilemanager imports this package. +var testIFaceBlackList = []string{"wt", "utun", "tun0"} + func TestWGIface_UpdateAddr(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+4) addr := "100.64.0.1/8" wgPort := 33100 - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -127,10 +131,7 @@ func getIfaceAddrs(ifaceName string) ([]net.Addr, error) { func Test_CreateInterface(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+1) wgIP := "10.99.99.1/32" - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, Address: wgaddr.MustParseWGAddress(wgIP), @@ -170,10 +171,7 @@ func Test_Close(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+2) wgIP := "10.99.99.2/32" wgPort := 33100 - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -215,10 +213,7 @@ func TestRecreation(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+2) wgIP := "10.99.99.2/32" wgPort := 33100 - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -288,10 +283,7 @@ func Test_ConfigureInterface(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+3) wgIP := "10.99.99.5/30" wgPort := 33100 - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, Address: wgaddr.MustParseWGAddress(wgIP), @@ -343,10 +335,7 @@ func Test_ConfigureInterface(t *testing.T) { func Test_UpdatePeer(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+4) wgIP := "10.99.99.9/30" - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -413,10 +402,7 @@ func Test_UpdatePeer(t *testing.T) { func Test_RemovePeer(t *testing.T) { ifaceName := fmt.Sprintf("utun%d", WgIntNumber+4) wgIP := "10.99.99.13/30" - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) opts := WGIFaceOpts{ IFaceName: ifaceName, @@ -477,10 +463,7 @@ func Test_ConnectPeers(t *testing.T) { peer2wgPort := 33200 keepAlive := 1 * time.Second - newNet, err := stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet := stdnet.NewNet(context.Background(), testIFaceBlackList, nil) guid := fmt.Sprintf("{%s}", uuid.New().String()) device.CustomWindowsGUIDString = strings.ToLower(guid) @@ -516,10 +499,7 @@ func Test_ConnectPeers(t *testing.T) { guid = fmt.Sprintf("{%s}", uuid.New().String()) device.CustomWindowsGUIDString = strings.ToLower(guid) - newNet, err = stdnet.NewNet(context.Background(), nil) - if err != nil { - t.Fatal(err) - } + newNet = stdnet.NewNet(context.Background(), testIFaceBlackList, nil) optsPeer2 := WGIFaceOpts{ IFaceName: peer2ifaceName, @@ -568,11 +548,14 @@ func Test_ConnectPeers(t *testing.T) { if err != nil { t.Fatal(err) } - // The peers use userspace WireGuard (stdnet transport). A tight busy-loop - // here starves the wireguard-go goroutines that process the handshake, so - // poll on a ticker instead and yield the CPU between checks. WireGuard also - // only retries a lost handshake initiation every REKEY_TIMEOUT (5s), which - // is why the overall wait can occasionally stretch to tens of seconds. + // On Linux with the kernel module both peers are kernel devices, elsewhere + // they run on wireguard-go. A tight busy-loop here would starve the + // wireguard-go goroutines that process the handshake, so poll on a ticker + // instead and yield the CPU between checks. WireGuard also only retries a + // lost handshake initiation every REKEY_TIMEOUT (5s), which is why the + // overall wait can occasionally stretch to tens of seconds. Each side sends + // its first initiation when its peer is configured, and the first one leaves + // before the other device knows the peer, so that one is always wasted. timeout := 30 * time.Second timeoutChannel := time.After(timeout) ticker := time.NewTicker(500 * time.Millisecond) @@ -590,13 +573,26 @@ func Test_ConnectPeers(t *testing.T) { select { case <-timeoutChannel: - t.Fatalf("waiting for peer handshake timeout after %s", timeout.String()) + // The counters tell whether initiations were sent at all, whether they + // arrived, and whether only one direction is working. + t.Fatalf("waiting for peer handshake timeout after %s\n%s\n%s", timeout.String(), + describePeer(peer1ifaceName, peer2Key.PublicKey().String()), + describePeer(peer2ifaceName, peer1Key.PublicKey().String())) case <-ticker.C: } } } +func describePeer(ifaceName, peerPubKey string) string { + peer, err := getPeer(ifaceName, peerPubKey) + if err != nil { + return fmt.Sprintf("%s: peer %s: %v", ifaceName, peerPubKey, err) + } + return fmt.Sprintf("%s: peer %s endpoint=%v tx=%d rx=%d last_handshake=%v", + ifaceName, peerPubKey, peer.Endpoint, peer.TransmitBytes, peer.ReceiveBytes, peer.LastHandshakeTime) +} + func getPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) { wg, err := wgctrl.New() if err != nil { diff --git a/client/iface/udpmux/mux.go b/client/iface/udpmux/mux.go index c5d2de4a5..3aa0b4d88 100644 --- a/client/iface/udpmux/mux.go +++ b/client/iface/udpmux/mux.go @@ -200,10 +200,7 @@ func (m *SingleSocketUDPMux) updateLocalAddresses() { } if len(networks) > 0 { if m.params.Net == nil { - var err error - if m.params.Net, err = stdnet.NewNet(context.Background(), nil); err != nil { - m.params.Logger.Errorf("failed to get create network: %v", err) - } + m.params.Net = stdnet.NewNet(context.Background(), nil, nil) } ips, err := localInterfaces(m.params.Net, m.params.InterfaceFilter, nil, networks, true) diff --git a/client/iface/wgproxy/ebpf/portlookup.go b/client/iface/wgproxy/ebpf/portlookup.go deleted file mode 100644 index fce8f1507..000000000 --- a/client/iface/wgproxy/ebpf/portlookup.go +++ /dev/null @@ -1,32 +0,0 @@ -package ebpf - -import ( - "fmt" - "net" -) - -var ( - portRangeStart = 3128 - portRangeEnd = portRangeStart + 100 -) - -type portLookup struct { -} - -func (pl portLookup) searchFreePort() (int, error) { - for i := portRangeStart; i <= portRangeEnd; i++ { - if pl.tryToBind(i) == nil { - return i, nil - } - } - return 0, fmt.Errorf("failed to bind free port for eBPF proxy") -} - -func (pl portLookup) tryToBind(port int) error { - l, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port)) - if err != nil { - return err - } - _ = l.Close() - return nil -} diff --git a/client/iface/wgproxy/ebpf/portlookup_test.go b/client/iface/wgproxy/ebpf/portlookup_test.go deleted file mode 100644 index a2e92fc79..000000000 --- a/client/iface/wgproxy/ebpf/portlookup_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package ebpf - -import ( - "fmt" - "net" - "testing" -) - -func Test_portLookup_searchFreePort(t *testing.T) { - pl := portLookup{} - _, err := pl.searchFreePort() - if err != nil { - t.Fatal(err) - } -} - -func Test_portLookup_on_allocated(t *testing.T) { - pl := portLookup{} - - portRangeStart = 4128 - portRangeEnd = portRangeStart + 100 - - allocatedPort, err := allocatePort(portRangeStart) - if err != nil { - t.Fatal(err) - } - defer allocatedPort.Close() - - fp, err := pl.searchFreePort() - if err != nil { - t.Fatal(err) - } - - if fp != (portRangeStart + 1) { - t.Errorf("invalid free port, expected: %d, got: %d", portRangeStart+1, fp) - } -} - -func allocatePort(port int) (net.PacketConn, error) { - c, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port)) - if err != nil { - return nil, err - } - return c, err -} diff --git a/client/iface/wgproxy/ebpf/proxy.go b/client/iface/wgproxy/ebpf/proxy.go deleted file mode 100644 index 91c741c0d..000000000 --- a/client/iface/wgproxy/ebpf/proxy.go +++ /dev/null @@ -1,243 +0,0 @@ -//go:build linux && !android - -package ebpf - -import ( - "context" - "fmt" - "net" - "sync" - - "github.com/hashicorp/go-multierror" - "github.com/pion/transport/v3" - log "github.com/sirupsen/logrus" - - nberrors "github.com/netbirdio/netbird/client/errors" - "github.com/netbirdio/netbird/client/iface/bufsize" - "github.com/netbirdio/netbird/client/iface/wgproxy/rawsocket" - "github.com/netbirdio/netbird/client/internal/ebpf" - ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager" - nbnet "github.com/netbirdio/netbird/client/net" -) - -const ( - loopbackAddr = "127.0.0.1" -) - -// WGEBPFProxy definition for proxy with EBPF support -type WGEBPFProxy struct { - localWGListenPort int - proxyPort int - mtu uint16 - - ebpfManager ebpfMgr.Manager - relayedConnStore map[uint16]net.Conn - relayedConnMutex sync.Mutex - - lastUsedPort uint16 - rawConnIPv4 net.PacketConn - rawConnIPv6 net.PacketConn - conn transport.UDPConn - - ctx context.Context - ctxCancel context.CancelFunc -} - -// NewWGEBPFProxy create new WGEBPFProxy instance -func NewWGEBPFProxy(wgPort int, mtu uint16) *WGEBPFProxy { - log.Debugf("instantiate ebpf proxy") - wgProxy := &WGEBPFProxy{ - localWGListenPort: wgPort, - mtu: mtu, - ebpfManager: ebpf.GetEbpfManagerInstance(), - relayedConnStore: make(map[uint16]net.Conn), - } - return wgProxy -} - -// Listen load ebpf program and listen the proxy -func (p *WGEBPFProxy) Listen() error { - pl := portLookup{} - proxyPort, err := pl.searchFreePort() - if err != nil { - return err - } - p.proxyPort = proxyPort - - // Prepare IPv4 raw socket (required) - p.rawConnIPv4, err = rawsocket.PrepareSenderRawSocketIPv4() - if err != nil { - return err - } - - // Prepare IPv6 raw socket (optional) - p.rawConnIPv6, err = rawsocket.PrepareSenderRawSocketIPv6() - if err != nil { - log.Warnf("failed to prepare IPv6 raw socket, continuing with IPv4 only: %v", err) - } - - err = p.ebpfManager.LoadWgProxy(proxyPort, p.localWGListenPort) - if err != nil { - if closeErr := p.rawConnIPv4.Close(); closeErr != nil { - log.Warnf("failed to close IPv4 raw socket: %v", closeErr) - } - if p.rawConnIPv6 != nil { - if closeErr := p.rawConnIPv6.Close(); closeErr != nil { - log.Warnf("failed to close IPv6 raw socket: %v", closeErr) - } - } - return err - } - - addr := net.UDPAddr{ - Port: proxyPort, - IP: net.ParseIP(loopbackAddr), - } - - p.ctx, p.ctxCancel = context.WithCancel(context.Background()) - - conn, err := nbnet.ListenUDP("udp", &addr) - if err != nil { - if cErr := p.Free(); cErr != nil { - log.Errorf("Failed to close the wgproxy: %s", cErr) - } - return err - } - p.conn = conn - - go p.proxyToRemote() - log.Infof("local wg proxy listening on: %d", proxyPort) - return nil -} - -// AddRelayedConn add new relayed connection for the proxy -func (p *WGEBPFProxy) AddRelayedConn(relayedConn net.Conn) (*net.UDPAddr, error) { - wgEndpointPort, err := p.storeRelayedConn(relayedConn) - if err != nil { - return nil, err - } - - log.Infof("relayed conn added to wg proxy store: %s, endpoint port: :%d", relayedConn.RemoteAddr(), wgEndpointPort) - - wgEndpoint := &net.UDPAddr{ - IP: net.ParseIP(loopbackAddr), - Port: int(wgEndpointPort), - } - return wgEndpoint, nil -} - -// Free resources except the remoteConns will be keep open. -func (p *WGEBPFProxy) Free() error { - log.Debugf("free up ebpf wg proxy") - if p.ctx != nil && p.ctx.Err() != nil { - //nolint - return nil - } - - p.ctxCancel() - - var result *multierror.Error - if p.conn != nil { - if err := p.conn.Close(); err != nil { - result = multierror.Append(result, err) - } - } - - if err := p.ebpfManager.FreeWGProxy(); err != nil { - result = multierror.Append(result, err) - } - - if p.rawConnIPv4 != nil { - if err := p.rawConnIPv4.Close(); err != nil { - result = multierror.Append(result, err) - } - } - - if p.rawConnIPv6 != nil { - if err := p.rawConnIPv6.Close(); err != nil { - result = multierror.Append(result, err) - } - } - return nberrors.FormatErrorOrNil(result) -} - -// GetProxyPort returns the proxy listening port. -func (p *WGEBPFProxy) GetProxyPort() uint16 { - return uint16(p.proxyPort) -} - -// proxyToRemote read messages from local WireGuard interface and forward it to remote conn -// From this go routine has only one instance. -func (p *WGEBPFProxy) proxyToRemote() { - buf := make([]byte, p.mtu+bufsize.WGBufferOverhead) - for p.ctx.Err() == nil { - if err := p.readAndForwardPacket(buf); err != nil { - if p.ctx.Err() != nil { - return - } - log.Errorf("failed to proxy packet to remote conn: %s", err) - } - } -} - -func (p *WGEBPFProxy) readAndForwardPacket(buf []byte) error { - n, addr, err := p.conn.ReadFromUDP(buf) - if err != nil { - return fmt.Errorf("failed to read UDP packet from WG: %w", err) - } - - p.relayedConnMutex.Lock() - conn, ok := p.relayedConnStore[uint16(addr.Port)] - p.relayedConnMutex.Unlock() - if !ok { - if p.ctx.Err() == nil { - log.Debugf("relayed conn not found by port because conn already has been closed: %d", addr.Port) - } - return nil - } - - if _, err := conn.Write(buf[:n]); err != nil { - return fmt.Errorf("forward local WG packet (%d) to remote relayed conn: %w", addr.Port, err) - } - return nil -} - -func (p *WGEBPFProxy) storeRelayedConn(relayedConn net.Conn) (uint16, error) { - p.relayedConnMutex.Lock() - defer p.relayedConnMutex.Unlock() - - np, err := p.nextFreePort() - if err != nil { - return np, err - } - p.relayedConnStore[np] = relayedConn - return np, nil -} - -func (p *WGEBPFProxy) removeRelayedConn(relayedConnID uint16) { - p.relayedConnMutex.Lock() - defer p.relayedConnMutex.Unlock() - - _, ok := p.relayedConnStore[relayedConnID] - if ok { - log.Debugf("remove relayed conn from store by port: %d", relayedConnID) - } - delete(p.relayedConnStore, relayedConnID) -} - -func (p *WGEBPFProxy) nextFreePort() (uint16, error) { - if len(p.relayedConnStore) == 65535 { - return 0, fmt.Errorf("reached maximum relayed connection numbers") - } -generatePort: - if p.lastUsedPort == 65535 { - p.lastUsedPort = 1 - } else { - p.lastUsedPort++ - } - - if _, ok := p.relayedConnStore[p.lastUsedPort]; ok { - goto generatePort - } - return p.lastUsedPort, nil -} diff --git a/client/iface/wgproxy/ebpf/proxy_test.go b/client/iface/wgproxy/ebpf/proxy_test.go deleted file mode 100644 index 228c06c9b..000000000 --- a/client/iface/wgproxy/ebpf/proxy_test.go +++ /dev/null @@ -1,56 +0,0 @@ -//go:build linux && !android - -package ebpf - -import ( - "testing" -) - -func TestWGEBPFProxy_connStore(t *testing.T) { - wgProxy := NewWGEBPFProxy(1, 1280) - - p, _ := wgProxy.storeRelayedConn(nil) - if p != 1 { - t.Errorf("invalid initial port: %d", wgProxy.lastUsedPort) - } - - numOfConns := 10 - for i := 0; i < numOfConns; i++ { - p, _ = wgProxy.storeRelayedConn(nil) - } - if p != uint16(numOfConns)+1 { - t.Errorf("invalid last used port: %d, expected: %d", p, numOfConns+1) - } - if len(wgProxy.relayedConnStore) != numOfConns+1 { - t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.relayedConnStore), numOfConns+1) - } -} - -func TestWGEBPFProxy_portCalculation_overflow(t *testing.T) { - wgProxy := NewWGEBPFProxy(1, 1280) - - _, _ = wgProxy.storeRelayedConn(nil) - wgProxy.lastUsedPort = 65535 - p, _ := wgProxy.storeRelayedConn(nil) - - if len(wgProxy.relayedConnStore) != 2 { - t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.relayedConnStore), 2) - } - - if p != 2 { - t.Errorf("invalid last used port: %d, expected: %d", p, 2) - } -} - -func TestWGEBPFProxy_portCalculation_maxConn(t *testing.T) { - wgProxy := NewWGEBPFProxy(1, 1280) - - for i := 0; i < 65535; i++ { - _, _ = wgProxy.storeRelayedConn(nil) - } - - _, err := wgProxy.storeRelayedConn(nil) - if err == nil { - t.Errorf("invalid relayed conn store calculation") - } -} diff --git a/client/iface/wgproxy/factory_kernel.go b/client/iface/wgproxy/factory_kernel.go index 7821df3de..0b2329b96 100644 --- a/client/iface/wgproxy/factory_kernel.go +++ b/client/iface/wgproxy/factory_kernel.go @@ -8,11 +8,13 @@ import ( log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/client/iface/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/iface/wgproxy/loopback" udpProxy "github.com/netbirdio/netbird/client/iface/wgproxy/udp" ) const ( + envDisableKernelWGProxy = "NB_DISABLE_KERNEL_WG_PROXY" + // envDisableEBPFWGProxy is a deprecated alias for envDisableKernelWGProxy. envDisableEBPFWGProxy = "NB_DISABLE_EBPF_WG_PROXY" ) @@ -20,7 +22,7 @@ type KernelFactory struct { wgPort int mtu uint16 - ebpfProxy *ebpf.WGEBPFProxy + loopbackProxy *loopback.Proxy } func NewKernelFactory(wgPort int, mtu uint16) *KernelFactory { @@ -29,55 +31,56 @@ func NewKernelFactory(wgPort int, mtu uint16) *KernelFactory { mtu: mtu, } - if isEBPFDisabled() { + if isKernelProxyDisabled() { log.Infof("WireGuard Proxy Factory will produce UDP proxy") - log.Infof("eBPF WireGuard proxy is disabled via %s environment variable", envDisableEBPFWGProxy) return f } - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, mtu) - if err := ebpfProxy.Listen(); err != nil { + loopbackProxy := loopback.NewProxy(wgPort, mtu) + if err := loopbackProxy.Listen(); err != nil { log.Infof("WireGuard Proxy Factory will produce UDP proxy") - log.Warnf("failed to initialize ebpf proxy, fallback to user space proxy: %s", err) + log.Warnf("failed to initialize loopback proxy, fallback to user space proxy: %s", err) return f } - log.Infof("WireGuard Proxy Factory will produce eBPF proxy") - f.ebpfProxy = ebpfProxy + log.Infof("WireGuard Proxy Factory will produce loopback proxy") + f.loopbackProxy = loopbackProxy return f } func (w *KernelFactory) GetProxy() Proxy { - if w.ebpfProxy == nil { + if w.loopbackProxy == nil { return udpProxy.NewWGUDPProxy(w.wgPort, w.mtu) } - return ebpf.NewProxyWrapper(w.ebpfProxy) -} - -// GetProxyPort returns the eBPF proxy port, or 0 if eBPF is not active. -func (w *KernelFactory) GetProxyPort() uint16 { - if w.ebpfProxy == nil { - return 0 - } - return w.ebpfProxy.GetProxyPort() + return loopback.NewProxyWrapper(w.loopbackProxy) } func (w *KernelFactory) Free() error { - if w.ebpfProxy == nil { + if w.loopbackProxy == nil { return nil } - return w.ebpfProxy.Free() + return w.loopbackProxy.Free() } -func isEBPFDisabled() bool { - val := os.Getenv(envDisableEBPFWGProxy) +func isKernelProxyDisabled() bool { + env := envDisableKernelWGProxy + val := os.Getenv(env) + if val == "" { + env = envDisableEBPFWGProxy + val = os.Getenv(env) + } if val == "" { return false } + disabled, err := strconv.ParseBool(val) if err != nil { - log.Warnf("failed to parse %s: %v", envDisableEBPFWGProxy, err) + log.Warnf("failed to parse %s: %v", env, err) return false } + + if disabled { + log.Infof("kernel WireGuard proxy is disabled via %s", env) + } return disabled } diff --git a/client/iface/wgproxy/factory_usp.go b/client/iface/wgproxy/factory_usp.go index bbd67e076..a1b1c34d7 100644 --- a/client/iface/wgproxy/factory_usp.go +++ b/client/iface/wgproxy/factory_usp.go @@ -24,11 +24,6 @@ func (w *USPFactory) GetProxy() Proxy { return proxyBind.NewProxyBind(w.bind, w.mtu) } -// GetProxyPort returns 0 as userspace WireGuard doesn't use a separate proxy port. -func (w *USPFactory) GetProxyPort() uint16 { - return 0 -} - func (w *USPFactory) Free() error { return nil } diff --git a/client/iface/wgproxy/loopback/addr.go b/client/iface/wgproxy/loopback/addr.go new file mode 100644 index 000000000..52feee295 --- /dev/null +++ b/client/iface/wgproxy/loopback/addr.go @@ -0,0 +1,70 @@ +//go:build linux && !android + +package loopback + +import ( + "fmt" + "net/netip" +) + +// Peer endpoints live in the upper half of 127.0.0.0/8. Everything in that +// range is delivered to the loopback device without any address or route being +// configured, and staying out of 127.0.0.0/9 keeps well-known squatters such as +// 127.0.0.53 (systemd-resolved) and 127.0.1.1 out of the way. +const ( + addrRangeBase uint32 = 0x7f800000 // 127.128.0.0 + addrRangeSize uint32 = 1 << 23 // /9 + addrRangePrefix = "127.128.0.0/9" +) + +// allocator hands out one loopback address per relayed connection. The address +// is the peer's identity: WireGuard sends to it, and the proxy recovers which +// peer a packet belongs to from the destination address. +type allocator struct { + cursor uint32 +} + +// next returns the first free address at or after the cursor, wrapping once. +// inUse reports whether an address is already handed out. +func (a *allocator) next(inUse func(netip.Addr) bool) (netip.Addr, error) { + for i := uint32(0); i < addrRangeSize; i++ { + a.cursor = (a.cursor + 1) % addrRangeSize + addr := addrFromOffset(a.cursor) + if !addr.IsValid() { + continue + } + if inUse(addr) { + continue + } + return addr, nil + } + return netip.Addr{}, fmt.Errorf("no free endpoint address in %s", addrRangePrefix) +} + +// addrFromOffset maps an offset in the range to an address, skipping the .0 and +// .255 hosts. They are unremarkable on loopback, but tools and firewall rules +// tend to treat them as network and broadcast addresses. +func addrFromOffset(offset uint32) netip.Addr { + last := offset & 0xff + if last == 0 || last == 0xff { + return netip.Addr{} + } + + v := addrRangeBase + offset + return netip.AddrFrom4([4]byte{ + byte(v >> 24), + byte(v >> 16), + byte(v >> 8), + byte(v), + }) +} + +// inRange reports whether addr is one this proxy could have handed out. +func inRange(addr netip.Addr) bool { + if !addr.Is4() { + return false + } + b := addr.As4() + v := uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]) + return v >= addrRangeBase && v < addrRangeBase+addrRangeSize && b[3] != 0 && b[3] != 0xff +} diff --git a/client/iface/wgproxy/loopback/addr_test.go b/client/iface/wgproxy/loopback/addr_test.go new file mode 100644 index 000000000..3755b7269 --- /dev/null +++ b/client/iface/wgproxy/loopback/addr_test.go @@ -0,0 +1,114 @@ +//go:build linux && !android + +package loopback + +import ( + "net/netip" + "testing" +) + +func TestAllocatorHandsOutDistinctAddresses(t *testing.T) { + var a allocator + taken := make(map[netip.Addr]bool) + + for i := 0; i < 1000; i++ { + addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] }) + if err != nil { + t.Fatalf("allocate %d: %v", i, err) + } + if taken[addr] { + t.Fatalf("address %s handed out twice", addr) + } + if !inRange(addr) { + t.Fatalf("address %s outside %s", addr, addrRangePrefix) + } + taken[addr] = true + } +} + +func TestAllocatorSkipsNetworkAndBroadcastHosts(t *testing.T) { + var a allocator + taken := make(map[netip.Addr]bool) + + // enough allocations to walk past a .255/.0 boundary + for i := 0; i < 600; i++ { + addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] }) + if err != nil { + t.Fatalf("allocate %d: %v", i, err) + } + last := addr.As4()[3] + if last == 0 || last == 255 { + t.Fatalf("address %s ends in .%d", addr, last) + } + taken[addr] = true + } +} + +func TestAllocatorReusesReleasedAddresses(t *testing.T) { + var a allocator + taken := make(map[netip.Addr]bool) + inUse := func(candidate netip.Addr) bool { return taken[candidate] } + alloc := func() netip.Addr { + t.Helper() + addr, err := a.next(inUse) + if err != nil { + t.Fatalf("allocate: %v", err) + } + taken[addr] = true + return addr + } + + first := alloc() + second := alloc() + delete(taken, first) + + // The cursor only moves forward, so a released address comes back after a + // wrap. Park the cursor near the end of the range instead of allocating + // 2^23 addresses: the next call takes the last usable address, and the one + // after that wraps past the skipped .255 and .0 hosts to the released one. + a.cursor = addrRangeSize - 3 + last := alloc() + if want := netip.MustParseAddr("127.255.255.254"); last != want { + t.Fatalf("expected the last usable address %s before the wrap, got %s", want, last) + } + + if reused := alloc(); reused != first { + t.Fatalf("expected the released address %s after the wrap, got %s", first, reused) + } + + // second is still held, so the allocator must step over it. + if next := alloc(); next == second { + t.Fatalf("allocator handed out %s while it was still in use", second) + } +} + +func TestInRange(t *testing.T) { + tests := []struct { + addr string + want bool + }{ + {"127.128.0.1", true}, + {"127.255.255.254", true}, + {"127.128.0.0", false}, // network host, never handed out + {"127.128.5.255", false}, // broadcast host, never handed out + {"127.127.255.255", false}, // below the range, where 127.0.0.53 and friends live + {"127.0.0.1", false}, + {"127.0.0.53", false}, + {"127.0.1.1", false}, + {"128.0.0.1", false}, + {"10.0.0.1", false}, + } + + for _, tc := range tests { + addr := netip.MustParseAddr(tc.addr) + if got := inRange(addr); got != tc.want { + t.Errorf("inRange(%s) = %v, want %v", tc.addr, got, tc.want) + } + } +} + +func TestInRangeIgnoresIPv6(t *testing.T) { + if inRange(netip.MustParseAddr("::1")) { + t.Error("inRange(::1) = true, want false") + } +} diff --git a/client/iface/wgproxy/loopback/proxy.go b/client/iface/wgproxy/loopback/proxy.go new file mode 100644 index 000000000..f9364c766 --- /dev/null +++ b/client/iface/wgproxy/loopback/proxy.go @@ -0,0 +1,291 @@ +//go:build linux && !android + +package loopback + +import ( + "context" + "fmt" + "net" + "net/netip" + "sync" + "syscall" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + "golang.org/x/net/ipv4" + "golang.org/x/sys/unix" + + nberrors "github.com/netbirdio/netbird/client/errors" + "github.com/netbirdio/netbird/client/iface/bufsize" + "github.com/netbirdio/netbird/client/iface/wgproxy/rawsocket" +) + +const ( + loopbackDevice = "lo" + + portRangeStart = 3128 + portRangeEnd = portRangeStart + 100 +) + +// Proxy forwards packets between relayed connections and a local kernel +// WireGuard instance. Every relayed peer gets its own loopback address as its +// WireGuard endpoint, so a single socket serves all of them: the destination +// address of an incoming packet identifies the peer. +type Proxy struct { + localWGListenPort int + mtu uint16 + proxyPort int + + conn *net.UDPConn + packetConn *ipv4.PacketConn + loIndex int + rawConnIPv4 net.PacketConn + rawConnIPv6 net.PacketConn + + relayedConnMutex sync.Mutex + relayedConnStore map[netip.Addr]net.Conn + addrs allocator + + ctx context.Context + ctxCancel context.CancelFunc +} + +// NewProxy creates a proxy for the WireGuard instance listening on wgPort. +func NewProxy(wgPort int, mtu uint16) *Proxy { + log.Debugf("instantiate loopback wg proxy") + return &Proxy{ + localWGListenPort: wgPort, + mtu: mtu, + relayedConnStore: make(map[netip.Addr]net.Conn), + } +} + +// Listen opens the shared socket and starts forwarding WireGuard packets to the +// relayed connections. +func (p *Proxy) Listen() error { + rawConnIPv4, err := rawsocket.PrepareSenderRawSocketIPv4() + if err != nil { + return fmt.Errorf("prepare IPv4 raw socket: %w", err) + } + p.rawConnIPv4 = rawConnIPv4 + + p.rawConnIPv6, err = rawsocket.PrepareSenderRawSocketIPv6() + if err != nil { + log.Warnf("failed to prepare IPv6 raw socket, continuing with IPv4 only: %v", err) + } + + loopback, err := net.InterfaceByName(loopbackDevice) + if err != nil { + if freeErr := p.Free(); freeErr != nil { + log.Errorf("failed to free the wgproxy: %s", freeErr) + } + return fmt.Errorf("look up %s: %w", loopbackDevice, err) + } + p.loIndex = loopback.Index + + if err := p.listen(); err != nil { + if freeErr := p.Free(); freeErr != nil { + log.Errorf("failed to free the wgproxy: %s", freeErr) + } + return err + } + + p.ctx, p.ctxCancel = context.WithCancel(context.Background()) + + go p.proxyToRemote() + log.Infof("local wg proxy listening on %s:%d", addrRangePrefix, p.proxyPort) + return nil +} + +// listen binds the shared socket on the first free port of the range. The bind +// has to be a wildcard one to receive every peer address in the range, so it is +// restricted to the loopback device: without that the port would be reachable +// on every interface. +func (p *Proxy) listen() error { + var lastErr error + for port := portRangeStart; port <= portRangeEnd; port++ { + err := p.listenOn(port) + if err == nil { + p.proxyPort = port + return nil + } + lastErr = err + } + return fmt.Errorf("bind proxy port in range %d-%d: %w", portRangeStart, portRangeEnd, lastErr) +} + +func (p *Proxy) listenOn(proxyPort int) error { + lc := net.ListenConfig{ + Control: func(_, _ string, c syscall.RawConn) error { + var sockErr error + if err := c.Control(func(fd uintptr) { + if err := unix.SetsockoptString(int(fd), unix.SOL_SOCKET, unix.SO_BINDTODEVICE, loopbackDevice); err != nil { + sockErr = fmt.Errorf("bind to %s: %w", loopbackDevice, err) + return + } + }); err != nil { + return fmt.Errorf("control socket: %w", err) + } + return sockErr + }, + } + + conn, err := lc.ListenPacket(context.Background(), "udp4", fmt.Sprintf(":%d", proxyPort)) + if err != nil { + return fmt.Errorf("listen on :%d: %w", proxyPort, err) + } + + udpConn, ok := conn.(*net.UDPConn) + if !ok { + if closeErr := conn.Close(); closeErr != nil { + log.Errorf("failed to close proxy conn: %s", closeErr) + } + return fmt.Errorf("unexpected conn type %T", conn) + } + + packetConn := ipv4.NewPacketConn(udpConn) + // the destination address carries the peer identity, the interface index is + // checked on receive as a second line of defense behind SO_BINDTODEVICE + if err := packetConn.SetControlMessage(ipv4.FlagDst|ipv4.FlagInterface, true); err != nil { + if closeErr := udpConn.Close(); closeErr != nil { + log.Errorf("failed to close proxy conn: %s", closeErr) + } + return fmt.Errorf("request destination address: %w", err) + } + + p.conn = udpConn + p.packetConn = packetConn + return nil +} + +// AddRelayedConn assigns an endpoint address to the relayed connection and +// returns the address WireGuard should send to, along with the key the +// connection is stored under. +func (p *Proxy) AddRelayedConn(relayedConn net.Conn) (*net.UDPAddr, netip.Addr, error) { + addr, err := p.storeRelayedConn(relayedConn) + if err != nil { + return nil, netip.Addr{}, err + } + + log.Infof("relayed conn added to wg proxy store: %s, endpoint address: %s", relayedConn.RemoteAddr(), addr) + + return &net.UDPAddr{ + IP: addr.AsSlice(), + Port: p.proxyPort, + }, addr, nil +} + +// Free releases the proxy resources. The relayed connections are left open. +func (p *Proxy) Free() error { + log.Debugf("free up loopback wg proxy") + if p.ctx != nil && p.ctx.Err() != nil { + //nolint + return nil + } + + if p.ctxCancel != nil { + p.ctxCancel() + } + + var result *multierror.Error + if p.conn != nil { + if err := p.conn.Close(); err != nil { + result = multierror.Append(result, err) + } + } + + if p.rawConnIPv4 != nil { + if err := p.rawConnIPv4.Close(); err != nil { + result = multierror.Append(result, err) + } + } + + if p.rawConnIPv6 != nil { + if err := p.rawConnIPv6.Close(); err != nil { + result = multierror.Append(result, err) + } + } + return nberrors.FormatErrorOrNil(result) +} + +// proxyToRemote reads packets from the local WireGuard instance and forwards +// them to the relayed connection the destination address belongs to. +func (p *Proxy) proxyToRemote() { + buf := make([]byte, p.mtu+bufsize.WGBufferOverhead) + for p.ctx.Err() == nil { + if err := p.readAndForwardPacket(buf); err != nil { + if p.ctx.Err() != nil { + return + } + log.Errorf("failed to proxy packet to remote conn: %s", err) + } + } +} + +func (p *Proxy) readAndForwardPacket(buf []byte) error { + n, cm, _, err := p.packetConn.ReadFrom(buf) + if err != nil { + return fmt.Errorf("read UDP packet from WG: %w", err) + } + + if cm == nil { + return fmt.Errorf("no control message on packet") + } + + if cm.IfIndex != p.loIndex { + log.Tracef("dropping packet received on interface %d instead of %s", cm.IfIndex, loopbackDevice) + return nil + } + + dst, ok := netip.AddrFromSlice(cm.Dst.To4()) + if !ok || !inRange(dst) { + log.Tracef("dropping packet for unexpected destination %s", cm.Dst) + return nil + } + + p.relayedConnMutex.Lock() + conn, ok := p.relayedConnStore[dst] + p.relayedConnMutex.Unlock() + if !ok { + if p.ctx.Err() == nil { + log.Debugf("relayed conn not found by address because conn already has been closed: %s", dst) + } + return nil + } + + if _, err := conn.Write(buf[:n]); err != nil { + return fmt.Errorf("forward local WG packet (%s) to remote relayed conn: %w", dst, err) + } + return nil +} + +func (p *Proxy) storeRelayedConn(relayedConn net.Conn) (netip.Addr, error) { + p.relayedConnMutex.Lock() + defer p.relayedConnMutex.Unlock() + + addr, err := p.addrs.next(func(a netip.Addr) bool { + _, ok := p.relayedConnStore[a] + return ok + }) + if err != nil { + return netip.Addr{}, err + } + + p.relayedConnStore[addr] = relayedConn + return addr, nil +} + +// removeRelayedConn releases an endpoint address. It only removes the entry +// while it still belongs to relayedConn, so a late release cannot take an +// address away from the peer it was handed to next. +func (p *Proxy) removeRelayedConn(addr netip.Addr, relayedConn net.Conn) { + p.relayedConnMutex.Lock() + defer p.relayedConnMutex.Unlock() + + if stored, ok := p.relayedConnStore[addr]; !ok || stored != relayedConn { + return + } + + log.Debugf("remove relayed conn from store by address: %s", addr) + delete(p.relayedConnStore, addr) +} diff --git a/client/iface/wgproxy/loopback/proxy_privileged_test.go b/client/iface/wgproxy/loopback/proxy_privileged_test.go new file mode 100644 index 000000000..6314fe2de --- /dev/null +++ b/client/iface/wgproxy/loopback/proxy_privileged_test.go @@ -0,0 +1,196 @@ +//go:build linux && !android && privileged + +package loopback + +import ( + "context" + "net" + "strconv" + "testing" + "time" +) + +const testWGPort = 51862 + +// relayEnd stands in for a relayed connection: the proxy writes what it read +// from WireGuard into it, and the test reads it back out here. +func relayEnd(t *testing.T) (proxySide net.Conn, testSide *net.UDPConn) { + t.Helper() + + testSide, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")}) + if err != nil { + t.Fatalf("relay listener: %v", err) + } + t.Cleanup(func() { + if err := testSide.Close(); err != nil { + t.Logf("close relay listener: %v", err) + } + }) + + proxySide, err = net.Dial("udp", testSide.LocalAddr().String()) + if err != nil { + t.Fatalf("relay conn: %v", err) + } + t.Cleanup(func() { + if err := proxySide.Close(); err != nil { + t.Logf("close relay conn: %v", err) + } + }) + + return proxySide, testSide +} + +// TestProxyDemuxesByDestinationAddress is the core of the design: one socket +// serves every peer, and the destination address decides which relayed +// connection a WireGuard packet belongs to. +func TestProxyDemuxesByDestinationAddress(t *testing.T) { + proxy := NewProxy(testWGPort, 1280) + if err := proxy.Listen(); err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { + if err := proxy.Free(); err != nil { + t.Errorf("free proxy: %v", err) + } + }() + + const peers = 3 + endpoints := make([]*net.UDPAddr, 0, peers) + readers := make([]*net.UDPConn, 0, peers) + for i := 0; i < peers; i++ { + proxySide, testSide := relayEnd(t) + endpoint, _, err := proxy.AddRelayedConn(proxySide) + if err != nil { + t.Fatalf("add relayed conn %d: %v", i, err) + } + if endpoint.Port != proxy.proxyPort { + t.Errorf("peer %d endpoint port = %d, want the shared proxy port %d", i, endpoint.Port, proxy.proxyPort) + } + endpoints = append(endpoints, endpoint) + readers = append(readers, testSide) + } + + // every peer must have its own address, otherwise they are indistinguishable + seen := make(map[string]bool, peers) + for i, endpoint := range endpoints { + if seen[endpoint.IP.String()] { + t.Fatalf("peer %d reuses endpoint address %s", i, endpoint.IP) + } + seen[endpoint.IP.String()] = true + } + + wgSock, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")}) + if err != nil { + t.Fatalf("wg socket: %v", err) + } + defer func() { + if err := wgSock.Close(); err != nil { + t.Logf("close wg socket: %v", err) + } + }() + + for i, endpoint := range endpoints { + payload := []byte{byte(i), 'p', 'k', 't'} + if _, err := wgSock.WriteTo(payload, endpoint); err != nil { + t.Fatalf("write to peer %d endpoint %s: %v", i, endpoint, err) + } + + buf := make([]byte, 1500) + if err := readers[i].SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set read deadline: %v", err) + } + n, _, err := readers[i].ReadFrom(buf) + if err != nil { + t.Fatalf("peer %d did not receive its packet: %v", i, err) + } + if string(buf[:n]) != string(payload) { + t.Errorf("peer %d got %q, want %q", i, buf[:n], payload) + } + + // no other peer may see it + for j, other := range readers { + if j == i { + continue + } + if err := other.SetReadDeadline(time.Now().Add(100 * time.Millisecond)); err != nil { + t.Fatalf("set read deadline: %v", err) + } + if _, _, err := other.ReadFrom(buf); err == nil { + t.Errorf("packet for peer %d also delivered to peer %d", i, j) + } + } + } +} + +// TestProxyDropsPacketsOutsideTheRange guards the wildcard bind: anything that +// is not addressed to a handed-out endpoint must not reach a relayed peer. +func TestProxyDropsPacketsOutsideTheRange(t *testing.T) { + proxy := NewProxy(testWGPort+1, 1280) + if err := proxy.Listen(); err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { + if err := proxy.Free(); err != nil { + t.Errorf("free proxy: %v", err) + } + }() + + proxySide, testSide := relayEnd(t) + if _, _, err := proxy.AddRelayedConn(proxySide); err != nil { + t.Fatalf("add relayed conn: %v", err) + } + + sender, err := net.Dial("udp", net.JoinHostPort("127.0.0.1", strconv.Itoa(proxy.proxyPort))) + if err != nil { + t.Fatalf("sender: %v", err) + } + defer func() { + if err := sender.Close(); err != nil { + t.Logf("close sender: %v", err) + } + }() + + if _, err := sender.Write([]byte("stray")); err != nil { + t.Fatalf("write stray packet: %v", err) + } + + buf := make([]byte, 1500) + if err := testSide.SetReadDeadline(time.Now().Add(500 * time.Millisecond)); err != nil { + t.Fatalf("set read deadline: %v", err) + } + if _, _, err := testSide.ReadFrom(buf); err == nil { + t.Error("packet addressed to 127.0.0.1 was forwarded to a relayed peer") + } +} + +// A wrapper that is closed before it starts forwarding still has to give its +// endpoint address back, otherwise the range leaks an address per attempt. +func TestClosingBeforeWorkReleasesTheAddress(t *testing.T) { + proxy := NewProxy(testWGPort+2, 1280) + if err := proxy.Listen(); err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { + if err := proxy.Free(); err != nil { + t.Errorf("free proxy: %v", err) + } + }() + + proxySide, _ := relayEnd(t) + wrapper := NewProxyWrapper(proxy) + if err := wrapper.AddRelayedConn(context.Background(), nil, proxySide); err != nil { + t.Fatalf("add relayed conn: %v", err) + } + + if got := len(proxy.relayedConnStore); got != 1 { + t.Fatalf("store holds %d entries after adding one conn, want 1", got) + } + + if err := wrapper.CloseConn(); err != nil { + t.Fatalf("close conn: %v", err) + } + + if got := len(proxy.relayedConnStore); got != 0 { + t.Errorf("store holds %d entries after close, want 0", got) + } +} diff --git a/client/iface/wgproxy/ebpf/wrapper.go b/client/iface/wgproxy/loopback/wrapper.go similarity index 84% rename from client/iface/wgproxy/ebpf/wrapper.go rename to client/iface/wgproxy/loopback/wrapper.go index f75e21aa6..a9cb1ab59 100644 --- a/client/iface/wgproxy/ebpf/wrapper.go +++ b/client/iface/wgproxy/loopback/wrapper.go @@ -1,6 +1,6 @@ //go:build linux && !android -package ebpf +package loopback import ( "context" @@ -8,6 +8,7 @@ import ( "fmt" "io" "net" + "net/netip" "sync" "github.com/google/gopacket" @@ -95,13 +96,14 @@ func NewPacketHeaders(localWGListenPort int, endpoint *net.UDPAddr) (*PacketHead // ProxyWrapper help to keep the remoteConn instance for net.Conn.Close function call type ProxyWrapper struct { - wgeBPFProxy *WGEBPFProxy + proxy *Proxy remoteConn net.Conn ctx context.Context cancel context.CancelFunc wgRelayedEndpointAddr *net.UDPAddr + peerAddr netip.Addr headers *PacketHeaders headerCurrentUsed *PacketHeaders rawConn net.PacketConn @@ -113,36 +115,44 @@ type ProxyWrapper struct { closeListener *listener.CloseListener } -func NewProxyWrapper(proxy *WGEBPFProxy) *ProxyWrapper { +func NewProxyWrapper(proxy *Proxy) *ProxyWrapper { return &ProxyWrapper{ - wgeBPFProxy: proxy, + proxy: proxy, pausedCond: sync.NewCond(&sync.Mutex{}), closeListener: listener.NewCloseListener(), } } func (p *ProxyWrapper) AddRelayedConn(ctx context.Context, _ *net.UDPAddr, remoteConn net.Conn) error { - addr, err := p.wgeBPFProxy.AddRelayedConn(remoteConn) + addr, peerAddr, err := p.proxy.AddRelayedConn(remoteConn) if err != nil { return fmt.Errorf("add relayed conn: %w", err) } - headers, err := NewPacketHeaders(p.wgeBPFProxy.localWGListenPort, addr) + // the endpoint address is otherwise only released by the forwarding + // goroutine, which never starts when the setup below fails + release := func() { p.proxy.removeRelayedConn(peerAddr, remoteConn) } + + headers, err := NewPacketHeaders(p.proxy.localWGListenPort, addr) if err != nil { + release() return fmt.Errorf("create packet sender: %w", err) } // Check if required raw connection is available - if !headers.isIPv4 && p.wgeBPFProxy.rawConnIPv6 == nil { + if !headers.isIPv4 && p.proxy.rawConnIPv6 == nil { + release() return errIPv6ConnNotAvailable } - if headers.isIPv4 && p.wgeBPFProxy.rawConnIPv4 == nil { + if headers.isIPv4 && p.proxy.rawConnIPv4 == nil { + release() return errIPv4ConnNotAvailable } p.remoteConn = remoteConn p.ctx, p.cancel = context.WithCancel(ctx) p.wgRelayedEndpointAddr = addr + p.peerAddr = peerAddr p.headers = headers p.rawConn = p.selectRawConn(headers) return nil @@ -193,18 +203,18 @@ func (p *ProxyWrapper) RedirectAs(endpoint *net.UDPAddr) { return } - header, err := NewPacketHeaders(p.wgeBPFProxy.localWGListenPort, endpoint) + header, err := NewPacketHeaders(p.proxy.localWGListenPort, endpoint) if err != nil { log.Errorf("failed to create packet headers: %s", err) return } // Check if required raw connection is available - if !header.isIPv4 && p.wgeBPFProxy.rawConnIPv6 == nil { + if !header.isIPv4 && p.proxy.rawConnIPv6 == nil { log.Error(errIPv6ConnNotAvailable) return } - if header.isIPv4 && p.wgeBPFProxy.rawConnIPv4 == nil { + if header.isIPv4 && p.proxy.rawConnIPv4 == nil { log.Error(errIPv4ConnNotAvailable) return } @@ -240,6 +250,10 @@ func (p *ProxyWrapper) CloseConn() error { p.closeListener.SetCloseListener(nil) + // releases the endpoint address for a wrapper that was never started, and + // is a no-op once the forwarding goroutine has released it + p.proxy.removeRelayedConn(p.peerAddr, p.remoteConn) + p.pausedCond.L.Lock() p.paused = false p.pausedCond.Signal() @@ -252,9 +266,9 @@ func (p *ProxyWrapper) CloseConn() error { } func (p *ProxyWrapper) proxyToLocal(ctx context.Context) { - defer p.wgeBPFProxy.removeRelayedConn(uint16(p.wgRelayedEndpointAddr.Port)) + defer p.proxy.removeRelayedConn(p.peerAddr, p.remoteConn) - buf := make([]byte, p.wgeBPFProxy.mtu+bufsize.WGBufferOverhead) + buf := make([]byte, p.proxy.mtu+bufsize.WGBufferOverhead) for { n, err := p.readFromRemote(ctx, buf) if err != nil { @@ -286,7 +300,7 @@ func (p *ProxyWrapper) readFromRemote(ctx context.Context, buf []byte) (int, err } p.closeListener.Notify() if !errors.Is(err, io.EOF) { - log.Errorf("failed to read from relayed conn (endpoint: :%d): %s", p.wgRelayedEndpointAddr.Port, err) + log.Errorf("failed to read from relayed conn (endpoint: %s): %s", p.wgRelayedEndpointAddr, err) } return 0, err } @@ -314,7 +328,7 @@ func (p *ProxyWrapper) sendPkg(data []byte, header *PacketHeaders) error { func (p *ProxyWrapper) selectRawConn(header *PacketHeaders) net.PacketConn { if header.isIPv4 { - return p.wgeBPFProxy.rawConnIPv4 + return p.proxy.rawConnIPv4 } - return p.wgeBPFProxy.rawConnIPv6 + return p.proxy.rawConnIPv6 } diff --git a/client/iface/wgproxy/proxy_linux_test.go b/client/iface/wgproxy/proxy_linux_test.go index e34dd3b6b..88d4588a5 100644 --- a/client/iface/wgproxy/proxy_linux_test.go +++ b/client/iface/wgproxy/proxy_linux_test.go @@ -9,25 +9,25 @@ import ( "github.com/netbirdio/netbird/client/iface/bind" "github.com/netbirdio/netbird/client/iface/wgaddr" bindproxy "github.com/netbirdio/netbird/client/iface/wgproxy/bind" - "github.com/netbirdio/netbird/client/iface/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/iface/wgproxy/loopback" "github.com/netbirdio/netbird/client/iface/wgproxy/udp" ) func seedProxies() ([]proxyInstance, error) { pl := make([]proxyInstance, 0) - ebpfProxy := ebpf.NewWGEBPFProxy(51831, 1280) - if err := ebpfProxy.Listen(); err != nil { - return nil, fmt.Errorf("failed to initialize ebpf proxy: %s", err) + loopbackProxy := loopback.NewProxy(51831, 1280) + if err := loopbackProxy.Listen(); err != nil { + return nil, fmt.Errorf("failed to initialize loopback proxy: %s", err) } - pEbpf := proxyInstance{ - name: "ebpf kernel proxy", - proxy: ebpf.NewProxyWrapper(ebpfProxy), + pLoopback := proxyInstance{ + name: "loopback kernel proxy", + proxy: loopback.NewProxyWrapper(loopbackProxy), wgPort: 51831, - closeFn: ebpfProxy.Free, + closeFn: loopbackProxy.Free, } - pl = append(pl, pEbpf) + pl = append(pl, pLoopback) pUDP := proxyInstance{ name: "udp kernel proxy", @@ -42,18 +42,18 @@ func seedProxies() ([]proxyInstance, error) { func seedProxyForProxyCloseByRemoteConn() ([]proxyInstance, error) { pl := make([]proxyInstance, 0) - ebpfProxy := ebpf.NewWGEBPFProxy(51831, 1280) - if err := ebpfProxy.Listen(); err != nil { - return nil, fmt.Errorf("failed to initialize ebpf proxy: %s", err) + loopbackProxy := loopback.NewProxy(51831, 1280) + if err := loopbackProxy.Listen(); err != nil { + return nil, fmt.Errorf("failed to initialize loopback proxy: %s", err) } - pEbpf := proxyInstance{ - name: "ebpf kernel proxy", - proxy: ebpf.NewProxyWrapper(ebpfProxy), + pLoopback := proxyInstance{ + name: "loopback kernel proxy", + proxy: loopback.NewProxyWrapper(loopbackProxy), wgPort: 51831, - closeFn: ebpfProxy.Free, + closeFn: loopbackProxy.Free, } - pl = append(pl, pEbpf) + pl = append(pl, pLoopback) pUDP := proxyInstance{ name: "udp kernel proxy", diff --git a/client/iface/wgproxy/redirect_test.go b/client/iface/wgproxy/redirect_test.go index f0d59cc64..47f571f2b 100644 --- a/client/iface/wgproxy/redirect_test.go +++ b/client/iface/wgproxy/redirect_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/netbirdio/netbird/client/iface/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/iface/wgproxy/loopback" "github.com/netbirdio/netbird/client/iface/wgproxy/udp" ) @@ -198,20 +198,20 @@ func testRedirectAs(t *testing.T, proxy Proxy, wgPort int, nbAddr, p2pEndpoint * } } -// TestRedirectAs_eBPF_IPv4 tests RedirectAs with eBPF proxy using IPv4 addresses -func TestRedirectAs_eBPF_IPv4(t *testing.T) { +// TestRedirectAs_Loopback_IPv4 tests RedirectAs with the loopback proxy using IPv4 addresses +func TestRedirectAs_Loopback_IPv4(t *testing.T) { wgPort := 51850 - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280) - if err := ebpfProxy.Listen(); err != nil { - t.Fatalf("failed to initialize ebpf proxy: %v", err) + loopbackProxy := loopback.NewProxy(wgPort, 1280) + if err := loopbackProxy.Listen(); err != nil { + t.Fatalf("failed to initialize loopback proxy: %v", err) } defer func() { - if err := ebpfProxy.Free(); err != nil { - t.Errorf("failed to free ebpf proxy: %v", err) + if err := loopbackProxy.Free(); err != nil { + t.Errorf("failed to free loopback proxy: %v", err) } }() - proxy := ebpf.NewProxyWrapper(ebpfProxy) + proxy := loopback.NewProxyWrapper(loopbackProxy) // NetBird UDP address of the remote peer nbAddr := &net.UDPAddr{ @@ -227,20 +227,20 @@ func TestRedirectAs_eBPF_IPv4(t *testing.T) { testRedirectAs(t, proxy, wgPort, nbAddr, p2pEndpoint) } -// TestRedirectAs_eBPF_IPv6 tests RedirectAs with eBPF proxy using IPv6 addresses -func TestRedirectAs_eBPF_IPv6(t *testing.T) { +// TestRedirectAs_Loopback_IPv6 tests RedirectAs with the loopback proxy using IPv6 addresses +func TestRedirectAs_Loopback_IPv6(t *testing.T) { wgPort := 51851 - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280) - if err := ebpfProxy.Listen(); err != nil { - t.Fatalf("failed to initialize ebpf proxy: %v", err) + loopbackProxy := loopback.NewProxy(wgPort, 1280) + if err := loopbackProxy.Listen(); err != nil { + t.Fatalf("failed to initialize loopback proxy: %v", err) } defer func() { - if err := ebpfProxy.Free(); err != nil { - t.Errorf("failed to free ebpf proxy: %v", err) + if err := loopbackProxy.Free(); err != nil { + t.Errorf("failed to free loopback proxy: %v", err) } }() - proxy := ebpf.NewProxyWrapper(ebpfProxy) + proxy := loopback.NewProxyWrapper(loopbackProxy) // NetBird UDP address of the remote peer nbAddr := &net.UDPAddr{ @@ -259,17 +259,17 @@ func TestRedirectAs_eBPF_IPv6(t *testing.T) { // TestRedirectAs_Multiple_Switches tests switching between multiple endpoints func TestRedirectAs_Multiple_Switches(t *testing.T) { wgPort := 51856 - ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280) - if err := ebpfProxy.Listen(); err != nil { - t.Fatalf("failed to initialize ebpf proxy: %v", err) + loopbackProxy := loopback.NewProxy(wgPort, 1280) + if err := loopbackProxy.Listen(); err != nil { + t.Fatalf("failed to initialize loopback proxy: %v", err) } defer func() { - if err := ebpfProxy.Free(); err != nil { - t.Errorf("failed to free ebpf proxy: %v", err) + if err := loopbackProxy.Free(); err != nil { + t.Errorf("failed to free loopback proxy: %v", err) } }() - proxy := ebpf.NewProxyWrapper(ebpfProxy) + proxy := loopback.NewProxyWrapper(loopbackProxy) ctx := context.Background() diff --git a/client/installer.nsis b/client/installer.nsis index eb2d7d5bd..304d38864 100644 --- a/client/installer.nsis +++ b/client/installer.nsis @@ -2,7 +2,7 @@ !define COMP_NAME "Netbird" !define WEB_SITE "Netbird.io" !define VERSION $%APPVER% -!define COPYRIGHT "Netbird Authors, 2022" +!define COPYRIGHT "Netbird Authors, 2022-2026" !define DESCRIPTION "Connect your devices into a secure WireGuard-based overlay network with SSO, MFA, and granular access controls." !define INSTALLER_NAME "netbird-installer.exe" !define MAIN_APP_EXE "Netbird" diff --git a/client/internal/auth/account_match_test.go b/client/internal/auth/account_match_test.go new file mode 100644 index 000000000..b879cf9ab --- /dev/null +++ b/client/internal/auth/account_match_test.go @@ -0,0 +1,113 @@ +package auth + +import ( + "encoding/base64" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTokenInfoMatchesAccount(t *testing.T) { + tests := []struct { + name string + token TokenInfo + hint string + match bool + }{ + { + name: "same account", + token: TokenInfo{EmailClaim: "user@example.com"}, + hint: "user@example.com", + match: true, + }, + { + name: "different account", + token: TokenInfo{EmailClaim: "other@example.com"}, + hint: "user@example.com", + match: false, + }, + { + name: "case differences are the same account", + token: TokenInfo{EmailClaim: "User@Example.com"}, + hint: "user@example.com", + match: true, + }, + { + name: "no hint leaves the choice to the IdP", + token: TokenInfo{EmailClaim: "other@example.com"}, + hint: "", + match: true, + }, + { + name: "token without an email claim is not judged", + token: TokenInfo{EmailClaim: ""}, + hint: "user@example.com", + match: true, + }, + { + name: "name fallback does not trigger matching", + token: TokenInfo{Email: "Some One"}, + hint: "user@example.com", + match: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.match, tc.token.MatchesAccount(tc.hint)) + }) + } +} + +func TestParseEmailFromIDToken(t *testing.T) { + tests := []struct { + name string + claims map[string]interface{} + wantValue string + wantFromEmail bool + wantErr bool + }{ + { + name: "email claim", + claims: map[string]interface{}{"email": "user@example.com", "name": "Some One"}, + wantValue: "user@example.com", + wantFromEmail: true, + }, + { + name: "name fallback", + claims: map[string]interface{}{"name": "Some One"}, + wantValue: "Some One", + }, + { + name: "neither claim present", + claims: map[string]interface{}{"sub": "abc"}, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + value, fromEmailClaim, err := parseEmailFromIDToken(idTokenWithClaims(t, tc.claims)) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantValue, value) + assert.Equal(t, tc.wantFromEmail, fromEmailClaim) + }) + } +} + +func TestRetryFlowForAccountUnsupportedFlow(t *testing.T) { + assert.Nil(t, RetryFlowForAccount(&DeviceAuthorizationFlow{})) +} + +func idTokenWithClaims(t *testing.T, claims map[string]interface{}) string { + t.Helper() + payload, err := json.Marshal(claims) + require.NoError(t, err) + return "header." + base64.RawURLEncoding.EncodeToString(payload) + ".signature" +} diff --git a/client/internal/auth/auth.go b/client/internal/auth/auth.go index 939df3a21..7c156b5fd 100644 --- a/client/internal/auth/auth.go +++ b/client/internal/auth/auth.go @@ -103,7 +103,7 @@ func (a *Auth) IsSSOSupported(ctx context.Context) (bool, error) { err := a.withRetry(ctx, func(client *mgm.GrpcClient) error { // Try PKCE flow first - _, err := a.getPKCEFlow(client) + _, err := a.getPKCEFlow(client, false) if err == nil { supportsSSO = true return nil @@ -136,9 +136,13 @@ func (a *Auth) IsSSOSupported(ctx context.Context) (bool, error) { return supportsSSO, err } -// GetOAuthFlow returns an OAuth flow (PKCE or Device) using the existing management connection -// This avoids creating a new connection to the management server -func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool, hint string) (OAuthFlow, error) { +// GetOAuthFlow returns an OAuth flow (PKCE or Device) using the existing management connection. +// This avoids creating a new connection to the management server. +// +// sessionExtend marks the flow as renewing an existing peer's session rather than +// logging one in; the server needs it to rule out a silent authorization that the +// IdP could answer from another account. See PKCEAuthorizationFlowRequest. +func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool, sessionExtend bool, hint string) (OAuthFlow, error) { var flow OAuthFlow err := a.withRetry(ctx, func(client *mgm.GrpcClient) error { @@ -153,7 +157,7 @@ func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool, hint stri } // Try PKCE flow first - pkceFlow, err := a.getPKCEFlow(client) + pkceFlow, err := a.getPKCEFlow(client, sessionExtend) if err != nil { // If PKCE not supported, try Device flow if s, ok := status.FromError(err); ok && (s.Code() == codes.NotFound || s.Code() == codes.Unimplemented) { @@ -240,8 +244,8 @@ func (a *Auth) Login(ctx context.Context, setupKey string, jwtToken string) (err } // getPKCEFlow retrieves PKCE authorization flow configuration and creates a flow instance -func (a *Auth) getPKCEFlow(client *mgm.GrpcClient) (*PKCEAuthorizationFlow, error) { - protoFlow, err := client.GetPKCEAuthorizationFlow() +func (a *Auth) getPKCEFlow(client *mgm.GrpcClient, sessionExtend bool) (*PKCEAuthorizationFlow, error) { + protoFlow, err := client.GetPKCEAuthorizationFlow(sessionExtend) if err != nil { if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound { log.Warnf("server couldn't find pkce flow, contact admin: %v", err) diff --git a/client/internal/auth/device_flow.go b/client/internal/auth/device_flow.go index 3592e589d..c2b8d34b8 100644 --- a/client/internal/auth/device_flow.go +++ b/client/internal/auth/device_flow.go @@ -308,10 +308,13 @@ func (d *DeviceAuthorizationFlow) WaitToken(ctx context.Context, info AuthFlowIn // callers store to send back as the login_hint. Without it a client // driven through the device flow — Android TV and tvOS — never binds // an account to its profile and every later login goes out blind. - if email, err := parseEmailFromIDToken(tokenInfo.IDToken); err != nil { + if email, fromEmailClaim, err := parseEmailFromIDToken(tokenInfo.IDToken); err != nil { log.Warnf("failed to parse email from ID token: %v", err) } else { tokenInfo.Email = email + if fromEmailClaim { + tokenInfo.EmailClaim = email + } } log.Infof("device flow: user authorization confirmed after %d polls in %s", polls, time.Since(start).Round(time.Second)) diff --git a/client/internal/auth/oauth.go b/client/internal/auth/oauth.go index 91329c98b..54ecb5d86 100644 --- a/client/internal/auth/oauth.go +++ b/client/internal/auth/oauth.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "runtime" + "strings" log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" @@ -25,6 +26,14 @@ type HTTPClient interface { Do(req *http.Request) (*http.Response, error) } +// accountPromptForcer is implemented by the PKCE flow only. The device code +// flow has no equivalent: RFC 8628 defines no prompt parameter, and the user +// confirms the code on a page that shows which account signs in, so a silent +// wrong-account answer is not the failure mode there. +type accountPromptForcer interface { + ForceAccountPrompt() +} + // AuthFlowInfo holds information for the OAuth 2.0 authorization flow type AuthFlowInfo struct { //nolint:revive DeviceCode string `json:"device_code"` @@ -49,6 +58,23 @@ type TokenInfo struct { ExpiresIn int `json:"expires_in"` UseIDToken bool `json:"-"` Email string `json:"-"` + EmailClaim string `json:"-"` +} + +// MatchesAccount reports whether the token belongs to the account a profile is +// bound to. A hint the IdP could not have acted on — no hint stored, or a token +// that carried no email claim — is reported as a match: the check exists to catch a +// login answered from the wrong account, not to block one it cannot judge. +// +// The comparison is case-insensitive. Local-parts are case-sensitive per RFC +// 5321, but no IdP in practice issues two accounts differing only in case, and +// an IdP that echoes a differently-cased address would otherwise fail every +// login. +func (t TokenInfo) MatchesAccount(hint string) bool { + if hint == "" || t.EmailClaim == "" { + return true + } + return strings.EqualFold(t.EmailClaim, hint) } // GetTokenToUse returns either the access or id token based on UseIDToken field @@ -63,19 +89,22 @@ func shouldUseDeviceFlow(force bool, isUnixDesktopClient bool) bool { return force || (runtime.GOOS == "linux" || runtime.GOOS == "freebsd") && !isUnixDesktopClient } -// NewOAuthFlow initializes and returns the appropriate OAuth flow based on the management configuration +// NewOAuthFlow initializes and returns the appropriate OAuth flow based on the management configuration. // // It starts by initializing the PKCE.If this process fails, it resorts to the Device Code Flow, // and if that also fails, the authentication process is deemed unsuccessful // -// On Linux distros without desktop environment support, it only tries to initialize the Device Code Flow -// forceDeviceCodeFlow can be used to skip PKCE and go directly to Device Code Flow (e.g., for Android TV) -func NewOAuthFlow(ctx context.Context, config *profilemanager.Config, isUnixDesktopClient bool, forceDeviceCodeFlow bool, hint string) (OAuthFlow, error) { +// On Linux distros without desktop environment support, it only tries to initialize the Device Code Flow. +// forceDeviceCodeFlow can be used to skip PKCE and go directly to Device Code Flow (e.g., for Android TV). +// +// sessionExtend marks the flow as renewing an existing peer's session rather than +// logging one in. See PKCEAuthorizationFlowRequest for what the server makes of it. +func NewOAuthFlow(ctx context.Context, config *profilemanager.Config, isUnixDesktopClient bool, forceDeviceCodeFlow bool, hint string, sessionExtend bool) (OAuthFlow, error) { if shouldUseDeviceFlow(forceDeviceCodeFlow, isUnixDesktopClient) { return authenticateWithDeviceCodeFlow(ctx, config, hint) } - pkceFlow, err := authenticateWithPKCEFlow(ctx, config, hint) + pkceFlow, err := authenticateWithPKCEFlow(ctx, config, hint, sessionExtend) if err != nil { log.Debugf("failed to initialize pkce authentication with error: %v\n", err) log.Debug("falling back to device code flow") @@ -85,14 +114,14 @@ func NewOAuthFlow(ctx context.Context, config *profilemanager.Config, isUnixDesk } // authenticateWithPKCEFlow initializes the Proof Key for Code Exchange flow auth flow -func authenticateWithPKCEFlow(ctx context.Context, config *profilemanager.Config, hint string) (OAuthFlow, error) { +func authenticateWithPKCEFlow(ctx context.Context, config *profilemanager.Config, hint string, sessionExtend bool) (OAuthFlow, error) { authClient, err := NewAuth(ctx, config.PrivateKey, config.ManagementURL, config) if err != nil { return nil, fmt.Errorf("failed to create auth client: %v", err) } defer authClient.Close() - pkceFlowInfo, err := authClient.getPKCEFlow(authClient.client) + pkceFlowInfo, err := authClient.getPKCEFlow(authClient.client, sessionExtend) if err != nil { return nil, fmt.Errorf("getting pkce authorization flow info failed with error: %v", err) } @@ -129,3 +158,21 @@ func authenticateWithDeviceCodeFlow(ctx context.Context, config *profilemanager. return deviceFlowInfo, nil } + +// RetryFlowForAccount returns a flow that asks the IdP to re-authenticate, for +// a login answered with an account other than the one hinted. Returns nil when +// the flow cannot ask — the caller then proceeds with the token it has. +// +// Proceeding rather than failing is deliberate. The hint is an email that may +// simply have changed since it was stored, and refusing the login would lock a +// user out of their own profile over a rename. The retry gives the account a +// chance to be corrected; the server still rejects a token that does not own +// the peer. +func RetryFlowForAccount(flow OAuthFlow) OAuthFlow { + forcer, ok := flow.(accountPromptForcer) + if !ok { + return nil + } + forcer.ForceAccountPrompt() + return flow +} diff --git a/client/internal/auth/pkce_flow.go b/client/internal/auth/pkce_flow.go index be64cc6a8..ebb0d4f1f 100644 --- a/client/internal/auth/pkce_flow.go +++ b/client/internal/auth/pkce_flow.go @@ -87,10 +87,11 @@ func validatePKCEConfig(config *PKCEAuthProviderConfig) error { // PKCEAuthorizationFlow implements the OAuthFlow interface for // the Authorization Code Flow with PKCE. type PKCEAuthorizationFlow struct { - providerConfig PKCEAuthProviderConfig - state string - codeVerifier string - oAuthConfig *oauth2.Config + providerConfig PKCEAuthProviderConfig + state string + codeVerifier string + oAuthConfig *oauth2.Config + forceAccountPrompt bool } // NewPKCEAuthorizationFlow returns new PKCE authorization code flow. @@ -153,11 +154,16 @@ func (p *PKCEAuthorizationFlow) RequestAuthInfo(ctx context.Context) (AuthFlowIn oauth2.SetAuthURLParam("code_challenge", codeChallenge), oauth2.SetAuthURLParam("audience", p.providerConfig.Audience), } + forceAccountPrompt := p.forceAccountPrompt + p.forceAccountPrompt = false + if !p.providerConfig.DisablePromptLogin { - switch p.providerConfig.LoginFlag { - case common.LoginFlagPromptLogin: + switch { + case forceAccountPrompt: params = append(params, oauth2.SetAuthURLParam("prompt", "login")) - case common.LoginFlagMaxAge0: + case p.providerConfig.LoginFlag == common.LoginFlagPromptLogin: + params = append(params, oauth2.SetAuthURLParam("prompt", "login")) + case p.providerConfig.LoginFlag == common.LoginFlagMaxAge0: params = append(params, oauth2.SetAuthURLParam("max_age", "0")) } } @@ -178,6 +184,20 @@ func (p *PKCEAuthorizationFlow) SetLoginHint(hint string) { p.providerConfig.LoginHint = hint } +// ForceAccountPrompt makes the next authorization request ask the IdP to +// re-authenticate instead of answering from the session it already holds. Used +// to retry a login that came back for an account other than the one hinted. +// +// The next RequestAuthInfo consumes the flag, so a flow that outlives its retry +// goes back to the configured behaviour instead of re-authenticating forever. +// +// DisablePromptLogin still wins: it is set for IdPs that break on prompt=login, +// where retrying with it would replace a wrong-account login with one that +// cannot complete at all. +func (p *PKCEAuthorizationFlow) ForceAccountPrompt() { + p.forceAccountPrompt = true +} + // WaitToken waits for the OAuth token in the PKCE Authorization Flow. // It starts an HTTP server to receive the OAuth token callback and waits for the token or an error. // Once the token is received, it is converted to TokenInfo and validated before returning. @@ -310,49 +330,48 @@ func (p *PKCEAuthorizationFlow) parseOAuthToken(token *oauth2.Token) (TokenInfo, return TokenInfo{}, fmt.Errorf("authentication failed: invalid access token - %w", err) } - email, err := parseEmailFromIDToken(tokenInfo.IDToken) + email, fromEmailClaim, err := parseEmailFromIDToken(tokenInfo.IDToken) if err != nil { log.Warnf("failed to parse email from ID token: %v", err) } else { tokenInfo.Email = email + if fromEmailClaim { + tokenInfo.EmailClaim = email + } } return tokenInfo, nil } // parseEmailFromIDToken extracts the email (or name) claim from an ID token -// without verifying its signature. The value is best-effort and used only as a -// UX convenience (login hint prefill and display); it never drives an -// authorization decision. The authoritative identity is established server-side -// from the signature-verified token. -func parseEmailFromIDToken(token string) (string, error) { +// without verifying its signature. The value is best-effort: it prefills the +// login hint and is displayed. Account matching (see MatchesAccount) only uses +// it when it came from the email claim, which fromEmailClaim reports. It never +// grants anything — the authoritative identity is established server-side from +// the signature-verified token. +func parseEmailFromIDToken(token string) (value string, fromEmailClaim bool, err error) { parts := strings.Split(token, ".") if len(parts) < 2 { - return "", fmt.Errorf("invalid token format") + return "", false, fmt.Errorf("invalid token format") } data, err := base64.RawURLEncoding.DecodeString(parts[1]) if err != nil { - return "", fmt.Errorf("failed to decode payload: %w", err) + return "", false, fmt.Errorf("failed to decode payload: %w", err) } var claims map[string]interface{} if err := json.Unmarshal(data, &claims); err != nil { - return "", fmt.Errorf("json unmarshal error: %w", err) + return "", false, fmt.Errorf("json unmarshal error: %w", err) } - var email string - if emailValue, ok := claims["email"].(string); ok { - email = emailValue - } else { - val, ok := claims["name"].(string) - if ok { - email = val - } else { - return "", fmt.Errorf("email or name field not found in token payload") - } + if email, ok := claims["email"].(string); ok { + return email, true, nil + } + if name, ok := claims["name"].(string); ok { + return name, false, nil } - return email, nil + return "", false, fmt.Errorf("email or name field not found in token payload") } func createCodeChallenge(codeVerifier string) string { diff --git a/client/internal/auth/pkce_flow_test.go b/client/internal/auth/pkce_flow_test.go index c487c13df..ccbca10e9 100644 --- a/client/internal/auth/pkce_flow_test.go +++ b/client/internal/auth/pkce_flow_test.go @@ -76,6 +76,32 @@ func TestPromptLogin(t *testing.T) { } } +func TestForceAccountPromptAppliesOnlyToTheRetry(t *testing.T) { + config := PKCEAuthProviderConfig{ + ClientID: "test-client-id", + Audience: "test-audience", + TokenEndpoint: "https://test-token-endpoint.com/token", + Scope: "openid email profile", + AuthorizationEndpoint: "https://test-auth-endpoint.com/authorize", + RedirectURLs: []string{"http://127.0.0.1:33992/"}, + UseIDToken: true, + LoginFlag: mgm.LoginFlagNone, + } + pkce, err := NewPKCEAuthorizationFlow(config) + require.NoError(t, err) + + pkce.ForceAccountPrompt() + + retry, err := pkce.RequestAuthInfo(context.Background()) + require.NoError(t, err) + require.Contains(t, retry.VerificationURIComplete, "prompt=login") + + next, err := pkce.RequestAuthInfo(context.Background()) + require.NoError(t, err) + require.NotContains(t, next.VerificationURIComplete, "prompt=login", + "the forced prompt outlived the retry it was armed for") +} + func TestIsPortInExcludedRange(t *testing.T) { tests := []struct { name string diff --git a/client/internal/auth/sessionwatch/watcher.go b/client/internal/auth/sessionwatch/watcher.go index e685c28d0..f71689df1 100644 --- a/client/internal/auth/sessionwatch/watcher.go +++ b/client/internal/auth/sessionwatch/watcher.go @@ -4,12 +4,20 @@ // T-WarningLead notification and a dismiss-gated T-FinalWarningLead // fallback dialog. // +// The deadline is an absolute wall-clock instant, so the watcher compares +// it against the wall clock on a ticker rather than arming a relative +// timer for it. A relative timer runs on the monotonic clock, which does +// not advance while a device is suspended: it fires once that much awake +// time has passed, which can be long after the deadline, and nothing +// re-evaluates when the device wakes up. Polling makes every tick after a +// resume (or after an NTP correction) see the real remaining time. +// // The watcher is idempotent: Update may be called as often as the network // map snapshots arrive. Repeating the same deadline is a no-op; a new -// deadline reschedules the timers and arms a fresh warning cycle. +// deadline starts a fresh warning cycle. // -// Warning firing is edge-detected. Each unique deadline value fires each -// warning callback at most once. +// Warning firing is edge-detected. Each unique deadline value publishes +// each warning at most once. package sessionwatch import ( @@ -28,9 +36,15 @@ const ( // maxDeadlineHorizon caps how far in the future an accepted deadline // can sit. A timestamp beyond this is almost certainly a protocol - // glitch, and silently arming a 100-year timer would hide the bug. + // glitch, and silently tracking a 100-year deadline would hide the bug. maxDeadlineHorizon = 10 * 365 * 24 * time.Hour + // defaultEvalInterval is how often the tracked deadline is compared + // against the wall clock. The leads are minutes, so a coarse tick costs + // nothing in accuracy and keeps the wakeup cheap on battery-powered + // devices. + defaultEvalInterval = 10 * time.Second + // WarningLead is how far before expiry the first (interactive) // warning fires. Drives the T-10 OS notification with // Extend/Dismiss actions. @@ -90,18 +104,23 @@ type StatusRecorder interface { // fallback T-FinalWarningLead dialog (suppressed when the user dismissed // the first one for the same deadline). Safe for concurrent use. type Watcher struct { - lead time.Duration - finalLead time.Duration + lead time.Duration + finalLead time.Duration + interval time.Duration + deadlineOnly bool // record the deadline, leave the warnings to the caller mu sync.Mutex current time.Time - timer *time.Timer - finalTimer *time.Timer - firedAt time.Time // deadline value the T-WarningLead callback last fired against - finalFiredAt time.Time // deadline value the T-FinalWarningLead callback last fired against - dismissedAt time.Time // deadline value the user dismissed via Dismiss(); gates fireFinal + firedAt time.Time // deadline value the T-WarningLead warning last published for + finalFiredAt time.Time // deadline value the T-FinalWarningLead warning last published for + dismissedAt time.Time // deadline value the user dismissed via Dismiss(); gates the final warning + announcedAt time.Time // deadline value the recorder has been told about; gates publishing closed bool recorder StatusRecorder + nowFn func() time.Time + stop chan struct{} // closed to stop the evaluation loop; nil while it is not running + done chan struct{} // closed by the loop on its way out + wake chan struct{} // buffered nudge asking the loop to evaluate before its next tick } // New returns a watcher with the package defaults WarningLead and @@ -113,27 +132,40 @@ func New(recorder StatusRecorder) *Watcher { } // NewWithLeads returns a watcher with custom lead times. Useful for tests. -// final must be strictly less than lead; otherwise both timers fire in the -// wrong order or simultaneously and the UI flow breaks. A zero final lead -// disables the final-warning timer entirely (see armTimerLocked) so a -// millisecond-scale deadline doesn't flush both timers in one tick. +// final must be strictly less than lead; otherwise the final warning takes +// over the whole warning window and the interactive notification never +// shows. A zero final lead disables the final warning entirely (see +// evaluate), leaving the interactive one as the only warning. func NewWithLeads(lead, final time.Duration, recorder StatusRecorder) *Watcher { return &Watcher{ lead: lead, finalLead: final, + interval: defaultEvalInterval, recorder: recorder, + nowFn: time.Now, } } +// NewDeadlineOnly returns a watcher that validates and records deadlines +// but publishes no warnings about them, and runs no evaluation loop to +// decide. Used where the deadline is handed on to something that schedules +// the warnings itself, such as the Android app. +func NewDeadlineOnly(recorder StatusRecorder) *Watcher { + w := New(recorder) + w.deadlineOnly = true + return w +} + // Update sets the latest deadline. Pass the zero time to clear (e.g. when // a Sync push from the server omits the field because login expiration // was disabled). // -// Same-value updates are no-ops. A different non-zero value cancels any -// pending timer, resets the "already fired" guards, and — when the -// deadline lies in the future — arms fresh warning timers. A deadline -// already in the past (within maxPastHorizon) is recorded as-is with no -// timers: the session has expired and consumers render it that way. +// Same-value updates are no-ops. A different non-zero value resets the +// "already fired" guards and starts a fresh warning cycle, evaluated +// immediately so a deadline that already sits inside a warning window +// warns without waiting for the next tick. A deadline already in the past +// (within maxPastHorizon) is recorded as-is and warns nothing: the session +// has expired and consumers render it that way. // // Returns one of the sentinel Err* values when the deadline fails the // sanity checks (pre-epoch, far future, or past beyond maxPastHorizon). @@ -153,7 +185,7 @@ func (w *Watcher) Update(deadline time.Time) error { return nil } - now := time.Now() + now := w.nowFn() switch { case deadline.Before(time.Unix(0, 0)): w.clearLocked() @@ -171,18 +203,22 @@ func (w *Watcher) Update(deadline time.Time) error { return nil } - w.stopTimerLocked() w.current = deadline - // Reset every per-deadline guard so a refreshed deadline arms a fresh + // Reset every per-deadline guard so a refreshed deadline starts a fresh // warning cycle: both edge triggers and the user Dismiss decision // (the user agreed to the old deadline expiring; a new deadline // restarts the contract). w.firedAt = time.Time{} w.finalFiredAt = time.Time{} w.dismissedAt = time.Time{} + w.announcedAt = time.Time{} - if deadline.After(now) { - w.armTimerLocked(deadline) + // Poll every accepted deadline, including one that reads as already + // expired: the clock may be running ahead of real time and get + // corrected later, and evaluate ignores a deadline that has genuinely + // passed anyway. + if !w.deadlineOnly { + w.startPollLocked() } recorder := w.recorder w.mu.Unlock() @@ -190,6 +226,28 @@ func (w *Watcher) Update(deadline time.Time) error { recorder.SetSessionExpiresAt(deadline) } log.Infof("auth session deadline set to: %s (in %s)", deadline.Format(time.RFC3339), time.Until(deadline).Round(time.Second)) + + // Open the gate only once the recorder knows the new deadline, so a + // warning that refers to it can never reach consumers before the state + // change itself. A tick landing in between finds the gate shut. + w.mu.Lock() + if w.closed || !w.current.Equal(deadline) { + w.mu.Unlock() + return nil + } + w.announcedAt = deadline + wake := w.wake + w.mu.Unlock() + + // Hand the evaluation to the loop rather than running it here, so a + // deadline that already sits inside a warning window is published at + // once without this goroutine ever touching the recorder. + if wake != nil { + select { + case wake <- struct{}{}: + default: + } + } return nil } @@ -202,9 +260,9 @@ func (w *Watcher) Deadline() time.Time { } // Dismiss records the user's "Dismiss" action against the current deadline -// and suppresses the upcoming final-warning callback for that deadline. -// Idempotent: repeated calls are no-ops. A subsequent Update with a fresh -// deadline resets the dismissal so the final-warning cycle re-arms. +// and suppresses the final warning for that deadline. Idempotent: repeated +// calls are no-ops. A subsequent Update with a fresh deadline resets the +// dismissal so the final-warning cycle starts over. // // No-op when the watcher holds no deadline or has been closed. func (w *Watcher) Dismiss() { @@ -217,35 +275,48 @@ func (w *Watcher) Dismiss() { return } w.dismissedAt = w.current - // Cancel the armed final-warning timer eagerly. fireFinal would also - // gate on dismissedAt, but stopping the timer avoids a wakeup with - // nothing to do and makes the intent visible. - if w.finalTimer != nil { - w.finalTimer.Stop() - w.finalTimer = nil - } log.Infof("auth session final-warning dismissed for deadline %s", w.current.Format(time.RFC3339)) } -// Close stops any pending timer. Update calls after Close are ignored. -// The recorder keeps its deadline: the watcher is engine-scoped and closes -// on every engine restart (network change, sleep/wake, stream errors) -// while the SSO deadline stays valid across those, so clearing here would -// blank the UI's "expires in" row on every transient reconnect. The -// client run loop clears the server-scoped recorder when it exits for -// real (Down, profile switch, permanent login failure). +// Close stops the evaluation loop and waits for it to exit. Update calls +// after Close are ignored. The recorder keeps its deadline: the watcher is +// engine-scoped and closes on every engine restart (network change, +// sleep/wake, stream errors) while the SSO deadline stays valid across +// those, so clearing here would blank the UI's "expires in" row on every +// transient reconnect. The client run loop clears the server-scoped +// recorder when it exits for real (Down, profile switch, permanent login +// failure). func (w *Watcher) Close() { w.mu.Lock() - defer w.mu.Unlock() if w.closed { + // A concurrent Close is already tearing down. w.done outlives it, + // so this caller waits for the same loop rather than returning + // while a warning is still on its way to the recorder. + done := w.done + w.mu.Unlock() + if done != nil { + <-done + } return } w.closed = true - w.stopTimerLocked() w.current = time.Time{} w.firedAt = time.Time{} w.finalFiredAt = time.Time{} w.dismissedAt = time.Time{} + w.announcedAt = time.Time{} + // Copy the channels out before releasing the lock: the loop takes w.mu + // on every tick, so waiting for it while holding the lock would + // deadlock. w.done stays on the receiver for the branch above. + stop, done := w.stop, w.done + w.stop, w.wake = nil, nil + w.mu.Unlock() + + if stop == nil { + return + } + close(stop) + <-done } // clearLocked drops the tracked deadline and notifies the recorder so @@ -257,11 +328,11 @@ func (w *Watcher) clearLocked() { w.mu.Unlock() return } - w.stopTimerLocked() w.current = time.Time{} w.firedAt = time.Time{} w.finalFiredAt = time.Time{} w.dismissedAt = time.Time{} + w.announcedAt = time.Time{} recorder := w.recorder w.mu.Unlock() if recorder != nil { @@ -270,87 +341,121 @@ func (w *Watcher) clearLocked() { log.Infof("auth session deadline cleared") } -func (w *Watcher) stopTimerLocked() { - if w.timer != nil { - w.timer.Stop() - w.timer = nil +// startPollLocked starts the evaluation loop unless it is already +// running. The loop starts lazily on the first accepted deadline, so a +// client whose server never publishes a session expiry never pays for a +// ticker, and it runs until Close: the watcher is engine-scoped, and a +// cleared deadline is normally followed by a fresh one on the next sync. +// Caller must hold w.mu. +func (w *Watcher) startPollLocked() { + if w.stop != nil { + return } - if w.finalTimer != nil { - w.finalTimer.Stop() - w.finalTimer = nil + stop := make(chan struct{}) + done := make(chan struct{}) + wake := make(chan struct{}, 1) + w.stop, w.done, w.wake = stop, done, wake + go w.poll(stop, done, wake, w.interval) +} + +// poll re-evaluates the tracked deadline every interval, and as soon as a +// new deadline is announced. It is the only caller of evaluate, so Close +// waiting for it to exit is enough to know no warning is still on its way +// to the recorder. Its channels and interval are passed in rather than +// read off the receiver, so Close can clear them without racing it. +func (w *Watcher) poll(stop <-chan struct{}, done chan<- struct{}, wake <-chan struct{}, interval time.Duration) { + defer close(done) + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-stop: + return + case <-wake: + w.evaluate() + case <-ticker.C: + w.evaluate() + } } } -func (w *Watcher) armTimerLocked(deadline time.Time) { - w.timer = armOneShotLocked(deadline.Add(-w.lead), func() { w.fire(deadline) }) - // finalLead <= 0 disables the final-warning timer entirely. Used by - // tests that predate the final-warning fallback so a millisecond-scale - // deadline does not flush both timers at once. - if w.finalLead > 0 { - w.finalTimer = armOneShotLocked(deadline.Add(-w.finalLead), func() { w.fireFinal(deadline) }) - } -} - -func (w *Watcher) fire(armedFor time.Time) { +// evaluate compares the tracked deadline against the wall clock and +// publishes whichever warning the remaining time calls for. Inside the +// final-warning window the interactive warning is stale, so the final one +// is published in its place: a device that resumes there never saw the +// T-WarningLead notification. +func (w *Watcher) evaluate() { w.mu.Lock() - if w.closed || !w.current.Equal(armedFor) { - // Deadline moved while we were waiting (e.g. a successful extend). - // The reschedule path armed a fresh timer; this one is stale. + if w.closed || w.deadlineOnly || w.current.IsZero() { w.mu.Unlock() return } - if !w.firedAt.IsZero() && w.firedAt.Equal(armedFor) { + + deadline := w.current + if !w.announcedAt.Equal(deadline) { + // Update is still on its way to the recorder with this deadline. w.mu.Unlock() return } - w.firedAt = armedFor + // Round(0) strips the monotonic reading so the comparison is wall + // clock on both sides, whether the deadline came off the wire or from + // a caller that derived it from time.Now. + remaining := deadline.Round(0).Sub(w.nowFn().Round(0)) + + switch { + case remaining <= 0: + // Already expired: the post-mortem SessionExpired flow owns it. + w.mu.Unlock() + case w.finalLead > 0 && remaining <= w.finalLead: + w.publishFinalLocked(deadline, remaining) + case remaining <= w.lead: + w.publishWarningLocked(deadline, remaining) + default: + w.mu.Unlock() + } +} + +// publishWarningLocked emits the interactive T-WarningLead warning, at +// most once per deadline value. Caller must hold w.mu; this helper +// releases it. +func (w *Watcher) publishWarningLocked(deadline time.Time, remaining time.Duration) { + if w.firedAt.Equal(deadline) { + w.mu.Unlock() + return + } + w.firedAt = deadline recorder := w.recorder w.mu.Unlock() if recorder == nil { return } - log.Infof("auth session expiry soon warning fired") - publishWarning(recorder, armedFor, false) + log.Infof("auth session expiry soon warning fired for deadline %s (in %s)", + deadline.Format(time.RFC3339), remaining.Round(time.Second)) + publishWarning(recorder, deadline, false) } -// fireFinal mirrors fire for the T-FinalWarningLead timer with an extra -// dismiss-gate: if the user dismissed the T-WarningLead notification for -// this deadline, the final warning is suppressed entirely. -func (w *Watcher) fireFinal(armedFor time.Time) { - w.mu.Lock() - if w.closed || !w.current.Equal(armedFor) { +// publishFinalLocked emits the final warning, at most once per deadline +// value and never once the user dismissed that deadline. It marks the +// interactive warning as handled too: the final window is open, so a +// "expires in WarningLead minutes" notification would be wrong. Caller +// must hold w.mu; this helper releases it. +func (w *Watcher) publishFinalLocked(deadline time.Time, remaining time.Duration) { + if w.finalFiredAt.Equal(deadline) || w.dismissedAt.Equal(deadline) { w.mu.Unlock() return } - if !w.finalFiredAt.IsZero() && w.finalFiredAt.Equal(armedFor) { - w.mu.Unlock() - return - } - if w.dismissedAt.Equal(armedFor) { - w.mu.Unlock() - log.Infof("auth session final-warning skipped (dismissed by user)") - return - } - w.finalFiredAt = armedFor + w.firedAt = deadline + w.finalFiredAt = deadline recorder := w.recorder w.mu.Unlock() if recorder == nil { return } - log.Infof("auth session final-warning fired") - publishWarning(recorder, armedFor, true) -} - -// armOneShotLocked schedules cb at fireAt. When fireAt is already in the -// past it dispatches on the next scheduler tick so a state-change recorder -// notification (invoked after w.mu is released) lands first. Caller must -// hold w.mu. -func armOneShotLocked(fireAt time.Time, cb func()) *time.Timer { - delay := time.Until(fireAt) - if delay <= 0 { - return time.AfterFunc(0, cb) - } - return time.AfterFunc(delay, cb) + log.Infof("auth session final-warning fired for deadline %s (in %s)", + deadline.Format(time.RFC3339), remaining.Round(time.Second)) + publishWarning(recorder, deadline, true) } // publishWarning composes the SystemEvent for a watcher-fired warning and diff --git a/client/internal/auth/sessionwatch/watcher_test.go b/client/internal/auth/sessionwatch/watcher_test.go index 4b49a94b6..f0cc3ed88 100644 --- a/client/internal/auth/sessionwatch/watcher_test.go +++ b/client/internal/auth/sessionwatch/watcher_test.go @@ -19,6 +19,9 @@ type fakeRecorder struct { mu sync.Mutex events []event lastDeadline time.Time + // setDelay stalls SetSessionExpiresAt, widening the window in which a + // concurrent evaluation could publish a warning out of order. + setDelay time.Duration } type eventKind int @@ -43,6 +46,12 @@ type event struct { // is the zero time, so an initial clear before any deadline is set emits // nothing — matching the real recorder. func (r *fakeRecorder) SetSessionExpiresAt(deadline time.Time) { + r.mu.Lock() + delay := r.setDelay + r.mu.Unlock() + // Stall without the lock held, so a concurrent publish can still record. + time.Sleep(delay) + r.mu.Lock() defer r.mu.Unlock() if r.lastDeadline.Equal(deadline) { @@ -116,10 +125,54 @@ func waitForEvents(t *testing.T, r *fakeRecorder, want int) []event { return nil } -// newWatcher builds a watcher with the final timer disabled (finalLead=0), +// testInterval keeps the ticker-driven tests fast; the leads they use are +// in the same millisecond scale. +const testInterval = 2 * time.Millisecond + +// newWatcher builds a watcher with the final warning disabled (finalLead=0), // matching the lead-only behaviour the pre-final-warning tests assume. func newWatcher(lead time.Duration, r *fakeRecorder) *Watcher { - return NewWithLeads(lead, 0, r) + return newWatcherWithLeads(lead, 0, r) +} + +// newWatcherWithLeads builds a watcher that evaluates on testInterval. The +// interval is set before Update, so the evaluation loop does not exist yet +// and the write cannot race it. +func newWatcherWithLeads(lead, final time.Duration, r StatusRecorder) *Watcher { + w := NewWithLeads(lead, final, r) + w.interval = testInterval + return w +} + +// fakeClock is the watcher's wall clock under test. Reads come from the +// evaluation goroutine while the test writes, so both go through the mutex. +type fakeClock struct { + mu sync.Mutex + t time.Time +} + +func newFakeClock(t time.Time) *fakeClock { + return &fakeClock{t: t} +} + +func (c *fakeClock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +// set jumps the clock, standing in for a resume from suspension or for an +// NTP correction. +func (c *fakeClock) set(t time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = t +} + +// settle waits out a handful of evaluation ticks so a test can assert that +// nothing was published. +func settle() { + time.Sleep(20 * testInterval) } func TestUpdateZeroBeforeAnythingIsNoop(t *testing.T) { @@ -410,7 +463,7 @@ func TestCloseWithoutDeadlineLeavesRecorderUntouched(t *testing.T) { func TestFinalWarningFiresAfterRegularWarning(t *testing.T) { r := &fakeRecorder{} // Warning fires at deadline-80ms, final at deadline-30ms. - w := NewWithLeads(80*time.Millisecond, 30*time.Millisecond, r) + w := newWatcherWithLeads(80*time.Millisecond, 30*time.Millisecond, r) defer w.Close() d := time.Now().Add(100 * time.Millisecond) @@ -446,7 +499,7 @@ func TestFinalWarningFiresAfterRegularWarning(t *testing.T) { func TestDismissSuppressesFinalWarning(t *testing.T) { r := &fakeRecorder{} - w := NewWithLeads(80*time.Millisecond, 30*time.Millisecond, r) + w := newWatcherWithLeads(80*time.Millisecond, 30*time.Millisecond, r) defer w.Close() d := time.Now().Add(100 * time.Millisecond) @@ -477,7 +530,7 @@ func TestDismissSuppressesFinalWarning(t *testing.T) { func TestDismissResetByNewDeadline(t *testing.T) { r := &fakeRecorder{} - w := NewWithLeads(80*time.Millisecond, 30*time.Millisecond, r) + w := newWatcherWithLeads(80*time.Millisecond, 30*time.Millisecond, r) defer w.Close() first := time.Now().Add(100 * time.Millisecond) @@ -507,7 +560,7 @@ func TestDismissResetByNewDeadline(t *testing.T) { func TestDismissBeforeUpdateIsNoop(t *testing.T) { r := &fakeRecorder{} - w := NewWithLeads(80*time.Millisecond, 30*time.Millisecond, r) + w := newWatcherWithLeads(80*time.Millisecond, 30*time.Millisecond, r) defer w.Close() // No deadline tracked yet; Dismiss must be a no-op (no panic, no state). @@ -527,3 +580,340 @@ func TestDismissBeforeUpdateIsNoop(t *testing.T) { } t.Fatalf("final-warning did not publish after no-op pre-Update Dismiss, events=%+v", r.snapshot()) } + +// The tests below drive the watcher's wall clock directly. The deadline sits +// an hour out in real time and the fake clock jumps, standing in for a device +// that was suspended and resumed somewhere inside — or past — the warning +// windows. The evaluation loop is what reacts to the jump, so these exercise +// the same path production takes on a resume. +func newResumeWatcher(t *testing.T, r *fakeRecorder) (*Watcher, *fakeClock, time.Time) { + t.Helper() + + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + start := time.Now() + clock := newFakeClock(start) + // Set before Update: the evaluation loop does not exist yet. + w.nowFn = clock.now + t.Cleanup(w.Close) + + deadline := start.Add(time.Hour).Round(0) + if err := w.Update(deadline); err != nil { + t.Fatalf("Update: %v", err) + } + return w, clock, deadline +} + +func TestResumeInsideWarningWindowWarns(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(-5 * time.Minute)) + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected the interactive warning after the resume, got %+v", events[1]) + } + if n := countWhere(events, event.isFinalWarning); n != 0 { + t.Fatalf("final-warning must wait for its own window, got %d: %+v", n, events) + } +} + +func TestResumeInsideFinalWindowSendsFinalWarningOnly(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(-time.Minute)) + + events := waitForEvents(t, r, 2) + if !events[1].isFinalWarning() { + t.Fatalf("expected the final warning after the resume, got %+v", events[1]) + } + settle() + if n := countWhere(r.snapshot(), event.isWarning); n != 0 { + t.Fatalf("the interactive warning is stale inside the final window, got %d: %+v", n, r.snapshot()) + } +} + +func TestResumePastDeadlinePublishesNothing(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(time.Minute)) + + settle() + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("an expired session must not warn, got %d publishes: %+v", n, r.snapshot()) + } +} + +func TestWarningPublishesOncePerDeadline(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(-5 * time.Minute)) + waitForEvents(t, r, 2) + + // Many ticks pass inside the same window. + settle() + if n := countWhere(r.snapshot(), event.isWarning); n != 1 { + t.Fatalf("expected exactly 1 warning publish across ticks, got %d: %+v", n, r.snapshot()) + } +} + +// TestWarningRecoversFromAClockRunningAhead covers the device that boots +// before NTP has corrected it: the deadline looks long gone, nothing is +// published, and the warning still arrives once the clock is fixed. +func TestWarningRecoversFromAClockRunningAhead(t *testing.T) { + r := &fakeRecorder{} + _, clock, deadline := newResumeWatcher(t, r) + + clock.set(deadline.Add(2 * time.Hour)) + settle() + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("a deadline that looks expired must not warn, got %d publishes: %+v", n, r.snapshot()) + } + + clock.set(deadline.Add(-5 * time.Minute)) + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected the warning once the clock was corrected, got %+v", events[1]) + } +} + +func TestResumeInsideFinalWindowRespectsDismiss(t *testing.T) { + r := &fakeRecorder{} + w, clock, deadline := newResumeWatcher(t, r) + + // The user dismissed the warning for this deadline before the device + // was suspended; resuming inside the final window must not reopen it. + w.Dismiss() + clock.set(deadline.Add(-time.Minute)) + + settle() + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("a dismissed deadline must not warn on resume, got %d: %+v", n, r.snapshot()) + } +} + +func TestCloseStopsTheEvaluationLoop(t *testing.T) { + r := &fakeRecorder{} + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + start := time.Now() + clock := newFakeClock(start) + w.nowFn = clock.now + + deadline := start.Add(time.Hour).Round(0) + if err := w.Update(deadline); err != nil { + t.Fatalf("Update: %v", err) + } + w.Close() + + clock.set(deadline.Add(-5 * time.Minute)) + settle() + + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("a closed watcher must not publish, got %d: %+v", n, r.snapshot()) + } +} + +func TestDeadlineOnlyRecordsDeadlineWithoutWarnings(t *testing.T) { + r := &fakeRecorder{} + w := NewDeadlineOnly(r) + defer w.Close() + + // With the default leads this deadline sits inside the final-warning + // window, so a watcher that warns at all would publish on the spot. + d := time.Now().Add(50 * time.Millisecond).Round(0) + if err := w.Update(d); err != nil { + t.Fatalf("Update: %v", err) + } + if got := r.deadline(); !got.Equal(d) { + t.Fatalf("expected recorder deadline %v, got %v", d, got) + } + + time.Sleep(100 * time.Millisecond) + + events := r.snapshot() + if got := countWhere(events, func(e event) bool { return e.kind == publish }); got != 0 { + t.Fatalf("expected no publish in deadline-only mode, got %d: %+v", got, events) + } + w.mu.Lock() + polling := w.stop != nil + w.mu.Unlock() + if polling { + t.Fatal("expected no evaluation loop in deadline-only mode") + } +} + +func TestDeadlineOnlyStillRejectsOutOfRangeDeadlines(t *testing.T) { + r := &fakeRecorder{} + w := NewDeadlineOnly(r) + defer w.Close() + + if err := w.Update(time.Now().Add(time.Hour)); err != nil { + t.Fatalf("Update: %v", err) + } + + err := w.Update(time.Now().Add(-maxPastHorizon - time.Hour)) + if !errors.Is(err, ErrDeadlineInPast) { + t.Fatalf("expected ErrDeadlineInPast, got %v", err) + } + if got := r.deadline(); !got.IsZero() { + t.Fatalf("expected recorder cleared after rejection, got %v", got) + } +} + +// TestClockAheadAtUpdateStillWarnsOnceCorrected covers a client that connects +// while its clock runs ahead of real time: the deadline reads as already +// expired when it arrives, so nothing publishes, and the warning must still +// come once NTP pulls the clock back. +func TestClockAheadAtUpdateStillWarnsOnceCorrected(t *testing.T) { + r := &fakeRecorder{} + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + deadline := time.Now().Add(time.Hour).Round(0) + // Two hours past the deadline from the device's point of view, well + // inside maxPastHorizon, so Update accepts and records it. + clock := newFakeClock(deadline.Add(2 * time.Hour)) + w.nowFn = clock.now + t.Cleanup(w.Close) + + if err := w.Update(deadline); err != nil { + t.Fatalf("Update: %v", err) + } + + settle() + if n := countWhere(r.snapshot(), func(e event) bool { return e.kind == publish }); n != 0 { + t.Fatalf("a deadline that reads as expired must not warn, got %d: %+v", n, r.snapshot()) + } + + clock.set(deadline.Add(-5 * time.Minute)) + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected the warning once the clock was corrected, got %+v", events[1]) + } +} + +// TestWarningNeverPrecedesTheDeadlineStateChange pins the ordering Update +// documents: consumers learn the new deadline before they see a warning that +// refers to it. The deadline here already sits inside the warning window, and +// the recorder stalls, so the evaluation loop gets many chances to publish +// while Update is still announcing. +func TestWarningNeverPrecedesTheDeadlineStateChange(t *testing.T) { + r := &fakeRecorder{setDelay: 50 * time.Millisecond} + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + deadline := time.Now().Add(time.Hour).Round(0) + clock := newFakeClock(deadline.Add(-5 * time.Minute)) + w.nowFn = clock.now + t.Cleanup(w.Close) + + if err := w.Update(deadline); err != nil { + t.Fatalf("Update: %v", err) + } + + events := waitForEvents(t, r, 2) + if events[0].kind != stateChange { + t.Fatalf("event[0] should be the deadline state change, got %+v", events) + } + if !events[1].isWarning() { + t.Fatalf("event[1] should be the warning, got %+v", events) + } +} + +// TestUpdateWakesTheLoopWithoutWaitingForATick pins the hand-off: Update +// publishes nothing itself, it nudges the evaluation loop, so a deadline that +// already sits inside a warning window is warned about at once even though the +// ticker here would not fire for an hour. +func TestUpdateWakesTheLoopWithoutWaitingForATick(t *testing.T) { + r := &fakeRecorder{} + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + w.interval = time.Hour + t.Cleanup(w.Close) + + d := time.Now().Add(5 * time.Minute).Round(0) + if err := w.Update(d); err != nil { + t.Fatalf("Update: %v", err) + } + + events := waitForEvents(t, r, 2) + if !events[1].isWarning() { + t.Fatalf("expected the warning on the wake-up, got %+v", events[1]) + } +} + +// blockingRecorder holds a publish open until the test releases it, so the +// evaluation loop can be parked mid-publish while Close is called. +type blockingRecorder struct { + fakeRecorder + entered chan struct{} + release chan struct{} +} + +func (r *blockingRecorder) PublishEvent( + severity cProto.SystemEvent_Severity, + category cProto.SystemEvent_Category, + message string, + userMessage string, + metadata map[string]string, +) { + select { + case r.entered <- struct{}{}: + default: + } + <-r.release + r.fakeRecorder.PublishEvent(severity, category, message, userMessage, metadata) +} + +// TestConcurrentCloseWaitsForTheLoop pins the contract for the caller that +// loses the race: Close returns only once the loop is done, whichever of the +// two calls got there first. +func TestConcurrentCloseWaitsForTheLoop(t *testing.T) { + r := &blockingRecorder{entered: make(chan struct{}, 1), release: make(chan struct{})} + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(r.release) }) } + + w := newWatcherWithLeads(WarningLead, FinalWarningLead, r) + w.interval = time.Hour + // Order matters: Close waits for the loop, which stays parked in the + // publish until the release, so a t.Fatal below would hang the cleanup + // instead of reporting the failure. + t.Cleanup(func() { + release() + w.Close() + }) + + d := time.Now().Add(5 * time.Minute).Round(0) + if err := w.Update(d); err != nil { + t.Fatalf("Update: %v", err) + } + + // The loop is now parked inside the warning publish. + select { + case <-r.entered: + case <-time.After(2 * time.Second): + t.Fatal("the loop never reached the publish") + } + + first := make(chan struct{}) + second := make(chan struct{}) + go func() { w.Close(); close(first) }() + go func() { w.Close(); close(second) }() + + select { + case <-first: + t.Fatal("Close returned while the loop was still publishing") + case <-second: + t.Fatal("Close returned while the loop was still publishing") + case <-time.After(100 * time.Millisecond): + } + + release() + for _, done := range []chan struct{}{first, second} { + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("Close did not return after the publish completed") + } + } +} diff --git a/client/internal/certproof/collect.go b/client/internal/certproof/collect.go new file mode 100644 index 000000000..65affca71 --- /dev/null +++ b/client/internal/certproof/collect.go @@ -0,0 +1,158 @@ +package certproof + +import ( + "context" + "crypto" + "crypto/sha256" + "crypto/x509" + "time" + + log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// Collect answers the certificate challenges in checks: for each challenge it picks a +// stored certificate that chains to the challenge's CAs and signs the nonce with its +// key. The same certificate is proven once even if several checks accept it. +func Collect(ctx context.Context, store Store, checks []*proto.Checks, peerKey []byte) []certposture.Proof { + challenges := certificateChallenges(checks) + if len(challenges) == 0 { + logNoChallenges(checks) + return nil + } + return CollectChallenges(ctx, store, challenges, peerKey) +} + +func logNoChallenges(checks []*proto.Checks) { + if len(checks) > 0 { + log.Debugf("certificate posture: %d posture checks received, none carries a certificate challenge", len(checks)) + } +} + +// CollectChallenges answers challenges already extracted from the posture checks, so a +// caller that ships them across a process boundary reuses the same matching and signing. +// Only nonces of the size management issues are signed, for a peer key of the size of +// ours, so the keys behind the store never sign arbitrary caller-chosen data. +func CollectChallenges(ctx context.Context, store Store, challenges []*proto.CertificateChallenge, peerKey []byte) []certposture.Proof { + if len(peerKey) != wgtypes.KeyLen { + log.Warnf("certificate posture: refusing to sign for a %d byte peer key", len(peerKey)) + return nil + } + challenges = wellFormed(challenges) + if len(challenges) == 0 { + return nil + } + log.Debugf("certificate posture: answering %d certificate challenges from store %T", len(challenges), store) + + candidates, err := store.Candidates(ctx) + if err != nil { + log.Warnf("failed loading certificates for posture checks: %v", err) + return nil + } + if len(candidates) == 0 { + log.Debug("certificate posture: certificate store holds no candidates, no proof will be sent") + return nil + } + log.Debugf("certificate posture: store holds %d candidate certificates", len(candidates)) + + now := time.Now() + proven := make(map[[sha256.Size]byte]struct{}) + var proofs []certposture.Proof + for i, challenge := range challenges { + roots, err := certposture.ParseCAs(challenge.GetCaCertificates()) + if err != nil { + log.Warnf("skipping certificate challenge with invalid CA certificates: %v", err) + continue + } + log.Debugf("certificate posture: challenge %d accepts %d CA certificates, nonce is %d bytes", i, len(challenge.GetCaCertificates()), len(challenge.GetNonce())) + + matched := false + for _, candidate := range candidates { + if len(candidate.Chain) == 0 { + continue + } + leaf := candidate.Chain[0] + chain, err := certposture.VerifiedChain(leaf, candidate.issuers(), roots, now) + if err != nil { + log.Debugf("certificate posture: challenge %d rejected %q issued by %q: %v", i, leaf.Subject, leaf.Issuer, err) + continue + } + matched = true + + // The same leaf can chain to different CAs for different challenges, and + // management checks each chain against each check's CAs, so a proof is + // deduplicated by its whole chain rather than by its leaf. + fingerprint := chainFingerprint(chain) + if _, done := proven[fingerprint]; done { + log.Debugf("certificate posture: challenge %d matched %q, already proven for an earlier challenge", i, leaf.Subject) + break + } + proof, err := prove(candidate.Signer, chain, challenge.GetNonce(), peerKey) + if err != nil { + log.Warnf("failed signing certificate proof for %s: %v", leaf.Subject, err) + continue + } + log.Debugf("certificate posture: challenge %d proven by %q with %s, signature %d bytes, chain of %d", i, leaf.Subject, proof.SigAlg, len(proof.Signature), len(proof.Chain)) + proven[fingerprint] = struct{}{} + proofs = append(proofs, proof) + break + } + if !matched { + log.Debugf("certificate posture: challenge %d matched none of the %d candidates", i, len(candidates)) + } + } + log.Debugf("certificate posture: %d challenges produced %d proofs", len(challenges), len(proofs)) + return proofs +} + +// HasChallenges reports whether any of checks asks for a certificate proof. +func HasChallenges(checks []*proto.Checks) bool { + return len(certificateChallenges(checks)) > 0 +} + +func certificateChallenges(checks []*proto.Checks) []*proto.CertificateChallenge { + var challenges []*proto.CertificateChallenge + for _, check := range checks { + if challenge := check.GetCertificateChallenge(); challenge != nil { + challenges = append(challenges, challenge) + } + } + return wellFormed(challenges) +} + +// wellFormed drops challenges whose nonce is not one management could have issued. +func wellFormed(challenges []*proto.CertificateChallenge) []*proto.CertificateChallenge { + var kept []*proto.CertificateChallenge + for _, challenge := range challenges { + if len(challenge.GetNonce()) != certposture.NonceSize { + log.Debugf("certificate posture: skipping challenge with a %d byte nonce", len(challenge.GetNonce())) + continue + } + kept = append(kept, challenge) + } + return kept +} + +func prove(signer crypto.Signer, chain []*x509.Certificate, nonce, peerKey []byte) (certposture.Proof, error) { + sigAlg, sig, err := certposture.Sign(signer, nonce, peerKey) + if err != nil { + return certposture.Proof{}, err + } + der := make([][]byte, 0, len(chain)) + for _, cert := range chain { + der = append(der, cert.Raw) + } + return certposture.Proof{Nonce: nonce, Chain: der, SigAlg: sigAlg, Signature: sig}, nil +} + +func chainFingerprint(chain []*x509.Certificate) [sha256.Size]byte { + buf := make([]byte, 0, len(chain)*sha256.Size) + for _, cert := range chain { + certHash := sha256.Sum256(cert.Raw) + buf = append(buf, certHash[:]...) + } + return sha256.Sum256(buf) +} diff --git a/client/internal/certproof/collect_darwin.go b/client/internal/certproof/collect_darwin.go new file mode 100644 index 000000000..92f0471e7 --- /dev/null +++ b/client/internal/certproof/collect_darwin.go @@ -0,0 +1,137 @@ +//go:build !ios + +package certproof + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +const helperTimeout = 30 * time.Second + +// userHelperBackoff holds off asking a user's keychain again after it proved nothing. +var userHelperBackoff = newHelperBackoff() + +// CollectProofs answers the certificate challenges in checks from every store this Mac +// can reach. The root daemon reads the System keychain itself, which is where MDM +// installs device identities, and reaches the console user's login keychain only by +// launching a helper into that user's session. A Mac sitting at the login window +// therefore yields device proofs alone. +func CollectProofs(ctx context.Context, checks []*proto.Checks, peerKey []byte, cfg Config) []certposture.Proof { + challenges := certificateChallenges(checks) + if len(challenges) == 0 { + logNoChallenges(checks) + return nil + } + + // A helper already runs inside the user's session, so it reads its own keychain + // directly and must never launch another one. + if os.Geteuid() != 0 { + return CollectChallenges(ctx, DefaultStore(), challenges, peerKey) + } + + proofs := CollectChallenges(ctx, DefaultStore(), challenges, peerKey) + if cfg.OwnerUnknown { + return proofs + } + + userProofs, err := collectAsConsoleUser(ctx, cfg.ProfileOwner, challenges, peerKey) + if err != nil { + log.Debugf("certificate posture: console user keychain unavailable: %v", err) + } + return mergeProofs(proofs, userProofs) +} + +// UserContext identifies the user whose keychain a collection would include: the console +// user when it owns the active profile, or empty when no user keychain would be asked. A +// change means a collection made earlier no longer reflects what this Mac can prove. +func UserContext(cfg Config) string { + if os.Geteuid() != 0 || cfg.OwnerUnknown { + return "" + } + user, ok := CurrentConsoleUser() + if !ok || !user.isOwner(cfg.ProfileOwner) { + return "" + } + return strconv.FormatUint(uint64(user.UID), 10) + ":" + user.Name +} + +// collectAsConsoleUser runs the helper inside the desktop session of the logged-in +// user. Dropping to their uid is not enough: keychain access is an XPC call to a +// per-session securityd, so the helper has to enter their Mach bootstrap namespace, +// which is what launchctl asuser does. +func collectAsConsoleUser(ctx context.Context, owner string, challenges []*proto.CertificateChallenge, peerKey []byte) ([]certposture.Proof, error) { + user, ok := CurrentConsoleUser() + if !ok { + return nil, nil + } + if !user.isOwner(owner) { + log.Debugf("certificate posture: console user %s does not own the active profile, no user keychain is asked", user.Name) + return nil, nil + } + + uid := strconv.FormatUint(uint64(user.UID), 10) + backoffKey := helperBackoffKey(uid, challenges) + if !userHelperBackoff.allow(backoffKey, time.Now()) { + log.Debugf("certificate posture: the keychain of uid %s proved nothing recently, not asking again yet", uid) + return nil, nil + } + + binary, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("resolve own binary: %w", err) + } + + parent := ctx + ctx, cancel := context.WithTimeout(ctx, helperTimeout) + defer cancel() + + // Absolute paths, because the daemon's PATH is configurable through the service + // environment, and sudo selects the user by uid so the name never has to round-trip. + cmd := exec.CommandContext(ctx, "/bin/launchctl", "asuser", uid, "/usr/bin/sudo", "-u", "#"+uid, "-H", "--", binary, "posture", "cert-proof") + killHelperGroupOnCancel(cmd) + + log.Debugf("certificate posture: asking the desktop session of uid %s to answer %d challenges", uid, len(challenges)) + proofs, err := runHelperCmd(cmd, helperRequest(challenges, peerKey)) + // A run that completed, or ran into the timeout waiting on a prompt nobody answered, + // tells whether the keychain proves anything. A launch or output failure, or a run the + // caller cut short, says nothing about it and must not hold off the next one. + timedOut := errors.Is(ctx.Err(), context.DeadlineExceeded) && parent.Err() == nil + if err == nil || timedOut { + userHelperBackoff.record(backoffKey, err == nil && len(proofs) > 0, time.Now()) + } + if err != nil { + return nil, fmt.Errorf("run helper as uid %s: %w", uid, err) + } + log.Debugf("certificate posture: desktop session of uid %s returned %d proofs", uid, len(proofs)) + return proofs, nil +} + +// helperStore is the store the helper reads: the user's login keychain alone. The +// session's search list also holds the System keychain, which the daemon reads itself, +// and using a System keychain key from the user's session would ask for an +// administrator's approval. +func helperStore() Store { + home, err := os.UserHomeDir() + if err != nil { + log.Debugf("certificate posture: no home directory, searching the default keychain list: %v", err) + return NewKeychainStore() + } + login := filepath.Join(home, "Library", "Keychains", "login.keychain-db") + if _, err := os.Stat(login); err != nil { + // Keychains created before macOS 10.12 keep the old file name. + login = filepath.Join(home, "Library", "Keychains", "login.keychain") + } + return NewKeychainStore(login) +} diff --git a/client/internal/certproof/collect_js.go b/client/internal/certproof/collect_js.go new file mode 100644 index 000000000..fd7c4c690 --- /dev/null +++ b/client/internal/certproof/collect_js.go @@ -0,0 +1,31 @@ +//go:build js + +package certproof + +import ( + "context" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// CollectProofs proves nothing in a browser: it has no TPM, certificate store or file +// store to sign with, so the stores stay out of the WebAssembly build. +func CollectProofs(context.Context, []*proto.Checks, []byte, Config) []certposture.Proof { + return nil +} + +// UserContext identifies the user whose certificates a collection would include. A +// browser has no per-user store, so it never changes. +func UserContext(Config) string { + return "" +} + +// DefaultStore is empty in a browser. +func DefaultStore() Store { + return Stores{} +} + +func helperStore() Store { + return DefaultStore() +} diff --git a/client/internal/certproof/collect_other.go b/client/internal/certproof/collect_other.go new file mode 100644 index 000000000..785d3db65 --- /dev/null +++ b/client/internal/certproof/collect_other.go @@ -0,0 +1,31 @@ +//go:build ((!darwin && !windows) || ios) && !js + +package certproof + +import ( + "context" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// CollectProofs answers the certificate challenges in checks from the PEM directory cfg +// names, joined by its PKCS#11 token when it names one. Only macOS and Windows keep +// per-user certificates out of reach of a privileged daemon, so every other platform +// reads its store in the daemon itself. +func CollectProofs(ctx context.Context, checks []*proto.Checks, peerKey []byte, cfg Config) []certposture.Proof { + return Collect(ctx, storeWithToken(cfg), checks, peerKey) +} + +// UserContext identifies the user whose certificates a collection would include. These +// platforms have no per-user store, so it never changes. +func UserContext(Config) string { + return "" +} + +// helperStore is the store the helper reads. Nothing launches a helper on these +// platforms, so it is the store the daemon reads, configured from the same environment, +// which lets an administrator check a setup by running the helper by hand. +func helperStore() Store { + return storeWithToken(Config{PKCS11: PKCS11FromEnv()}) +} diff --git a/client/internal/certproof/collect_test.go b/client/internal/certproof/collect_test.go new file mode 100644 index 000000000..e16ee3668 --- /dev/null +++ b/client/internal/certproof/collect_test.go @@ -0,0 +1,258 @@ +package certproof + +import ( + "context" + "crypto/rand" + "crypto/x509" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/certposture/certtest" + "github.com/netbirdio/netbird/shared/management/proto" +) + +var peerKey = []byte("peer-public-key-aaaaaaaaaaaaaaaa") + +func TestCollect_ProvesOneMatchingCertificatePerChallenge(t *testing.T) { + corpCA := certtest.NewCA(t, "corp-root") + otherCA := certtest.NewCA(t, "other-root") + unrelatedCA := certtest.NewCA(t, "unrelated-root") + + dir := storeDir(t) + deviceKey := certtest.ECDSAKey(t) + device := corpCA.Issue(t, deviceKey, "device") + writeFile(t, dir, "device.pem", certtest.CertPEM(device)+certtest.KeyPEM(t, deviceKey)) + + otherKey := certtest.RSAKey(t) + writeFile(t, dir, "other.crt", certtest.CertPEM(otherCA.Issue(t, otherKey, "other"))) + writeFile(t, dir, "other.key", certtest.KeyPEM(t, otherKey)) + + writeFile(t, dir, "keyless.crt", certtest.CertPEM(corpCA.Issue(t, certtest.ECDSAKey(t), "keyless"))) + writeFile(t, dir, "notes.txt", "ignored") + + challenger := certposture.NewChallenger([]byte("secret")) + nonce := challenger.Nonce(peerKey, time.Now()) + challenge := func(cas ...string) *proto.Checks { + return &proto.Checks{CertificateChallenge: &proto.CertificateChallenge{Nonce: nonce, CaCertificates: cas}} + } + checks := []*proto.Checks{ + {Files: []string{"/usr/bin/agent"}}, + challenge(corpCA.PEM), + challenge(corpCA.PEM), + challenge(otherCA.PEM), + challenge(unrelatedCA.PEM), + challenge("not a pem"), + } + + proofs := Collect(context.Background(), NewFileStore(dir), checks, peerKey) + + require.Len(t, proofs, 2) + var subjects []string + for _, p := range proofs { + chain, err := challenger.Verify(p, peerKey, time.Now()) + require.NoError(t, err) + subjects = append(subjects, chain[0].Subject.CommonName) + } + assert.ElementsMatch(t, []string{"device", "other"}, subjects) +} + +func TestCollect_NothingToProve(t *testing.T) { + dir := storeDir(t) + key := certtest.ECDSAKey(t) + ca := certtest.NewCA(t, "root") + writeFile(t, dir, "device.pem", certtest.CertPEM(ca.Issue(t, key, "device"))+certtest.KeyPEM(t, key)) + + tests := []struct { + name string + store Store + checks []*proto.Checks + }{ + {"no checks", NewFileStore(dir), nil}, + {"files only", NewFileStore(dir), []*proto.Checks{{Files: []string{"/bin/x"}}}}, + {"challenge without nonce", NewFileStore(dir), []*proto.Checks{{CertificateChallenge: &proto.CertificateChallenge{CaCertificates: []string{ca.PEM}}}}}, + {"missing store dir", NewFileStore(filepath.Join(dir, "missing")), []*proto.Checks{{CertificateChallenge: &proto.CertificateChallenge{Nonce: make([]byte, certposture.NonceSize), CaCertificates: []string{ca.PEM}}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Nil(t, Collect(context.Background(), tt.store, tt.checks, peerKey)) + }) + } +} + +func TestFileStore_ChainWithIntermediate(t *testing.T) { + root := certtest.NewCA(t, "root") + intermediate := certtest.NewIntermediate(t, root, "intermediate") + key := certtest.ECDSAKey(t) + leaf := intermediate.Issue(t, key, "device") + + dir := storeDir(t) + writeFile(t, dir, "device.pem", certtest.CertPEM(leaf)+certtest.CertPEM(intermediate.Cert)+certtest.KeyPEM(t, key)) + + candidates, err := NewFileStore(dir).Candidates(context.Background()) + require.NoError(t, err) + require.Len(t, candidates, 1) + require.Len(t, candidates[0].Chain, 2) + + roots, err := certposture.ParseCAs([]string{root.PEM}) + require.NoError(t, err) + assert.NoError(t, certposture.VerifyChain(candidates[0].Chain, roots, time.Now())) +} + +// storeDir is a PEM directory the store accepts: t.TempDir follows the umask, which +// leaves the directory group-writable on systems with a user-private group umask. +func storeDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.Chmod(dir, 0o700)) + return dir +} + +func writeFile(t *testing.T, dir, name, content string) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600)) +} + +func TestCollectChallenges_RefusesMalformedInput(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + dir := storeDir(t) + key := certtest.ECDSAKey(t) + writeFile(t, dir, "device.pem", certtest.CertPEM(ca.Issue(t, key, "device"))+certtest.KeyPEM(t, key)) + store := NewFileStore(dir) + nonce := certposture.NewChallenger([]byte("secret")).Nonce(peerKey, time.Now()) + + tests := []struct { + name string + nonce []byte + peerKey []byte + want int + }{ + {"issued nonce and peer key are signed", nonce, peerKey, 1}, + {"short nonce is not signed", nonce[:8], peerKey, 0}, + {"oversized nonce is not signed", append(append([]byte{}, nonce...), 0), peerKey, 0}, + {"short peer key is not signed", nonce, peerKey[:16], 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + challenges := []*proto.CertificateChallenge{{Nonce: tt.nonce, CaCertificates: []string{ca.PEM}}} + assert.Len(t, CollectChallenges(context.Background(), store, challenges, tt.peerKey), tt.want, + "the device key signs only what management could have issued") + }) + } +} + +func TestFileStore_SkipsKeyOfAnotherCertificate(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + dir := storeDir(t) + + // A stale key next to a renewed certificate, sorted before the good pair, must not + // produce a proof that management rejects and stop the search there. + writeFile(t, dir, "a-renewed.crt", certtest.CertPEM(ca.Issue(t, certtest.ECDSAKey(t), "renewed"))) + writeFile(t, dir, "a-renewed.key", certtest.KeyPEM(t, certtest.ECDSAKey(t))) + goodKey := certtest.ECDSAKey(t) + good := ca.Issue(t, goodKey, "good") + writeFile(t, dir, "b-good.pem", certtest.CertPEM(good)+certtest.KeyPEM(t, goodKey)) + + candidates, err := NewFileStore(dir).Candidates(context.Background()) + require.NoError(t, err) + require.Len(t, candidates, 1, "only the certificate whose key matches is a candidate") + assert.True(t, good.Equal(candidates[0].Chain[0]), "the matching pair is kept") + + challenger := certposture.NewChallenger([]byte("secret")) + now := time.Now() + nonce := challenger.Nonce(peerKey, now) + proofs := CollectChallenges(context.Background(), NewFileStore(dir), []*proto.CertificateChallenge{{Nonce: nonce, CaCertificates: []string{ca.PEM}}}, peerKey) + require.Len(t, proofs, 1) + _, err = challenger.Verify(proofs[0], peerKey, now) + assert.NoError(t, err, "the proof sent is one management accepts") +} + +// staticStore hands out fixed candidates, for scenarios no on-disk layout can express. +type staticStore []Candidate + +func (s staticStore) Candidates(context.Context) ([]Candidate, error) { return s, nil } + +func TestCollectChallenges_RoutesAroundExpiredCopyOfRenewedIntermediate(t *testing.T) { + root := certtest.NewCA(t, "root") + intermediate := certtest.NewIntermediate(t, root, "issuing-ca") + + // Renewing a CA with the same key pair leaves two certificates with the same + // subject and key in the store. The expired one sorts first here. + expiredTmpl := *intermediate.Cert + expiredTmpl.SerialNumber = big.NewInt(1) + expiredTmpl.NotBefore = time.Now().Add(-72 * time.Hour) + expiredTmpl.NotAfter = time.Now().Add(-48 * time.Hour) + der, err := x509.CreateCertificate(rand.Reader, &expiredTmpl, root.Cert, intermediate.Key.Public(), root.Key) + require.NoError(t, err) + expired, err := x509.ParseCertificate(der) + require.NoError(t, err) + + key := certtest.ECDSAKey(t) + leaf := intermediate.Issue(t, key, "device") + pool := []*x509.Certificate{expired, intermediate.Cert} + + chain := buildChain(leaf, pool) + require.Len(t, chain, 2) + require.True(t, expired.Equal(chain[1]), "precondition: the first-match chain runs through the expired copy") + + challenger := certposture.NewChallenger([]byte("secret")) + now := time.Now() + nonce := challenger.Nonce(peerKey, now) + store := staticStore{{Chain: chain, Signer: key, Intermediates: pool}} + + proofs := CollectChallenges(context.Background(), store, []*proto.CertificateChallenge{{Nonce: nonce, CaCertificates: []string{root.PEM}}}, peerKey) + + require.Len(t, proofs, 1, "a valid path through the renewed intermediate exists, so the challenge is answered") + verified, err := challenger.Verify(proofs[0], peerKey, now) + require.NoError(t, err) + assert.True(t, intermediate.Cert.Equal(verified[1]), "the proof carries the valid intermediate, not the expired copy") + assert.True(t, certposture.ChainMatchesCAs(certposture.EncodeChainPEM(verified), []string{root.PEM}, now), + "management's own check accepts the chain the proof carries") +} + +func TestCollectChallenges_ProvesALeafOncePerDistinctChain(t *testing.T) { + rootA := certtest.NewCA(t, "root-a") + rootB := certtest.NewCA(t, "root-b") + issuer := certtest.NewIntermediate(t, rootA, "issuing-ca") + + // The same issuing CA cross-signed by a second root: one leaf, two valid paths. + crossTmpl := *issuer.Cert + crossTmpl.SerialNumber = big.NewInt(2) + der, err := x509.CreateCertificate(rand.Reader, &crossTmpl, rootB.Cert, issuer.Key.Public(), rootB.Key) + require.NoError(t, err) + cross, err := x509.ParseCertificate(der) + require.NoError(t, err) + + key := certtest.ECDSAKey(t) + leaf := issuer.Issue(t, key, "device") + pool := []*x509.Certificate{issuer.Cert, cross} + store := staticStore{{Chain: buildChain(leaf, pool), Signer: key, Intermediates: pool}} + + challenger := certposture.NewChallenger([]byte("secret")) + now := time.Now() + nonce := challenger.Nonce(peerKey, now) + challenges := []*proto.CertificateChallenge{ + {Nonce: nonce, CaCertificates: []string{rootA.PEM}}, + {Nonce: nonce, CaCertificates: []string{rootB.PEM}}, + {Nonce: nonce, CaCertificates: []string{rootA.PEM}}, + } + + proofs := CollectChallenges(context.Background(), store, challenges, peerKey) + + require.Len(t, proofs, 2, "one proof per distinct chain, the repeated root-a challenge reuses the first") + for _, root := range []*certtest.CA{rootA, rootB} { + matched := false + for _, p := range proofs { + chain, err := challenger.Verify(p, peerKey, now) + require.NoError(t, err) + matched = matched || certposture.ChainMatchesCAs(certposture.EncodeChainPEM(chain), []string{root.PEM}, now) + } + assert.True(t, matched, "management can match a chain for the check that trusts %s", root.Cert.Subject.CommonName) + } +} diff --git a/client/internal/certproof/collect_windows.go b/client/internal/certproof/collect_windows.go new file mode 100644 index 000000000..8e5695539 --- /dev/null +++ b/client/internal/certproof/collect_windows.go @@ -0,0 +1,108 @@ +package certproof + +import ( + "context" + "fmt" + "os" + "os/exec" + "syscall" + "time" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +const helperTimeout = 30 * time.Second + +// CollectProofs answers the certificate challenges in checks from every store this +// machine can reach. The service reads the local machine store itself, where AD and +// Intune enrol device certificates, and reaches the signed-in user's store by launching +// a helper with that session's token. A machine at the sign-in screen therefore proves +// device certificates alone. +func CollectProofs(ctx context.Context, checks []*proto.Checks, peerKey []byte, cfg Config) []certposture.Proof { + challenges := certificateChallenges(checks) + if len(challenges) == 0 { + logNoChallenges(checks) + return nil + } + + proofs := CollectChallenges(ctx, DefaultStore(), challenges, peerKey) + + // The helper already runs as the signed-in user, and an ordinary process has no + // right to a session token, so only the service goes looking for one. + if !runningAsLocalSystem() || cfg.OwnerUnknown { + return proofs + } + + userProofs, err := collectAsDesktopUser(ctx, cfg.ProfileOwner, challenges, peerKey) + if err != nil { + log.Debugf("certificate posture: user certificate store unavailable: %v", err) + } + return mergeProofs(proofs, userProofs) +} + +// UserContext identifies the session whose store a collection would include: a session +// of the profile owner, or empty when no user store would be asked. A change means a +// collection made earlier no longer reflects what this machine can prove. +func UserContext(cfg Config) string { + if !runningAsLocalSystem() || cfg.OwnerUnknown { + return "" + } + user, ok := CurrentDesktopUser(cfg.ProfileOwner) + if !ok { + return "" + } + defer user.Close() + return fmt.Sprintf("%d:%s", user.Session, user.Name) +} + +// helperStore is the store the helper reads. It runs as the signed-in user, so it wants +// that user's store rather than the machine store the service already read. +func helperStore() Store { + return NewUserStore() +} + +// collectAsDesktopUser runs the helper inside the interactive session of the signed-in +// user. Unlike a keychain on macOS, a Windows service can assume a user identity +// directly, so the session token goes straight into the child process. +func collectAsDesktopUser(ctx context.Context, owner string, challenges []*proto.CertificateChallenge, peerKey []byte) ([]certposture.Proof, error) { + user, ok := CurrentDesktopUser(owner) + if !ok { + return nil, nil + } + defer user.Close() + + binary, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("resolve own binary: %w", err) + } + + // The user's own environment, not the service's: the service environment may carry + // secrets such as a setup key that the signed-in user must not be able to read. + env, err := user.Token.Environ(false) + if err != nil { + return nil, fmt.Errorf("build environment of %s: %w", user.Name, err) + } + + ctx, cancel := context.WithTimeout(ctx, helperTimeout) + defer cancel() + + cmd := exec.CommandContext(ctx, binary, "posture", "cert-proof") + cmd.Env = env + cmd.SysProcAttr = &syscall.SysProcAttr{ + Token: syscall.Token(user.Token), + HideWindow: true, + CreationFlags: windows.CREATE_NO_WINDOW, + } + + log.Debugf("certificate posture: asking session %d to answer %d challenges", user.Session, len(challenges)) + proofs, err := runHelperCmd(cmd, helperRequest(challenges, peerKey)) + if err != nil { + return nil, fmt.Errorf("run helper in session %d: %w", user.Session, err) + } + log.Debugf("certificate posture: session %d returned %d proofs", user.Session, len(proofs)) + return proofs, nil +} diff --git a/client/internal/certproof/collector.go b/client/internal/certproof/collector.go new file mode 100644 index 000000000..2931f4b00 --- /dev/null +++ b/client/internal/certproof/collector.go @@ -0,0 +1,189 @@ +package certproof + +import ( + "context" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +const ( + collectTimeout = 45 * time.Second + + // lostAfter is how many deadlines an abandoned collection is waited for before a new + // one may start beside it. A store that is merely slow finishes well within that and + // releases its slot; only one wedged for good is worked around. + lostAfter = 10 + + // maxInFlight bounds how many abandoned collections may pile up. A store that never + // answers would otherwise leak a goroutine on every collection for the life of the + // daemon. + maxInFlight = 2 +) + +// collectSlots tracks the collections running in the process. Collectors share one by +// default: a collection abandoned in a token, TPM or keychain call keeps running after +// the engine that started it has stopped, and the next engine must account for it. +type collectSlots struct { + mu sync.Mutex + next uint64 + // running holds the start time of each collection still running, by slot id. + running map[uint64]time.Time +} + +var processSlots collectSlots + +// Collector runs CollectProofs with a deadline and, in the normal case, one collection +// at a time in the process. Token, TPM and keychain calls cannot be interrupted, and on +// Linux there is not even a process to kill, so a collection that overruns is abandoned +// rather than awaited. A single wedged call would then refuse every later collection for +// the life of the daemon, so once an abandoned collection has been running for lostAfter +// deadlines another may start beside it, up to maxInFlight. The zero value is ready to +// use. +type Collector struct { + // slots overrides the process-wide slots when set. + slots *collectSlots + // timeout overrides collectTimeout when set. + timeout time.Duration + // now overrides the clock when set. + now func() time.Time +} + +// Collect answers the certificate challenges in checks, returning no proofs when there +// are no challenges, when previous collections still hold the slots, or when this one +// does not finish in time. Missing proofs fail the certificate check on management. +func (c *Collector) Collect(ctx context.Context, checks []*proto.Checks, peerKey []byte, cfg Config) []certposture.Proof { + return c.collect(ctx, checks, func(ctx context.Context) []certposture.Proof { + return CollectProofs(ctx, checks, peerKey, cfg) + }) +} + +func (c *Collector) collect(ctx context.Context, checks []*proto.Checks, run func(context.Context) []certposture.Proof) []certposture.Proof { + if len(certificateChallenges(checks)) == 0 { + return nil + } + slots := c.slotsInUse() + slot, ok := slots.start(c.clock(), time.Duration(lostAfter)*c.deadline()) + if !ok { + return nil + } + + ctx, cancel := context.WithTimeout(ctx, c.deadline()) + defer cancel() + + done := make(chan []certposture.Proof, 1) + go func() { + // The slot is freed before the result is delivered, so a caller that starts the + // next collection right after this one returned is not turned away. + proofs := run(ctx) + slots.finish(slot) + done <- proofs + }() + + select { + case proofs := <-done: + return proofs + case <-ctx.Done(): + log.Warnf("certificate posture: proof collection did not finish within %s, sending no proofs", c.deadline()) + return nil + } +} + +// Stuck reports whether lost collections hold every slot. No collection starts again +// until one of them returns, which for a store that never answers means a restart. +func (c *Collector) Stuck() bool { + return c.slotsInUse().allLost(c.clock(), time.Duration(lostAfter)*c.deadline()) +} + +func (c *Collector) slotsInUse() *collectSlots { + if c.slots != nil { + return c.slots + } + return &processSlots +} + +func (c *Collector) clock() time.Time { + if c.now != nil { + return c.now() + } + return time.Now() +} + +func (c *Collector) deadline() time.Duration { + if c.timeout > 0 { + return c.timeout + } + return collectTimeout +} + +// start claims a slot at now, reporting its id and whether the caller may collect. A +// collection that has held a slot for lost is taken to be wedged. +func (s *collectSlots) start(now time.Time, lost time.Duration) (uint64, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.running) > 0 { + oldest := s.oldest() + switch { + case len(s.running) >= maxInFlight: + log.Warnf("certificate posture: %d proof collections are wedged, sending no proofs", len(s.running)) + return 0, false + case now.Sub(oldest) < lost: + log.Warnf("certificate posture: previous proof collection is still running, sending no proofs") + return 0, false + default: + log.Warnf("certificate posture: a proof collection has been stuck since %s, starting another", oldest.Format(time.RFC3339)) + } + } + if s.running == nil { + s.running = make(map[uint64]time.Time, maxInFlight) + } + s.next++ + s.running[s.next] = now + return s.next, true +} + +// finish releases the slot with the given id. +func (s *collectSlots) finish(slot uint64) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.running, slot) +} + +// allLost reports whether every slot is held by a collection that has run for lost. +func (s *collectSlots) allLost(now time.Time, lost time.Duration) bool { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.running) < maxInFlight { + return false + } + for _, started := range s.running { + if now.Sub(started) < lost { + return false + } + } + return true +} + +// idle reports whether no collection is running. +func (s *collectSlots) idle() bool { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.running) == 0 +} + +// oldest returns the start time of the longest-running collection. The caller holds mu +// and has checked that one is running. +func (s *collectSlots) oldest() time.Time { + var oldest time.Time + for _, started := range s.running { + if oldest.IsZero() || started.Before(oldest) { + oldest = started + } + } + return oldest +} diff --git a/client/internal/certproof/collector_test.go b/client/internal/certproof/collector_test.go new file mode 100644 index 000000000..a28c47da2 --- /dev/null +++ b/client/internal/certproof/collector_test.go @@ -0,0 +1,207 @@ +package certproof + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +var challengeChecks = []*proto.Checks{{CertificateChallenge: &proto.CertificateChallenge{ + Nonce: certposture.NewChallenger([]byte("secret")).Nonce(peerKey, time.Now()), +}}} + +func TestCollector_SkipsChecksWithoutChallenges(t *testing.T) { + var c Collector + called := false + + proofs := c.collect(context.Background(), []*proto.Checks{{Files: []string{"/bin/agent"}}}, func(context.Context) []certposture.Proof { + called = true + return nil + }) + + assert.Nil(t, proofs) + assert.False(t, called, "no store is touched when no check carries a challenge") +} + +func TestCollector_ReturnsProofs(t *testing.T) { + var c Collector + want := []certposture.Proof{{Nonce: []byte("nonce")}} + + proofs := c.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { return want }) + + assert.Equal(t, want, proofs, "a collection that finishes in time is returned as is") +} + +func TestCollector_AbandonsStuckCollection(t *testing.T) { + c := Collector{timeout: 50 * time.Millisecond, slots: &collectSlots{}} + release := make(chan struct{}) + finished := make(chan struct{}) + + // A token or keychain call that ignores its context and blocks well past the deadline. + stuck := func(context.Context) []certposture.Proof { + defer close(finished) + <-release + return []certposture.Proof{{Nonce: []byte("late")}} + } + + start := time.Now() + proofs := c.collect(context.Background(), challengeChecks, stuck) + assert.Nil(t, proofs, "an overrunning collection yields no proofs") + assert.Less(t, time.Since(start), time.Second, "the caller is released at the deadline, not when the call returns") + + called := false + proofs = c.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { + called = true + return nil + }) + assert.Nil(t, proofs) + assert.False(t, called, "no second collection starts while the first is still running") + + close(release) + <-finished + require.Eventually(t, func() bool { return c.slots.idle() }, time.Second, 5*time.Millisecond, "the collector frees up once the stuck call returns") + + want := []certposture.Proof{{Nonce: []byte("nonce")}} + assert.Equal(t, want, c.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { return want }), + "collection works again after the stuck call returned") +} + +func TestCollector_CancelsContextAtDeadline(t *testing.T) { + c := Collector{timeout: 20 * time.Millisecond, slots: &collectSlots{}} + cancelled := make(chan struct{}) + + c.collect(context.Background(), challengeChecks, func(ctx context.Context) []certposture.Proof { + <-ctx.Done() + close(cancelled) + return nil + }) + + select { + case <-cancelled: + case <-time.After(time.Second): + t.Fatal("a collection that honours its context, like the helper process, must see it cancelled") + } +} + +// TestCollector_SingleFlightAcrossCollectors: each engine has its own Collector, and a +// collection stuck in a token call outlives the engine that started it, so the next +// engine's Collector must not start another one until it has finished. +func TestCollector_SingleFlightAcrossCollectors(t *testing.T) { + shared := &collectSlots{} + first := Collector{timeout: 20 * time.Millisecond, slots: shared} + second := Collector{timeout: time.Second, slots: shared} + + release := make(chan struct{}) + stuck := func(context.Context) []certposture.Proof { + <-release + return nil + } + assert.Nil(t, first.collect(context.Background(), challengeChecks, stuck), "the first collection is abandoned at its deadline") + + called := false + proofs := second.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { + called = true + return []certposture.Proof{{}} + }) + assert.Nil(t, proofs, "no proofs while the abandoned collection still runs") + assert.False(t, called, "a second collection does not start on top of the abandoned one") + + close(release) + require.Eventually(t, func() bool { return shared.idle() }, time.Second, 5*time.Millisecond) + assert.Len(t, second.collect(context.Background(), challengeChecks, func(context.Context) []certposture.Proof { return []certposture.Proof{{}} }), 1, + "collections resume once the abandoned one finished") +} + +// TestCollector_ZeroValuesShareTheProcessFlag: the zero value uses the process-wide flag. +func TestCollector_ZeroValuesShareTheProcessFlag(t *testing.T) { + var a, b Collector + assert.Same(t, a.slotsInUse(), b.slotsInUse(), "separate Collectors share the process slots") +} + +// TestCollector_LostCollectionDoesNotBlockForever: a store call that never answers, such +// as a wedged token on Linux where there is no process to kill, must not refuse every +// later collection. Once it has run for lostAfter deadlines another starts beside it, +// and no more than maxInFlight ever run. +func TestCollector_LostCollectionDoesNotBlockForever(t *testing.T) { + now := time.Now() + slots := &collectSlots{} + c := Collector{timeout: 10 * time.Millisecond, slots: slots, now: func() time.Time { return now }} + + release := make(chan struct{}) + defer close(release) + var started atomic.Int32 + wedged := func(context.Context) []certposture.Proof { + started.Add(1) + <-release + return nil + } + + c.collect(context.Background(), challengeChecks, wedged) + waitStarted(t, &started, 1) + c.collect(context.Background(), challengeChecks, wedged) + assert.Equal(t, int32(1), started.Load(), "a second collection is refused while the first may still finish") + assert.False(t, c.Stuck(), "one slow collection is not stuck yet") + + now = now.Add(time.Duration(lostAfter) * c.deadline()) + c.collect(context.Background(), challengeChecks, wedged) + waitStarted(t, &started, 2) + assert.False(t, c.Stuck(), "the second collection may still return, so the collector is not stuck yet") + + now = now.Add(time.Duration(lostAfter) * c.deadline()) + c.collect(context.Background(), challengeChecks, wedged) + assert.Equal(t, int32(2), started.Load(), "no more than maxInFlight collections run") + assert.True(t, c.Stuck(), "with every slot held by a lost collection the collector reports itself stuck") +} + +// TestCollector_LostWindowFollowsTheOldestRunningCollection: A wedges, B starts beside it +// once A counts as lost, then A returns. B is recent, so a third collection must wait for +// B to be lost in its own right rather than inherit A's start time. +func TestCollector_LostWindowFollowsTheOldestRunningCollection(t *testing.T) { + now := time.Now() + slots := &collectSlots{} + c := Collector{timeout: 10 * time.Millisecond, slots: slots, now: func() time.Time { return now }} + lost := time.Duration(lostAfter) * c.deadline() + + releaseA := make(chan struct{}) + releaseB := make(chan struct{}) + defer close(releaseB) + var started atomic.Int32 + blockOn := func(release chan struct{}) func(context.Context) []certposture.Proof { + return func(context.Context) []certposture.Proof { + started.Add(1) + <-release + return nil + } + } + + c.collect(context.Background(), challengeChecks, blockOn(releaseA)) + waitStarted(t, &started, 1) + now = now.Add(lost) + c.collect(context.Background(), challengeChecks, blockOn(releaseB)) + waitStarted(t, &started, 2) + + close(releaseA) + require.Eventually(t, func() bool { + slots.mu.Lock() + defer slots.mu.Unlock() + return len(slots.running) == 1 + }, time.Second, 5*time.Millisecond, "A frees its slot once it returns") + + c.collect(context.Background(), challengeChecks, blockOn(make(chan struct{}))) + assert.Equal(t, int32(2), started.Load(), "a third collection waits while B is recent") +} + +// waitStarted waits for the collections the test launched to have started. collect may +// return at its deadline before the goroutine running the store call was scheduled. +func waitStarted(t *testing.T, started *atomic.Int32, n int32) { + t.Helper() + require.Eventually(t, func() bool { return started.Load() == n }, time.Second, time.Millisecond, + "%d collections should have started", n) +} diff --git a/client/internal/certproof/consoleuser_darwin.go b/client/internal/certproof/consoleuser_darwin.go new file mode 100644 index 000000000..a5182c7fd --- /dev/null +++ b/client/internal/certproof/consoleuser_darwin.go @@ -0,0 +1,123 @@ +//go:build !ios + +package certproof + +import ( + "bytes" + "fmt" + "strconv" + "sync" + + "github.com/ebitengine/purego" + log "github.com/sirupsen/logrus" +) + +const ( + systemConfigurationFramework = "/System/Library/Frameworks/SystemConfiguration.framework/SystemConfiguration" + + encodingUTF8 = 0x08000100 + consoleNameBufSize = 256 +) + +var ( + consoleOnce sync.Once + consoleErr error + + scDynamicStoreCopyConsoleUser func(store uintptr, uid, gid *uint32) uintptr + cfStringGetCString func(str uintptr, buffer *byte, size int, encoding uint32) bool +) + +// ConsoleUser is the account whose desktop session owns the display. Its login keychain +// is the only user keychain a NetBird daemon can reach, and only while it is logged in. +type ConsoleUser struct { + Name string + UID uint32 + GID uint32 +} + +// CurrentConsoleUser reports the user sitting at the desktop. The second return value is +// false when nobody is: at the login window macOS either reports no console user at all +// or attributes the session to root, and neither has a login keychain to offer. +func CurrentConsoleUser() (ConsoleUser, bool) { + if err := loadConsoleUser(); err != nil { + log.Debugf("console user lookup unavailable: %v", err) + return ConsoleUser{}, false + } + + var uid, gid uint32 + name := scDynamicStoreCopyConsoleUser(0, &uid, &gid) + if name == 0 { + log.Debug("no console user is logged in, no login keychain is reachable") + return ConsoleUser{}, false + } + defer release(name) + + user := ConsoleUser{Name: cfString(name), UID: uid, GID: gid} + if !user.hasDesktop() { + log.Debugf("console session belongs to %q uid=%d, which is not a desktop login, no login keychain is reachable", user.Name, user.UID) + return ConsoleUser{}, false + } + return user, true +} + +// hasDesktop reports whether the console session is a real user desktop. The login +// window runs as root and some macOS releases name it "loginwindow" instead. +func (u ConsoleUser) hasDesktop() bool { + switch u.Name { + case "", "root", "loginwindow": + return false + } + return u.UID != 0 +} + +// isOwner reports whether the console user is owner, the account of the active profile, +// which is recorded as a short user name or, for an account without one, a numeric uid. +// With no owner the console user counts, as macOS has a single console user. +func (u ConsoleUser) isOwner(owner string) bool { + return owner == "" || owner == u.Name || owner == strconv.FormatUint(uint64(u.UID), 10) +} + +func cfString(str uintptr) string { + buf := make([]byte, consoleNameBufSize) + if !cfStringGetCString(str, &buf[0], len(buf), encodingUTF8) { + return "" + } + if end := bytes.IndexByte(buf, 0); end >= 0 { + return string(buf[:end]) + } + return string(buf) +} + +// loadConsoleUser resolves the console user symbols. It loads the keychain bindings +// first because CFRelease is resolved there and released strings depend on it. +func loadConsoleUser() error { + if err := loadKeychain(); err != nil { + return err + } + consoleOnce.Do(func() { consoleErr = resolveConsoleUser() }) + return consoleErr +} + +func resolveConsoleUser() error { + systemConfiguration, err := purego.Dlopen(systemConfigurationFramework, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return fmt.Errorf("open %s: %w", systemConfigurationFramework, err) + } + coreFoundation, err := purego.Dlopen(coreFoundationFramework, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return fmt.Errorf("open %s: %w", coreFoundationFramework, err) + } + + symbol, err := purego.Dlsym(systemConfiguration, "SCDynamicStoreCopyConsoleUser") + if err != nil { + return fmt.Errorf("resolve SCDynamicStoreCopyConsoleUser: %w", err) + } + purego.RegisterFunc(&scDynamicStoreCopyConsoleUser, symbol) + + symbol, err = purego.Dlsym(coreFoundation, "CFStringGetCString") + if err != nil { + return fmt.Errorf("resolve CFStringGetCString: %w", err) + } + purego.RegisterFunc(&cfStringGetCString, symbol) + return nil +} diff --git a/client/internal/certproof/consoleuser_darwin_test.go b/client/internal/certproof/consoleuser_darwin_test.go new file mode 100644 index 000000000..9c1aaa22c --- /dev/null +++ b/client/internal/certproof/consoleuser_darwin_test.go @@ -0,0 +1,51 @@ +//go:build !ios + +package certproof + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConsoleUser_OnlyADesktopSessionCanBeValidated(t *testing.T) { + tests := []struct { + name string + user ConsoleUser + desktop bool + }{ + {"logged in user", ConsoleUser{Name: "maycon", UID: 501, GID: 20}, true}, + {"login window as root", ConsoleUser{Name: "root", UID: 0}, false}, + {"login window by name", ConsoleUser{Name: "loginwindow", UID: 0}, false}, + {"named user still at uid 0", ConsoleUser{Name: "admin", UID: 0}, false}, + {"no console user", ConsoleUser{}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.desktop, tt.user.hasDesktop(), "only a real desktop session offers a login keychain") + }) + } +} + +// CurrentConsoleUser runs against the real SystemConfiguration framework. A machine with +// a desktop open must report a non-root user; a headless runner must report none. +func TestCurrentConsoleUser_AgreesWithItself(t *testing.T) { + user, ok := CurrentConsoleUser() + if !ok { + t.Log("no console user, running headless") + return + } + assert.NotEmpty(t, user.Name, "a console user must have a name") + assert.NotZero(t, user.UID, "a desktop session never belongs to uid 0") + assert.True(t, user.hasDesktop(), "a reported console user must be a desktop session") +} + +func TestConsoleUser_IsOwner(t *testing.T) { + user := ConsoleUser{Name: "maycon", UID: 501, GID: 20} + + assert.True(t, user.isOwner(""), "a profile without owner accepts the single console user") + assert.True(t, user.isOwner("maycon"), "the owner by short name") + assert.True(t, user.isOwner("501"), "the owner recorded as a numeric uid") + assert.False(t, user.isOwner("viktor"), "another account's profile must not read this user's keychain") + assert.False(t, user.isOwner("502"), "another uid is another account") +} diff --git a/client/internal/certproof/desktopuser_windows.go b/client/internal/certproof/desktopuser_windows.go new file mode 100644 index 000000000..6f50aae4f --- /dev/null +++ b/client/internal/certproof/desktopuser_windows.go @@ -0,0 +1,186 @@ +package certproof + +import ( + "fmt" + "strings" + "unsafe" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" +) + +const ( + noActiveSession = 0xFFFFFFFF + servicesSession = 0 + + // wtsCurrentServer is WTS_CURRENT_SERVER_HANDLE; wtsActive and wtsDisconnected are + // WTSActive and WTSDisconnected of WTS_CONNECTSTATE_CLASS. None is exported by + // x/sys/windows. + wtsCurrentServer = windows.Handle(0) + wtsActive = 0 + wtsDisconnected = 4 +) + +// DesktopUser is an interactive session and the account signed into it. The user's +// certificate store is readable only from a process running as that account, because +// its private keys are protected against the user profile rather than the machine. +type DesktopUser struct { + Session uint32 + Name string + Token windows.Token +} + +// Close releases the session token. +func (u DesktopUser) Close() { + if err := u.Token.Close(); err != nil { + log.Debugf("failed closing desktop session token: %v", err) + } +} + +// CurrentDesktopUser returns a token for the session whose certificate store should be +// asked: a session of owner, the account the active profile belongs to, with the +// physical console preferred over remote sessions. With no owner only the console user +// counts. Picking any signed-in user instead would let whoever else is logged in to a +// terminal server or VDI host decide the result. The second return is false when no +// such session exists, and only machine certificates can then be proven. +// +// Obtaining the token needs SE_TCB_NAME, which the LocalSystem service has and an +// ordinary process does not. +func CurrentDesktopUser(owner string) (DesktopUser, bool) { + console := windows.WTSGetActiveConsoleSessionId() + if owner == "" { + return consoleUser(console) + } + + sessions, err := userSessions(console) + if err != nil { + log.Debugf("cannot enumerate terminal sessions: %v", err) + return DesktopUser{}, false + } + var found DesktopUser + var ok bool + for _, session := range sessions { + user, signedIn := desktopUser(session) + if !signedIn { + continue + } + switch { + case !sameAccountName(user.Name, owner): + user.Close() + case !ok: + found, ok = user, true + case strings.EqualFold(user.Name, found.Name): + // Another session of the same account; the first one in preference order wins. + user.Close() + default: + // An owner recorded without a domain matches accounts of several domains here. + // Picking one would let another domain's user answer for the owner. + log.Debugf("certificate posture: profile owner %s matches both %s and %s, no user certificate store is used", owner, found.Name, user.Name) + user.Close() + found.Close() + return DesktopUser{}, false + } + } + if !ok { + log.Debugf("certificate posture: profile owner %s has no signed-in session, no user certificate store is reachable", owner) + } + return found, ok +} + +func consoleUser(console uint32) (DesktopUser, bool) { + if console == noActiveSession || console == servicesSession { + log.Debug("no console session, no user certificate store is reachable") + return DesktopUser{}, false + } + user, ok := desktopUser(console) + if !ok { + log.Debugf("console session %d has nobody signed in, no user certificate store is reachable", console) + } + return user, ok +} + +// sameAccountName compares DOMAIN\account names case-insensitively, as Windows does, and an +// owner given without a domain against the account part alone. The session side comes from +// the session token's own SID, which Windows resolves from its cache of signed-in users. +// Resolving the owner name to a SID instead would ask the domain controller, which on a +// laptop that cannot reach it yet blocks for tens of seconds, past the collection deadline. +func sameAccountName(sessionName, owner string) bool { + if strings.EqualFold(sessionName, owner) { + return true + } + if strings.Contains(owner, `\`) { + return false + } + _, account, found := strings.Cut(sessionName, `\`) + return found && strings.EqualFold(account, owner) +} + +func desktopUser(session uint32) (DesktopUser, bool) { + var token windows.Token + if err := windows.WTSQueryUserToken(session, &token); err != nil { + log.Debugf("no user token for session %d: %v", session, err) + return DesktopUser{}, false + } + + name, err := tokenAccount(token) + if err != nil { + log.Debugf("session %d token has no readable account: %v", session, err) + if closeErr := token.Close(); closeErr != nil { + log.Debugf("failed closing session token: %v", closeErr) + } + return DesktopUser{}, false + } + return DesktopUser{Session: session, Name: name, Token: token}, true +} + +func tokenAccount(token windows.Token) (string, error) { + user, err := token.GetTokenUser() + if err != nil { + return "", fmt.Errorf("read token user: %w", err) + } + account, domain, _, err := user.User.Sid.LookupAccount("") + if err != nil { + return "", fmt.Errorf("look up account: %w", err) + } + if domain == "" { + return account, nil + } + return domain + `\` + account, nil +} + +// userSessions lists the sessions a user can be signed in to, console first, then +// active remote sessions, then disconnected ones, whose user is still signed in. Session +// 0 is skipped: it hosts services and never belongs to an interactive user. +func userSessions(console uint32) ([]uint32, error) { + var info *windows.WTS_SESSION_INFO + var count uint32 + if err := windows.WTSEnumerateSessions(wtsCurrentServer, 0, 1, &info, &count); err != nil { + return nil, fmt.Errorf("enumerate sessions: %w", err) + } + defer windows.WTSFreeMemory(uintptr(unsafe.Pointer(info))) + + var sessions []uint32 + if console != noActiveSession && console != servicesSession { + sessions = append(sessions, console) + } + for _, state := range []uint32{wtsActive, wtsDisconnected} { + for _, session := range unsafe.Slice(info, count) { + if session.SessionID == servicesSession || session.SessionID == console || session.State != state { + continue + } + sessions = append(sessions, session.SessionID) + } + } + return sessions, nil +} + +// runningAsLocalSystem reports whether this process is the service. The helper runs as +// the signed-in user and must read its own store rather than launching another helper. +func runningAsLocalSystem() bool { + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + log.Debugf("failed reading own token user: %v", err) + return false + } + return user.User.Sid.IsWellKnown(windows.WinLocalSystemSid) +} diff --git a/client/internal/certproof/desktopuser_windows_test.go b/client/internal/certproof/desktopuser_windows_test.go new file mode 100644 index 000000000..95d29eb5f --- /dev/null +++ b/client/internal/certproof/desktopuser_windows_test.go @@ -0,0 +1,61 @@ +package certproof + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSameAccountName(t *testing.T) { + tests := []struct { + session, owner string + want bool + }{ + {`CORP\alice`, `CORP\alice`, true}, + {`CORP\alice`, `corp\ALICE`, true}, + {`CORP\alice`, `alice`, true}, + {`CORP\alice`, `OTHER\alice`, false}, + {`CORP\alice`, `bob`, false}, + {`alice`, `alice`, true}, + {`CORP\alice`, `CORP\alic`, false}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, sameAccountName(tt.session, tt.owner), "session %q, owner %q", tt.session, tt.owner) + } +} + +// CurrentDesktopUser against the real session manager: an owner no session belongs to +// must never yield a session, whoever else is signed in. +func TestCurrentDesktopUser_UnknownOwnerHasNoSession(t *testing.T) { + user, ok := CurrentDesktopUser(`NO-SUCH-DOMAIN\no-such-user-netbird`) + if ok { + user.Close() + } + assert.False(t, ok, "a profile owner without a session must not borrow another user's store") +} + +// TestCurrentDesktopUser_FindsTheOwnersSession runs against a real machine as LocalSystem, +// with NB_TEST_DESKTOP_OWNER naming an account that is signed in. That account's session +// must be found, and only that account's. +func TestCurrentDesktopUser_FindsTheOwnersSession(t *testing.T) { + owner := os.Getenv("NB_TEST_DESKTOP_OWNER") + if owner == "" { + t.Skip("set NB_TEST_DESKTOP_OWNER to a signed-in account and run as LocalSystem") + } + user, ok := CurrentDesktopUser(owner) + require.True(t, ok, "the signed-in owner %s must have a session", owner) + defer user.Close() + t.Logf("owner %s resolved to session %d as %s", owner, user.Session, user.Name) + assert.True(t, sameAccountName(user.Name, owner), "the session found belongs to %s, not %s", owner, user.Name) + assert.NotZero(t, user.Session, "session 0 hosts services and never belongs to the owner") + + console, consoleOK := CurrentDesktopUser("") + if consoleOK { + defer console.Close() + t.Logf("without an owner the console user counts: session %d as %s", console.Session, console.Name) + } else { + t.Log("without an owner nobody counts: no user at the console") + } +} diff --git a/client/internal/certproof/doc.go b/client/internal/certproof/doc.go new file mode 100644 index 000000000..c8dc62f0c --- /dev/null +++ b/client/internal/certproof/doc.go @@ -0,0 +1,26 @@ +// Package certproof answers certificate posture challenges: it finds the certificates a +// peer holds a private key for and signs the challenge nonce, bound to the peer's +// WireGuard key, with that key. Management verifies the signature and the chain against +// the CAs of the check, so only a holder of the key passes and a proof made for one peer +// cannot be replayed by another. Keys in the OS stores, the TPM and a PKCS#11 token are +// used through the platform, which signs; a plain PEM key file is the exception, parsed +// and used in the daemon's memory. +// +// Machine stores are read by the daemon itself: the Windows LocalMachine store, the macOS +// System keychain, and on Linux a directory of PEM files, TSS2 key files the TPM signs +// with, and a PKCS#11 token. On Linux the directory and the token URI come from the +// daemon's environment (NB_CERT_STORE_DIR, NB_CERT_PKCS11_URI), as does the token PIN +// (NB_TPM_PIN); none of them is read from the profile config. +// +// User stores cannot be read by a privileged daemon, so it starts this binary as +// "netbird posture cert-proof" inside the user's session and receives only signatures +// and chains on its stdout. On macOS uid is not what unlocks a login keychain, the +// session's bootstrap namespace is, which is why the helper enters it through launchctl +// asuser. On Windows a service opening CURRENT_USER silently reads its own empty hive, so +// the helper runs with the session's token instead. Its output is untrusted: sizes are +// capped and only proofs for requested nonces are kept. +// +// A user proof shows that some process in that user's session could use a key whose +// certificate chains to the CA. It does not show which binary signed, or that the key is +// hardware-bound; device stores are the stronger signal. +package certproof diff --git a/client/internal/certproof/helper.go b/client/internal/certproof/helper.go new file mode 100644 index 000000000..81a72ac70 --- /dev/null +++ b/client/internal/certproof/helper.go @@ -0,0 +1,66 @@ +package certproof + +import ( + "context" + "encoding/json" + "fmt" + "io" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +// HelperRequest is the work the daemon hands to a helper running in a user session. The +// peer key binds every signature to this machine, so a proof cannot be replayed onto +// another peer. +type HelperRequest struct { + PeerKey []byte `json:"peerKey"` + Challenges []HelperChallenge `json:"challenges"` +} + +// HelperChallenge is one certificate challenge in the form the helper needs. +type HelperChallenge struct { + Nonce []byte `json:"nonce"` + CACertificates []string `json:"caCertificates"` +} + +// HelperResponse carries the proofs the helper produced from its own keychain. +type HelperResponse struct { + Proofs []certposture.Proof `json:"proofs"` +} + +// RunHelper answers the challenges on in from the store of the user running this +// process and writes the proofs to out. It is the child half of the console user +// lookup: the daemon cannot read a login keychain, so it launches this in the user's +// session instead. Only the signature crosses back, never the private key. +func RunHelper(ctx context.Context, in io.Reader, out io.Writer) error { + return runHelper(ctx, helperStore(), in, out) +} + +func runHelper(ctx context.Context, store Store, in io.Reader, out io.Writer) error { + var req HelperRequest + if err := json.NewDecoder(in).Decode(&req); err != nil { + return fmt.Errorf("decode helper request: %w", err) + } + + challenges := make([]*proto.CertificateChallenge, 0, len(req.Challenges)) + for _, challenge := range req.Challenges { + challenges = append(challenges, &proto.CertificateChallenge{ + Nonce: challenge.Nonce, + CaCertificates: challenge.CACertificates, + }) + } + + var proofs []certposture.Proof + if len(challenges) > 0 { + proofs = CollectChallenges(ctx, store, challenges, req.PeerKey) + } + log.Debugf("certificate posture helper: answering %d challenges with %d proofs", len(challenges), len(proofs)) + + if err := json.NewEncoder(out).Encode(HelperResponse{Proofs: proofs}); err != nil { + return fmt.Errorf("encode helper response: %w", err) + } + return nil +} diff --git a/client/internal/certproof/helper_backoff.go b/client/internal/certproof/helper_backoff.go new file mode 100644 index 000000000..3372b026b --- /dev/null +++ b/client/internal/certproof/helper_backoff.go @@ -0,0 +1,78 @@ +package certproof + +import ( + "crypto/sha256" + "encoding/hex" + "slices" + "strings" + "sync" + "time" + + "github.com/netbirdio/netbird/shared/management/proto" +) + +// helperQuietPeriod is how long a user whose helper proved nothing is not asked again +// for the same CAs. On macOS each helper run may show a keychain prompt, so retrying on +// every collection would put up a new one each time the user ignored or denied the last. +const helperQuietPeriod = time.Hour + +// helperBackoff remembers, per user and set of CAs asked about, until when the helper +// is not launched again because its last run proved nothing. +type helperBackoff struct { + mu sync.Mutex + until map[string]time.Time +} + +func newHelperBackoff() *helperBackoff { + return &helperBackoff{until: map[string]time.Time{}} +} + +// allow reports whether the helper may be launched for key now. +func (b *helperBackoff) allow(key string, now time.Time) bool { + b.mu.Lock() + defer b.mu.Unlock() + return !now.Before(b.until[key]) +} + +// record stores the outcome of a helper run for key: one that proved something clears +// the back-off, one that proved nothing starts it. Expired entries are dropped, so the +// map holds only users currently held off. +func (b *helperBackoff) record(key string, proven bool, now time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + for other, until := range b.until { + if !now.Before(until) { + delete(b.until, other) + } + } + if proven { + delete(b.until, key) + return + } + b.until[key] = now.Add(helperQuietPeriod) +} + +// helperBackoffKey identifies a user and the set of CAs the challenges accept, in any +// order, leaving out the nonces, which rotate without changing what the user is asked to +// prove. +func helperBackoffKey(user string, challenges []*proto.CertificateChallenge) string { + sets := make([]string, 0, len(challenges)) + for _, challenge := range challenges { + cas := make([]string, 0, len(challenge.GetCaCertificates())) + for _, ca := range challenge.GetCaCertificates() { + sum := sha256.Sum256([]byte(ca)) + cas = append(cas, hex.EncodeToString(sum[:])) + } + slices.Sort(cas) + sets = append(sets, strings.Join(cas, ",")) + } + slices.Sort(sets) + + h := sha256.New() + h.Write([]byte(user)) + for _, set := range sets { + h.Write([]byte{0}) + h.Write([]byte(set)) + } + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/client/internal/certproof/helper_backoff_test.go b/client/internal/certproof/helper_backoff_test.go new file mode 100644 index 000000000..0d3cafea1 --- /dev/null +++ b/client/internal/certproof/helper_backoff_test.go @@ -0,0 +1,69 @@ +package certproof + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/netbirdio/netbird/shared/management/proto" +) + +// TestHelperBackoff: a user whose helper proved nothing, for one ignoring or denying a +// keychain prompt, is not asked again for the same CAs within the quiet period, while a +// success or other CAs are asked at once. +func TestHelperBackoff(t *testing.T) { + now := time.Now() + b := newHelperBackoff() + key := helperBackoffKey("501", []*proto.CertificateChallenge{{CaCertificates: []string{"ca-a"}}}) + + assert.True(t, b.allow(key, now), "a user never asked is asked") + + b.record(key, false, now) + assert.False(t, b.allow(key, now.Add(helperQuietPeriod-time.Second)), "no new prompt within the quiet period") + assert.True(t, b.allow(key, now.Add(helperQuietPeriod)), "asked again once the quiet period passed") + + other := helperBackoffKey("501", []*proto.CertificateChallenge{{CaCertificates: []string{"ca-b"}}}) + assert.True(t, b.allow(other, now), "a check for other CAs is asked at once") + assert.True(t, b.allow(helperBackoffKey("502", []*proto.CertificateChallenge{{CaCertificates: []string{"ca-a"}}}), now), "another user is asked at once") + + b.record(key, true, now) + assert.True(t, b.allow(key, now), "a run that proved something clears the quiet period") +} + +// TestHelperBackoffKey_IgnoresNonces: nonces rotate every window without changing what +// the user is asked to prove, so they do not reset the quiet period. +func TestHelperBackoffKey_IgnoresNonces(t *testing.T) { + a := helperBackoffKey("501", []*proto.CertificateChallenge{{Nonce: []byte("one"), CaCertificates: []string{"ca"}}}) + b := helperBackoffKey("501", []*proto.CertificateChallenge{{Nonce: []byte("two"), CaCertificates: []string{"ca"}}}) + assert.Equal(t, a, b, "the key does not depend on the nonce") +} + +// TestHelperBackoffKey_IgnoresOrder: the same CAs, listed in another order or with the +// challenges reordered, are the same question to the user. +func TestHelperBackoffKey_IgnoresOrder(t *testing.T) { + a := helperBackoffKey("501", []*proto.CertificateChallenge{ + {CaCertificates: []string{"ca-1", "ca-2"}}, + {CaCertificates: []string{"ca-3"}}, + }) + b := helperBackoffKey("501", []*proto.CertificateChallenge{ + {CaCertificates: []string{"ca-3"}}, + {CaCertificates: []string{"ca-2", "ca-1"}}, + }) + assert.Equal(t, a, b, "the key does not depend on the order of CAs or challenges") + + c := helperBackoffKey("501", []*proto.CertificateChallenge{{CaCertificates: []string{"ca-1", "ca-2", "ca-3"}}}) + assert.NotEqual(t, a, c, "the same CAs grouped into other challenges are another question") +} + +// TestHelperBackoff_PrunesExpired: entries whose quiet period passed are dropped when the +// next outcome is recorded, so the map does not grow with every user and CA set seen. +func TestHelperBackoff_PrunesExpired(t *testing.T) { + now := time.Now() + b := newHelperBackoff() + b.record("old", false, now) + b.record("new", false, now.Add(helperQuietPeriod)) + + assert.NotContains(t, b.until, "old", "an expired entry is pruned") + assert.Contains(t, b.until, "new", "a current entry is kept") +} diff --git a/client/internal/certproof/helper_kill_other.go b/client/internal/certproof/helper_kill_other.go new file mode 100644 index 000000000..2ececb27e --- /dev/null +++ b/client/internal/certproof/helper_kill_other.go @@ -0,0 +1,10 @@ +//go:build !unix && !windows + +package certproof + +import "os/exec" + +// startHelper starts cmd. +func startHelper(cmd *exec.Cmd) (func(), error) { + return func() {}, cmd.Start() +} diff --git a/client/internal/certproof/helper_kill_unix.go b/client/internal/certproof/helper_kill_unix.go new file mode 100644 index 000000000..e434d4962 --- /dev/null +++ b/client/internal/certproof/helper_kill_unix.go @@ -0,0 +1,35 @@ +//go:build unix + +package certproof + +import ( + "errors" + "os/exec" + "syscall" +) + +// killHelperGroupOnCancel puts cmd in a process group of its own and kills the whole +// group when cmd's context ends. The helper may run below launchers such as launchctl +// and sudo, which do not pass a kill on, and it keeps the output pipes open, so killing +// only the direct child would leave the helper running and Wait blocked until the +// helper exits on its own, for one waiting on a keychain prompt nobody answers. +func killHelperGroupOnCancel(cmd *exec.Cmd) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.Setpgid = true + cmd.Cancel = func() error { + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if errors.Is(err, syscall.ESRCH) { + return nil + } + return err + } + cmd.WaitDelay = helperWaitDelay +} + +// startHelper starts cmd. Killing the helper with everything below it is arranged by +// killHelperGroupOnCancel where the helper runs under a launcher. +func startHelper(cmd *exec.Cmd) (func(), error) { + return func() {}, cmd.Start() +} diff --git a/client/internal/certproof/helper_kill_windows.go b/client/internal/certproof/helper_kill_windows.go new file mode 100644 index 000000000..1e143c132 --- /dev/null +++ b/client/internal/certproof/helper_kill_windows.go @@ -0,0 +1,77 @@ +//go:build windows + +package certproof + +import ( + "fmt" + "os/exec" + "unsafe" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" +) + +// startHelper starts cmd inside a job object that is terminated when cmd's context ends +// and closed, killing whatever is left in it, once the helper has been awaited. Killing +// only the helper on Windows leaves its descendants running, and one that holds the +// output pipe would block Wait until it exits, so WaitDelay bounds that wait as well. +func startHelper(cmd *exec.Cmd) (func(), error) { + job, err := newKillOnCloseJob() + if err != nil { + return nil, err + } + closeJob := func() { + if err := windows.CloseHandle(job); err != nil { + log.Debugf("failed to close certificate proof helper job: %v", err) + } + } + + cmd.Cancel = func() error { + return windows.TerminateJobObject(job, 1) + } + cmd.WaitDelay = helperWaitDelay + if err := cmd.Start(); err != nil { + closeJob() + return nil, err + } + + // A helper outside the job is still killed directly and bounded by WaitDelay; only + // its descendants would outlive it. + if err := assignToJob(job, cmd.Process.Pid); err != nil { + log.Debugf("failed to put certificate proof helper %d in its job: %v", cmd.Process.Pid, err) + cmd.Cancel = func() error { return cmd.Process.Kill() } + } + return closeJob, nil +} + +// newKillOnCloseJob creates a job object whose processes are killed when its last handle +// is closed. +func newKillOnCloseJob() (windows.Handle, error) { + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return 0, fmt.Errorf("create job object: %w", err) + } + info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{ + BasicLimitInformation: windows.JOBOBJECT_BASIC_LIMIT_INFORMATION{ + LimitFlags: windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + }, + } + if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(info))); err != nil { + _ = windows.CloseHandle(job) + return 0, fmt.Errorf("set job object limits: %w", err) + } + return job, nil +} + +// assignToJob puts the process pid into job. +func assignToJob(job windows.Handle, pid int) error { + process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(pid)) + if err != nil { + return fmt.Errorf("open process: %w", err) + } + defer func() { + _ = windows.CloseHandle(process) + }() + return windows.AssignProcessToJobObject(job, process) +} diff --git a/client/internal/certproof/helper_kill_windows_test.go b/client/internal/certproof/helper_kill_windows_test.go new file mode 100644 index 000000000..4778d253b --- /dev/null +++ b/client/internal/certproof/helper_kill_windows_test.go @@ -0,0 +1,29 @@ +//go:build windows + +package certproof + +import ( + "context" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRunHelperCmd_KillsHelperTree: the helper's child holds stdout and runs for a minute, +// like a helper stuck below a launcher. Killing only the helper would leave Wait blocked +// on the child's pipe until WaitDelay gives up; terminating the job ends both at once. +func TestRunHelperCmd_KillsHelperTree(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + + cmd := exec.CommandContext(ctx, "cmd.exe", "/c", "ping -n 60 127.0.0.1") + + start := time.Now() + _, err := runHelperCmd(cmd, HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}}) + + require.Error(t, err) + assert.Less(t, time.Since(start), helperWaitDelay, "the whole tree is killed at the timeout, not left for WaitDelay") +} diff --git a/client/internal/certproof/helper_run.go b/client/internal/certproof/helper_run.go new file mode 100644 index 000000000..52c9db775 --- /dev/null +++ b/client/internal/certproof/helper_run.go @@ -0,0 +1,126 @@ +package certproof + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os/exec" + "strings" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" +) + +const ( + maxHelperStdout = 1 << 20 + maxHelperStderr = 4 << 10 + + // helperWaitDelay bounds how long a helper's output is awaited after it was killed. + helperWaitDelay = 2 * time.Second +) + +var errHelperOutputTooLarge = errors.New("helper output exceeds the size limit") + +// runHelperCmd feeds req to the helper process cmd and returns the proofs it answered. +// The helper runs as an unprivileged user who can control its output, so both streams +// are capped and only proofs for a nonce req asked about are kept, one per challenge. +func runHelperCmd(cmd *exec.Cmd, req HelperRequest) ([]certposture.Proof, error) { + payload, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("encode helper request: %w", err) + } + + stdout := &cappedBuffer{limit: maxHelperStdout} + stderr := &cappedBuffer{limit: maxHelperStderr} + cmd.Stdin = bytes.NewReader(payload) + cmd.Stdout = stdout + cmd.Stderr = stderr + + release, err := startHelper(cmd) + if err != nil { + return nil, fmt.Errorf("start helper: %w", err) + } + defer release() + if err := cmd.Wait(); err != nil { + return nil, fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String())) + } + logHelperStderr(stderr.String()) + if stdout.truncated { + return nil, errHelperOutputTooLarge + } + + var resp HelperResponse + if err := json.Unmarshal(stdout.Bytes(), &resp); err != nil { + return nil, fmt.Errorf("decode helper response: %w", err) + } + return requestedProofs(req, resp.Proofs), nil +} + +// logHelperStderr records what a helper that succeeded wrote to stderr, such as a user +// certificate it could not sign with, which is otherwise only visible in the user's +// session. Each line is quoted because the helper's user controls its content. +func logHelperStderr(stderr string) { + for _, line := range strings.Split(strings.TrimSpace(stderr), "\n") { + if line = strings.TrimSpace(line); line != "" { + log.Debugf("certificate posture helper: %q", line) + } + } +} + +// requestedProofs keeps the proofs whose nonce belongs to one of req's challenges, at +// most as many as req has challenges. +func requestedProofs(req HelperRequest, proofs []certposture.Proof) []certposture.Proof { + var kept []certposture.Proof + for _, proof := range proofs { + if len(kept) == len(req.Challenges) { + break + } + if !req.asked(proof.Nonce) { + log.Debugf("certificate posture: dropping helper proof for a nonce that was not requested") + continue + } + kept = append(kept, proof) + } + return kept +} + +func (r HelperRequest) asked(nonce []byte) bool { + for _, challenge := range r.Challenges { + if len(challenge.Nonce) > 0 && bytes.Equal(challenge.Nonce, nonce) { + return true + } + } + return false +} + +// cappedBuffer keeps the first limit bytes written to it and discards the rest, so a +// misbehaving child cannot grow the parent's memory without bound. The buffer is a +// named field rather than embedded: an embedded bytes.Buffer would promote ReadFrom, +// which io.Copy prefers over Write, bypassing the cap. +type cappedBuffer struct { + buf bytes.Buffer + limit int + truncated bool +} + +func (b *cappedBuffer) Write(p []byte) (int, error) { + if room := b.limit - b.buf.Len(); room < len(p) { + b.truncated = true + if room > 0 { + b.buf.Write(p[:room]) + } + return len(p), nil + } + return b.buf.Write(p) +} + +func (b *cappedBuffer) Bytes() []byte { + return b.buf.Bytes() +} + +func (b *cappedBuffer) String() string { + return b.buf.String() +} diff --git a/client/internal/certproof/helper_run_test.go b/client/internal/certproof/helper_run_test.go new file mode 100644 index 000000000..ea1246e93 --- /dev/null +++ b/client/internal/certproof/helper_run_test.go @@ -0,0 +1,119 @@ +//go:build !windows && !js + +package certproof + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" +) + +// fakeHelper is a child process that drains its stdin and then prints script's output, +// standing in for a helper running in a user session the daemon cannot trust. +func fakeHelper(t *testing.T, script string) *exec.Cmd { + t.Helper() + return exec.Command("/bin/sh", "-c", "cat >/dev/null; "+script) +} + +func printJSON(t *testing.T, v any) string { + t.Helper() + out, err := json.Marshal(v) + require.NoError(t, err) + return fmt.Sprintf("printf '%%s' '%s'", out) +} + +func TestRunHelperCmd_KeepsOnlyRequestedProofs(t *testing.T) { + req := HelperRequest{PeerKey: peerKey, Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + resp := HelperResponse{Proofs: []certposture.Proof{ + {Nonce: []byte("injected"), Signature: []byte("x")}, + {Nonce: []byte("asked"), Signature: []byte("first")}, + {Nonce: []byte("asked"), Signature: []byte("second")}, + }} + + proofs, err := runHelperCmd(fakeHelper(t, printJSON(t, resp)), req) + + require.NoError(t, err) + require.Len(t, proofs, 1, "a helper may not return more proofs than challenges or proofs for nonces it was not asked about") + assert.Equal(t, []byte("first"), proofs[0].Signature, "the first proof for the requested nonce is kept") +} + +func TestRunHelperCmd_RejectsOversizedOutput(t *testing.T) { + req := HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + script := fmt.Sprintf("head -c %d /dev/zero", maxHelperStdout+1) + + _, err := runHelperCmd(fakeHelper(t, script), req) + + assert.ErrorIs(t, err, errHelperOutputTooLarge) +} + +func TestRunHelperCmd_CapsStderrInError(t *testing.T) { + req := HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + script := fmt.Sprintf("head -c %d /dev/zero | tr '\\0' 'a' >&2; exit 3", 10*maxHelperStderr) + + _, err := runHelperCmd(fakeHelper(t, script), req) + + require.Error(t, err) + assert.LessOrEqual(t, len(err.Error()), maxHelperStderr+100, "a chatty helper must not blow up the daemon's error or log line") + assert.True(t, strings.Contains(err.Error(), "exit status 3"), "the exit status is kept: %v", err) +} + +// TestRunHelperCmd_LogsStderrOfSuccessfulHelper covers a helper that answers but warns, +// for one about a certificate it could not sign with: the warning reaches the daemon's +// log, quoted so a helper cannot forge log lines with embedded newlines. +func TestRunHelperCmd_LogsStderrOfSuccessfulHelper(t *testing.T) { + var logged bytes.Buffer + log.SetOutput(&logged) + level := log.GetLevel() + log.SetLevel(log.DebugLevel) + t.Cleanup(func() { + log.SetOutput(os.Stderr) + log.SetLevel(level) + }) + + req := HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + script := "printf 'failed signing certificate proof for CN=user\\nforged line\\n' >&2; " + printJSON(t, HelperResponse{}) + + _, err := runHelperCmd(fakeHelper(t, script), req) + + require.NoError(t, err) + assert.Contains(t, logged.String(), "failed signing certificate proof for CN=user", "the helper's warning reaches the daemon log") + assert.NotContains(t, logged.String(), "\nforged line", "helper output cannot start a log line of its own") +} + +// TestRunHelperCmd_KillsHelperBelowLauncher stands in for launchctl and sudo starting the +// helper: the direct child spawns a grandchild that holds stdout and never exits, like a +// helper waiting on a keychain prompt. The timeout must end both, not wait for the +// grandchild. +func TestRunHelperCmd_KillsHelperBelowLauncher(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", "sleep 30 & cat >/dev/null; wait") + killHelperGroupOnCancel(cmd) + + start := time.Now() + _, err := runHelperCmd(cmd, HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}}) + + require.Error(t, err) + assert.Less(t, time.Since(start), 5*time.Second, "the helper is killed at the timeout, not awaited until it exits") +} + +func TestRunHelperCmd_RejectsGarbage(t *testing.T) { + req := HelperRequest{Challenges: []HelperChallenge{{Nonce: []byte("asked")}}} + + _, err := runHelperCmd(fakeHelper(t, "printf 'not json'"), req) + + assert.ErrorContains(t, err, "decode helper response") +} diff --git a/client/internal/certproof/helper_spawn.go b/client/internal/certproof/helper_spawn.go new file mode 100644 index 000000000..968cc3679 --- /dev/null +++ b/client/internal/certproof/helper_spawn.go @@ -0,0 +1,63 @@ +//go:build (darwin && !ios) || windows + +package certproof + +import ( + "crypto/sha256" + "crypto/x509" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/proto" +) + +func helperRequest(challenges []*proto.CertificateChallenge, peerKey []byte) HelperRequest { + req := HelperRequest{PeerKey: peerKey, Challenges: make([]HelperChallenge, 0, len(challenges))} + for _, challenge := range challenges { + req.Challenges = append(req.Challenges, HelperChallenge{ + Nonce: challenge.GetNonce(), + CACertificates: challenge.GetCaCertificates(), + }) + } + return req +} + +// mergeProofs appends the user session proofs to the device proofs, dropping a leaf that +// both stores hold so the same certificate is proven once. +func mergeProofs(device, user []certposture.Proof) []certposture.Proof { + if len(user) == 0 { + return device + } + + seen := make(map[[sha256.Size]byte]struct{}, len(device)) + for _, proof := range device { + if len(proof.Chain) > 0 { + seen[sha256.Sum256(proof.Chain[0])] = struct{}{} + } + } + + merged := device + for _, proof := range user { + if len(proof.Chain) == 0 { + continue + } + fingerprint := sha256.Sum256(proof.Chain[0]) + if _, done := seen[fingerprint]; done { + continue + } + seen[fingerprint] = struct{}{} + merged = append(merged, proof) + logUserProof(proof) + } + return merged +} + +func logUserProof(proof certposture.Proof) { + leaf, err := x509.ParseCertificate(proof.Chain[0]) + if err != nil { + log.Debugf("certificate posture: user proof carries an unparsable leaf: %v", err) + return + } + log.Debugf("certificate posture: signed-in user proved %q issued by %q", leaf.Subject, leaf.Issuer) +} diff --git a/client/internal/certproof/helper_spawn_test.go b/client/internal/certproof/helper_spawn_test.go new file mode 100644 index 000000000..338a5f66a --- /dev/null +++ b/client/internal/certproof/helper_spawn_test.go @@ -0,0 +1,56 @@ +//go:build (darwin && !ios) || windows + +package certproof + +import ( + "crypto/sha256" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/certposture/certtest" + "github.com/netbirdio/netbird/shared/management/proto" +) + +func TestMergeProofs_ProvesACertificateHeldByBothStoresOnce(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + shared := ca.Issue(t, certtest.ECDSAKey(t), "shared") + userOnly := ca.Issue(t, certtest.ECDSAKey(t), "user-only") + + device := []certposture.Proof{{Chain: [][]byte{shared.Raw}}} + user := []certposture.Proof{{Chain: [][]byte{shared.Raw}}, {Chain: [][]byte{userOnly.Raw}}, {}} + + merged := mergeProofs(device, user) + + require.Len(t, merged, 2, "the shared leaf is proven once and the empty chain is dropped") + assert.Equal(t, shared.Raw, merged[0].Chain[0], "the device proof keeps its place") + assert.Equal(t, userOnly.Raw, merged[1].Chain[0], "the user-only certificate is appended") +} + +func TestMergeProofs_KeepsDeviceProofsWhenNoUserSession(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + device := []certposture.Proof{{Chain: [][]byte{ca.Issue(t, certtest.ECDSAKey(t), "device").Raw}}} + + merged := mergeProofs(device, nil) + + require.Len(t, merged, 1, "a machine with nobody signed in still sends its device proof") + assert.Equal(t, sha256.Sum256(device[0].Chain[0]), sha256.Sum256(merged[0].Chain[0]), "the device proof is unchanged") +} + +func TestHelperRequest_CarriesEveryChallenge(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + challenges := []*proto.CertificateChallenge{ + {Nonce: []byte("first"), CaCertificates: []string{ca.PEM}}, + {Nonce: []byte("second")}, + } + + req := helperRequest(challenges, peerKey) + + require.Len(t, req.Challenges, 2, "every challenge must reach the helper") + assert.Equal(t, peerKey, req.PeerKey, "the peer key binds the signature to this machine") + assert.Equal(t, []byte("first"), req.Challenges[0].Nonce, "the nonce must survive unchanged") + assert.Equal(t, []string{ca.PEM}, req.Challenges[0].CACertificates, "the accepted CAs must survive unchanged") + assert.Empty(t, req.Challenges[1].CACertificates, "a challenge without CAs stays without CAs") +} diff --git a/client/internal/certproof/helper_test.go b/client/internal/certproof/helper_test.go new file mode 100644 index 000000000..37dadead2 --- /dev/null +++ b/client/internal/certproof/helper_test.go @@ -0,0 +1,63 @@ +package certproof + +import ( + "bytes" + "context" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/certposture/certtest" +) + +func TestRunHelper_ProofSurvivesTheProcessBoundary(t *testing.T) { + ca := certtest.NewCA(t, "corp-root") + dir := storeDir(t) + key := certtest.ECDSAKey(t) + writeFile(t, dir, "device.pem", certtest.CertPEM(ca.Issue(t, key, "device"))+certtest.KeyPEM(t, key)) + + challenger := certposture.NewChallenger([]byte("secret")) + nonce := challenger.Nonce(peerKey, time.Now()) + request, err := json.Marshal(HelperRequest{ + PeerKey: peerKey, + Challenges: []HelperChallenge{{Nonce: nonce, CACertificates: []string{ca.PEM}}}, + }) + require.NoError(t, err, "request must encode") + + var stdout bytes.Buffer + require.NoError(t, runHelper(context.Background(), NewFileStore(dir), bytes.NewReader(request), &stdout)) + + var resp HelperResponse + require.NoError(t, json.Unmarshal(stdout.Bytes(), &resp), "helper must emit decodable JSON") + require.Len(t, resp.Proofs, 1, "the matching certificate should produce one proof") + + // Verify exactly as management does, so the proof is proven to survive the encode, + // the process boundary and the decode intact. + chain, err := challenger.Verify(resp.Proofs[0], peerKey, time.Now()) + require.NoError(t, err, "the decoded proof must verify against the issued nonce") + assert.Equal(t, "device", chain[0].Subject.CommonName, "the proven leaf should be the device certificate") +} + +func TestRunHelper_NoChallengesYieldsEmptyResponse(t *testing.T) { + request, err := json.Marshal(HelperRequest{PeerKey: peerKey}) + require.NoError(t, err) + + var stdout bytes.Buffer + require.NoError(t, runHelper(context.Background(), NewFileStore(storeDir(t)), bytes.NewReader(request), &stdout)) + + var resp HelperResponse + require.NoError(t, json.Unmarshal(stdout.Bytes(), &resp), "an empty request must still emit valid JSON") + assert.Empty(t, resp.Proofs, "no challenges should produce no proofs") +} + +func TestRunHelper_RejectsMalformedRequest(t *testing.T) { + var stdout bytes.Buffer + err := runHelper(context.Background(), NewFileStore(storeDir(t)), bytes.NewReader([]byte("not json")), &stdout) + + require.Error(t, err, "a malformed request must fail rather than emit an empty proof set") + assert.Empty(t, stdout.String(), "nothing should be written to stdout on a decode failure") +} diff --git a/client/internal/certproof/keychain_darwin.go b/client/internal/certproof/keychain_darwin.go new file mode 100644 index 000000000..2bffdc81e --- /dev/null +++ b/client/internal/certproof/keychain_darwin.go @@ -0,0 +1,483 @@ +//go:build !ios + +package certproof + +import ( + "bytes" + "context" + "crypto" + "crypto/x509" + "errors" + "fmt" + "io" + "os" + "sync" + "unsafe" + + "github.com/ebitengine/purego" + log "github.com/sirupsen/logrus" +) + +const ( + securityFramework = "/System/Library/Frameworks/Security.framework/Security" + coreFoundationFramework = "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation" + + errSecItemNotFound = -25300 + errSecInteractionNotAllowed = -25308 +) + +// errKeyNeedsApproval reports a key whose access list does not include netbird, so using +// it needs the user's approval, which a daemon has no UI to ask for. +var errKeyNeedsApproval = errors.New("the key's access control requires user approval for netbird; " + + "import the identity with netbird allowed (security import -k " + + "-T /Applications/NetBird.app/Contents/MacOS/netbird), " + + "or set AllowAllAppsAccess in the MDM certificate payload, which allows every application") + +var ( + keychainOnce sync.Once + keychainErr error + + secItemCopyMatching func(query uintptr, result *uintptr) int32 + secIdentityCopyCertificate func(identity uintptr, cert *uintptr) int32 + secIdentityCopyPrivateKey func(identity uintptr, key *uintptr) int32 + secCertificateCopyData func(cert uintptr) uintptr + secKeyCreateSignature func(key, algorithm, data uintptr, err *uintptr) uintptr + + secKeychainOpen func(path *byte, keychain *uintptr) int32 + secKeychainCopySearchList func(searchList *uintptr) int32 + secKeychainGetPath func(keychain uintptr, pathLength *uint32, path *byte) int32 + cfArrayCreate func(alloc uintptr, values *uintptr, count int, callBacks uintptr) uintptr + + cfDictionaryCreate func(alloc uintptr, keys, values *uintptr, count int, keyCallBacks, valueCallBacks uintptr) uintptr + cfArrayGetCount func(array uintptr) int + cfArrayGetValueAtIndex func(array uintptr, index int) uintptr + cfDataCreate func(alloc uintptr, data *byte, length int) uintptr + cfDataGetLength func(data uintptr) int + cfDataGetBytePtr func(data uintptr) unsafe.Pointer + cfErrorGetCode func(err uintptr) int + cfRelease func(ref uintptr) + + kSecClass, kSecClassIdentity, kSecClassCertificate, kSecMatchLimit, kSecMatchLimitAll, kSecReturnRef uintptr + kSecMatchSearchList uintptr + kSecKeyAlgorithmECDSASHA256, kSecKeyAlgorithmECDSASHA384, kSecKeyAlgorithmRSAPSSSHA256 uintptr + kCFBooleanTrue, kCFTypeDictionaryKeyCallBacks, kCFTypeDictionaryValueCallBacks, kCFTypeArrayCallBacks uintptr +) + +// systemKeychain is where MDM installs device identities. +const systemKeychain = "/Library/Keychains/System.keychain" + +// DefaultStore is the System keychain for the root daemon, so it never touches a user's +// keychain, whose keys would ask for approval in a session the daemon has no UI in. Run +// by an ordinary user, it is that user's keychain search list. +func DefaultStore() Store { + if os.Geteuid() == 0 { + return NewKeychainStore(systemKeychain) + } + return NewKeychainStore() +} + +// KeychainStore yields the identities of a set of keychains, or of the process's keychain +// search list when none is named, reached through purego so the client keeps building +// with CGO_ENABLED=0. +type KeychainStore struct { + keychains []string +} + +// NewKeychainStore returns a store that searches the keychain files at paths, or the +// process's keychain search list when no path is given. +func NewKeychainStore(paths ...string) *KeychainStore { + return &KeychainStore{keychains: paths} +} + +func (s *KeychainStore) Candidates(_ context.Context) ([]Candidate, error) { + if err := loadKeychain(); err != nil { + return nil, err + } + searchList, done, err := searchListOf(s.keychains) + if err != nil { + return nil, err + } + defer done() + + var leaves []*x509.Certificate + err = eachIdentity(searchList, func(_ uintptr, der []byte) (bool, error) { + cert, err := x509.ParseCertificate(der) + if err != nil { + log.Warnf("skipping keychain identity: %v", err) + return false, nil + } + log.Debugf("keychain identity: subject=%q issuer=%q serial=%s expires=%s", cert.Subject, cert.Issuer, cert.SerialNumber, cert.NotAfter) + leaves = append(leaves, cert) + return false, nil + }) + if err != nil { + return nil, err + } + // The certificate query runs even without identities: it separates a keychain that is + // readable but holds no identity from one the process cannot read at all. + pool, err := keychainCertificates(searchList) + if err != nil { + return nil, err + } + if len(leaves) == 0 { + log.Debugf("keychain search list holds no identities usable for certificate posture, but %d readable certificates: an identity needs its private key in the same keychain", len(pool)) + return nil, nil + } + log.Debugf("keychain search list holds %d identities and %d certificates for chain building", len(leaves), len(pool)) + + candidates := make([]Candidate, 0, len(leaves)) + for _, leaf := range leaves { + chain := buildChain(leaf, pool) + log.Debugf("keychain candidate %q issued by %q built a chain of %d certificates", leaf.Subject, leaf.Issuer, len(chain)) + if len(chain) == 1 && leaf.CheckSignatureFrom(leaf) != nil { + log.Debugf("keychain candidate %q has no issuer in the keychain, its proof carries the leaf alone and only verifies if the challenge supplies %q", leaf.Subject, leaf.Issuer) + } + candidates = append(candidates, Candidate{Chain: chain, Signer: &keychainSigner{leaf: leaf, keychains: s.keychains}, Intermediates: pool}) + } + return candidates, nil +} + +// keychainSigner holds only the certificate; the identity is looked up again at signing +// time so no keychain references outlive a call. +type keychainSigner struct { + leaf *x509.Certificate + keychains []string +} + +func (s *keychainSigner) Public() crypto.PublicKey { + return s.leaf.PublicKey +} + +func (s *keychainSigner) Sign(_ io.Reader, digest []byte, opts crypto.SignerOpts) ([]byte, error) { + scheme, err := schemeFor(s.leaf.PublicKey, opts) + if err != nil { + return nil, err + } + log.Debugf("signing certificate posture challenge with keychain key of %q", s.leaf.Subject) + + searchList, done, err := searchListOf(s.keychains) + if err != nil { + return nil, err + } + defer done() + + algorithm := keychainAlgorithm(scheme) + var signature []byte + err = eachIdentity(searchList, func(identity uintptr, der []byte) (bool, error) { + if !bytes.Equal(der, s.leaf.Raw) { + return false, nil + } + signature, err = signWithIdentity(identity, algorithm, digest) + return true, err + }) + if err != nil { + return nil, err + } + if signature == nil { + return nil, errors.New("certificate is no longer in the keychain") + } + log.Debugf("keychain signed certificate posture challenge for %q, %d bytes", s.leaf.Subject, len(signature)) + return signature, nil +} + +func keychainAlgorithm(scheme sigScheme) uintptr { + switch scheme { + case schemeECDSASHA384: + return kSecKeyAlgorithmECDSASHA384 + case schemeRSAPSSSHA256: + return kSecKeyAlgorithmRSAPSSSHA256 + default: + return kSecKeyAlgorithmECDSASHA256 + } +} + +func signWithIdentity(identity, algorithm uintptr, digest []byte) ([]byte, error) { + var key uintptr + if status := secIdentityCopyPrivateKey(identity, &key); status != 0 { + return nil, fmt.Errorf("SecIdentityCopyPrivateKey: %d", status) + } + defer release(key) + + data := cfDataCreate(0, &digest[0], len(digest)) + if data == 0 { + return nil, errors.New("CFDataCreate returned NULL") + } + defer release(data) + + var cfErr uintptr + signature := secKeyCreateSignature(key, algorithm, data, &cfErr) + if signature == 0 { + if cfErr == 0 { + return nil, errors.New("SecKeyCreateSignature failed without a CFError") + } + defer release(cfErr) + code := cfErrorGetCode(cfErr) + if code == errSecInteractionNotAllowed { + return nil, fmt.Errorf("SecKeyCreateSignature: CFError %d: %w", code, errKeyNeedsApproval) + } + return nil, fmt.Errorf("SecKeyCreateSignature: CFError %d", code) + } + defer release(signature) + return dataBytes(signature), nil +} + +// searchListOf opens the keychain files at paths as a CFArray for kSecMatchSearchList, +// or yields 0, the process's own search list, when paths is empty. The caller calls done +// once it no longer uses the list. +func searchListOf(paths []string) (searchList uintptr, done func(), err error) { + if len(paths) == 0 { + return 0, func() {}, nil + } + list, err := openSearchList(paths) + if err != nil { + return 0, nil, err + } + return list, func() { release(list) }, nil +} + +// openSearchList opens the keychain files at paths, at least one, as a CFArray. The +// caller releases the array. +func openSearchList(paths []string) (uintptr, error) { + refs := make([]uintptr, 0, len(paths)) + defer func() { + for _, ref := range refs { + release(ref) + } + }() + for _, path := range paths { + cpath := append([]byte(path), 0) + var keychain uintptr + if status := secKeychainOpen(&cpath[0], &keychain); status != 0 { + return 0, fmt.Errorf("SecKeychainOpen %s: %d", path, status) + } + refs = append(refs, keychain) + } + // The array retains the keychains, so the references opened here are released. + list := cfArrayCreate(0, &refs[0], len(refs), kCFTypeArrayCallBacks) + if list == 0 { + return 0, errors.New("CFArrayCreate returned NULL") + } + return list, nil +} + +// eachIdentity calls fn with every identity in searchList, or in the process's search +// list when it is 0, and its certificate. An identity whose certificate cannot be read is +// skipped rather than ending the walk. +func eachIdentity(searchList uintptr, fn func(identity uintptr, der []byte) (bool, error)) error { + return eachMatching(searchList, kSecClassIdentity, "identity", func(identity uintptr) (bool, error) { + var cert uintptr + if status := secIdentityCopyCertificate(identity, &cert); status != 0 { + log.Debugf("skipping keychain identity: SecIdentityCopyCertificate: %d", status) + return false, nil + } + der := certificateDER(cert) + release(cert) + if der == nil { + log.Debug("skipping keychain identity whose certificate has no DER data") + return false, nil + } + return fn(identity, der) + }) +} + +func keychainCertificates(searchList uintptr) ([]*x509.Certificate, error) { + var certs []*x509.Certificate + var unparsable int + err := eachMatching(searchList, kSecClassCertificate, "certificate", func(item uintptr) (bool, error) { + if cert, err := x509.ParseCertificate(certificateDER(item)); err == nil { + certs = append(certs, cert) + return false, nil + } + unparsable++ + return false, nil + }) + log.Debugf("keychain holds %d parsable certificates, %d unparsable", len(certs), unparsable) + return certs, err +} + +func eachMatching(searchList, class uintptr, name string, fn func(item uintptr) (bool, error)) error { + keys := []uintptr{kSecClass, kSecMatchLimit, kSecReturnRef} + values := []uintptr{class, kSecMatchLimitAll, kCFBooleanTrue} + if searchList != 0 { + keys = append(keys, kSecMatchSearchList) + values = append(values, searchList) + } + query := cfDictionaryCreate(0, &keys[0], &values[0], len(keys), kCFTypeDictionaryKeyCallBacks, kCFTypeDictionaryValueCallBacks) + defer release(query) + + var items uintptr + switch status := secItemCopyMatching(query, &items); status { + case 0: + case errSecItemNotFound: + log.Debugf("keychain %s query returned errSecItemNotFound (%d): the search list holds no item of this class", name, errSecItemNotFound) + return nil + default: + log.Debugf("keychain %s query returned OSStatus %d", name, status) + return fmt.Errorf("SecItemCopyMatching: %d", status) + } + defer release(items) + + n := cfArrayGetCount(items) + log.Debugf("keychain %s query returned %d items", name, n) + for i := 0; i < n; i++ { + if stop, err := fn(cfArrayGetValueAtIndex(items, i)); stop || err != nil { + return err + } + } + return nil +} + +// certificateDER returns the DER form of cert, or nil when SecCertificateCopyData +// returns NULL, which it does for an object that is not a valid certificate. +func certificateDER(cert uintptr) []byte { + data := secCertificateCopyData(cert) + if data == 0 { + return nil + } + defer release(data) + return dataBytes(data) +} + +func dataBytes(data uintptr) []byte { + n := cfDataGetLength(data) + if n <= 0 { + return nil + } + return bytes.Clone(unsafe.Slice((*byte)(cfDataGetBytePtr(data)), n)) +} + +// release drops a CoreFoundation reference. CFRelease crashes the process on NULL, and +// several Security calls return NULL on failure, so every release goes through here. +func release(ref uintptr) { + if ref != 0 { + cfRelease(ref) + } +} + +func loadKeychain() error { + keychainOnce.Do(func() { + if keychainErr = resolveKeychain(); keychainErr != nil { + log.Debugf("macOS keychain unavailable for certificate posture: %v", keychainErr) + return + } + log.Debugf("macOS Security framework loaded for certificate posture, running as uid=%d euid=%d", os.Getuid(), os.Geteuid()) + logSearchList() + }) + return keychainErr +} + +// logSearchList reports the keychains the process searches. The root daemon sees the +// System keychain and System Roots, never a user's login keychain. +func logSearchList() { + if secKeychainCopySearchList == nil || secKeychainGetPath == nil { + log.Debug("keychain search list diagnostics unavailable on this macOS version") + return + } + + var list uintptr + if status := secKeychainCopySearchList(&list); status != 0 { + log.Debugf("SecKeychainCopySearchList returned OSStatus %d", status) + return + } + defer release(list) + + n := cfArrayGetCount(list) + log.Debugf("keychain search list contains %d keychains", n) + for i := 0; i < n; i++ { + log.Debugf("keychain search list[%d]: %s", i, keychainPath(cfArrayGetValueAtIndex(list, i))) + } +} + +func keychainPath(keychain uintptr) string { + path := make([]byte, 1024) + length := uint32(len(path)) + if status := secKeychainGetPath(keychain, &length, &path[0]); status != 0 { + return fmt.Sprintf("", status) + } + return string(path[:length]) +} + +func resolveKeychain() error { + security, err := purego.Dlopen(securityFramework, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return fmt.Errorf("open %s: %w", securityFramework, err) + } + coreFoundation, err := purego.Dlopen(coreFoundationFramework, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return fmt.Errorf("open %s: %w", coreFoundationFramework, err) + } + + for _, fn := range []struct { + ptr any + lib uintptr + name string + }{ + {&secItemCopyMatching, security, "SecItemCopyMatching"}, + {&secIdentityCopyCertificate, security, "SecIdentityCopyCertificate"}, + {&secIdentityCopyPrivateKey, security, "SecIdentityCopyPrivateKey"}, + {&secCertificateCopyData, security, "SecCertificateCopyData"}, + {&secKeyCreateSignature, security, "SecKeyCreateSignature"}, + {&secKeychainOpen, security, "SecKeychainOpen"}, + {&cfArrayCreate, coreFoundation, "CFArrayCreate"}, + {&cfDictionaryCreate, coreFoundation, "CFDictionaryCreate"}, + {&cfArrayGetCount, coreFoundation, "CFArrayGetCount"}, + {&cfArrayGetValueAtIndex, coreFoundation, "CFArrayGetValueAtIndex"}, + {&cfDataCreate, coreFoundation, "CFDataCreate"}, + {&cfDataGetLength, coreFoundation, "CFDataGetLength"}, + {&cfDataGetBytePtr, coreFoundation, "CFDataGetBytePtr"}, + {&cfErrorGetCode, coreFoundation, "CFErrorGetCode"}, + {&cfRelease, coreFoundation, "CFRelease"}, + } { + symbol, err := purego.Dlsym(fn.lib, fn.name) + if err != nil { + return fmt.Errorf("resolve %s: %w", fn.name, err) + } + purego.RegisterFunc(fn.ptr, symbol) + } + + for _, global := range []struct { + ptr *uintptr + lib uintptr + name string + deref bool + }{ + {&kSecClass, security, "kSecClass", true}, + {&kSecClassIdentity, security, "kSecClassIdentity", true}, + {&kSecClassCertificate, security, "kSecClassCertificate", true}, + {&kSecMatchLimit, security, "kSecMatchLimit", true}, + {&kSecMatchLimitAll, security, "kSecMatchLimitAll", true}, + {&kSecReturnRef, security, "kSecReturnRef", true}, + {&kSecMatchSearchList, security, "kSecMatchSearchList", true}, + {&kSecKeyAlgorithmECDSASHA256, security, "kSecKeyAlgorithmECDSASignatureDigestX962SHA256", true}, + {&kSecKeyAlgorithmECDSASHA384, security, "kSecKeyAlgorithmECDSASignatureDigestX962SHA384", true}, + {&kSecKeyAlgorithmRSAPSSSHA256, security, "kSecKeyAlgorithmRSASignatureDigestPSSSHA256", true}, + {&kCFBooleanTrue, coreFoundation, "kCFBooleanTrue", true}, + {&kCFTypeDictionaryKeyCallBacks, coreFoundation, "kCFTypeDictionaryKeyCallBacks", false}, + {&kCFTypeDictionaryValueCallBacks, coreFoundation, "kCFTypeDictionaryValueCallBacks", false}, + {&kCFTypeArrayCallBacks, coreFoundation, "kCFTypeArrayCallBacks", false}, + } { + addr, err := purego.Dlsym(global.lib, global.name) + if err != nil { + return fmt.Errorf("resolve %s: %w", global.name, err) + } + if global.deref { + addr = **(**uintptr)(unsafe.Pointer(&addr)) + } + *global.ptr = addr + } + + resolveOptional(security, "SecKeychainCopySearchList", &secKeychainCopySearchList) + resolveOptional(security, "SecKeychainGetPath", &secKeychainGetPath) + return nil +} + +// resolveOptional binds a diagnostic-only symbol, leaving it nil when the framework no +// longer exports it so keychain lookups keep working without it. +func resolveOptional(lib uintptr, name string, ptr any) { + symbol, err := purego.Dlsym(lib, name) + if err != nil { + log.Debugf("keychain diagnostics: %s unavailable: %v", name, err) + return + } + purego.RegisterFunc(ptr, symbol) +} diff --git a/client/internal/certproof/keystore.go b/client/internal/certproof/keystore.go new file mode 100644 index 000000000..7ce676b4c --- /dev/null +++ b/client/internal/certproof/keystore.go @@ -0,0 +1,77 @@ +package certproof + +import ( + "crypto" + "crypto/ecdsa" + "crypto/rsa" + "crypto/x509" + "encoding/asn1" + "errors" + "fmt" + "math/big" + "slices" +) + +var errUnsupportedScheme = errors.New("unsupported signature scheme for OS keystore") + +type sigScheme int + +const ( + schemeECDSASHA256 sigScheme = iota + 1 + schemeECDSASHA384 + schemeRSAPSSSHA256 +) + +// schemeFor maps a crypto.Signer request onto the schemes the OS keystores perform. +func schemeFor(pub crypto.PublicKey, opts crypto.SignerOpts) (sigScheme, error) { + switch pub.(type) { + case *ecdsa.PublicKey: + switch opts.HashFunc() { + case crypto.SHA256: + return schemeECDSASHA256, nil + case crypto.SHA384: + return schemeECDSASHA384, nil + } + case *rsa.PublicKey: + if pss, ok := opts.(*rsa.PSSOptions); ok && pss.Hash == crypto.SHA256 { + return schemeRSAPSSSHA256, nil + } + } + return 0, fmt.Errorf("%w: %T with %v", errUnsupportedScheme, pub, opts.HashFunc()) +} + +// buildChain extends leaf with the issuers found in pool up to a self-signed certificate. +func buildChain(leaf *x509.Certificate, pool []*x509.Certificate) []*x509.Certificate { + chain := []*x509.Certificate{leaf} + current := leaf + for current.CheckSignatureFrom(current) != nil { + issuer := issuerIn(current, pool, chain) + if issuer == nil { + break + } + chain = append(chain, issuer) + current = issuer + } + return chain +} + +func issuerIn(cert *x509.Certificate, pool, seen []*x509.Certificate) *x509.Certificate { + for _, candidate := range pool { + if slices.ContainsFunc(seen, candidate.Equal) { + continue + } + if cert.CheckSignatureFrom(candidate) == nil { + return candidate + } + } + return nil +} + +// ecdsaSignatureASN1 converts the fixed-width r||s form emitted by CNG into the DER form Go verifies. +func ecdsaSignatureASN1(raw []byte) ([]byte, error) { + if len(raw) == 0 || len(raw)%2 != 0 { + return nil, errors.New("malformed raw ECDSA signature") + } + half := len(raw) / 2 + return asn1.Marshal(struct{ R, S *big.Int }{new(big.Int).SetBytes(raw[:half]), new(big.Int).SetBytes(raw[half:])}) +} diff --git a/client/internal/certproof/keystore_test.go b/client/internal/certproof/keystore_test.go new file mode 100644 index 000000000..592d7414f --- /dev/null +++ b/client/internal/certproof/keystore_test.go @@ -0,0 +1,87 @@ +package certproof + +import ( + "crypto" + "crypto/ecdsa" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/shared/management/certposture" + "github.com/netbirdio/netbird/shared/management/certposture/certtest" +) + +func TestBuildChain_FollowsIssuersThroughThePool(t *testing.T) { + root := certtest.NewCA(t, "root") + intermediate := certtest.NewIntermediate(t, root, "intermediate") + unrelated := certtest.NewCA(t, "unrelated") + leaf := intermediate.Issue(t, certtest.ECDSAKey(t), "device") + pool := []*x509.Certificate{unrelated.Cert, root.Cert, leaf, intermediate.Cert} + + chain := buildChain(leaf, pool) + + require.Equal(t, []*x509.Certificate{leaf, intermediate.Cert, root.Cert}, chain) + roots, err := certposture.ParseCAs([]string{root.PEM}) + require.NoError(t, err) + assert.NoError(t, certposture.VerifyChain(chain, roots, time.Now())) +} + +func TestBuildChain_StopsWhereThePoolEnds(t *testing.T) { + root := certtest.NewCA(t, "root") + intermediate := certtest.NewIntermediate(t, root, "intermediate") + leaf := intermediate.Issue(t, certtest.ECDSAKey(t), "device") + + assert.Equal(t, []*x509.Certificate{leaf}, buildChain(leaf, nil)) + assert.Equal(t, []*x509.Certificate{leaf, intermediate.Cert}, buildChain(leaf, []*x509.Certificate{intermediate.Cert})) +} + +func TestSchemeFor(t *testing.T) { + ecKey := certtest.ECDSAKey(t) + rsaKey := certtest.RSAKey(t) + pss := &rsa.PSSOptions{SaltLength: rsa.PSSSaltLengthEqualsHash, Hash: crypto.SHA256} + + tests := []struct { + name string + pub crypto.PublicKey + opts crypto.SignerOpts + want sigScheme + }{ + {"ecdsa sha256", ecKey.Public(), crypto.SHA256, schemeECDSASHA256}, + {"ecdsa sha384", ecKey.Public(), crypto.SHA384, schemeECDSASHA384}, + {"rsa pss sha256", rsaKey.Public(), pss, schemeRSAPSSSHA256}, + {"rsa pkcs1v15", rsaKey.Public(), crypto.SHA256, 0}, + {"ed25519", certtest.Ed25519Key(t).Public(), crypto.Hash(0), 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := schemeFor(tc.pub, tc.opts) + if tc.want == 0 { + assert.ErrorIs(t, err, errUnsupportedScheme) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestECDSASignatureASN1(t *testing.T) { + key := certtest.ECDSAKey(t).(*ecdsa.PrivateKey) + digest := sha256.Sum256([]byte("nonce")) + r, s, err := ecdsa.Sign(rand.Reader, key, digest[:]) + require.NoError(t, err) + raw := append(r.FillBytes(make([]byte, 32)), s.FillBytes(make([]byte, 32))...) + + der, err := ecdsaSignatureASN1(raw) + require.NoError(t, err) + assert.True(t, ecdsa.VerifyASN1(&key.PublicKey, digest[:], der)) + + _, err = ecdsaSignatureASN1(raw[:63]) + assert.Error(t, err) +} diff --git a/client/internal/certproof/pinlatch.go b/client/internal/certproof/pinlatch.go new file mode 100644 index 000000000..79e396b11 --- /dev/null +++ b/client/internal/certproof/pinlatch.go @@ -0,0 +1,54 @@ +package certproof + +import ( + "crypto/sha256" + "encoding/binary" + "errors" + "sync" +) + +// errPINRejectedBefore is returned instead of logging in with a PIN the token already +// refused: every failed login counts towards the token's lockout, which for a TPM is +// shared with everything else on the machine, and proofs are collected on every sync. +var errPINRejectedBefore = errors.New("PKCS#11 token rejected this PIN before, not trying it again") + +// errPINNeedsToken refuses a PIN that names no token to log in to. +var errPINNeedsToken = errors.New("a PKCS#11 PIN needs the token named in NB_CERT_PKCS11_URI, as token=