diff --git a/.git-branches.toml b/.git-branches.toml index d1818090f..4c34d7928 100644 --- a/.git-branches.toml +++ b/.git-branches.toml @@ -3,7 +3,7 @@ [branches] main = "main" perennials = [] -perennial-regex = "" +perennial-regex = "^release-" [create] new-branch-type = "feature" diff --git a/.github/workflows/check-license-dependencies.yml b/.github/workflows/check-license-dependencies.yml index 17c9fdc8d..81d293e4f 100644 --- a/.github/workflows/check-license-dependencies.yml +++ b/.github/workflows/check-license-dependencies.yml @@ -2,7 +2,7 @@ name: Check License Dependencies on: push: - branches: [main] + branches: [main, "release-*"] paths: - "go.mod" - "go.sum" diff --git a/.github/workflows/frontend-ui.yml b/.github/workflows/frontend-ui.yml index 552ccef29..014c5c2ae 100644 --- a/.github/workflows/frontend-ui.yml +++ b/.github/workflows/frontend-ui.yml @@ -10,6 +10,7 @@ on: push: branches: - main + - "release-*" paths: - "client/ui/frontend/**" - "client/ui/i18n/**" diff --git a/.github/workflows/golang-test-darwin.yml b/.github/workflows/golang-test-darwin.yml index 420749a0e..c17d8e775 100644 --- a/.github/workflows/golang-test-darwin.yml +++ b/.github/workflows/golang-test-darwin.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: concurrency: diff --git a/.github/workflows/golang-test-freebsd.yml b/.github/workflows/golang-test-freebsd.yml index 9c795e783..65c39147a 100644 --- a/.github/workflows/golang-test-freebsd.yml +++ b/.github/workflows/golang-test-freebsd.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: concurrency: diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml index 0af506bba..004b78b3e 100644 --- a/.github/workflows/golang-test-linux.yml +++ b/.github/workflows/golang-test-linux.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: concurrency: diff --git a/.github/workflows/golang-test-windows.yml b/.github/workflows/golang-test-windows.yml index 50a5ba4d6..fb7b745d2 100644 --- a/.github/workflows/golang-test-windows.yml +++ b/.github/workflows/golang-test-windows.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: env: diff --git a/.github/workflows/install-script-test.yml b/.github/workflows/install-script-test.yml index 1514caedc..61709501c 100644 --- a/.github/workflows/install-script-test.yml +++ b/.github/workflows/install-script-test.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: paths: - "release_files/install.sh" diff --git a/.github/workflows/mobile-build-validation.yml b/.github/workflows/mobile-build-validation.yml index 44e912c73..322f129c9 100644 --- a/.github/workflows/mobile-build-validation.yml +++ b/.github/workflows/mobile-build-validation.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: concurrency: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 727bff45a..4d1945451 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,6 +6,7 @@ on: - "v*" branches: - main + - "release-*" pull_request: env: @@ -254,15 +255,23 @@ jobs: id: tag_and_push_images if: | (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || - (github.event_name == 'push' && github.ref == 'refs/heads/main') + (github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release-'))) run: | set -euo pipefail + # $GITHUB_REF / $GITHUB_EVENT_NAME are read from the runner + # environment rather than substituted into this script with the + # workflow expression syntax: branch names may legally contain + # $(…), and interpolating github.ref would execute it. resolve_tags() { - if [[ "${{ github.event_name }}" == "pull_request" ]]; then + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then echo "pr-${{ github.event.pull_request.number }}" - else + elif [[ "$GITHUB_REF" == "refs/heads/main" ]]; then echo "main sha-$(git rev-parse --short HEAD)" + else + # Release branches get an immutable sha-* tag only — the floating + # "main" tag must never move from a release branch. + echo "sha-$(git rev-parse --short HEAD)" fi } diff --git a/.github/workflows/sync-tag.yml b/.github/workflows/sync-tag.yml index d99f88b54..088e538d5 100644 --- a/.github/workflows/sync-tag.yml +++ b/.github/workflows/sync-tag.yml @@ -9,21 +9,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }} cancel-in-progress: true -# Receiving workflows (cloud sync-tag, mobile bump-netbird) expect the short -# tag form (e.g. v0.30.0), not refs/tags/v0.30.0 — github.ref_name, not github.ref. +# The receiving bump-netbird workflows expect the short tag form +# (e.g. v0.30.0), not refs/tags/v0.30.0 — github.ref_name, not github.ref. jobs: - trigger_sync_tag: - runs-on: ubuntu-latest - steps: - - name: Trigger release tag sync - uses: benc-uk/workflow-dispatch@31e2b3319479a63f0ab15bf800eff9e913504e26 # v1.3.2 - with: - workflow: sync-tag.yml - ref: main - repo: ${{ secrets.UPSTREAM_REPO }} - token: ${{ secrets.NC_GITHUB_TOKEN }} - inputs: '{ "tag": "${{ github.ref_name }}" }' - trigger_android_bump: runs-on: ubuntu-latest if: github.event.created && !github.event.deleted && startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-') diff --git a/.github/workflows/test-infrastructure-files.yml b/.github/workflows/test-infrastructure-files.yml index 729214d9e..1313379ee 100644 --- a/.github/workflows/test-infrastructure-files.yml +++ b/.github/workflows/test-infrastructure-files.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: paths: - "infrastructure_files/**" diff --git a/.github/workflows/ui-translations.yml b/.github/workflows/ui-translations.yml new file mode 100644 index 000000000..7d3b12f2d --- /dev/null +++ b/.github/workflows/ui-translations.yml @@ -0,0 +1,42 @@ +name: UI Translations + +on: + pull_request: + paths: + - "client/ui/i18n/locales/**" + - "client/ui/i18n/check-translations.mjs" + - ".github/workflows/ui-translations.yml" + push: + branches: + - main + paths: + - "client/ui/i18n/locales/**" + - "client/ui/i18n/check-translations.mjs" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }} + cancel-in-progress: true + +jobs: + check-translations: + name: Check translation key parity + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "22" + + # English (en) is the source of truth for translation keys; every other + # locale declared in _index.json must carry the exact same key set. + - name: Check translation key parity + run: node client/ui/i18n/check-translations.mjs diff --git a/.github/workflows/wasm-build-validation.yml b/.github/workflows/wasm-build-validation.yml index e8a12cdaf..5f21472e5 100644 --- a/.github/workflows/wasm-build-validation.yml +++ b/.github/workflows/wasm-build-validation.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - "release-*" pull_request: concurrency: diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 8dd05a192..c5d260376 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -468,6 +468,13 @@ checksum: - glob: ./infrastructure_files/migrate-to-enterprise.sh release: + # The signing pipeline (netbirdio/sign-pipelines, dispatched by + # trigger_signer) marks the release latest once the Windows and macOS + # artifacts are signed. Without this override goreleaser marks it latest + # at publish time, while those artifacts are still unsigned. + make_latest: false + # Mark x.y.z-rc.* and other prerelease tags as prereleases on GitHub. + prerelease: auto extra_files: - glob: ./infrastructure_files/getting-started-with-zitadel.sh - glob: ./release_files/install.sh diff --git a/.goreleaser_ui.yaml b/.goreleaser_ui.yaml index ca5148823..1c5bc41ac 100644 --- a/.goreleaser_ui.yaml +++ b/.goreleaser_ui.yaml @@ -96,6 +96,7 @@ nfpms: - netbird (>= 0.75.0) - libgtk-4-1 (>= 4.14) - libwebkitgtk-6.0-4 + - xdg-utils - maintainer: Netbird description: Netbird client UI. @@ -119,6 +120,7 @@ nfpms: - netbird >= 0.75.0 - (gtk4 >= 4.14 or libgtk-4-1 >= 4.14) - (webkitgtk6.0 or libwebkitgtk-6_0-4) + - xdg-utils rpm: signature: @@ -142,3 +144,11 @@ uploads: target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }} username: dev@wiretrustee.com method: PUT + +release: + # Uploads into the release created by the main .goreleaser.yaml run. + # make_latest stays false everywhere: the signing pipeline + # (netbirdio/sign-pipelines) marks the release latest after the Windows + # and macOS artifacts are signed. + make_latest: false + prerelease: auto diff --git a/.goreleaser_ui_darwin.yaml b/.goreleaser_ui_darwin.yaml index 47b991344..8ca0e8da6 100644 --- a/.goreleaser_ui_darwin.yaml +++ b/.goreleaser_ui_darwin.yaml @@ -43,3 +43,11 @@ checksum: name_template: "{{ .ProjectName }}_darwin_checksums.txt" changelog: disable: true + +release: + # Uploads into the release created by the main .goreleaser.yaml run. + # make_latest stays false everywhere: the signing pipeline + # (netbirdio/sign-pipelines) marks the release latest after the Windows + # and macOS artifacts are signed. + make_latest: false + prerelease: auto diff --git a/.goreleaser_ui_gtk3.yaml b/.goreleaser_ui_gtk3.yaml index d0a7de6e2..a9b2ca650 100644 --- a/.goreleaser_ui_gtk3.yaml +++ b/.goreleaser_ui_gtk3.yaml @@ -43,19 +43,17 @@ archives: - netbird-ui-gtk3 nfpms: - # Same package_name as the GTK4 packages -- the two are mutually-exclusive - # alternatives served from separate repo paths (see uploads below); a given - # distro points at exactly one of them. The file names must still differ: - # the Debian pool is shared storage keyed by file name, so a default-named - # gtk3 .deb would overwrite the stable one. + # Mutually-exclusive alternative to the GTK4 netbird-ui package -- both + # ship the same /usr/bin/netbird-ui from the shared stable/yum repos, so + # this one carries its own name and conflicts with the GTK4 package. - maintainer: Netbird description: Netbird client UI. homepage: https://netbird.io/ license: BSD-3-Clause vendor: NetBird id: netbird_ui_deb_gtk3 - package_name: netbird-ui - file_name_template: "{{ .PackageName }}-gtk3_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + package_name: netbird-ui-gtk3 + file_name_template: "{{ .PackageName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" builds: - netbird-ui-gtk3 formats: @@ -67,10 +65,15 @@ nfpms: dst: /usr/share/applications/org.wails.netbird.desktop - src: client/ui/build/appicon.png dst: /usr/share/pixmaps/netbird.png + conflicts: + - netbird-ui + replaces: + - netbird-ui dependencies: - netbird (>= 0.75.0) - libgtk-3-0 - libwebkit2gtk-4.1-0 + - xdg-utils - maintainer: Netbird description: Netbird client UI. @@ -78,8 +81,8 @@ nfpms: license: BSD-3-Clause vendor: NetBird id: netbird_ui_rpm_gtk3 - package_name: netbird-ui - file_name_template: "{{ .PackageName }}-gtk3_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + package_name: netbird-ui-gtk3 + file_name_template: "{{ .PackageName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" builds: - netbird-ui-gtk3 formats: @@ -91,10 +94,15 @@ nfpms: dst: /usr/share/applications/org.wails.netbird.desktop - src: client/ui/build/appicon.png dst: /usr/share/pixmaps/netbird.png + # No `replaces` here: nfpm maps it to rpm Obsoletes, which would make + # dnf swap installed GTK4 netbird-ui packages for this one on upgrade. + conflicts: + - netbird-ui dependencies: - netbird >= 0.75.0 - (gtk3 or libgtk-3-0) - (webkit2gtk4.1 or libwebkit2gtk-4_1-0) + - xdg-utils rpm: signature: @@ -109,23 +117,28 @@ changelog: disable: true uploads: - # The gtk3 packages reuse the netbird-ui package name, so they live in - # dedicated repo paths (deb distribution `gtk3`, yum path `yum-gtk3`) that - # legacy distros point their repo config at. - - name: debian-gtk3 + - name: debian skip: "{{ .Env.SKIP_PUBLISH }}" ids: - netbird_ui_deb_gtk3 mode: archive - target: https://pkgs.wiretrustee.com/debian/pool/{{ .ArtifactName }};deb.distribution=gtk3;deb.component=main;deb.architecture={{ if .Arm }}armhf{{ else }}{{ .Arch }}{{ end }};deb.package= + target: https://pkgs.wiretrustee.com/debian/pool/{{ .ArtifactName }};deb.distribution=stable;deb.component=main;deb.architecture={{ if .Arm }}armhf{{ else }}{{ .Arch }}{{ end }};deb.package= username: dev@wiretrustee.com method: PUT - - name: yum-gtk3 + - name: yum skip: "{{ .Env.SKIP_PUBLISH }}" ids: - netbird_ui_rpm_gtk3 mode: archive - target: https://pkgs.wiretrustee.com/yum-gtk3/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }} + target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }} username: dev@wiretrustee.com method: PUT + +release: + # Uploads into the release created by the main .goreleaser.yaml run. + # make_latest stays false everywhere: the signing pipeline + # (netbirdio/sign-pipelines) marks the release latest after the Windows + # and macOS artifacts are signed. + make_latest: false + prerelease: auto diff --git a/AGENTS.md b/AGENTS.md index 95b02a91d..5497acb15 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # NetBird Agent Guidelines -**NetBird** is an open-source connectivity platform: a WireGuard®-based overlay +**NetBird** is an open source connectivity platform: a WireGuard®-based overlay network with a control plane. The **agent** (`client/`) runs on user machines as a privileged daemon and manages the WireGuard interface, routing, firewall, and DNS. **Management** (`management/`) is the control plane and REST/gRPC API, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db5097a48..9dea37ec8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,6 +112,7 @@ aligns with our security standards and design expectations. - [Test suite](#test-suite) - [Checklist before submitting a PR](#checklist-before-submitting-a-pr) - [When we close a PR](#when-we-close-a-pr) + - [Translations](#translations) - [Other project repositories](#other-project-repositories) - [Contributor License Agreement](#contributor-license-agreement) @@ -478,7 +479,7 @@ go test -race ./client/internal/dns/... ## Checklist before submitting a PR -As a critical network service and open-source project, we must enforce a few +As a critical network service and open source project, we must enforce a few things before submitting a pull request. The [pull request template](/.github/pull_request_template.md) mirrors this list — fill it in rather than deleting it. @@ -612,6 +613,17 @@ A closed PR is not a rejected idea. Take it back to the [discussion](https://github.com/netbirdio/netbird/discussions), settle the approach, and reopen the work from there. +## Translations + +Desktop UI translations are not contributed through pull requests. Translate on +[Crowdin](https://crowdin.com/project/netbird) instead: no ticket needed, just +join the project and pick your language. Crowdin syncs with this repository and +opens the service PRs itself, so hand-edited locale files would conflict with +the next sync. Style, terminology, and review guidance live in +[client/ui/i18n/TRANSLATING.md](client/ui/i18n/TRANSLATING.md). To request a +language the project does not offer yet, ask on the Crowdin project page or in +a [discussion](https://github.com/netbirdio/netbird/discussions). + ## Other project repositories NetBird project is composed of 3 main repositories: diff --git a/README.md b/README.md index 40c6b9ed5..3bcb4a035 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ In November 2022, NetBird joined the [StartUpSecure program](https://www.forschu ![CISPA_Logo_BLACK_EN_RZ_RGB (1)](https://user-images.githubusercontent.com/700848/203091324-c6d311a0-22b5-4b05-a288-91cbc6cdcc46.png) ### Acknowledgements -We build on open-source technologies like [WireGuard®](https://www.wireguard.com/), [Pion ICE](https://github.com/pion/ice), and [Rosenpass](https://rosenpass.eu). We greatly appreciate the work these projects are doing, and we'd love it if you could support them too (e.g., by starring or contributing). +We build on open source technologies like [WireGuard®](https://www.wireguard.com/), [Pion ICE](https://github.com/pion/ice), and [Rosenpass](https://rosenpass.eu). We greatly appreciate the work these projects are doing, and we'd love it if you could support them too (e.g., by starring or contributing). ### Legal This repository is licensed under the BSD-3-Clause license, which applies to all parts of the repository except for the directories management/, signal/ and relay/. diff --git a/SECURITY.md b/SECURITY.md index bdf88d670..cbcc975ba 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -14,7 +14,7 @@ Report security issues one of these two ways: on this repository. This is the preferred route: it keeps the discussion, the draft advisory, and the credit in one place. - **Email** — `security@netbird.io`. -If the finding affects NetBird Cloud or our hosted infrastructure rather than the open-source code, email us rather than +If the finding affects NetBird Cloud or our hosted infrastructure rather than the open source code, email us rather than filing a repository report. ### What to include diff --git a/client/android/client.go b/client/android/client.go index 154bd8484..a21348f27 100644 --- a/client/android/client.go +++ b/client/android/client.go @@ -15,6 +15,7 @@ import ( log "github.com/sirupsen/logrus" + nbAnonymize "github.com/netbirdio/netbird/client/anonymize" "github.com/netbirdio/netbird/client/iface/device" "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/debug" @@ -32,6 +33,13 @@ import ( types "github.com/netbirdio/netbird/upload-server/types" ) +// AnonymizeLevelDefault and AnonymizeLevelStrict are the accepted +// anonymizeLevel values for DebugBundle. +const ( + AnonymizeLevelDefault = nbAnonymize.LevelDefaultString + AnonymizeLevelStrict = nbAnonymize.LevelStrictString +) + // ConnectionListener export internal Listener for mobile type ConnectionListener interface { peer.Listener @@ -278,8 +286,10 @@ func (c *Client) GetTunSettings() (*TunSettings, error) { } // DebugBundle generates a debug bundle, uploads it, and returns the upload key. -// It works both with and without a running engine. -func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (string, error) { +// It works both with and without a running engine. anonymizeLevel is "default" +// or "strict"; strict also anonymizes internal IP ranges, peer names, and +// WireGuard public keys, and implies anonymize. +func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool, anonymizeLevel string) (string, error) { cfg, cacheDir, cc := c.stateSnapshot() // If the engine hasn't been started, load config from disk @@ -298,6 +308,7 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (strin InternalConfig: cfg, StatusRecorder: c.recorder, TempDir: cacheDir, + StatePath: platformFiles.StateFilePath(), } if cc != nil { @@ -321,6 +332,7 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (strin deps, debug.BundleConfig{ Anonymize: anonymize, + AnonymizeLevel: nbAnonymize.ParseLevel(anonymizeLevel), IncludeSystemInfo: true, }, ) diff --git a/client/android/login.go b/client/android/login.go index 3f367b97f..897b1561e 100644 --- a/client/android/login.go +++ b/client/android/login.go @@ -204,8 +204,9 @@ func (a *Auth) foregroundGetTokenInfo(authClient *auth.Auth, urlOpener URLOpener return nil, fmt.Errorf("failed to get OAuth flow: %v", err) } - // An empty hint is deliberate, not a fallback: a fresh or logged-out profile - // leaves the choice to the IdP, which is how accounts get switched. + // An empty hint is deliberate, not a fallback: a fresh profile leaves the + // choice to the IdP. Switching accounts is done by switching or removing + // profiles, not by logging out — logout keeps the email. if a.cfgPath != "" { if hint := readProfileEmail(a.cfgPath); hint != "" { if setter, ok := oAuthFlow.(loginHintSetter); ok { diff --git a/client/android/profile_manager.go b/client/android/profile_manager.go index 3197124d7..20d585d6a 100644 --- a/client/android/profile_manager.go +++ b/client/android/profile_manager.go @@ -22,7 +22,8 @@ type Profile struct { ID string Name string // Email is the account this profile last logged in with, "" if it never - // completed an SSO login or was logged out. See profile_state.go. + // completed an SSO login. Kept across logouts; cleared when the profile is + // removed. See profile_state.go. Email string IsActive bool } @@ -200,11 +201,9 @@ func (pm *ProfileManager) LogoutProfile(id string) error { return fmt.Errorf("failed to save config: %w", err) } - // Not fatal: a stale hint costs an account switch, not the logout itself. - if err := removeProfileEmail(configPath); err != nil { - log.Warnf("failed to clear stored account email for profile %s: %v", id, err) - } - + // The stored account email is kept on purpose, matching the desktop and CLI + // logout semantics: the next login passes it as the login_hint so the IdP + // preselects the account. Removing the profile is what deletes it. log.Infof("logged out from profile: %s", id) return nil } @@ -224,11 +223,24 @@ func (pm *ProfileManager) RenameProfile(id string, newName string) error { // RemoveProfile deletes a profile func (pm *ProfileManager) RemoveProfile(id string) error { + configPath, err := pm.getProfileConfigPath(id) + if err != nil { + return err + } + // Use ServiceManager (removes profile from profiles/ directory) if err := pm.serviceMgr.RemoveProfile(profilemanager.ID(id), androidUsername); err != nil { return fmt.Errorf("failed to remove profile: %w", err) } + // The account file is this package's, not the ServiceManager's, so it must + // go here. The default profile has a fixed filename, so a recreated one + // would otherwise inherit the deleted profile's email as its login_hint. + // Not fatal: the profile itself is gone. + if err := removeProfileEmail(configPath); err != nil { + log.Warnf("failed to remove stored account email for profile %s: %v", id, err) + } + log.Infof("removed profile: %s", id) return nil } diff --git a/client/android/profile_state.go b/client/android/profile_state.go index 3f0a09701..0063b587f 100644 --- a/client/android/profile_state.go +++ b/client/android/profile_state.go @@ -90,10 +90,10 @@ func writeProfileEmail(configPath string, email string) error { return nil } -// removeProfileEmail drops the stored account email. Called on logout: while the -// email is on disk it goes out as a login_hint, which would steer the next login -// straight back into the account just logged out of. Mirrors the desktop UI's -// RemoveProfileState call. +// removeProfileEmail drops the stored account email. Called on profile removal, +// not on logout: a logged-out profile keeps its email so the next login passes +// it as the login_hint, matching the desktop and CLI semantics. Mirrors the +// desktop UI's RemoveProfileState call. func removeProfileEmail(configPath string) error { accountPath, err := profileAccountPathFor(configPath) if err != nil { diff --git a/client/android/profile_state_test.go b/client/android/profile_state_test.go index 623e16c3b..82a1c2a87 100644 --- a/client/android/profile_state_test.go +++ b/client/android/profile_state_test.go @@ -127,10 +127,10 @@ func TestWriteThenReadProfileEmail(t *testing.T) { t.Fatalf("remove: %v", err) } if got := readProfileEmail(configPath); got != "" { - t.Errorf("expected no email after logout, got %q", got) + t.Errorf("expected no email after removal, got %q", got) } - // Logout may run on a never-logged-in profile, so a second remove must pass. + // Removal may run on a never-logged-in profile, so a second remove must pass. if err := removeProfileEmail(configPath); err != nil { t.Fatalf("second remove should be a no-op: %v", err) } diff --git a/client/anonymize/anonymize.go b/client/anonymize/anonymize.go index c140cef89..c5d43ed55 100644 --- a/client/anonymize/anonymize.go +++ b/client/anonymize/anonymize.go @@ -2,6 +2,7 @@ package anonymize import ( "crypto/rand" + "encoding/base64" "fmt" "math/big" "net" @@ -15,13 +16,88 @@ import ( const anonTLD = ".domain" +// Level selects how much the anonymizer redacts. Levels are ordered: a higher +// level redacts strictly more. On the wire (protos, flags) levels travel as +// their string form. +type Level int + +const ( + // LevelDefault anonymizes public IP addresses, IPv6 ULA, domains, and MAC + // addresses. Internal IPv4 ranges (RFC 1918, CGNAT, link-local) are + // preserved so support can reason about the real topology. + LevelDefault Level = iota + // LevelStrict additionally anonymizes internal IP ranges, peer names, and + // WireGuard public keys. + LevelStrict +) + +// LevelDefaultString and LevelStrictString are the wire forms of the levels, +// for boundaries that pass levels as strings (flags, protos, mobile bindings). +const ( + LevelDefaultString = "default" + LevelStrictString = "strict" +) + +// ParseLevel maps s to a Level. Empty means LevelDefault; anything +// unrecognized maps to LevelStrict so an unknown request never yields less +// anonymization than intended. +func ParseLevel(s string) Level { + switch strings.ToLower(s) { + case "", LevelDefaultString: + return LevelDefault + default: + return LevelStrict + } +} + +// String returns the wire form of the level: "default" or "strict". +func (l Level) String() string { + if l >= LevelStrict { + return LevelStrictString + } + return LevelDefaultString +} + +// protectedDomains are NetBird-operated suffixes that stay recognizable in an +// anonymized bundle. At LevelStrict the labels in front of them (the peer +// name) are still replaced, except under netbird.io, which only hosts +// NetBird infrastructure (api, signal, flow), never peer names. +var protectedDomains = []string{"netbird.io", "netbird.selfhosted", "netbird.cloud", "netbird.stage"} + +const infraDomain = "netbird.io" + +var ( + macColonRegex = regexp.MustCompile(`\b[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}\b`) + macDashRegex = regexp.MustCompile(`\b[0-9a-fA-F]{2}(?:-[0-9a-fA-F]{2}){5}\b`) + wgKeyRegex = regexp.MustCompile(`\b[A-Za-z0-9+/]{43}=`) +) + type Anonymizer struct { ipAnonymizer map[netip.Addr]netip.Addr domainAnonymizer map[string]string - currentAnonIPv4 netip.Addr - currentAnonIPv6 netip.Addr - startAnonIPv4 netip.Addr - startAnonIPv6 netip.Addr + // domainOrder caches the keys of domainAnonymizer sorted longest-first + // for AnonymizeString; it is rebuilt when the map gains entries. + domainOrder []string + labelAnonymizer map[string]string + labelAnonymized map[string]struct{} + labelCounter uint32 + macAnonymizer map[string]string + macCounter uint32 + wgKeyAnonymizer map[string]string + wgKeyAnonymized map[string]struct{} + currentAnonIPv4 netip.Addr + currentAnonIPv6 netip.Addr + startAnonIPv4 netip.Addr + startAnonIPv6 netip.Addr + + // LevelStrict also anonymizes internal ranges (RFC 1918, CGNAT, + // link-local), replacing them from the dedicated internal pools below so + // a reader can still tell an internal address from a public one. + level Level + currentAnonInternalIPv4 netip.Addr + currentAnonInternalIPv6 netip.Addr + startAnonInternalIPv4 netip.Addr + startAnonInternalIPv6 netip.Addr domainKeyRegex *regexp.Regexp } @@ -32,25 +108,50 @@ func DefaultAddresses() (netip.Addr, netip.Addr) { return netip.AddrFrom4([4]byte{198, 51, 100, 0}), netip.MustParseAddr("2001:db8:ffff::") } +// InternalAddresses returns the pool starts used in strict mode for internal +// ranges. Both are reserved ranges that cannot collide with real addressing: +// 198.18.0.0 (RFC 2544 benchmarking), 2001:db8:1:: (RFC 3849 documentation). +func InternalAddresses() (netip.Addr, netip.Addr) { + return netip.AddrFrom4([4]byte{198, 18, 0, 0}), netip.MustParseAddr("2001:db8:1::") +} + func NewAnonymizer(startIPv4, startIPv6 netip.Addr) *Anonymizer { + internalIPv4, internalIPv6 := InternalAddresses() return &Anonymizer{ ipAnonymizer: map[netip.Addr]netip.Addr{}, domainAnonymizer: map[string]string{}, + labelAnonymizer: map[string]string{}, + labelAnonymized: map[string]struct{}{}, + macAnonymizer: map[string]string{}, + wgKeyAnonymizer: map[string]string{}, + wgKeyAnonymized: map[string]struct{}{}, currentAnonIPv4: startIPv4, currentAnonIPv6: startIPv6, startAnonIPv4: startIPv4, startAnonIPv6: startIPv6, + level: LevelDefault, + currentAnonInternalIPv4: internalIPv4, + currentAnonInternalIPv6: internalIPv6, + startAnonInternalIPv4: internalIPv4, + startAnonInternalIPv6: internalIPv6, + domainKeyRegex: regexp.MustCompile(`\bdomain=([^\s,:"]+)`), } } +// SetLevel selects the anonymization level. The zero value of a new +// Anonymizer is LevelDefault. +func (a *Anonymizer) SetLevel(level Level) { + a.level = level +} + func (a *Anonymizer) AnonymizeIP(ip netip.Addr) netip.Addr { + // Normalize 4-in-6 addresses so ::ffff:192.168.1.1 classifies and maps + // like 192.168.1.1. + ip = ip.Unmap() + if ip.IsLoopback() || - ip.IsLinkLocalUnicast() || - ip.IsLinkLocalMulticast() || - ip.IsInterfaceLocalMulticast() || - (ip.Is4() && ip.IsPrivate()) || ip.IsUnspecified() || ip.IsMulticast() || isWellKnown(ip) || @@ -59,18 +160,100 @@ func (a *Anonymizer) AnonymizeIP(ip netip.Addr) netip.Addr { return ip } + if isInternal(ip) && a.level < LevelStrict { + return ip + } + if _, ok := a.ipAnonymizer[ip]; !ok { - if ip.Is4() { - a.ipAnonymizer[ip] = a.currentAnonIPv4 - a.currentAnonIPv4 = a.currentAnonIPv4.Next() - } else { - a.ipAnonymizer[ip] = a.currentAnonIPv6 - a.currentAnonIPv6 = a.currentAnonIPv6.Next() - } + a.ipAnonymizer[ip] = a.nextAnonIP(ip) } return a.ipAnonymizer[ip] } +func (a *Anonymizer) nextAnonIP(ip netip.Addr) netip.Addr { + // At the strict level, internal addresses (including IPv6 ULA, matched + // by IsPrivate) come from the internal pools so they remain recognizable + // as internal without disclosing the real values. + if a.level >= LevelStrict && (isInternal(ip) || ip.IsPrivate()) { + if ip.Is4() { + anon := a.currentAnonInternalIPv4 + a.currentAnonInternalIPv4 = a.currentAnonInternalIPv4.Next() + return anon + } + anon := a.currentAnonInternalIPv6 + a.currentAnonInternalIPv6 = a.currentAnonInternalIPv6.Next() + return anon + } + + if ip.Is4() { + anon := a.currentAnonIPv4 + a.currentAnonIPv4 = a.currentAnonIPv4.Next() + return anon + } + anon := a.currentAnonIPv6 + a.currentAnonIPv6 = a.currentAnonIPv6.Next() + return anon +} + +// AnonymizeMAC replaces a MAC address with a consistent placeholder from the +// locally administered range starting at 02:00:00:00:00:01, at every +// anonymization level. Broadcast, multicast, all-zero, and already assigned +// placeholder addresses are preserved. The colon and dash spellings of the +// same address share one placeholder; the output keeps the input's separator. +func (a *Anonymizer) AnonymizeMAC(mac string) string { + hw, err := net.ParseMAC(mac) + if err != nil || len(hw) != 6 { + return mac + } + + if isWellKnownMAC(hw) || a.isAnonymizedMAC(hw) { + return mac + } + + key := hw.String() + anon, ok := a.macAnonymizer[key] + if !ok { + a.macCounter++ + anon = fmt.Sprintf("02:00:00:%02x:%02x:%02x", byte(a.macCounter>>16), byte(a.macCounter>>8), byte(a.macCounter)) + a.macAnonymizer[key] = anon + } + + if strings.Contains(mac, "-") { + anon = strings.ReplaceAll(anon, ":", "-") + } + return anon +} + +// isAnonymizedMAC reports whether hw is a placeholder this anonymizer already +// handed out, so a second pass over anonymized output leaves it unchanged. +func (a *Anonymizer) isAnonymizedMAC(hw net.HardwareAddr) bool { + if hw[0] != 0x02 || hw[1] != 0 || hw[2] != 0 { + return false + } + value := uint32(hw[3])<<16 | uint32(hw[4])<<8 | uint32(hw[5]) + return value <= a.macCounter +} + +// AnonymizeWGKey replaces a WireGuard public key with a consistent random +// placeholder of the same shape. Keys are only anonymized at LevelStrict; +// placeholders already handed out pass through unchanged. +func (a *Anonymizer) AnonymizeWGKey(key string) string { + if a.level < LevelStrict || !looksLikeWGKey(key) { + return key + } + if _, ok := a.wgKeyAnonymized[key]; ok { + return key + } + + anon, ok := a.wgKeyAnonymizer[key] + if !ok { + anon = generateAnonymousKey() + a.wgKeyAnonymizer[key] = anon + a.wgKeyAnonymized[anon] = struct{}{} + } + return anon +} + func (a *Anonymizer) AnonymizeUDPAddr(addr net.UDPAddr) net.UDPAddr { // Convert IP to netip.Addr ip, ok := netip.AddrFromSlice(addr.IP) @@ -89,12 +272,12 @@ func (a *Anonymizer) AnonymizeUDPAddr(addr net.UDPAddr) net.UDPAddr { // isInAnonymizedRange checks if an IP is within the range of already assigned anonymized IPs func (a *Anonymizer) isInAnonymizedRange(ip netip.Addr) bool { - if ip.Is4() && ip.Compare(a.startAnonIPv4) >= 0 && ip.Compare(a.currentAnonIPv4) <= 0 { - return true - } else if !ip.Is4() && ip.Compare(a.startAnonIPv6) >= 0 && ip.Compare(a.currentAnonIPv6) <= 0 { - return true + if ip.Is4() { + return inPoolRange(ip, a.startAnonIPv4, a.currentAnonIPv4) || + inPoolRange(ip, a.startAnonInternalIPv4, a.currentAnonInternalIPv4) } - return false + return inPoolRange(ip, a.startAnonIPv6, a.currentAnonIPv6) || + inPoolRange(ip, a.startAnonInternalIPv6, a.currentAnonInternalIPv6) } func (a *Anonymizer) AnonymizeIPString(ip string) string { @@ -118,14 +301,23 @@ func (a *Anonymizer) AnonymizeDomain(domain string) string { baseDomain = domain[:len(domain)-1] } - if strings.HasSuffix(baseDomain, "netbird.io") || - strings.HasSuffix(baseDomain, "netbird.selfhosted") || - strings.HasSuffix(baseDomain, "netbird.cloud") || - strings.HasSuffix(baseDomain, "netbird.stage") || - strings.HasSuffix(baseDomain, anonTLD) { + if strings.HasSuffix(baseDomain, anonTLD) { return domain } + // A reverse zone names an address prefix, so it follows the address rules, + // which also keeps its digit labels intact. + if zone, ok := a.anonymizeReverseZone(baseDomain); ok { + return withTrailingDot(zone, hasDot) + } + + if suffix := protectedSuffix(baseDomain); suffix != "" { + if a.level < LevelStrict || baseDomain == suffix || suffix == infraDomain { + return domain + } + return withTrailingDot(a.anonymizePeerName(baseDomain, suffix), hasDot) + } + parts := strings.Split(baseDomain, ".") if len(parts) < 2 { return domain @@ -141,12 +333,53 @@ func (a *Anonymizer) AnonymizeDomain(domain string) string { } result := strings.Replace(baseDomain, baseForLookup, anonymized, 1) - if hasDot { - result += "." + if a.level >= LevelStrict && len(parts) > 2 { + prefix := strings.TrimSuffix(baseDomain, "."+baseForLookup) + result = a.anonymizeLabels(prefix, "host") + "." + anonymized + // The full mapping feeds AnonymizeString so seeded FQDNs are caught + // in log lines as a whole, labels included. + a.domainAnonymizer[baseDomain] = result + } + return withTrailingDot(result, hasDot) +} + +// anonymizePeerName replaces the labels in front of a protected suffix with +// numbered peer placeholders, keeping the suffix, and records the full +// mapping for string replacement in logs. The numbering keeps a peer +// recognizable across the whole bundle without disclosing its name. +func (a *Anonymizer) anonymizePeerName(baseDomain, suffix string) string { + prefix := strings.TrimSuffix(baseDomain, "."+suffix) + result := a.anonymizeLabels(prefix, "peer") + "." + suffix + if result != baseDomain { + a.domainAnonymizer[baseDomain] = result } return result } +// anonymizeLabels replaces each dot-separated label with a consistent +// numbered placeholder ("-"). Wildcard labels and +// placeholders already handed out pass through unchanged. +func (a *Anonymizer) anonymizeLabels(prefix, placeholder string) string { + labels := strings.Split(prefix, ".") + for i, label := range labels { + if label == "*" { + continue + } + if _, ok := a.labelAnonymized[label]; ok { + continue + } + anon, ok := a.labelAnonymizer[label] + if !ok { + a.labelCounter++ + anon = fmt.Sprintf("%s-%d", placeholder, a.labelCounter) + a.labelAnonymizer[label] = anon + a.labelAnonymized[anon] = struct{}{} + } + labels[i] = anon + } + return strings.Join(labels, ".") +} + func (a *Anonymizer) AnonymizeURI(uri string) string { u, err := url.Parse(uri) if err != nil { @@ -178,17 +411,75 @@ func (a *Anonymizer) AnonymizeString(str string) string { ipv4Regex := regexp.MustCompile(`\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b`) ipv6Regex := regexp.MustCompile(`\b([0-9a-fA-F:]+:+[0-9a-fA-F]{0,4})(?:%[0-9a-zA-Z]+)?(?:\/[0-9]{1,3})?(?::[0-9]{1,5})?\b`) + // Reverse zones go first and are then held out of the passes below: their + // labels are digits, which the address patterns would otherwise consume. + str, restoreZones := a.replaceReverseZones(str) + str = ipv4Regex.ReplaceAllStringFunc(str, a.AnonymizeIPString) str = ipv6Regex.ReplaceAllStringFunc(str, a.AnonymizeIPString) - for domain, anonDomain := range a.domainAnonymizer { - str = strings.ReplaceAll(str, domain, anonDomain) + for _, domain := range a.sortedDomains() { + str = strings.ReplaceAll(str, domain, a.domainAnonymizer[domain]) } str = a.AnonymizeSchemeURI(str) str = a.AnonymizeDNSLogLine(str) - return str + // MAC handling runs after the IP passes so preserved IPv6 addresses are + // already out of the way; the separator guard skips matches embedded in a + // longer colon- or dash-separated sequence (such as an IPv6 tail). + str = a.anonymizeMACsInString(str, macColonRegex, ':') + str = a.anonymizeMACsInString(str, macDashRegex, '-') + + if a.level >= LevelStrict { + str = wgKeyRegex.ReplaceAllStringFunc(str, a.AnonymizeWGKey) + } + + return restoreZones(str) +} + +// sortedDomains returns the domain mappings longest-first, so a full-FQDN +// mapping (strict level) is applied before the base-domain mapping it +// contains. The order is rebuilt only when domainAnonymizer has grown. +func (a *Anonymizer) sortedDomains() []string { + if len(a.domainOrder) == len(a.domainAnonymizer) { + return a.domainOrder + } + + a.domainOrder = a.domainOrder[:0] + for domain := range a.domainAnonymizer { + a.domainOrder = append(a.domainOrder, domain) + } + slices.SortFunc(a.domainOrder, func(x, y string) int { + if d := len(y) - len(x); d != 0 { + return d + } + return strings.Compare(x, y) + }) + return a.domainOrder +} + +// anonymizeMACsInString replaces MAC addresses matched by re, skipping +// matches that directly adjoin another sep so a six-group run inside a longer +// separated sequence is left alone. +func (a *Anonymizer) anonymizeMACsInString(str string, re *regexp.Regexp, sep byte) string { + matches := re.FindAllStringIndex(str, -1) + if len(matches) == 0 { + return str + } + + var b strings.Builder + last := 0 + for _, m := range matches { + if (m[0] > 0 && str[m[0]-1] == sep) || (m[1] < len(str) && str[m[1]] == sep) { + continue + } + b.WriteString(str[last:m[0]]) + b.WriteString(a.AnonymizeMAC(str[m[0]:m[1]])) + last = m[1] + } + b.WriteString(str[last:]) + return b.String() } // AnonymizeSchemeURI finds and anonymizes URIs with ws, wss, rel, rels, stun, stuns, turn, and turns schemes. @@ -239,10 +530,79 @@ func isWellKnown(addr netip.Addr) bool { "128.0.0.0", "8000::", // 2nd split subnet for default routes } - if slices.Contains(wellKnown, addr.String()) { + return slices.Contains(wellKnown, addr.String()) +} + +// isInternal reports whether ip identifies a host only within the local +// network: IPv4 private (RFC 1918), CGNAT (RFC 6598), and link-local (v4 and +// v6). These are preserved at the default level so support can reason about +// the real topology, and replaced from the internal pools at the strict +// level. IPv6 ULA is deliberately not internal: its random global ID uniquely +// fingerprints the network, so it is anonymized at every level. +func isInternal(ip netip.Addr) bool { + return (ip.Is4() && ip.IsPrivate()) || + ip.IsLinkLocalUnicast() || + isCGNAT(ip) +} + +func inPoolRange(ip, start, current netip.Addr) bool { + return ip.Compare(start) >= 0 && ip.Compare(current) <= 0 +} + +// isWellKnownMAC reports whether hw carries no stable host identity: all-zero +// or a group address (broadcast and multicast). +func isWellKnownMAC(hw net.HardwareAddr) bool { + if hw[0]&1 == 1 { return true } + for _, b := range hw { + if b != 0 { + return false + } + } + return true +} +// looksLikeWGKey reports whether s has the shape of a WireGuard key: +// 44 base64 characters decoding to 32 bytes. +func looksLikeWGKey(s string) bool { + if len(s) != 44 || s[43] != '=' { + return false + } + decoded, err := base64.StdEncoding.DecodeString(s) + return err == nil && len(decoded) == 32 +} + +func generateAnonymousKey() string { + buf := make([]byte, 32) + if _, err := rand.Read(buf); err != nil { + return strings.Repeat("A", 43) + "=" + } + return base64.StdEncoding.EncodeToString(buf) +} + +// protectedSuffix returns the protected NetBird suffix baseDomain ends with, +// or empty. The match is label-anchored so an unrelated domain that merely +// ends in the same characters is not preserved. +func protectedSuffix(baseDomain string) string { + for _, d := range protectedDomains { + if baseDomain == d || strings.HasSuffix(baseDomain, "."+d) { + return d + } + } + return "" +} + +func withTrailingDot(domain string, hasDot bool) string { + if hasDot { + return domain + "." + } + return domain +} + +// isCGNAT reports whether addr is in 100.64.0.0/10 (RFC 6598), the range +// NetBird assigns overlay peer addresses from. +func isCGNAT(addr netip.Addr) bool { cgnatRangeStart := netip.AddrFrom4([4]byte{100, 64, 0, 0}) cgnatRange := netip.PrefixFrom(cgnatRangeStart, 10) diff --git a/client/anonymize/anonymize_test.go b/client/anonymize/anonymize_test.go index 852315fa1..7c3c7bcf8 100644 --- a/client/anonymize/anonymize_test.go +++ b/client/anonymize/anonymize_test.go @@ -1,8 +1,11 @@ package anonymize_test import ( + "bytes" + "encoding/base64" "net/netip" "regexp" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -44,6 +47,301 @@ func TestAnonymizeIP(t *testing.T) { } } +func TestParseLevel(t *testing.T) { + tests := []struct { + input string + expect anonymize.Level + }{ + {"", anonymize.LevelDefault}, + {"default", anonymize.LevelDefault}, + {"DEFAULT", anonymize.LevelDefault}, + {"strict", anonymize.LevelStrict}, + {"STRICT", anonymize.LevelStrict}, + // Unknown values must never yield less anonymization than requested. + {"garbage", anonymize.LevelStrict}, + } + + for _, tc := range tests { + t.Run("input="+tc.input, func(t *testing.T) { + assert.Equal(t, tc.expect, anonymize.ParseLevel(tc.input), "parsed level should match") + }) + } +} + +func TestAnonymizeIP_DefaultLevelInternalRanges(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + tests := []struct { + name string + ip string + expect string + }{ + {"RFC1918 10/8", "10.1.2.3", "10.1.2.3"}, + {"RFC1918 172.16/12", "172.16.5.5", "172.16.5.5"}, + {"RFC1918 192.168/16", "192.168.1.1", "192.168.1.1"}, + {"CGNAT", "100.64.0.5", "100.64.0.5"}, + {"IPv4 link-local", "169.254.1.1", "169.254.1.1"}, + {"IPv6 link-local", "fe80::1", "fe80::1"}, + // ULA is anonymized even at the default level: its random global ID + // uniquely fingerprints the network, unlike shared RFC 1918 space. + {"IPv6 ULA", "fd12:3456:789a::1", "2001:db8:ffff::"}, + // 4-in-6 addresses classify like their unmapped IPv4 form. + {"4-in-6 RFC1918", "::ffff:192.168.1.1", "192.168.1.1"}, + {"4-in-6 CGNAT", "::ffff:100.64.0.5", "100.64.0.5"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := anonymizer.AnonymizeIP(netip.MustParseAddr(tc.ip)) + assert.Equal(t, tc.expect, result.String(), "default level should preserve internal ranges except ULA") + }) + } +} + +func TestAnonymizeIP_StrictLevel(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + // Order matters: internal pool addresses are assigned sequentially. + tests := []struct { + name string + ip string + expect string + }{ + {"RFC1918 192.168/16", "192.168.1.1", "198.18.0.0"}, + {"Second RFC1918", "192.168.1.2", "198.18.0.1"}, + {"Repeated RFC1918", "192.168.1.1", "198.18.0.0"}, + {"RFC1918 10/8", "10.1.2.3", "198.18.0.2"}, + {"RFC1918 172.16/12", "172.16.5.5", "198.18.0.3"}, + {"CGNAT", "100.64.0.5", "198.18.0.4"}, + {"IPv4 link-local", "169.254.1.1", "198.18.0.5"}, + {"Public IPv4 uses public pool", "1.2.3.4", "198.51.100.0"}, + {"IPv6 link-local", "fe80::1", "2001:db8:1::"}, + {"IPv6 ULA", "fd12:3456:789a::1", "2001:db8:1::1"}, + {"Public IPv6 uses public pool", "2607:f8b0:4005:805::200e", "2001:db8:ffff::"}, + {"Loopback IPv4", "127.0.0.1", "127.0.0.1"}, + {"Loopback IPv6", "::1", "::1"}, + {"Unspecified", "0.0.0.0", "0.0.0.0"}, + {"Multicast", "224.0.0.251", "224.0.0.251"}, + {"Well known resolver", "8.8.8.8", "8.8.8.8"}, + {"Well known split marker", "128.0.0.0", "128.0.0.0"}, + {"In internal pool range", "198.18.0.3", "198.18.0.3"}, + {"In public pool range", "198.51.100.0", "198.51.100.0"}, + {"4-in-6 repeated RFC1918", "::ffff:192.168.1.1", "198.18.0.0"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := anonymizer.AnonymizeIP(netip.MustParseAddr(tc.ip)) + assert.Equal(t, tc.expect, result.String(), "strict level should replace internal ranges from the internal pools") + }) + } +} + +func TestAnonymizeString_StrictInternalIPs(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + input := "route 10.20.30.0/24 via 192.168.1.1 dev eth0 src 100.64.0.7" + firstPass := anonymizer.AnonymizeString(input) + secondPass := anonymizer.AnonymizeString(firstPass) + + assert.NotContains(t, firstPass, "10.20.30.0", "private network address should be anonymized") + assert.NotContains(t, firstPass, "192.168.1.1", "private gateway should be anonymized") + assert.NotContains(t, firstPass, "100.64.0.7", "CGNAT address should be anonymized") + assert.Contains(t, firstPass, "/24", "prefix length should be preserved") + assert.Equal(t, firstPass, secondPass, "second pass should not further anonymize the string") +} + +func TestAnonymizeMAC(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + first := anonymizer.AnonymizeMAC("aa:bb:cc:dd:ee:0f") + assert.Equal(t, "02:00:00:00:00:01", first, "first MAC should get the first placeholder") + assert.Equal(t, first, anonymizer.AnonymizeMAC("aa:bb:cc:dd:ee:0f"), "repeated MAC should map to the same placeholder") + assert.Equal(t, first, anonymizer.AnonymizeMAC("AA:BB:CC:DD:EE:0F"), "case should not affect the mapping") + assert.Equal(t, "02-00-00-00-00-01", anonymizer.AnonymizeMAC("AA-BB-CC-DD-EE-0F"), "dash form should keep its separator but share the mapping") + + second := anonymizer.AnonymizeMAC("10:22:33:44:55:66") + assert.Equal(t, "02:00:00:00:00:02", second, "second distinct MAC should get the next placeholder") + + tests := []struct { + name string + mac string + }{ + {"Broadcast", "ff:ff:ff:ff:ff:ff"}, + {"IPv4 multicast", "01:00:5e:00:00:fb"}, + {"IPv6 multicast", "33:33:00:00:00:01"}, + {"All zero", "00:00:00:00:00:00"}, + {"Assigned placeholder", "02:00:00:00:00:01"}, + {"Invalid", "not-a-mac"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.mac, anonymizer.AnonymizeMAC(tc.mac), "should be preserved") + }) + } +} + +func TestAnonymizeString_MACAddresses(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + tests := []struct { + name string + input string + expect string + }{ + { + name: "nftables ether rule", + input: "ether saddr aa:bb:cc:dd:ee:ff drop", + expect: "ether saddr 02:00:00:00:00:01 drop", + }, + { + name: "Windows dash form", + input: "Physical Address : AA-BB-CC-DD-EE-FF", + expect: "Physical Address : 02-00-00-00-00-01", + }, + { + name: "IPv6 address tail is not treated as MAC", + input: "addr fe80:0:11:22:33:44:55:66 scope link", + expect: "addr fe80:0:11:22:33:44:55:66 scope link", + }, + { + name: "broadcast MAC preserved", + input: "dst ff:ff:ff:ff:ff:ff type ARP", + expect: "dst ff:ff:ff:ff:ff:ff type ARP", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := anonymizer.AnonymizeString(tc.input) + assert.Equal(t, tc.expect, result, "MAC addresses should be anonymized at every level") + assert.Equal(t, result, anonymizer.AnonymizeString(result), "second pass should not change the result") + }) + } +} + +func TestAnonymizeWGKey(t *testing.T) { + key := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0x42}, 32)) + + t.Run("default level preserves keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + assert.Equal(t, key, anonymizer.AnonymizeWGKey(key), "default level should not touch WireGuard keys") + }) + + t.Run("strict level replaces keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + anon := anonymizer.AnonymizeWGKey(key) + assert.NotEqual(t, key, anon, "strict level should replace the key") + assert.Regexp(t, `^[A-Za-z0-9+/]{43}=$`, anon, "placeholder should keep the WireGuard key shape") + assert.Equal(t, anon, anonymizer.AnonymizeWGKey(key), "repeated key should map to the same placeholder") + assert.Equal(t, anon, anonymizer.AnonymizeWGKey(anon), "an assigned placeholder should pass through unchanged") + + assert.Equal(t, "not-a-key", anonymizer.AnonymizeWGKey("not-a-key"), "non-key values should be preserved") + }) +} + +func TestAnonymizeString_WGKeys(t *testing.T) { + key := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0x42}, 32)) + input := "peer " + key + " handshake completed" + + t.Run("default level preserves keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + assert.Equal(t, input, anonymizer.AnonymizeString(input), "default level should not touch WireGuard keys in strings") + }) + + t.Run("strict level replaces keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + firstPass := anonymizer.AnonymizeString(input) + assert.NotContains(t, firstPass, key, "the key should not survive strict anonymization") + assert.Equal(t, anonymizer.AnonymizeWGKey(key), extractKey(t, firstPass), "string replacement should be consistent with AnonymizeWGKey") + assert.Equal(t, firstPass, anonymizer.AnonymizeString(firstPass), "second pass should not change the result") + }) +} + +func extractKey(t *testing.T, logLine string) string { + t.Helper() + fields := strings.Fields(logLine) + require.Len(t, fields, 4, "log line should keep its structure") + return fields[1] +} + +func TestAnonymizeDomain_StrictLevel(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + t.Run("netbird peer name", func(t *testing.T) { + result := anonymizer.AnonymizeDomain("my-laptop.netbird.cloud") + assert.Regexp(t, `^peer-\d+\.netbird\.cloud$`, result, "peer name should be anonymized, suffix kept") + assert.NotContains(t, result, "my-laptop", "the peer name should not survive") + assert.Equal(t, result, anonymizer.AnonymizeDomain("my-laptop.netbird.cloud"), "repeated domain should map consistently") + assert.Equal(t, result, anonymizer.AnonymizeDomain(result), "an anonymized domain should pass through unchanged") + }) + + t.Run("bare netbird domain", func(t *testing.T) { + assert.Equal(t, "netbird.cloud", anonymizer.AnonymizeDomain("netbird.cloud"), "the bare protected suffix should be preserved") + }) + + t.Run("netbird infrastructure preserved", func(t *testing.T) { + assert.Equal(t, "api.netbird.io", anonymizer.AnonymizeDomain("api.netbird.io"), + "netbird.io hosts infrastructure, not peer names, and should stay readable") + }) + + t.Run("leading labels of other domains", func(t *testing.T) { + result := anonymizer.AnonymizeDomain("host1.corp.example.com") + assert.Regexp(t, `^host-\d+\.host-\d+\.anon-[a-zA-Z0-9]+\.domain$`, result, "every label should be anonymized") + for _, label := range []string{"host1", "corp", "example"} { + assert.NotContains(t, result, label, "no original label should survive") + } + assert.Equal(t, result, anonymizer.AnonymizeDomain("host1.corp.example.com"), "repeated domain should map consistently") + }) + + t.Run("same label maps consistently across domains", func(t *testing.T) { + first := anonymizer.AnonymizeDomain("shared.one.com") + second := anonymizer.AnonymizeDomain("shared.two.com") + assert.Equal(t, strings.Split(first, ".")[0], strings.Split(second, ".")[0], "the shared host label should get one placeholder") + }) + + t.Run("wildcard label preserved", func(t *testing.T) { + result := anonymizer.AnonymizeDomain("*.example.com") + assert.Regexp(t, `^\*\.anon-[a-zA-Z0-9]+\.domain$`, result, "the wildcard label should stay a wildcard") + }) +} + +func TestAnonymizeDomain_DefaultLevelKeepsPeerNames(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + assert.Equal(t, "my-laptop.netbird.cloud", anonymizer.AnonymizeDomain("my-laptop.netbird.cloud"), + "default level should preserve netbird FQDNs including the peer name") + assert.Regexp(t, `^sub\.anon-[a-zA-Z0-9]+\.domain$`, anonymizer.AnonymizeDomain("sub.example.com"), + "default level should keep subdomain labels") +} + +func TestAnonymizeString_StrictPeerNames(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + // Seed like the bundle generator does from the status: base first, then + // the full FQDN, so replacement must prefer the longer mapping. + anonBase := anonymizer.AnonymizeDomain("example.com") + anonPeer := anonymizer.AnonymizeDomain("peer1.netbird.cloud") + anonHost := anonymizer.AnonymizeDomain("host1.example.com") + + logLine := "connected to peer1.netbird.cloud via host1.example.com endpoint" + firstPass := anonymizer.AnonymizeString(logLine) + assert.NotContains(t, firstPass, "peer1", "the peer name should not survive in logs") + assert.NotContains(t, firstPass, "host1", "the host label should not survive in logs") + assert.Contains(t, firstPass, anonPeer, "the seeded peer mapping should be applied") + assert.Contains(t, firstPass, anonHost, "the seeded host mapping should be applied, not just the base mapping") + assert.NotContains(t, firstPass, "host1."+anonBase, "the base mapping must not preempt the longer FQDN mapping") + assert.Equal(t, firstPass, anonymizer.AnonymizeString(firstPass), "second pass should not change the result") +} + func TestAnonymizeDNSLogLine(t *testing.T) { anonymizer := anonymize.NewAnonymizer(netip.Addr{}, netip.Addr{}) tests := []struct { diff --git a/client/anonymize/reverse_zone.go b/client/anonymize/reverse_zone.go new file mode 100644 index 000000000..b521b71b7 --- /dev/null +++ b/client/anonymize/reverse_zone.go @@ -0,0 +1,174 @@ +package anonymize + +import ( + "encoding/hex" + "net/netip" + "regexp" + "strconv" + "strings" +) + +const ( + reverseZoneSuffixV4 = ".in-addr.arpa" + reverseZoneSuffixV6 = ".ip6.arpa" + + v6Nibbles = 32 + v4Octets = 4 +) + +// reverseZoneRegexes match a reverse zone or a full reverse name in free text. +// They are applied before the address passes of AnonymizeString, whose IPv4 +// pattern would otherwise consume the digit labels of a zone and replace parts +// of it with unrelated addresses. +var reverseZoneRegexes = []*regexp.Regexp{ + regexp.MustCompile(`(?:[0-9]{1,3}\.){1,4}in-addr\.arpa\b`), + regexp.MustCompile(`(?:[0-9a-fA-F]\.){1,32}ip6\.arpa\b`), +} + +// anonymizeReverseZone maps a reverse zone to the zone of the anonymized form +// of the prefix it encodes, so it follows the address rules rather than the +// domain ones: the zone of an address that is preserved is preserved too, and +// the zone of one that is replaced names the replacement. This keeps a reverse +// zone recognizable as such, and consistent with the addresses it belongs to +// elsewhere in the same output. It reports false for anything that is not a +// reverse zone. +func (a *Anonymizer) anonymizeReverseZone(domain string) (string, bool) { + prefix, labelCount, suffix, ok := parseReverseZone(domain) + if !ok { + return "", false + } + + anonymized := a.AnonymizeIP(prefix) + if anonymized == prefix { + return domain, true + } + + return reverseZoneName(anonymized, labelCount) + suffix, true +} + +// replaceReverseZones anonymizes every reverse zone in str and swaps each one +// for a placeholder, returning a function that puts the anonymized zones back. +// The placeholders carry no dots, digits or colons, so no later pass matches +// them. +func (a *Anonymizer) replaceReverseZones(str string) (string, func(string) string) { + var zones []string + + for _, re := range reverseZoneRegexes { + str = re.ReplaceAllStringFunc(str, func(match string) string { + zone, ok := a.anonymizeReverseZone(match) + if !ok { + return match + } + + zones = append(zones, zone) + return reverseZonePlaceholder(len(zones) - 1) + }) + } + + if len(zones) == 0 { + return str, func(s string) string { return s } + } + + return str, func(s string) string { + for i, zone := range zones { + s = strings.ReplaceAll(s, reverseZonePlaceholder(i), zone) + } + return s + } +} + +func reverseZonePlaceholder(index int) string { + return "\x00reversezone" + strconv.Itoa(index) + "\x00" +} + +// parseReverseZone turns a reverse zone into the address of the prefix its +// labels spell backwards, padding the absent low-order part with zeroes, and +// returns the label count and zone suffix so the name can be rebuilt. +func parseReverseZone(domain string) (netip.Addr, int, string, bool) { + lower := strings.ToLower(domain) + + switch { + case strings.HasSuffix(lower, reverseZoneSuffixV4): + labels := strings.Split(strings.TrimSuffix(lower, reverseZoneSuffixV4), ".") + addr, ok := reverseZoneAddrV4(labels) + return addr, len(labels), reverseZoneSuffixV4, ok + case strings.HasSuffix(lower, reverseZoneSuffixV6): + labels := strings.Split(strings.TrimSuffix(lower, reverseZoneSuffixV6), ".") + addr, ok := reverseZoneAddrV6(labels) + return addr, len(labels), reverseZoneSuffixV6, ok + default: + return netip.Addr{}, 0, "", false + } +} + +func reverseZoneAddrV4(labels []string) (netip.Addr, bool) { + if len(labels) == 0 || len(labels) > v4Octets { + return netip.Addr{}, false + } + + var octets [v4Octets]byte + for i, label := range labels { + octet, err := strconv.ParseUint(label, 10, 8) + if err != nil { + return netip.Addr{}, false + } + octets[len(labels)-1-i] = byte(octet) + } + + return netip.AddrFrom4(octets), true +} + +func reverseZoneAddrV6(labels []string) (netip.Addr, bool) { + if len(labels) == 0 || len(labels) > v6Nibbles { + return netip.Addr{}, false + } + + nibbles := make([]byte, 0, v6Nibbles) + for i := len(labels) - 1; i >= 0; i-- { + if len(labels[i]) != 1 || !isHexDigit(labels[i][0]) { + return netip.Addr{}, false + } + nibbles = append(nibbles, labels[i][0]) + } + for len(nibbles) < v6Nibbles { + nibbles = append(nibbles, '0') + } + + var groups []string + for i := 0; i < len(nibbles); i += 4 { + groups = append(groups, string(nibbles[i:i+4])) + } + + addr, err := netip.ParseAddr(strings.Join(groups, ":")) + if err != nil { + return netip.Addr{}, false + } + + return addr, true +} + +// reverseZoneName spells the first labelCount labels of addr backwards, the +// inverse of parseReverseZone, without the zone suffix. +func reverseZoneName(addr netip.Addr, labelCount int) string { + labels := make([]string, 0, labelCount) + + if addr.Is4() { + octets := addr.As4() + for i := labelCount - 1; i >= 0; i-- { + labels = append(labels, strconv.Itoa(int(octets[i]))) + } + return strings.Join(labels, ".") + } + + address := addr.As16() + nibbles := hex.EncodeToString(address[:]) + for i := labelCount - 1; i >= 0; i-- { + labels = append(labels, string(nibbles[i])) + } + + return strings.Join(labels, ".") +} + +func isHexDigit(c byte) bool { + return c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F' +} diff --git a/client/anonymize/reverse_zone_test.go b/client/anonymize/reverse_zone_test.go new file mode 100644 index 000000000..8c3b8954a --- /dev/null +++ b/client/anonymize/reverse_zone_test.go @@ -0,0 +1,171 @@ +package anonymize + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newLeveledAnonymizer(level Level) *Anonymizer { + a := NewAnonymizer(DefaultAddresses()) + a.SetLevel(level) + return a +} + +// TestAnonymizeDomainReverseZone covers reverse zones going through the address +// rules instead of the domain ones, so a zone stays a zone and an address that +// is preserved keeps the zone that names it. +func TestAnonymizeDomainReverseZone(t *testing.T) { + // 100.64.0.0/10 is the overlay range, which is CGNAT: preserved at the + // default level and replaced from the internal pool at the strict one + const overlayZone = "64.100.in-addr.arpa" + + t.Run("overlay zone preserved at the default level", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + assert.Equal(t, overlayZone, a.AnonymizeDomain(overlayZone), "should keep the zone of a preserved address") + }) + + t.Run("private zone preserved at the default level", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + assert.Equal(t, "168.192.in-addr.arpa", a.AnonymizeDomain("168.192.in-addr.arpa"), "should keep the zone of a private address") + }) + + t.Run("overlay zone replaced at the strict level", func(t *testing.T) { + a := newLeveledAnonymizer(LevelStrict) + + got := a.AnonymizeDomain(overlayZone) + require.True(t, strings.HasSuffix(got, reverseZoneSuffixV4), "should stay a reverse zone, got %q", got) + assert.NotEqual(t, overlayZone, got, "should replace the encoded prefix") + assert.Len(t, strings.Split(strings.TrimSuffix(got, reverseZoneSuffixV4), "."), 2, + "should keep the label count, got %q", got) + }) + + t.Run("public zone replaced at the default level", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + got := a.AnonymizeDomain("113.0.203.in-addr.arpa") + require.True(t, strings.HasSuffix(got, reverseZoneSuffixV4), "should stay a reverse zone, got %q", got) + assert.NotEqual(t, "113.0.203.in-addr.arpa", got, "should replace a public prefix") + }) + + t.Run("zone of an address keeps that address mapping", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + anonymizedAddr := a.AnonymizeIPString("203.0.113.7") + got := a.AnonymizeDomain("7.113.0.203.in-addr.arpa") + + octets := strings.Split(anonymizedAddr, ".") + want := octets[3] + "." + octets[2] + "." + octets[1] + "." + octets[0] + reverseZoneSuffixV4 + assert.Equal(t, want, got, "should name the same replacement as the address itself") + }) + + t.Run("ipv6 nibble labels stay single digits", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + zone := "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.0.0.0" + reverseZoneSuffixV6 + got := a.AnonymizeDomain(zone) + + require.True(t, strings.HasSuffix(got, reverseZoneSuffixV6), "should stay a reverse zone, got %q", got) + labels := strings.Split(strings.TrimSuffix(got, reverseZoneSuffixV6), ".") + assert.Len(t, labels, 28, "should keep every nibble label, got %q", got) + for _, label := range labels { + assert.Len(t, label, 1, "nibble label %q should stay a single digit", label) + } + }) + + t.Run("trailing dot is kept", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + assert.Equal(t, "64.100.in-addr.arpa.", a.AnonymizeDomain("64.100.in-addr.arpa."), "should keep the trailing dot") + }) + + t.Run("a domain that only looks like a zone is anonymized as a domain", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + got := a.AnonymizeDomain("not-a-zone.in-addr.arpa") + assert.NotContains(t, got, "in-addr.arpa", "should fall back to domain anonymization") + }) +} + +// TestAnonymizeStringReverseZone verifies that a zone inside free text, such as +// a DNS log line, is not chewed up by the address passes. The IPv4 pattern +// matches any run of dotted digits, which a reverse zone is made of. +func TestAnonymizeStringReverseZone(t *testing.T) { + t.Run("ipv6 zone survives the address passes", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + zone := "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.0.0.0" + reverseZoneSuffixV6 + got := a.AnonymizeString("question: domain=" + zone + " type=PTR") + + assert.Contains(t, got, "type=PTR", "should keep the rest of the line") + assert.NotContains(t, got, "198.51.100", "should not rewrite nibble labels as an address") + + labels := strings.Split(strings.TrimSuffix(strings.TrimPrefix(got, "question: domain="), reverseZoneSuffixV6+" type=PTR"), ".") + assert.Len(t, labels, 28, "should keep every nibble label, got %q", got) + }) + + t.Run("preserved ipv4 zone is untouched", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + line := "reverse zone 64.100.in-addr.arpa registered" + assert.Equal(t, line, a.AnonymizeString(line), "should keep the zone of a preserved address") + }) + + t.Run("public ipv4 zone is replaced consistently", func(t *testing.T) { + a := newLeveledAnonymizer(LevelDefault) + + got := a.AnonymizeString("zone 113.0.203.in-addr.arpa and address 203.0.113.7") + assert.NotContains(t, got, "113.0.203.in-addr.arpa", "should replace the zone") + assert.NotContains(t, got, "203.0.113.7", "should replace the address") + assert.Contains(t, got, reverseZoneSuffixV4, "should keep the zone suffix") + }) +} + +func TestParseReverseZone(t *testing.T) { + tests := []struct { + name string + zone string + addr string + labels int + }{ + {name: "v4 two labels", zone: "0.100" + reverseZoneSuffixV4, addr: "100.0.0.0", labels: 2}, + {name: "v4 three labels", zone: "1.168.192" + reverseZoneSuffixV4, addr: "192.168.1.0", labels: 3}, + {name: "v4 full address", zone: "7.113.0.203" + reverseZoneSuffixV4, addr: "203.0.113.7", labels: 4}, + { + name: "v6 prefix", + zone: "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.0.0.0" + reverseZoneSuffixV6, + addr: "2::", + labels: 28, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + addr, labels, suffix, ok := parseReverseZone(tc.zone) + require.True(t, ok, "should decode the reverse zone") + assert.Equal(t, tc.addr, addr.String(), "should decode to the encoded prefix") + assert.Equal(t, tc.labels, labels, "should count the labels") + assert.Equal(t, tc.zone, reverseZoneName(addr, labels)+suffix, "should re-encode to the original zone") + }) + } +} + +func TestParseReverseZoneRejectsNonZones(t *testing.T) { + tests := []string{ + "example.com", + "in-addr.arpa", + "x.100" + reverseZoneSuffixV4, + "256" + reverseZoneSuffixV4, + "1.2.3.4.5" + reverseZoneSuffixV4, + "ab" + reverseZoneSuffixV6, + "g" + reverseZoneSuffixV6, + } + + for _, zone := range tests { + t.Run(zone, func(t *testing.T) { + _, _, _, ok := parseReverseZone(zone) + assert.False(t, ok, "should reject %q", zone) + }) + } +} diff --git a/client/cmd/debug.go b/client/cmd/debug.go index 7ddc3afc4..893b1e248 100644 --- a/client/cmd/debug.go +++ b/client/cmd/debug.go @@ -27,8 +27,8 @@ import ( const errCloseConnection = "Failed to close connection: %v" var ( - logFileCount uint32 - systemInfoFlag bool + logFileCount uint32 + systemInfoFlag bool uploadBundleFlag bool uploadBundleURLFlag string uploadBundleInsecureFlag bool @@ -156,6 +156,11 @@ func debugConfigDump(cmd *cobra.Command, _ []string) error { // request. Returns an error if the RPC fails or if the daemon reports // an upload failure reason. func debugBundle(cmd *cobra.Command, _ []string) error { + anonymizeEnabled, anonymizeLevel, err := effectiveAnonymize() + if err != nil { + return err + } + conn, err := getClient(cmd) if err != nil { return err @@ -168,10 +173,11 @@ func debugBundle(cmd *cobra.Command, _ []string) error { client := proto.NewDaemonServiceClient(conn) request := &proto.DebugBundleRequest{ - Anonymize: anonymizeFlag, - SystemInfo: systemInfoFlag, - LogFileCount: logFileCount, - CliVersion: version.NetbirdVersion(), + Anonymize: anonymizeEnabled, + AnonymizeLevel: anonymizeLevel.String(), + SystemInfo: systemInfoFlag, + LogFileCount: logFileCount, + CliVersion: version.NetbirdVersion(), } if uploadBundleFlag { request.UploadURL = uploadBundleURLFlag @@ -229,6 +235,11 @@ func runForDuration(cmd *cobra.Command, args []string) error { return fmt.Errorf("invalid duration format: %v", err) } + anonymizeEnabled, anonymizeLevel, err := effectiveAnonymize() + if err != nil { + return err + } + conn, err := getClient(cmd) if err != nil { return err @@ -368,10 +379,11 @@ func runForDuration(cmd *cobra.Command, args []string) error { cmd.Println("Creating debug bundle...") request := &proto.DebugBundleRequest{ - Anonymize: anonymizeFlag, - SystemInfo: systemInfoFlag, - LogFileCount: logFileCount, - CliVersion: version.NetbirdVersion(), + Anonymize: anonymizeEnabled, + AnonymizeLevel: anonymizeLevel.String(), + SystemInfo: systemInfoFlag, + LogFileCount: logFileCount, + CliVersion: version.NetbirdVersion(), } if uploadBundleFlag { request.UploadURL = uploadBundleURLFlag diff --git a/client/cmd/login.go b/client/cmd/login.go index a53cb6d5f..6aa019896 100644 --- a/client/cmd/login.go +++ b/client/cmd/login.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "os/user" - "runtime" "strings" log "github.com/sirupsen/logrus" @@ -121,7 +120,7 @@ func doDaemonLogin(ctx context.Context, cmd *cobra.Command, providedSetupKey str loginRequest := proto.LoginRequest{ SetupKey: providedSetupKey, ManagementUrl: managementURL, - IsUnixDesktopClient: isUnixRunningDesktop(), + IsUnixDesktopClient: util.HasGraphicalSession(), Hostname: hostName, DnsLabels: dnsLabelsReq, ProfileName: &handle, @@ -189,7 +188,8 @@ func doExtendSession(ctx context.Context, cmd *cobra.Command) error { client := proto.NewDaemonServiceClient(conn) - req := &proto.RequestExtendAuthSessionRequest{} + // the CLI runs in the user's session, the daemon does not: tell it what we can see + req := &proto.RequestExtendAuthSessionRequest{HasGraphicalSession: util.HasGraphicalSession()} // Pre-fill the IdP login hint from the active profile so the user // doesn't have to retype their email. Best-effort: we still proceed // without a hint if the lookup fails. @@ -408,7 +408,7 @@ func foregroundGetTokenInfo(ctx context.Context, cmd *cobra.Command, config *pro hint = profileState.Email } - oAuthFlow, err := auth.NewOAuthFlow(ctx, config, isUnixRunningDesktop(), false, hint) + oAuthFlow, err := auth.NewOAuthFlow(ctx, config, util.HasGraphicalSession(), false, hint) if err != nil { return nil, err } @@ -458,14 +458,6 @@ func openURL(cmd *cobra.Command, verificationURIComplete, userCode string, noBro } } -// isUnixRunningDesktop checks if a Linux OS is running desktop environment -func isUnixRunningDesktop() bool { - if runtime.GOOS != "linux" && runtime.GOOS != "freebsd" { - return false - } - return os.Getenv("DESKTOP_SESSION") != "" || os.Getenv("XDG_CURRENT_DESKTOP") != "" -} - func setEnvAndFlags(cmd *cobra.Command) error { SetFlagsFromEnvVars(rootCmd) diff --git a/client/cmd/root.go b/client/cmd/root.go index ebaae7e3e..ccad78942 100644 --- a/client/cmd/root.go +++ b/client/cmd/root.go @@ -21,6 +21,7 @@ import ( "github.com/spf13/pflag" "google.golang.org/grpc" + "github.com/netbirdio/netbird/client/anonymize" daddr "github.com/netbirdio/netbird/client/internal/daemonaddr" "github.com/netbirdio/netbird/client/internal/profilemanager" ) @@ -69,6 +70,7 @@ var ( autoConnectDisabled bool extraIFaceBlackList []string anonymizeFlag bool + anonymizeLevelFlag string dnsRouteInterval time.Duration // lazyConnEnabled is the parse target for the deprecated --enable-lazy-connection // flag. The flag is inert; the value is no longer read (use NB_LAZY_CONN instead). @@ -156,7 +158,8 @@ func init() { rootCmd.MarkFlagsMutuallyExclusive("setup-key", "setup-key-file") rootCmd.PersistentFlags().StringVar(&preSharedKey, preSharedKeyFlag, "", "Sets WireGuard PreSharedKey property. If set, then only peers that have the same key can communicate.") rootCmd.PersistentFlags().StringVarP(&hostName, "hostname", "n", "", "Sets a custom hostname for the device") - rootCmd.PersistentFlags().BoolVarP(&anonymizeFlag, "anonymize", "A", false, "anonymize IP addresses and non-netbird.io domains in logs and status output") + rootCmd.PersistentFlags().BoolVarP(&anonymizeFlag, "anonymize", "A", false, "anonymize public IP addresses, MAC addresses, and non-netbird.io domains in logs and status output; private, CGNAT, and link-local IP ranges are kept (see --anonymize-level strict)") + rootCmd.PersistentFlags().StringVar(&anonymizeLevelFlag, "anonymize-level", "", "anonymization level: \"default\" or \"strict\"; strict also anonymizes private, CGNAT, and link-local IP ranges, peer names, and WireGuard public keys. Setting this flag implies --anonymize") rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", profilemanager.DefaultConfigPath, "Overrides the default profile file location") rootCmd.AddCommand(upCmd) @@ -293,6 +296,19 @@ var CLIBackOffSettings = &backoff.ExponentialBackOff{ Clock: backoff.SystemClock, } +// effectiveAnonymize resolves the --anonymize and --anonymize-level flags: +// setting a level implies anonymization, and an invalid level is rejected. +func effectiveAnonymize() (bool, anonymize.Level, error) { + if anonymizeLevelFlag == "" { + return anonymizeFlag, anonymize.LevelDefault, nil + } + level := anonymize.ParseLevel(anonymizeLevelFlag) + if !strings.EqualFold(anonymizeLevelFlag, level.String()) { + return false, anonymize.LevelDefault, fmt.Errorf("invalid anonymize level %q: use %q or %q", anonymizeLevelFlag, anonymize.LevelDefault.String(), anonymize.LevelStrict.String()) + } + return true, level, nil +} + func getSetupKey() (string, error) { if setupKeyPath != "" && setupKey == "" { return getSetupKeyFromFile(setupKeyPath) diff --git a/client/cmd/status.go b/client/cmd/status.go index c4057ed82..f2e5bcc66 100644 --- a/client/cmd/status.go +++ b/client/cmd/status.go @@ -121,8 +121,14 @@ func statusFunc(cmd *cobra.Command, args []string) error { sessionExpiresAt = ts.AsTime().UTC() } + anonymizeEnabled, anonymizeLevel, err := effectiveAnonymize() + if err != nil { + return err + } + var outputInformationHolder = nbstatus.ConvertToStatusOutputOverview(resp.GetFullStatus(), nbstatus.ConvertOptions{ - Anonymize: anonymizeFlag, + Anonymize: anonymizeEnabled, + AnonymizeLevel: anonymizeLevel, DaemonVersion: resp.GetDaemonVersion(), DaemonStatus: nbstatus.ParseDaemonStatus(status), StatusFilter: statusFilter, diff --git a/client/cmd/up.go b/client/cmd/up.go index 142bcf6bd..9f4fa8c33 100644 --- a/client/cmd/up.go +++ b/client/cmd/up.go @@ -21,8 +21,8 @@ import ( "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/profilemanager" - "github.com/netbirdio/netbird/client/proto" nbnet "github.com/netbirdio/netbird/client/net" + "github.com/netbirdio/netbird/client/proto" "github.com/netbirdio/netbird/client/server" "github.com/netbirdio/netbird/client/system" "github.com/netbirdio/netbird/shared/management/domain" @@ -626,7 +626,7 @@ func setupLoginRequest(providedSetupKey string, customDNSAddressConverted []byte NatExternalIPs: natExternalIPs, CleanNATExternalIPs: natExternalIPs != nil && len(natExternalIPs) == 0, CustomDNSAddress: customDNSAddressConverted, - IsUnixDesktopClient: isUnixRunningDesktop(), + IsUnixDesktopClient: util.HasGraphicalSession(), Hostname: hostName, ExtraIFaceBlacklist: extraIFaceBlackList, DnsLabels: dnsLabels, diff --git a/client/firewall/iptables/acl_linux.go b/client/firewall/iptables/acl_linux.go index 4b4cebf9c..89d1ebf7c 100644 --- a/client/firewall/iptables/acl_linux.go +++ b/client/firewall/iptables/acl_linux.go @@ -42,6 +42,7 @@ type aclManager struct { optionalEntries map[string][]entry ipsetStore *ipsetStore v6 bool + ipsetSupported bool stateManager *statemanager.Manager } @@ -60,6 +61,8 @@ func newAclManager(iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*acl func (m *aclManager) init(stateManager *statemanager.Manager) error { m.stateManager = stateManager + m.ipsetSupported = m.probeIPSetSupport() + m.seedInitialEntries() m.seedInitialOptionalEntries() @@ -91,6 +94,12 @@ func (m *aclManager) AddPeerFiltering( if m.v6 && ipsetName != "" { ipsetName += "-v6" } + // When the kernel lacks the required ipset hash module, fall back to + // per-IP iptables rules (pre-0.68 behavior) so ACLs keep working instead + // of silently leaving the chain empty. + if ipsetName != "" && !m.ipsetSupported { + ipsetName = "" + } proto := protoForFamily(protocol, m.v6) specs := filterRuleSpecs(ip, proto, sPort, dPort, action, ipsetName) @@ -498,6 +507,40 @@ func transformIPsetName(ipsetName string, sPort, dPort *firewall.Port, action fi } } +// probeIPSetSupport checks whether the kernel can create the ipset type used for +// ACL rules. On kernels lacking the required ipset hash module, ipset creation +// fails (e.g. "invalid argument"), which would otherwise leave the ACL chain +// empty and silently drop all policy-permitted inbound traffic. When unsupported, +// the manager falls back to per-IP iptables rules. +func (m *aclManager) probeIPSetSupport() bool { + // Use a unique name so concurrent processes don't collide and we only ever + // destroy the set we created ourselves. ipset names are limited to 31 chars, + // so use a short random suffix. + probeName := "nb-probe-" + uuid.New().String()[:8] + + opts := ipset.CreateOptions{ + Replace: true, + } + if m.v6 { + opts.Family = ipset.FamilyIPV6 + } + + if err := ipset.Create(probeName, ipset.TypeHashNet, opts); err != nil { + log.Warnf("ipset is not available (failed to create probe set: %v); "+ + "falling back to per-IP iptables ACL rules. Ensure the kernel provides "+ + "the ipset hash:net module (ip_set_hash_net) for better performance with large rule sets", err) + return false + } + + defer func() { + if err := ipset.Destroy(probeName); err != nil { + log.Debugf("destroy ipset probe set %q: %v", probeName, err) + } + }() + + return true +} + func (m *aclManager) createIPSet(name string) error { opts := ipset.CreateOptions{ Replace: true, diff --git a/client/firewall/iptables/dnat_refcount_linux_test.go b/client/firewall/iptables/dnat_refcount_linux_test.go new file mode 100644 index 000000000..681bc0b99 --- /dev/null +++ b/client/firewall/iptables/dnat_refcount_linux_test.go @@ -0,0 +1,240 @@ +//go:build privileged + +package iptables + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + fw "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/wgaddr" +) + +func iptRefcountIfaceV4() *iFaceMock { + return &iFaceMock{ + NameFunc: func() string { return "wt-refcount" }, + AddressFunc: func() wgaddr.Address { + return wgaddr.Address{ + IP: netip.MustParseAddr("10.20.0.1"), + Network: netip.MustParsePrefix("10.20.0.0/24"), + } + }, + } +} + +func iptRefcountIfaceDual() *iFaceMock { + return &iFaceMock{ + NameFunc: func() string { return "wt-refcount" }, + AddressFunc: func() wgaddr.Address { + return wgaddr.Address{ + IP: netip.MustParseAddr("10.20.0.1"), + Network: netip.MustParsePrefix("10.20.0.0/24"), + IPv6: netip.MustParseAddr("fd00::1"), + IPv6Net: netip.MustParsePrefix("fd00::/64"), + } + }, + } +} + +func newIptRefcountManager(t *testing.T, dual bool) *Manager { + t.Helper() + var ifMock *iFaceMock + if dual { + ifMock = iptRefcountIfaceDual() + } else { + ifMock = iptRefcountIfaceV4() + } + m, err := Create(ifMock, iface.DefaultMTU) + require.NoError(t, err, "create manager") + require.NoError(t, m.Init(nil), "init manager") + t.Cleanup(func() { + require.NoError(t, m.Close(nil), "close manager") + }) + return m +} + +func iptDnatV4(port uint16) fw.ForwardRule { + return fw.ForwardRule{ + Protocol: fw.ProtocolTCP, + DestinationPort: fw.Port{Values: []uint16{port}}, + TranslatedAddress: netip.MustParseAddr("10.20.0.2"), + TranslatedPort: fw.Port{Values: []uint16{80}}, + } +} + +func iptDnatV6(port uint16) fw.ForwardRule { + return fw.ForwardRule{ + Protocol: fw.ProtocolTCP, + DestinationPort: fw.Port{Values: []uint16{port}}, + TranslatedAddress: netip.MustParseAddr("fd00::2"), + TranslatedPort: fw.Port{Values: []uint16{80}}, + } +} + +// TestIptablesRouting_RepeatedEnableSingleReference verifies that EnableRouting +// (called on every network-map update) holds at most one reference per family +// and a single DisableRouting drops both back to zero. +func TestIptablesRouting_RepeatedEnableSingleReference(t *testing.T) { + m := newIptRefcountManager(t, true) + state := m.router.ipFwdState + + require.NoError(t, m.EnableRouting(), "first enable") + require.NoError(t, m.EnableRouting(), "second enable") + require.NoError(t, m.EnableRouting(), "third enable") + v4, v6 := state.Counts() + assert.Equal(t, 1, v4, "repeated enable holds a single v4 reference") + assert.Equal(t, 1, v6, "repeated enable holds a single v6 reference") + + require.NoError(t, m.DisableRouting(), "disable") + v4, v6 = state.Counts() + assert.Equal(t, 0, v4, "single disable releases the v4 reference") + assert.Equal(t, 0, v6, "single disable releases the v6 reference") +} + +// TestIptablesRouting_DisableKeepsDNATReference verifies that an unpaired +// DisableRouting does not release references held by active DNAT rules. +func TestIptablesRouting_DisableKeepsDNATReference(t *testing.T) { + m := newIptRefcountManager(t, true) + state := m.router.ipFwdState + + r1, err := m.AddDNATRule(iptDnatV6(9095)) + require.NoError(t, err, "add v6 dnat") + + require.NoError(t, m.DisableRouting(), "unpaired disable") + _, v6 := state.Counts() + assert.Equal(t, 1, v6, "DNAT-held reference survives unpaired DisableRouting") + + require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat") + _, v6 = state.Counts() + assert.Equal(t, 0, v6, "delete releases the DNAT reference") +} + +// TestIptablesDNAT_RefcountBalancedV4 covers a Balanced Add/Delete pair on v4. +func TestIptablesDNAT_RefcountBalancedV4(t *testing.T) { + m := newIptRefcountManager(t, false) + state := m.router.ipFwdState + + r1, err := m.AddDNATRule(iptDnatV4(7081)) + require.NoError(t, err, "add v4 dnat 1") + v4, v6 := state.Counts() + assert.Equal(t, 1, v4, "v4 refcount after first add") + assert.Equal(t, 0, v6, "v6 refcount unchanged") + + r2, err := m.AddDNATRule(iptDnatV4(7082)) + require.NoError(t, err, "add v4 dnat 2") + v4, v6 = state.Counts() + assert.Equal(t, 2, v4, "v4 refcount after second add") + assert.Equal(t, 0, v6, "v6 refcount unchanged") + + require.NoError(t, m.DeleteDNATRule(r1)) + v4, v6 = state.Counts() + assert.Equal(t, 1, v4, "v4 refcount after first delete") + assert.Equal(t, 0, v6, "v6 refcount unchanged") + + require.NoError(t, m.DeleteDNATRule(r2)) + v4, v6 = state.Counts() + assert.Equal(t, 0, v4, "v4 refcount after second delete") + assert.Equal(t, 0, v6, "v6 refcount unchanged") +} + +// TestIptablesDNAT_RefcountBalancedV6 checks the v6 path increments v6 only and +// decrements back to zero. +func TestIptablesDNAT_RefcountBalancedV6(t *testing.T) { + m := newIptRefcountManager(t, true) + require.NotNil(t, m.router6, "v6 router") + require.Same(t, m.router.ipFwdState, m.router6.ipFwdState, "shared state") + state := m.router.ipFwdState + + r1, err := m.AddDNATRule(iptDnatV6(9081)) + require.NoError(t, err, "add v6 dnat 1") + v4, v6 := state.Counts() + assert.Equal(t, 0, v4) + assert.Equal(t, 1, v6, "v6 refcount after first add") + + r2, err := m.AddDNATRule(iptDnatV6(9082)) + require.NoError(t, err, "add v6 dnat 2") + v4, v6 = state.Counts() + assert.Equal(t, 0, v4, "v4 refcount unchanged") + assert.Equal(t, 2, v6, "v6 refcount after second add") + + require.NoError(t, m.DeleteDNATRule(r1)) + v4, v6 = state.Counts() + assert.Equal(t, 0, v4, "v4 refcount unchanged") + assert.Equal(t, 1, v6, "v6 refcount after first delete") + + require.NoError(t, m.DeleteDNATRule(r2)) + v4, v6 = state.Counts() + assert.Equal(t, 0, v4) + assert.Equal(t, 0, v6, "v6 refcount after second delete") +} + +// TestIptablesDNAT_DuplicateAddNoLeak verifies the duplicate-rule path returns +// without bumping the refcount. +func TestIptablesDNAT_DuplicateAddNoLeak(t *testing.T) { + m := newIptRefcountManager(t, true) + state := m.router.ipFwdState + + rule := iptDnatV4(7083) + r1, err := m.AddDNATRule(rule) + require.NoError(t, err) + v4, _ := state.Counts() + assert.Equal(t, 1, v4) + + _, err = m.AddDNATRule(rule) + require.NoError(t, err, "duplicate add") + v4, _ = state.Counts() + assert.Equal(t, 1, v4, "duplicate add must not increment") + + require.NoError(t, m.DeleteDNATRule(r1)) + v4, _ = state.Counts() + assert.Equal(t, 0, v4, "single delete must drop to zero") +} + +// TestIptablesDNAT_DeleteMissingNoUnderflow verifies Delete on an unknown rule +// neither errors nor releases the refcount. +func TestIptablesDNAT_DeleteMissingNoUnderflow(t *testing.T) { + m := newIptRefcountManager(t, true) + state := m.router.ipFwdState + + phantom := iptDnatV4(7099) + require.NoError(t, m.DeleteDNATRule(&phantom), "delete missing v4") + v4, v6 := state.Counts() + assert.Equal(t, 0, v4) + assert.Equal(t, 0, v6) + + phantom6 := iptDnatV6(9099) + require.NoError(t, m.DeleteDNATRule(&phantom6), "delete missing v6") + v4, v6 = state.Counts() + assert.Equal(t, 0, v4) + assert.Equal(t, 0, v6) + + r1, err := m.AddDNATRule(iptDnatV4(7100)) + require.NoError(t, err) + v4, _ = state.Counts() + assert.Equal(t, 1, v4, "real add still increments after phantom delete") + require.NoError(t, m.DeleteDNATRule(r1)) +} + +// TestIptablesDNAT_DoubleDeleteNoUnderflow verifies a second Delete on the same +// rule is a no-op. +func TestIptablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) { + m := newIptRefcountManager(t, true) + state := m.router.ipFwdState + + r1, err := m.AddDNATRule(iptDnatV6(9083)) + require.NoError(t, err) + _, v6 := state.Counts() + assert.Equal(t, 1, v6) + + require.NoError(t, m.DeleteDNATRule(r1), "first delete") + _, v6 = state.Counts() + assert.Equal(t, 0, v6) + + require.NoError(t, m.DeleteDNATRule(r1), "second delete must be no-op") + _, v6 = state.Counts() + assert.Equal(t, 0, v6, "double delete must not underflow") +} diff --git a/client/firewall/iptables/manager_linux.go b/client/firewall/iptables/manager_linux.go index 696537dd8..aa052d933 100644 --- a/client/firewall/iptables/manager_linux.go +++ b/client/firewall/iptables/manager_linux.go @@ -89,7 +89,7 @@ func (m *Manager) createIPv6Components(wgIface iFaceMapper, mtu uint16) error { } // Share the same IP forwarding state with the v4 router, since - // EnableIPForwarding controls both v4 and v6 sysctls. + // Forwarding refcounter is per-family but shared between v4 and v6 routers. m.router6.ipFwdState = m.router.ipFwdState m.aclMgr6, err = newAclManager(ip6Client, wgIface) @@ -402,17 +402,12 @@ func (m *Manager) SetLogLevel(log.Level) { } func (m *Manager) EnableRouting() error { - if err := m.router.ipFwdState.RequestForwarding(); err != nil { - return fmt.Errorf("enable IP forwarding: %w", err) - } - return nil + // v6 only when the overlay actually has v6. + return m.router.ipFwdState.RequestRouting(m.router6 != nil) } func (m *Manager) DisableRouting() error { - if err := m.router.ipFwdState.ReleaseForwarding(); err != nil { - return fmt.Errorf("disable IP forwarding: %w", err) - } - return nil + return m.router.ipFwdState.ReleaseRouting() } // AddDNATRule adds a DNAT rule diff --git a/client/firewall/iptables/manager_linux_test.go b/client/firewall/iptables/manager_linux_test.go index 7b0989f6c..2c3c1a08e 100644 --- a/client/firewall/iptables/manager_linux_test.go +++ b/client/firewall/iptables/manager_linux_test.go @@ -291,3 +291,40 @@ func TestIptablesCreatePerformance(t *testing.T) { }) } } + +// TestIptablesACLIPSetFallback verifies that when the kernel lacks ipset support, +// the ACL manager falls back to per-IP iptables rules (-s ) instead of +// silently leaving the chain empty. See discussion #6125. +func TestIptablesACLIPSetFallback(t *testing.T) { + ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + require.NoError(t, err) + + // Use Create()/Init() so the router-owned chains (chainRTFWDIN/OUT) are + // created before the ACL manager's createDefaultChains() references them. + manager, err := Create(ifaceMock, iface.DefaultMTU) + require.NoError(t, err) + require.NoError(t, manager.Init(nil)) + + aclMgr := manager.aclMgr + // Simulate a kernel without the ipset hash module. + aclMgr.ipsetSupported = false + + defer func() { + require.NoError(t, manager.Close(nil)) + }() + + ip := netip.MustParseAddr("10.20.0.42") + port := &fw.Port{Values: []uint16{22}} + + rules, err := aclMgr.AddPeerFiltering(nil, ip.AsSlice(), "tcp", nil, port, fw.ActionAccept, "nb0000001") + require.NoError(t, err, "AddPeerFiltering should succeed via fallback") + require.NotEmpty(t, rules) + + rule := rules[0].(*Rule) + require.Empty(t, rule.ipsetName, "fallback rule must not reference an ipset") + require.Contains(t, strings.Join(rule.specs, " "), "-s 10.20.0.42", "fallback rule must match by source IP") + require.NotContains(t, strings.Join(rule.specs, " "), "--match-set", "fallback rule must not use ipset matching") + + // The rule must actually be present in the ACL chain (not silently dropped). + checkRuleSpecs(t, ipv4Client, rule.chain, true, rule.specs...) +} diff --git a/client/firewall/iptables/router_linux.go b/client/firewall/iptables/router_linux.go index 42d305f5c..01b18570c 100644 --- a/client/firewall/iptables/router_linux.go +++ b/client/firewall/iptables/router_linux.go @@ -102,7 +102,7 @@ func newRouter(iptablesClient *iptables.IPTables, wgIface iFaceMapper, mtu uint1 wgIface: wgIface, mtu: mtu, v6: iptablesClient.Proto() == iptables.ProtocolIPv6, - ipFwdState: ipfwdstate.NewIPForwardingState(), + ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()), } r.ipsetCounter = refcounter.New( @@ -770,10 +770,6 @@ func (r *router) updateState() { } func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - if err := r.ipFwdState.RequestForwarding(); err != nil { - return nil, err - } - ruleKey := rule.ID() if _, exists := r.rules[ruleKey+dnatSuffix]; exists { return rule, nil @@ -840,18 +836,34 @@ func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { for key, ruleInfo := range rules { if err := r.iptablesClient.Append(ruleInfo.table, ruleInfo.chain, ruleInfo.rule...); err != nil { - if rollbackErr := r.rollbackRules(rules); rollbackErr != nil { - log.Errorf("rollback failed: %v", rollbackErr) - } + r.cleanupFailedDNATAdd(rules) return nil, fmt.Errorf("add rule %s: %w", key, err) } r.rules[key] = ruleInfo.rule } + if err := r.ipFwdState.RequestForwarding(r.v6); err != nil { + r.cleanupFailedDNATAdd(rules) + return nil, fmt.Errorf("enable forwarding: %w", err) + } + r.updateState() return rule, nil } +// cleanupFailedDNATAdd removes the bookkeeping written by a partially applied +// AddDNATRule before rolling back the kernel rules, so no entries remain that +// never got a forwarding refcount. rollbackRules re-adds entries it failed to +// remove from the kernel. +func (r *router) cleanupFailedDNATAdd(rules map[string]ruleInfo) { + for key := range rules { + delete(r.rules, key) + } + if err := r.rollbackRules(rules); err != nil { + log.Errorf("rollback failed: %v", err) + } +} + func (r *router) rollbackRules(rules map[string]ruleInfo) error { var merr *multierror.Error for key, ruleInfo := range rules { @@ -868,32 +880,47 @@ func (r *router) rollbackRules(rules map[string]ruleInfo) error { } func (r *router) DeleteDNATRule(rule firewall.Rule) error { - if err := r.ipFwdState.ReleaseForwarding(); err != nil { - log.Errorf("%v", err) - } - ruleKey := rule.ID() + _, hadDNAT := r.rules[ruleKey+dnatSuffix] + _, hadSNAT := r.rules[ruleKey+snatSuffix] + _, hadFWD := r.rules[ruleKey+fwdSuffix] + if !hadDNAT && !hadSNAT && !hadFWD { + return nil + } + var merr *multierror.Error if dnatRule, exists := r.rules[ruleKey+dnatSuffix]; exists { if err := r.iptablesClient.Delete(tableNat, chainRTRDR, dnatRule...); err != nil { merr = multierror.Append(merr, fmt.Errorf("delete DNAT rule: %w", err)) + } else { + delete(r.rules, ruleKey+dnatSuffix) } - delete(r.rules, ruleKey+dnatSuffix) } if snatRule, exists := r.rules[ruleKey+snatSuffix]; exists { if err := r.iptablesClient.Delete(tableNat, chainRTNAT, snatRule...); err != nil { merr = multierror.Append(merr, fmt.Errorf("delete SNAT rule: %w", err)) + } else { + delete(r.rules, ruleKey+snatSuffix) } - delete(r.rules, ruleKey+snatSuffix) } if fwdRule, exists := r.rules[ruleKey+fwdSuffix]; exists { if err := r.iptablesClient.Delete(tableFilter, chainRTFWDOUT, fwdRule...); err != nil { merr = multierror.Append(merr, fmt.Errorf("delete forward rule: %w", err)) + } else { + delete(r.rules, ruleKey+fwdSuffix) + } + } + + // Release the refcount only once all rules are gone from the kernel. On + // partial failure the failed entries stay in r.rules so a retry can remove + // them and release then. + if merr == nil { + if err := r.ipFwdState.ReleaseForwarding(r.v6); err != nil { + log.Errorf("%v", err) } - delete(r.rules, ruleKey+fwdSuffix) } r.updateState() diff --git a/client/firewall/nftables/dnat_refcount_linux_test.go b/client/firewall/nftables/dnat_refcount_linux_test.go new file mode 100644 index 000000000..86079676f --- /dev/null +++ b/client/firewall/nftables/dnat_refcount_linux_test.go @@ -0,0 +1,249 @@ +//go:build privileged + +package nftables + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + fw "github.com/netbirdio/netbird/client/firewall/manager" + "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/iface/wgaddr" +) + +func nftRefcountIfaceV4() *iFaceMock { + return &iFaceMock{ + NameFunc: func() string { return "wt-refcount" }, + AddressFunc: func() wgaddr.Address { + return wgaddr.Address{ + IP: netip.MustParseAddr("100.96.0.1"), + Network: netip.MustParsePrefix("100.96.0.0/16"), + } + }, + } +} + +func nftRefcountIfaceDual() *iFaceMock { + return &iFaceMock{ + NameFunc: func() string { return "wt-refcount" }, + AddressFunc: func() wgaddr.Address { + return wgaddr.Address{ + IP: netip.MustParseAddr("100.96.0.1"), + Network: netip.MustParsePrefix("100.96.0.0/16"), + IPv6: netip.MustParseAddr("fd00::1"), + IPv6Net: netip.MustParsePrefix("fd00::/64"), + } + }, + } +} + +func newNftRefcountManager(t *testing.T, dual bool) *Manager { + t.Helper() + if check() != NFTABLES { + t.Skip("nftables not supported on this system") + } + var ifMock *iFaceMock + if dual { + ifMock = nftRefcountIfaceDual() + } else { + ifMock = nftRefcountIfaceV4() + } + m, err := Create(ifMock, iface.DefaultMTU) + require.NoError(t, err, "create manager") + require.NoError(t, m.Init(nil), "init manager") + t.Cleanup(func() { + require.NoError(t, m.Close(nil), "close manager") + }) + return m +} + +func dnatV4(port uint16) fw.ForwardRule { + return fw.ForwardRule{ + Protocol: fw.ProtocolTCP, + DestinationPort: fw.Port{Values: []uint16{port}}, + TranslatedAddress: netip.MustParseAddr("100.96.0.2"), + TranslatedPort: fw.Port{Values: []uint16{80}}, + } +} + +func dnatV6(port uint16) fw.ForwardRule { + return fw.ForwardRule{ + Protocol: fw.ProtocolTCP, + DestinationPort: fw.Port{Values: []uint16{port}}, + TranslatedAddress: netip.MustParseAddr("fd00::2"), + TranslatedPort: fw.Port{Values: []uint16{80}}, + } +} + +// TestNftablesDNAT_RefcountBalancedV4 verifies that Add/Delete pairs leave the +// v4 refcount at zero. +func TestNftablesDNAT_RefcountBalancedV4(t *testing.T) { + m := newNftRefcountManager(t, false) + state := m.router.ipFwdState + + r1, err := m.AddDNATRule(dnatV4(8081)) + require.NoError(t, err, "add v4 dnat 1") + v4, v6 := state.Counts() + assert.Equal(t, 1, v4, "v4 refcount after first add") + assert.Equal(t, 0, v6, "v6 refcount unchanged") + + r2, err := m.AddDNATRule(dnatV4(8082)) + require.NoError(t, err, "add v4 dnat 2") + v4, v6 = state.Counts() + assert.Equal(t, 2, v4, "v4 refcount after second add") + assert.Equal(t, 0, v6, "v6 refcount unchanged") + + require.NoError(t, m.DeleteDNATRule(r1), "delete v4 dnat 1") + v4, v6 = state.Counts() + assert.Equal(t, 1, v4, "v4 refcount after first delete") + assert.Equal(t, 0, v6, "v6 refcount unchanged") + + require.NoError(t, m.DeleteDNATRule(r2), "delete v4 dnat 2") + v4, v6 = state.Counts() + assert.Equal(t, 0, v4, "v4 refcount after second delete") + assert.Equal(t, 0, v6, "v6 refcount unchanged") +} + +// TestNftablesDNAT_RefcountBalancedV6 verifies the v6 path increments v6 only +// and decrements back to zero on Delete. +func TestNftablesDNAT_RefcountBalancedV6(t *testing.T) { + m := newNftRefcountManager(t, true) + require.NotNil(t, m.router6, "v6 router") + require.Same(t, m.router.ipFwdState, m.router6.ipFwdState, "shared state") + state := m.router.ipFwdState + + r1, err := m.AddDNATRule(dnatV6(9091)) + require.NoError(t, err, "add v6 dnat 1") + v4, v6 := state.Counts() + assert.Equal(t, 0, v4, "v4 refcount unchanged") + assert.Equal(t, 1, v6, "v6 refcount after first add") + + r2, err := m.AddDNATRule(dnatV6(9092)) + require.NoError(t, err, "add v6 dnat 2") + v4, v6 = state.Counts() + assert.Equal(t, 0, v4) + assert.Equal(t, 2, v6, "v6 refcount after second add") + + require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat 1") + v4, v6 = state.Counts() + assert.Equal(t, 0, v4, "v4 refcount unchanged") + assert.Equal(t, 1, v6, "v6 refcount after first delete") + + require.NoError(t, m.DeleteDNATRule(r2), "delete v6 dnat 2") + v4, v6 = state.Counts() + assert.Equal(t, 0, v4) + assert.Equal(t, 0, v6, "v6 refcount after second delete") +} + +// TestNftablesDNAT_DuplicateAddNoLeak verifies that a duplicate Add (same +// ForwardRule) does not double-increment the refcount. +func TestNftablesDNAT_DuplicateAddNoLeak(t *testing.T) { + m := newNftRefcountManager(t, true) + state := m.router.ipFwdState + + rule := dnatV4(8083) + r1, err := m.AddDNATRule(rule) + require.NoError(t, err, "add v4 dnat") + v4, _ := state.Counts() + assert.Equal(t, 1, v4) + + // duplicate add: same rule ID, must be a no-op for the refcount. + _, err = m.AddDNATRule(rule) + require.NoError(t, err, "duplicate add") + v4, _ = state.Counts() + assert.Equal(t, 1, v4, "duplicate add must not increment") + + require.NoError(t, m.DeleteDNATRule(r1), "delete v4 dnat") + v4, _ = state.Counts() + assert.Equal(t, 0, v4, "single delete must drop to zero") +} + +// TestNftablesDNAT_DeleteMissingNoUnderflow verifies deleting a rule that was +// never added does not underflow the refcount. +func TestNftablesDNAT_DeleteMissingNoUnderflow(t *testing.T) { + m := newNftRefcountManager(t, true) + state := m.router.ipFwdState + + // Construct a Rule reference for something never added. The router stores + // rules by ID(), and DeleteDNATRule looks them up in r.rules; a missing + // entry must be a no-op rather than calling Release. + phantom := dnatV4(8099) + require.NoError(t, m.DeleteDNATRule(&phantom), "delete missing v4 dnat") + v4, v6 := state.Counts() + assert.Equal(t, 0, v4, "v4 refcount unaffected by missing delete") + assert.Equal(t, 0, v6, "v6 refcount unaffected") + + phantom6 := dnatV6(9099) + require.NoError(t, m.DeleteDNATRule(&phantom6), "delete missing v6 dnat") + v4, v6 = state.Counts() + assert.Equal(t, 0, v4) + assert.Equal(t, 0, v6, "v6 refcount unaffected by missing delete") + + // And after a phantom delete, a real add still results in count=1. + r1, err := m.AddDNATRule(dnatV4(8100)) + require.NoError(t, err, "add v4 dnat after phantom delete") + v4, _ = state.Counts() + assert.Equal(t, 1, v4, "real add still increments after phantom delete") + require.NoError(t, m.DeleteDNATRule(r1)) +} + +// TestNftablesRouting_RepeatedEnableSingleReference verifies that EnableRouting +// (called on every network-map update) holds at most one reference per family +// and a single DisableRouting drops both back to zero. +func TestNftablesRouting_RepeatedEnableSingleReference(t *testing.T) { + m := newNftRefcountManager(t, true) + state := m.router.ipFwdState + + require.NoError(t, m.EnableRouting(), "first enable") + require.NoError(t, m.EnableRouting(), "second enable") + require.NoError(t, m.EnableRouting(), "third enable") + v4, v6 := state.Counts() + assert.Equal(t, 1, v4, "repeated enable holds a single v4 reference") + assert.Equal(t, 1, v6, "repeated enable holds a single v6 reference") + + require.NoError(t, m.DisableRouting(), "disable") + v4, v6 = state.Counts() + assert.Equal(t, 0, v4, "single disable releases the v4 reference") + assert.Equal(t, 0, v6, "single disable releases the v6 reference") +} + +// TestNftablesRouting_DisableKeepsDNATReference verifies that an unpaired +// DisableRouting does not release references held by active DNAT rules. +func TestNftablesRouting_DisableKeepsDNATReference(t *testing.T) { + m := newNftRefcountManager(t, true) + state := m.router.ipFwdState + + r1, err := m.AddDNATRule(dnatV6(9095)) + require.NoError(t, err, "add v6 dnat") + + require.NoError(t, m.DisableRouting(), "unpaired disable") + _, v6 := state.Counts() + assert.Equal(t, 1, v6, "DNAT-held reference survives unpaired DisableRouting") + + require.NoError(t, m.DeleteDNATRule(r1), "delete v6 dnat") + _, v6 = state.Counts() + assert.Equal(t, 0, v6, "delete releases the DNAT reference") +} + +// TestNftablesDNAT_DoubleDeleteNoUnderflow verifies that deleting the same rule +// twice does not underflow the refcount (the second delete is a no-op). +func TestNftablesDNAT_DoubleDeleteNoUnderflow(t *testing.T) { + m := newNftRefcountManager(t, true) + state := m.router.ipFwdState + + r1, err := m.AddDNATRule(dnatV6(9093)) + require.NoError(t, err) + _, v6 := state.Counts() + assert.Equal(t, 1, v6) + + require.NoError(t, m.DeleteDNATRule(r1), "first delete") + _, v6 = state.Counts() + assert.Equal(t, 0, v6) + + require.NoError(t, m.DeleteDNATRule(r1), "second delete must be no-op") + _, v6 = state.Counts() + assert.Equal(t, 0, v6, "double delete must not underflow") +} diff --git a/client/firewall/nftables/manager_linux.go b/client/firewall/nftables/manager_linux.go index fdc7c2f3c..984b1c3ba 100644 --- a/client/firewall/nftables/manager_linux.go +++ b/client/firewall/nftables/manager_linux.go @@ -105,8 +105,8 @@ func (m *Manager) createIPv6Components(tableName string, wgIface iFaceMapper, mt return fmt.Errorf("create v6 router: %w", err) } - // Share the same IP forwarding state with the v4 router, since - // EnableIPForwarding controls both v4 and v6 sysctls. + // Share the per-family forwarding refcounter with the v4 router so a v4 + // rule and a v6 rule against the same state machine cooperate cleanly. m.router6.ipFwdState = m.router.ipFwdState m.aclManager6, err = newAclManager(workTable6, wgIface, chainNameRoutingFw) @@ -530,17 +530,12 @@ func (m *Manager) SetLogLevel(log.Level) { } func (m *Manager) EnableRouting() error { - if err := m.router.ipFwdState.RequestForwarding(); err != nil { - return fmt.Errorf("enable IP forwarding: %w", err) - } - return nil + // v6 only when the overlay actually has v6. + return m.router.ipFwdState.RequestRouting(m.router6 != nil) } func (m *Manager) DisableRouting() error { - if err := m.router.ipFwdState.ReleaseForwarding(); err != nil { - return fmt.Errorf("disable IP forwarding: %w", err) - } - return nil + return m.router.ipFwdState.ReleaseRouting() } // Flush rule/chain/set operations from the buffer diff --git a/client/firewall/nftables/router_linux.go b/client/firewall/nftables/router_linux.go index dfb94c514..d3e031c5f 100644 --- a/client/firewall/nftables/router_linux.go +++ b/client/firewall/nftables/router_linux.go @@ -93,7 +93,7 @@ func newRouter(workTable *nftables.Table, wgIface iFaceMapper, mtu uint16) (*rou rules: make(map[string]*nftables.Rule), af: familyForAddr(workTable.Family == nftables.TableFamilyIPv4), wgIface: wgIface, - ipFwdState: ipfwdstate.NewIPForwardingState(), + ipFwdState: ipfwdstate.NewIPForwardingState(wgIface.Name()), mtu: mtu, } @@ -1553,10 +1553,6 @@ func (r *router) refreshRulesMap() error { } func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { - if err := r.ipFwdState.RequestForwarding(); err != nil { - return nil, err - } - ruleKey := rule.ID() if _, exists := r.rules[ruleKey+dnatSuffix]; exists { return rule, nil @@ -1567,7 +1563,18 @@ func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { return nil, fmt.Errorf("convert protocol to number: %w", err) } + // Request forwarding before queueing rules: addDnatRedirect/addDnatMasq + // buffer netlink messages on r.conn that the next caller's Flush would + // commit if we returned without flushing them ourselves. + v6 := r.af.tableFamily == nftables.TableFamilyIPv6 + if err := r.ipFwdState.RequestForwarding(v6); err != nil { + return nil, fmt.Errorf("enable forwarding: %w", err) + } + if err := r.addDnatRedirect(rule, protoNum, ruleKey); err != nil { + if rerr := r.ipFwdState.ReleaseForwarding(v6); rerr != nil { + log.Warnf("rollback forwarding refcount: %v", rerr) + } return nil, err } @@ -1579,6 +1586,11 @@ func (r *router) AddDNATRule(rule firewall.ForwardRule) (firewall.Rule, error) { // TODO: find chains with drop policies and add rules there if err := r.conn.Flush(); err != nil { + if rerr := r.ipFwdState.ReleaseForwarding(v6); rerr != nil { + log.Warnf("rollback forwarding refcount: %v", rerr) + } + delete(r.rules, ruleKey+dnatSuffix) + delete(r.rules, ruleKey+snatSuffix) return nil, fmt.Errorf("flush rules: %w", err) } @@ -1781,16 +1793,18 @@ func (r *router) addDnatMasq(rule firewall.ForwardRule, protoNum uint8, ruleKey } func (r *router) DeleteDNATRule(rule firewall.Rule) error { - if err := r.ipFwdState.ReleaseForwarding(); err != nil { - log.Errorf("%v", err) - } - ruleKey := rule.ID() if err := r.refreshRulesMap(); err != nil { return fmt.Errorf(refreshRulesMapError, err) } + _, hadDNAT := r.rules[ruleKey+dnatSuffix] + _, hadSNAT := r.rules[ruleKey+snatSuffix] + if !hadDNAT && !hadSNAT { + return nil + } + var merr *multierror.Error var needsFlush bool @@ -1822,9 +1836,16 @@ func (r *router) DeleteDNATRule(rule firewall.Rule) error { } } + // Release the refcount only once the rules are gone from the kernel. On + // failure (including the refreshRulesMap error above) the rules and their + // map entries remain, keeping forwarding on until a retry removes them. if merr == nil { delete(r.rules, ruleKey+dnatSuffix) delete(r.rules, ruleKey+snatSuffix) + + if err := r.ipFwdState.ReleaseForwarding(r.af.tableFamily == nftables.TableFamilyIPv6); err != nil { + log.Errorf("%v", err) + } } return nberrors.FormatErrorOrNil(merr) diff --git a/client/iface/bind/ice_bind.go b/client/iface/bind/ice_bind.go index 156450c61..2d35b9c6f 100644 --- a/client/iface/bind/ice_bind.go +++ b/client/iface/bind/ice_bind.go @@ -22,6 +22,16 @@ import ( nbnet "github.com/netbirdio/netbird/client/net" ) +const ( + // wgMsgTypeHandshakeInitiation is the lowest WireGuard message type. + wgMsgTypeHandshakeInitiation uint32 = 1 + // wgMsgTypeTransport is the highest WireGuard message type. + wgMsgTypeTransport uint32 = 4 + // wgMinMsgSize is the smallest WireGuard message: transport data with an empty + // payload, which is what a keepalive is. + wgMinMsgSize = 32 +) + type receiverCreator struct { iceBind *ICEBind } @@ -216,8 +226,15 @@ func (s *ICEBind) createReceiverFn(pc wgConn.BatchReader, conn *net.UDPConn, rxO for i := 0; i < numMsgs; i++ { msg := &(*msgs)[i] - // todo: handle err - if ok, _ := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr); ok { + if ok, err := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr); ok { + if err != nil { + log.Debugf("failed to handle STUN packet from %s: %v", msg.Addr, err) + } + // WireGuard reuses sizes and eps across reads and only skips a slot + // whose size is below the minimum message size. Leaving a consumed + // slot untouched makes it process this buffer again under the + // previous packet's length and endpoint. + sizes[i] = 0 continue } sizes[i] = msg.N @@ -271,11 +288,16 @@ func (s *ICEBind) createOrUpdateMux() { func (s *ICEBind) filterOutStunMessages(buffers [][]byte, n int, addr net.Addr) (bool, error) { for i := range buffers { - if !stun.IsMessage(buffers[i]) { + if n > len(buffers[i]) { + continue + } + pkt := buffers[i][:n] + + if isWireGuardMsg(pkt) || !stun.IsMessage(pkt) { continue } - msg, err := s.parseSTUNMessage(buffers[i][:n]) + msg, err := s.parseSTUNMessage(pkt) if err != nil { buffers[i] = []byte{} return true, err @@ -347,18 +369,34 @@ func putMessages(msgs *[]ipv6.Message, msgsPool *sync.Pool) { msgsPool.Put(msgs) } -func isTransportPkg(buffers [][]byte, n int) bool { - // The first buffer should contain at least 4 bytes for type - if len(buffers[0]) < 4 { - return true +// isWireGuardMsg reports whether the packet carries a WireGuard message header: a +// little-endian uint32 message type in the range 1..4, which leaves the three bytes +// after the type byte zero, in a packet long enough to hold any WireGuard message. +// +// A well formed STUN message cannot take that shape. Its length field sits in the two +// bytes the type must leave zero, and for a message of at least wgMinMsgSize bytes that +// field holds at least 12, so the two framings do not overlap. The test has to be this +// tight because stun.IsMessage only looks at the magic cookie, which in a WireGuard +// message overlaps the receiver index: a session whose index happens to equal the cookie +// would otherwise have all of its inbound data misrouted to the STUN handler until the +// next rekey. +func isWireGuardMsg(pkt []byte) bool { + if len(pkt) < wgMinMsgSize { + return false } - // WireGuard packet type is a little-endian uint32 at start - packetType := binary.LittleEndian.Uint32(buffers[0][:4]) - - // Check if packetType matches known WireGuard message types - if packetType == 4 && n > 32 { - return true - } - return false + msgType := binary.LittleEndian.Uint32(pkt[:4]) + return msgType >= wgMsgTypeHandshakeInitiation && msgType <= wgMsgTypeTransport +} + +// isTransportPkg reports whether the packet is WireGuard transport data carrying a +// payload, which is what counts as peer activity. A keepalive holds no payload and is +// exactly wgMinMsgSize bytes. +func isTransportPkg(buffers [][]byte, n int) bool { + if n < 4 || n > len(buffers[0]) { + return false + } + + msgType := binary.LittleEndian.Uint32(buffers[0][:4]) + return msgType == wgMsgTypeTransport && n > wgMinMsgSize } diff --git a/client/iface/bind/stun_filter_test.go b/client/iface/bind/stun_filter_test.go new file mode 100644 index 000000000..0e118e0fd --- /dev/null +++ b/client/iface/bind/stun_filter_test.go @@ -0,0 +1,215 @@ +//go:build !js + +package bind + +import ( + "encoding/binary" + "net" + "testing" + "time" + + "github.com/pion/stun/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/ipv4" + wgConn "golang.zx2c4.com/wireguard/conn" +) + +// magicCookieBytes is the STUN magic cookie as it appears on the wire. In a +// WireGuard message the same offset holds the receiver (or sender) index, which is +// a random uint32, so a session can draw exactly this value. +var magicCookieBytes = []byte{0x21, 0x12, 0xA4, 0x42} + +const testBufSize = 1500 + +// wgMsg builds a WireGuard message of the given type and size, with the index field +// at bytes 4:8 set to index. +func wgMsg(msgType uint32, size int, index []byte) []byte { + pkt := make([]byte, size) + binary.LittleEndian.PutUint32(pkt[:4], msgType) + copy(pkt[4:8], index) + return pkt +} + +// intoBuffer copies pkt into a full-size receive buffer, the way the kernel read +// does, so tests see the same buffer/length split as the hot path. +func intoBuffer(pkt []byte) [][]byte { + buf := make([]byte, testBufSize) + copy(buf, pkt) + return [][]byte{buf} +} + +func TestFilterOutStunMessages_PassesWireGuardWithCookieShapedIndex(t *testing.T) { + tests := []struct { + name string + msgType uint32 + size int + }{ + {"transport data", wgMsgTypeTransport, 128}, + {"keepalive", wgMsgTypeTransport, wgMinMsgSize}, + {"handshake initiation", wgMsgTypeHandshakeInitiation, 148}, + {"handshake response", 2, 92}, + {"cookie reply", 3, 64}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pkt := wgMsg(tc.msgType, tc.size, magicCookieBytes) + require.True(t, stun.IsMessage(pkt), "precondition: pion sees this as STUN") + + buffers := intoBuffer(pkt) + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, tc.size, &net.UDPAddr{}) + assert.NoError(t, err) + assert.False(t, filtered, "WireGuard message must be handed to WireGuard, not the STUN handler") + assert.Len(t, buffers[0], testBufSize, "buffer must be left intact for WireGuard") + }) + } +} + +func TestFilterOutStunMessages_FiltersRealSTUNMessage(t *testing.T) { + msg, err := stun.Build(stun.BindingRequest, stun.TransactionID, stun.Fingerprint) + require.NoError(t, err) + + buffers := intoBuffer(msg.Raw) + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, len(msg.Raw), &net.UDPAddr{}) + assert.NoError(t, err) + assert.True(t, filtered, "STUN message must be consumed by the STUN handler") + assert.Empty(t, buffers[0], "consumed buffer must be emptied so WireGuard does not see it") +} + +// TestIsWireGuardMsg_DisjointFromSTUN locks the invariant the filter relies on: a +// well formed STUN message long enough to be a WireGuard message always has a +// non-zero length field, so it cannot be mistaken for a WireGuard header. +func TestIsWireGuardMsg_DisjointFromSTUN(t *testing.T) { + types := []stun.MessageType{ + stun.BindingRequest, + stun.BindingSuccess, + stun.BindingError, + {Method: stun.MethodBinding, Class: stun.ClassIndication}, + } + + for _, msgType := range types { + // Long enough that the length guard is not what makes this pass. + msg, err := stun.Build(msgType, stun.TransactionID, + stun.NewUsername("remoteUfrag:localUfrag"), stun.Fingerprint) + require.NoError(t, err) + require.GreaterOrEqual(t, len(msg.Raw), wgMinMsgSize, "precondition: %s", msgType) + assert.False(t, isWireGuardMsg(msg.Raw), + "%s must not look like a WireGuard message", msgType) + } +} + +func TestIsWireGuardMsg(t *testing.T) { + tests := []struct { + name string + pkt []byte + want bool + }{ + {"transport data", wgMsg(wgMsgTypeTransport, 128, nil), true}, + {"handshake initiation", wgMsg(wgMsgTypeHandshakeInitiation, 148, nil), true}, + {"unknown type 5", wgMsg(5, 128, nil), false}, + {"type 0", wgMsg(0, 128, nil), false}, + {"non-zero reserved byte", []byte{0x04, 0x00, 0x01, 0x00}, false}, + {"too short", []byte{0x04, 0x00, 0x00}, false}, + {"empty", nil, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isWireGuardMsg(tc.pkt), "wrong classification for %s", tc.name) + }) + } +} + +// TestFilterOutStunMessages_IgnoresBytesBeyondPacket guards against classifying on +// buffer contents left over from an earlier, longer packet. +func TestFilterOutStunMessages_IgnoresBytesBeyondPacket(t *testing.T) { + buf := make([]byte, testBufSize) + copy(buf[4:8], magicCookieBytes) + buffers := [][]byte{buf} + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, 2, &net.UDPAddr{}) + assert.NoError(t, err) + assert.False(t, filtered, "a 2 byte packet must not be classified from stale buffer bytes") +} + +// TestReceiveFn_ClearsSizeOfConsumedPacket covers the accounting WireGuard relies +// on: sizes is reused across reads, so a slot whose packet was consumed as STUN must +// be reported as empty. Otherwise WireGuard reprocesses the same buffer under the +// previous packet's length, which for a WireGuard-shaped packet means it is handled +// twice. +func TestReceiveFn_ClearsSizeOfConsumedPacket(t *testing.T) { + conn := listenUDP(t, "udp4", "127.0.0.1:0") + defer conn.Close() + + recvFn := receiverCreator{setupICEBind(t)}.CreateReceiverFn( + ipv4.NewPacketConn(conn), conn, false, createMsgPool(), + ) + + msg, err := stun.Build(stun.BindingRequest, stun.TransactionID, stun.Fingerprint) + require.NoError(t, err) + + sender := listenUDP(t, "udp4", "127.0.0.1:0") + defer sender.Close() + _, err = sender.WriteTo(msg.Raw, conn.LocalAddr()) + require.NoError(t, err) + + require.NoError(t, conn.SetReadDeadline(time.Now().Add(3*time.Second))) + + bufs := [][]byte{make([]byte, 1500)} + // A leftover size from an earlier read, which is what makes the missing reset + // observable. + sizes := []int{148} + eps := make([]wgConn.Endpoint, 1) + + n, err := recvFn(bufs, sizes, eps) + require.NoError(t, err) + require.Equal(t, 1, n) + assert.Zero(t, sizes[0], "consumed STUN packet must not leave a size behind for WireGuard") +} + +func TestIsTransportPkg(t *testing.T) { + tests := []struct { + name string + pkt []byte + n int + want bool + }{ + {"transport data with payload", wgMsg(wgMsgTypeTransport, 128, nil), 128, true}, + {"keepalive", wgMsg(wgMsgTypeTransport, wgMinMsgSize, nil), wgMinMsgSize, false}, + {"handshake initiation", wgMsg(wgMsgTypeHandshakeInitiation, 148, nil), 148, false}, + {"stale type bytes beyond packet", wgMsg(wgMsgTypeTransport, 128, nil), 2, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isTransportPkg(intoBuffer(tc.pkt), tc.n), + "wrong activity classification for %s", tc.name) + }) + } +} + +// TestFilterOutStunMessages_ConsumesSTUNWithWireGuardShapedType covers the one STUN +// encoding whose leading bytes collide with a WireGuard message type: method 0x080 as a +// request encodes to 0x0200, so the type byte reads as a handshake response and the byte +// after it is zero. Only the length check keeps such a message out of WireGuard's hands. +// pion implements no method in that range, so this is a synthetic worst case rather than +// traffic ICE produces. +func TestFilterOutStunMessages_ConsumesSTUNWithWireGuardShapedType(t *testing.T) { + msg, err := stun.Build(stun.NewType(stun.Method(0x080), stun.ClassRequest), stun.TransactionID) + require.NoError(t, err) + require.Equal(t, []byte{0x02, 0x00, 0x00, 0x00}, msg.Raw[:4], + "precondition: the leading bytes read as a WireGuard message type") + + buffers := intoBuffer(msg.Raw) + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, len(msg.Raw), &net.UDPAddr{}) + assert.NoError(t, err) + assert.True(t, filtered, "STUN message must be consumed despite its WireGuard-shaped type") +} diff --git a/client/installer.nsis b/client/installer.nsis index 71699071b..eb2d7d5bd 100644 --- a/client/installer.nsis +++ b/client/installer.nsis @@ -22,8 +22,6 @@ !define UI_REG_APP_PATH "Software\Microsoft\Windows\CurrentVersion\App Paths\${UI_APP_EXE}" !define UI_UNINSTALL_PATH "Software\Microsoft\Windows\CurrentVersion\Uninstall\${UI_APP_NAME}" -!define AUTOSTART_REG_KEY "Software\Microsoft\Windows\CurrentVersion\Run" - !define NETBIRD_DATA_DIR "$COMMONPROGRAMDATA\Netbird" Unicode True @@ -228,13 +226,6 @@ WriteRegStr ${REG_ROOT} "${UNINSTALL_PATH}" "Publisher" "${COMP_NAME}" WriteRegStr ${REG_ROOT} "${UI_REG_APP_PATH}" "" "$INSTDIR\${UI_APP_EXE}" -; Autostart is owned by the UI's per-user setting (HKCU\...\Run via Wails), -; not the installer. Drop the machine-wide entry older installers wrote so the -; toggle is the single source of truth. HKCU is left untouched -- it may hold -; the user's own toggle state, which must survive upgrades. -DetailPrint "Removing installer-managed autostart registry entry if present..." -DeleteRegValue HKLM "${AUTOSTART_REG_KEY}" "${APP_NAME}" - EnVar::SetHKLM EnVar::AddValueEx "path" "$INSTDIR" @@ -299,15 +290,6 @@ ExecWait '"$INSTDIR\${MAIN_APP_EXE}" service uninstall' DetailPrint "Terminating Netbird UI process..." ExecWait `taskkill /im ${UI_APP_EXE}.exe /f` -; Remove autostart registry entries -DetailPrint "Removing autostart registry entries if they exist..." -; Legacy machine-wide entry written by older installers. -DeleteRegValue HKLM "${AUTOSTART_REG_KEY}" "${APP_NAME}" -; Per-user entry the UI toggle writes via Wails (value name is the lowercase -; app-name slug). Uninstall removes the app, so drop it too. -DeleteRegValue HKCU "${AUTOSTART_REG_KEY}" "${APP_NAME}" -DeleteRegValue HKCU "${AUTOSTART_REG_KEY}" "netbird" - ; Handle data deletion based on checkbox DetailPrint "Checking if user requested data deletion..." ${If} $DeleteDataEnabled == "1" diff --git a/client/internal/debug/debug.go b/client/internal/debug/debug.go index 0f81844f6..1d31c75ca 100644 --- a/client/internal/debug/debug.go +++ b/client/internal/debug/debug.go @@ -34,9 +34,8 @@ import ( "github.com/netbirdio/netbird/shared/netiputil" ) -const readmeContent = `Netbird debug bundle -This debug bundle contains the following files. -If the --anonymize flag is set, the files are anonymized to protect sensitive information. +const readmeContent = `This debug bundle contains the following files. +If anonymization is enabled (--anonymize / --anonymize-level), the files are anonymized to protect sensitive information. status.txt: Anonymized status information of the NetBird client. client.log: Most recent, anonymized client log file of the NetBird client. @@ -52,6 +51,7 @@ nftables.txt: Anonymized nftables rules with packet counters across all families sysctls.txt: Forwarding, reverse-path filter, source-validation, and conntrack accounting sysctl values that the NetBird client may read or modify, if --system-info flag was provided (Linux only). resolv.conf: DNS resolver configuration from /etc/resolv.conf (Unix systems only), if --system-info flag was provided. scutil_dns.txt: DNS configuration from scutil --dns (macOS only), if --system-info flag was provided. +dns_windows.txt: Anonymized NRPT rules and policy table in effect, DNS client policy, and per-interface and per-adapter DNS configuration (Windows only), if --system-info flag was provided. resolved_domains.txt: Anonymized resolved domain IP addresses from the status recorder. config.txt: Anonymized configuration information of the NetBird client. network_map.json: Anonymized sync response containing peer configurations, routes, DNS settings, and firewall rules. @@ -70,21 +70,34 @@ capture.pcap: Packet capture in pcap format. Only present when capture was runni Anonymization Process -The files in this bundle have been anonymized to protect sensitive information. Here's how the anonymization was applied: +The files in this bundle have been anonymized to protect sensitive information. The level applied to this bundle is recorded at the top of this file. Here's how the anonymization was applied: IP Addresses -IPv4 addresses are replaced with addresses starting from 198.51.100.0 -IPv6 addresses are replaced with addresses starting from 100:: +Default level: +- Public IPv4 addresses are replaced with addresses starting from 198.51.100.0 +- Public IPv6 addresses are replaced with addresses starting from 2001:db8:ffff:: +- IPv6 unique local addresses (fc00::/7) are anonymized as well: their random global ID uniquely identifies the network. +- IP addresses from internal IPv4 ranges and well-known addresses are not anonymized (e.g. 8.8.8.8, 100.64.0.0/10, addresses starting with 192.168., 172.16., 10., 169.254., fe80::). + +Strict level (--anonymize-level strict), in addition to the default level: +- Private (RFC 1918), CGNAT (100.64.0.0/10), and link-local (169.254.0.0/16, fe80::/10) addresses are anonymized too. +- Internal IPv4 addresses are replaced with addresses starting from 198.18.0.0 and internal IPv6 addresses with addresses starting from 2001:db8:1::, so internal addresses remain distinguishable from public ones. +- Addresses are mapped in order of first appearance: subnet structure, allocation scheme, and gateway conventions are not preserved. Prefix lengths of networks are preserved. +- Peer names in front of NetBird domains are replaced with numbered placeholders (e.g. peer-1.netbird.cloud), and subdomain labels of other domains with host-N placeholders. +- WireGuard public keys are replaced with consistent placeholder keys. -IP addresses from non public ranges and well known addresses are not anonymized (e.g. 8.8.8.8, 100.64.0.0/10, addresses starting with 192.168., 172.16., 10., etc.). Reoccuring IP addresses are replaced with the same anonymized address. Note: The anonymized IP addresses in the status file do not match those in the log and routes files. However, the anonymized IP addresses are consistent within the status file and across the routes and log files. +MAC Addresses +MAC addresses are replaced at every anonymization level with consistent placeholders counting up from 02:00:00:00:00:01. Broadcast, multicast, and all-zero addresses are kept. At the default level a preserved IPv6 link-local address may still embed a MAC address (EUI-64); the strict level anonymizes those addresses. + Domains All domain names (except for the netbird domains) are replaced with randomly generated strings ending in ".domain". Anonymized domains are consistent across all files in the bundle. Reoccuring domain names are replaced with the same anonymized domain. +At the strict level, the peer name labels in front of netbird domains are anonymized as well. Sync Response The network_map.json file contains the following anonymized information: @@ -225,6 +238,13 @@ scutil_dns.txt (macOS only): - Shows DNS configuration for all network interfaces - Includes search domains, nameservers, and DNS resolver settings - All IP addresses and domain names are anonymized + +dns_windows.txt (Windows only): +- Lists the NRPT rules of both policy stores, the local one and the group policy one, marking the rules the client created +- Follows them with the policy table the resolver has loaded, which differs from the rules while a change has not been picked up yet +- Includes the DNS client group policy, the global TCP/IP and Dnscache parameters, and the DNS values of every interface that has any +- Ends with the resolver configuration in effect per adapter, from GetAdaptersAddresses +- All IP addresses and domain names are anonymized ` const ( @@ -281,6 +301,7 @@ type BundleGenerator struct { cliVersion string anonymize bool + anonymizeLevel anonymize.Level includeSystemInfo bool logFileCount uint32 @@ -288,7 +309,10 @@ type BundleGenerator struct { } type BundleConfig struct { - Anonymize bool + Anonymize bool + // AnonymizeLevel selects how much the anonymizer redacts. + // anonymize.LevelStrict implies Anonymize. + AnonymizeLevel anonymize.Level IncludeSystemInfo bool LogFileCount uint32 } @@ -327,8 +351,11 @@ func NewBundleGenerator(deps GeneratorDependencies, cfg BundleConfig) *BundleGen uiLogOpener = openLogFile } + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(cfg.AnonymizeLevel) + return &BundleGenerator{ - anonymizer: anonymize.NewAnonymizer(anonymize.DefaultAddresses()), + anonymizer: anonymizer, internalConfig: deps.InternalConfig, statusRecorder: deps.StatusRecorder, @@ -345,7 +372,8 @@ func NewBundleGenerator(deps GeneratorDependencies, cfg BundleConfig) *BundleGen daemonVersion: deps.DaemonVersion, cliVersion: deps.CliVersion, - anonymize: cfg.Anonymize, + anonymize: cfg.Anonymize || cfg.AnonymizeLevel >= anonymize.LevelStrict, + anonymizeLevel: cfg.AnonymizeLevel, includeSystemInfo: cfg.IncludeSystemInfo, logFileCount: logFileCount, } @@ -485,7 +513,13 @@ func (g *BundleGenerator) addSystemInfo() { } func (g *BundleGenerator) addReadme() error { - readmeReader := strings.NewReader(readmeContent) + level := "none (anonymization disabled)" + if g.anonymize { + level = g.anonymizeLevel.String() + } + header := fmt.Sprintf("Netbird debug bundle\nAnonymization level applied to this bundle: %s\n", level) + + readmeReader := strings.NewReader(header + readmeContent) if err := g.addFileToZip(readmeReader, "README.txt"); err != nil { return fmt.Errorf("add README file to zip: %w", err) } @@ -507,9 +541,10 @@ func (g *BundleGenerator) addStatus() error { fullStatus := g.statusRecorder.GetFullStatus() protoFullStatus := nbstatus.ToProtoFullStatus(fullStatus) overview := nbstatus.ConvertToStatusOutputOverview(protoFullStatus, nbstatus.ConvertOptions{ - Anonymize: g.anonymize, - ProfileName: profName, - DaemonVersion: g.daemonVersion, + Anonymize: g.anonymize, + AnonymizeLevel: g.anonymizeLevel, + ProfileName: profName, + DaemonVersion: g.daemonVersion, }) overview.CliVersion = g.cliVersion statusOutput := overview.FullDetailSummary() @@ -662,7 +697,7 @@ func (g *BundleGenerator) addCommonConfigFields(configContent *strings.Builder) configContent.WriteString("NetBird Client Configuration:\n\n") if key, err := wgtypes.ParseKey(g.internalConfig.PrivateKey); err == nil { - configContent.WriteString(fmt.Sprintf("PublicKey: %s\n", key.PublicKey().String())) + configContent.WriteString(fmt.Sprintf("PublicKey: %s\n", g.anonymizer.AnonymizeWGKey(key.PublicKey().String()))) } configContent.WriteString(fmt.Sprintf("WgIface: %s\n", g.internalConfig.WgIface)) configContent.WriteString(fmt.Sprintf("WgPort: %d\n", g.internalConfig.WgPort)) @@ -952,6 +987,11 @@ func (g *BundleGenerator) addUpdateLogs() error { } baseName := filepath.Base(logFile) + data, err = g.anonymizeBytes(data) + if err != nil { + log.Warnf("skipping update log file %s: %v", baseName, err) + continue + } if err := g.addFileToZip(bytes.NewReader(data), filepath.Join("update-logs", baseName)); err != nil { return fmt.Errorf("add update log file %s to zip: %w", baseName, err) } @@ -979,6 +1019,13 @@ func (g *BundleGenerator) addCorruptedStateFiles() error { } fileName := filepath.Base(match) + // Corrupted state files usually fail structured JSON anonymization, + // so run them through the string anonymizer instead. + data, err = g.anonymizeBytes(data) + if err != nil { + log.Warnf("skipping corrupted state file %s: %v", fileName, err) + continue + } if err := g.addFileToZip(bytes.NewReader(data), "corrupted_states/"+fileName); err != nil { log.Warnf("Failed to add corrupted state file %s to zip: %v", fileName, err) continue @@ -990,6 +1037,27 @@ func (g *BundleGenerator) addCorruptedStateFiles() error { return nil } +// anonymizeBytes runs raw file content through the string anonymizer line by +// line when anonymization is enabled. It errors instead of returning partial +// content, so a caller never adds an unanonymized fallback to the bundle. +func (g *BundleGenerator) anonymizeBytes(data []byte) ([]byte, error) { + if !g.anonymize { + return data, nil + } + + var buf bytes.Buffer + scanner := bufio.NewScanner(bytes.NewReader(data)) + scanner.Buffer(make([]byte, 1024*1024), 1024*1024) + for scanner.Scan() { + buf.WriteString(g.anonymizer.AnonymizeString(scanner.Text())) + buf.WriteByte('\n') + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("anonymize content: %w", err) + } + return buf.Bytes(), nil +} + func (g *BundleGenerator) addMetrics() error { if g.clientMetrics == nil { log.Debugf("skipping metrics in debug bundle: no metrics collector") @@ -1462,6 +1530,7 @@ func anonymizeRemotePeer(peer *mgmProto.RemotePeerConfig, anonymizer *anonymize. } peer.Fqdn = anonymizer.AnonymizeDomain(peer.Fqdn) + peer.WgPubKey = anonymizer.AnonymizeWGKey(peer.WgPubKey) anonymizeSSHConfig(peer.SshConfig) } diff --git a/client/internal/debug/debug_linux.go b/client/internal/debug/debug_linux.go index 40d864eda..a36c0c0e7 100644 --- a/client/internal/debug/debug_linux.go +++ b/client/internal/debug/debug_linux.go @@ -844,6 +844,10 @@ func collectSysctls() string { []string{"net.ipv4.conf.all.src_valid_mark", "net.ipv4.conf.default.src_valid_mark"}, listInterfaceSysctls("ipv4", "src_valid_mark")..., )) + writeSysctlGroup(&builder, "accept_ra", append( + []string{"net.ipv6.conf.all.accept_ra", "net.ipv6.conf.default.accept_ra"}, + listInterfaceSysctls("ipv6", "accept_ra")..., + )) writeSysctlGroup(&builder, "conntrack", []string{ "net.netfilter.nf_conntrack_acct", "net.netfilter.nf_conntrack_tcp_loose", diff --git a/client/internal/debug/debug_nonunix.go b/client/internal/debug/debug_nonunix.go index 18d017050..adc9b9649 100644 --- a/client/internal/debug/debug_nonunix.go +++ b/client/internal/debug/debug_nonunix.go @@ -1,4 +1,4 @@ -//go:build !unix +//go:build !unix && !windows package debug diff --git a/client/internal/debug/debug_windows.go b/client/internal/debug/debug_windows.go new file mode 100644 index 000000000..e88940fd3 --- /dev/null +++ b/client/internal/debug/debug_windows.go @@ -0,0 +1,443 @@ +//go:build windows + +package debug + +import ( + "encoding/hex" + "errors" + "fmt" + "net/netip" + "strings" + "unsafe" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" + "golang.org/x/sys/windows/registry" + + nbdns "github.com/netbirdio/netbird/client/internal/dns" +) + +const dnsInfoFileName = "dns_windows.txt" + +const ( + gpoDNSClientRoot = `SOFTWARE\Policies\Microsoft\Windows NT\DNSClient` + tcpipParamsPath = `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters` + dnscacheParams = `SYSTEM\CurrentControlSet\Services\Dnscache\Parameters` +) + +// interfaceDNSValues are the per-interface values that decide how a name is +// resolved and registered. Everything the DNS host manager writes is in here, +// so a bundle shows both what we set and what it replaced. +var interfaceDNSValues = []string{ + "NameServer", + "DhcpNameServer", + "Domain", + "DhcpDomain", + "SearchList", + "RegistrationEnabled", + "DisableDynamicUpdate", + "MaxNumberOfAddressesToRegister", + "EnableDHCP", +} + +// addDNSInfo collects and adds DNS configuration information to the archive +func (g *BundleGenerator) addDNSInfo() error { + if err := g.addFileToZip(strings.NewReader(g.collectDNSInfo()), dnsInfoFileName); err != nil { + return fmt.Errorf("add DNS info to zip: %w", err) + } + + return nil +} + +// collectDNSInfo renders the report. Everything below it reaches the platform +// through COM and through lazily resolved procedures, which panic when a +// procedure is missing rather than returning an error, and a debug bundle is not +// allowed to take the daemon down. The panic is contained here, and whatever was +// collected before it is kept and reported with it. +func (g *BundleGenerator) collectDNSInfo() (content string) { + var sb strings.Builder + + defer func() { + if r := recover(); r != nil { + log.Errorf("collecting Windows DNS configuration panicked: %v", r) + fmt.Fprintf(&sb, "\nerror: collection stopped: %v\n", r) + } + content = sb.String() + }() + + sb.WriteString("Windows DNS configuration\n") + sb.WriteString("=========================\n") + + adapters, adaptersErr := adapterAddresses() + + g.writeNRPTRules(&sb, "NRPT rules, local policy store", nbdns.DNSPolicyConfigRoot) + g.writeNRPTRules(&sb, "NRPT rules, group policy store", nbdns.GPODNSPolicyConfigRoot) + g.writeEffectiveNRPTPolicies(&sb) + g.writeRegistryKey(&sb, "DNS client group policy", gpoDNSClientRoot) + g.writeRegistryKey(&sb, "Global TCP/IP parameters", tcpipParamsPath) + g.writeRegistryKey(&sb, "Dnscache parameters", dnscacheParams) + g.writeInterfaceDNS(&sb, "Per-interface DNS, IPv4", nbdns.InterfaceConfigPath, adapterNames(adapters)) + g.writeInterfaceDNS(&sb, "Per-interface DNS, IPv6", nbdns.InterfaceConfigPathV6, adapterNames(adapters)) + g.writeAdapterDNS(&sb, adapters, adaptersErr) + + return sb.String() +} + +// writeNRPTRules lists every rule in a policy store, ours and any other +// product's, since a foreign rule for the same namespace decides resolution +// just as ours does. Rules the client wrote are marked. +func (g *BundleGenerator) writeNRPTRules(sb *strings.Builder, title, root string) { + writeSection(sb, title, root) + + names, err := subKeyNames(root) + if err != nil { + fmt.Fprintf(sb, "error: %v\n", err) + return + } + + if len(names) == 0 { + sb.WriteString("no rules\n") + return + } + + for _, name := range names { + owner := "" + if strings.HasPrefix(strings.ToLower(name), strings.ToLower(nbdns.NRPTKeyPrefix)) { + owner = " (netbird)" + } + fmt.Fprintf(sb, "%s%s\n", name, owner) + g.writeValues(sb, root+`\`+name, nil, " ") + } +} + +// writeEffectiveNRPTPolicies reports the table the resolver answers from, which +// the registry cannot show: a rule is written before it is loaded, and it keeps +// being enforced after its key is gone until the resolver reloads its policy. +func (g *BundleGenerator) writeEffectiveNRPTPolicies(sb *strings.Builder) { + writeSection(sb, "NRPT policy table in effect", nrptPolicyClass+"."+nrptPolicyMethod+" in "+nrptPolicyNamespace) + + entries, err := effectiveNRPTPolicies() + if err != nil { + fmt.Fprintf(sb, "error: %v\n", err) + return + } + + if len(entries) == 0 { + sb.WriteString("no policies\n") + return + } + + for _, entry := range entries { + fmt.Fprintf(sb, "%s\n", g.anonymizeValue("Namespace", entry.namespace)) + for _, value := range entry.values { + fmt.Fprintf(sb, " %s: %s\n", value.name, g.anonymizeValue(value.name, value.value)) + } + } +} + +// writeInterfaceDNS reports the DNS values of every interface that has any, so +// the netbird interface can be compared against the physical ones. The registry +// keys the values by GUID, so each is named from the adapter list; a GUID with +// no adapter is a leftover key of an interface that no longer exists. +func (g *BundleGenerator) writeInterfaceDNS(sb *strings.Builder, title, root string, names map[string]string) { + writeSection(sb, title, root) + + guids, err := subKeyNames(root) + if err != nil { + fmt.Fprintf(sb, "error: %v\n", err) + return + } + + var reported int + for _, guid := range guids { + var iface strings.Builder + g.writeValues(&iface, root+`\`+guid, interfaceDNSValues, " ") + if iface.Len() == 0 { + continue + } + + name, ok := names[strings.ToLower(guid)] + if !ok { + name = "no adapter with this GUID" + } + + reported++ + fmt.Fprintf(sb, "%s (%s)\n%s", guid, name, iface.String()) + } + + if reported == 0 { + sb.WriteString("no interface holds DNS values\n") + } +} + +// writeRegistryKey reports the values of a single key, without its subkeys. +func (g *BundleGenerator) writeRegistryKey(sb *strings.Builder, title, path string) { + writeSection(sb, title, path) + + var values strings.Builder + g.writeValues(&values, path, nil, "") + if values.Len() == 0 { + sb.WriteString("no values\n") + return + } + + sb.WriteString(values.String()) +} + +// writeValues renders the values of a key. A nil names list reports every +// value, otherwise only those named and present. +func (g *BundleGenerator) writeValues(sb *strings.Builder, path string, names []string, indent string) { + k, err := registry.OpenKey(registry.LOCAL_MACHINE, path, registry.QUERY_VALUE) + switch { + case errors.Is(err, registry.ErrNotExist), errors.Is(err, windows.ERROR_PATH_NOT_FOUND): + // an absent key is the normal state for the GPO store and for + // interfaces without DNS settings + log.Debugf("HKEY_LOCAL_MACHINE\\%s does not exist", path) + return + case err != nil: + fmt.Fprintf(sb, "%serror: open HKEY_LOCAL_MACHINE\\%s: %v\n", indent, path, err) + return + } + defer closeKey(k) + + if names == nil { + names, err = k.ReadValueNames(-1) + if err != nil { + fmt.Fprintf(sb, "%serror: read value names: %v\n", indent, err) + return + } + } + + for _, name := range names { + value, err := readRegistryValue(k, name) + switch { + case errors.Is(err, registry.ErrNotExist): + // the caller asks for a fixed set of values, most of which a + // given interface does not carry + continue + case err != nil: + // report rather than omit: a value that is there but cannot be + // read reads as unset otherwise + fmt.Fprintf(sb, "%s%s: error: %v\n", indent, name, err) + continue + } + + fmt.Fprintf(sb, "%s%s: %s\n", indent, name, g.anonymizeValue(name, value)) + } +} + +// anonymizeValue redacts a registry value according to what its name says it +// holds. Domains and addresses are handled per entry rather than by the string +// pass: the pass only replaces domains something else in the bundle already +// seeded, and its address regex would eat the digit labels of a reverse zone. +func (g *BundleGenerator) anonymizeValue(name, value string) string { + if !g.anonymize || value == "" { + return value + } + + switch { + case holdsDomains(name): + return joinValueEntries(splitValueEntries(value), g.anonymizeDomain) + case holdsAddresses(name): + return joinValueEntries(splitValueEntries(value), g.anonymizer.AnonymizeIPString) + default: + return g.anonymizer.AnonymizeString(value) + } +} + +// holdsDomains reports whether a value name holds domains: the domain list of +// an NRPT rule (Name) or of the policy table (Namespace), a search list, the +// DNS suffix values of the TCP/IP and policy keys, which all end in "Domain" +// (Domain, DhcpDomain, NV Domain, ICSDomain), and a proxy host name. +func holdsDomains(name string) bool { + lower := strings.ToLower(name) + return lower == "name" || lower == "namespace" || lower == "searchlist" || + strings.HasSuffix(lower, "domain") || strings.HasSuffix(lower, "proxyname") +} + +// holdsAddresses reports whether a value name holds DNS server addresses +// (NameServer, DhcpNameServer, GenericDNSServers, NameServers). +func holdsAddresses(name string) bool { + lower := strings.ToLower(name) + return strings.Contains(lower, "nameserver") || strings.Contains(lower, "dnsserver") +} + +// adapterNames maps adapter GUIDs, as the registry keys the interfaces, to the +// names an operator sees. +func adapterNames(adapters []*windows.IpAdapterAddresses) map[string]string { + names := make(map[string]string, len(adapters)) + for _, adapter := range adapters { + guid := windows.BytePtrToString(adapter.AdapterName) + names[strings.ToLower(guid)] = windows.UTF16PtrToString(adapter.FriendlyName) + } + return names +} + +// writeAdapterDNS reports the resolver configuration in effect per adapter, +// which is what the resolver uses for a name no NRPT rule matches. +func (g *BundleGenerator) writeAdapterDNS(sb *strings.Builder, adapters []*windows.IpAdapterAddresses, err error) { + writeSection(sb, "Adapter DNS configuration", "GetAdaptersAddresses") + + if err != nil { + fmt.Fprintf(sb, "error: %v\n", err) + return + } + + for _, adapter := range adapters { + name := windows.UTF16PtrToString(adapter.FriendlyName) + suffix := g.anonymizeDomain(windows.UTF16PtrToString(adapter.DnsSuffix)) + + fmt.Fprintf(sb, "%s (index %d, oper status %d)\n", name, adapter.IfIndex, adapter.OperStatus) + fmt.Fprintf(sb, " DNS suffix: %s\n", suffix) + + var servers []string + for server := adapter.FirstDnsServerAddress; server != nil; server = server.Next { + addr, ok := netip.AddrFromSlice(server.Address.IP()) + if !ok { + continue + } + + addr = addr.Unmap() + if g.anonymize { + addr = g.anonymizer.AnonymizeIP(addr) + } + servers = append(servers, addr.String()) + } + + fmt.Fprintf(sb, " DNS servers: %s\n", strings.Join(servers, ", ")) + } +} + +// anonymizeDomain anonymizes a single domain, keeping the leading dot an NRPT +// match domain carries. +func (g *BundleGenerator) anonymizeDomain(entry string) string { + if !g.anonymize { + return entry + } + + domain, dot := strings.CutPrefix(entry, ".") + if domain == "" { + return entry + } + + anonymized := g.anonymizer.AnonymizeDomain(domain) + if dot { + anonymized = "." + anonymized + } + return anonymized +} + +// splitValueEntries splits a registry value that holds a list. The separator +// differs per value: a REG_MULTI_SZ arrives joined with ", ", a SearchList is +// comma separated and a NameServer may use commas or spaces. +func splitValueEntries(value string) []string { + return strings.FieldsFunc(value, func(r rune) bool { + return r == ',' || r == ';' || r == ' ' || r == '\t' + }) +} + +func joinValueEntries(entries []string, anonymize func(string) string) string { + for i, entry := range entries { + entries[i] = anonymize(entry) + } + return strings.Join(entries, ", ") +} + +func writeSection(sb *strings.Builder, title, source string) { + fmt.Fprintf(sb, "\n%s\n%s\n%s\n", title, strings.Repeat("-", len(title)), source) +} + +func subKeyNames(root string) ([]string, error) { + k, err := registry.OpenKey(registry.LOCAL_MACHINE, root, registry.ENUMERATE_SUB_KEYS) + if err != nil { + return nil, fmt.Errorf("open HKEY_LOCAL_MACHINE\\%s: %w", root, err) + } + defer closeKey(k) + + names, err := k.ReadSubKeyNames(-1) + if err != nil { + return nil, fmt.Errorf("read subkey names: %w", err) + } + + return names, nil +} + +// readRegistryValue renders a value as text regardless of its type, so an +// unexpected type in a policy key still shows up instead of being dropped. +func readRegistryValue(k registry.Key, name string) (string, error) { + _, valueType, err := k.GetValue(name, nil) + if err != nil { + return "", fmt.Errorf("get value %s: %w", name, err) + } + + switch valueType { + case registry.SZ, registry.EXPAND_SZ: + value, _, err := k.GetStringValue(name) + if err != nil { + return "", fmt.Errorf("get string value %s: %w", name, err) + } + return value, nil + case registry.MULTI_SZ: + values, _, err := k.GetStringsValue(name) + if err != nil { + return "", fmt.Errorf("get strings value %s: %w", name, err) + } + return strings.Join(values, ", "), nil + case registry.DWORD, registry.QWORD: + value, _, err := k.GetIntegerValue(name) + if err != nil { + return "", fmt.Errorf("get integer value %s: %w", name, err) + } + return fmt.Sprintf("%d (0x%x)", value, value), nil + case registry.BINARY: + value, _, err := k.GetBinaryValue(name) + if err != nil { + return "", fmt.Errorf("get binary value %s: %w", name, err) + } + return hex.EncodeToString(value), nil + default: + return fmt.Sprintf("", valueType), nil + } +} + +// adapterAddresses returns the adapter list including DNS servers. The call +// reports the size it needs, so grow the buffer and retry until it fits. +func adapterAddresses() (adapters []*windows.IpAdapterAddresses, err error) { + // GetAdaptersAddresses is resolved on first use and panics when it is + // missing, so this reports it as an error and leaves the rest of the + // report intact. + defer func() { + if r := recover(); r != nil { + adapters, err = nil, fmt.Errorf("GetAdaptersAddresses: %v", r) + } + }() + + const flags = windows.GAA_FLAG_SKIP_ANYCAST | windows.GAA_FLAG_SKIP_MULTICAST + + size := uint32(15000) + for range 3 { + buf := make([]byte, size) + first := (*windows.IpAdapterAddresses)(unsafe.Pointer(&buf[0])) + + err := windows.GetAdaptersAddresses(windows.AF_UNSPEC, flags, 0, first, &size) + if errors.Is(err, windows.ERROR_BUFFER_OVERFLOW) { + continue + } + if err != nil { + return nil, fmt.Errorf("GetAdaptersAddresses: %w", err) + } + + for adapter := first; adapter != nil; adapter = adapter.Next { + adapters = append(adapters, adapter) + } + return adapters, nil + } + + return nil, fmt.Errorf("GetAdaptersAddresses: buffer kept growing") +} + +func closeKey(k registry.Key) { + if err := k.Close(); err != nil { + log.Debugf("close registry key: %v", err) + } +} diff --git a/client/internal/debug/debug_windows_test.go b/client/internal/debug/debug_windows_test.go new file mode 100644 index 000000000..47df3f6f9 --- /dev/null +++ b/client/internal/debug/debug_windows_test.go @@ -0,0 +1,146 @@ +//go:build windows + +package debug + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/anonymize" +) + +func newDNSValueGenerator(level anonymize.Level) *BundleGenerator { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(level) + + return &BundleGenerator{ + anonymize: true, + anonymizeLevel: level, + anonymizer: anonymizer, + } +} + +// TestAnonymizeValueByName covers the value kinds of the DNS registry keys. The +// names decide the treatment, because the string pass alone replaces only +// domains another part of the bundle already seeded. +func TestAnonymizeValueByName(t *testing.T) { + tests := []struct { + name string + valueName string + value string + assert func(t *testing.T, got string) + }{ + { + name: "NRPT match domains keep the leading dot", + valueName: "Name", + value: ".internal.example.com, .corp.example.org", + assert: func(t *testing.T, got string) { + t.Helper() + for _, entry := range strings.Split(got, ", ") { + assert.True(t, strings.HasPrefix(entry, "."), "entry %q should keep its leading dot", entry) + assert.NotContains(t, entry, "example", "entry %q should not keep the original domain", entry) + } + }, + }, + { + name: "any value name ending in Domain is treated as a domain", + valueName: "ICSDomain", + value: "mshome.net", + assert: func(t *testing.T, got string) { + t.Helper() + assert.NotContains(t, got, "mshome", "should anonymize a domain suffix value") + }, + }, + { + name: "search list is a comma separated domain list", + valueName: "SearchList", + value: "corp.example.com,branch.example.com", + assert: func(t *testing.T, got string) { + t.Helper() + assert.NotContains(t, got, "example", "should anonymize every search domain") + assert.Len(t, strings.Split(got, ", "), 2, "should keep both search domains") + }, + }, + { + name: "name servers are anonymized as addresses", + valueName: "DhcpNameServer", + value: "203.0.113.10 8.8.8.8", + assert: func(t *testing.T, got string) { + t.Helper() + assert.NotContains(t, got, "203.0.113.10", "should anonymize a public resolver address") + // well-known resolvers stay readable at every level + assert.Contains(t, got, "8.8.8.8", "should keep a well-known resolver address") + }, + }, + { + name: "opaque values are left to the string pass", + valueName: "DataBasePath", + value: `%SystemRoot%\System32\drivers\etc`, + assert: func(t *testing.T, got string) { + t.Helper() + assert.Equal(t, `%SystemRoot%\System32\drivers\etc`, got, "should not alter a path") + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := newDNSValueGenerator(anonymize.LevelDefault) + tc.assert(t, g.anonymizeValue(tc.valueName, tc.value)) + }) + } +} + +// TestParseNRPTPolicyTable parses the MOF text of the policy table out +// parameters, as the provider on a client with one NRPT rule renders it. +func TestParseNRPTPolicyTable(t *testing.T) { + const text = `[abstract] +class __PARAMETERS +{ + [Out, EmbeddedInstance("DnsClientPolicyConfiguration"): ToSubClass, ID(2): DisableOverride ToInstance] DnsClientPolicyConfiguration cmdletOutput[] = { +instance of DnsClientPolicyConfiguration +{ + DirectAccessProxyType = "NoProxy"; + DirectAccessQueryIPsecRequired = FALSE; + NameEncoding = "Utf8WithoutMapping"; + Namespace = ".0.100.in-addr.arpa"; +}, +instance of DnsClientPolicyConfiguration +{ + DirectAccessProxyType = "NoProxy"; + NameEncoding = "Utf8WithoutMapping"; + NameServers = {"100.0.255.254", "100.0.255.253"}; + Namespace = ".nb.internal"; +}}; + [in] boolean Effective; + [out] uint32 ReturnValue = 0; +}; +` + + entries := parseNRPTPolicyTable(text) + require.Len(t, entries, 2, "should parse both embedded instances") + + assert.Equal(t, ".0.100.in-addr.arpa", entries[0].namespace, "should read the namespace of the first instance") + assert.Equal(t, ".nb.internal", entries[1].namespace, "should read the namespace of the second instance") + + assert.Equal(t, []registryValue{ + {name: "DirectAccessProxyType", value: "NoProxy"}, + {name: "DirectAccessQueryIPsecRequired", value: "FALSE"}, + {name: "NameEncoding", value: "Utf8WithoutMapping"}, + }, entries[0].values, "should keep the remaining values in order") + + assert.Contains(t, entries[1].values, registryValue{name: "NameServers", value: "100.0.255.254, 100.0.255.253"}, + "should flatten a MOF array") + + for _, value := range entries[1].values { + assert.NotContains(t, value.name, "ReturnValue", "should not read the class level parameters as values") + } +} + +func TestParseNRPTPolicyTableEmpty(t *testing.T) { + assert.Empty(t, parseNRPTPolicyTable(""), "should parse no entries from empty text") + assert.Empty(t, parseNRPTPolicyTable("class __PARAMETERS\n{\n};\n"), "should parse no entries from a table with no instances") +} diff --git a/client/internal/debug/nrpt_windows.go b/client/internal/debug/nrpt_windows.go new file mode 100644 index 000000000..6b6e0e29a --- /dev/null +++ b/client/internal/debug/nrpt_windows.go @@ -0,0 +1,317 @@ +//go:build windows + +package debug + +import ( + "errors" + "fmt" + "runtime" + "strings" + "time" + + "github.com/go-ole/go-ole" + "github.com/go-ole/go-ole/oleutil" + log "github.com/sirupsen/logrus" +) + +const ( + // The NRPT policy table is reachable through the CIM class that backs + // Get-DnsClientNrptPolicy. Unlike the rules in the registry, the table is + // what the resolver currently has loaded, which is the only way to tell an + // applied rule from one that is merely written, in either direction. + nrptPolicyNamespace = `root\Microsoft\Windows\DNS` + nrptPolicyClass = "PS_DnsClientNrptPolicy" + nrptPolicyMethod = "Get" + + // The class has no instances, so the table comes from the out parameters + // of a static method call, rendered as MOF text: the embedded instances + // arrive as a safe array of objects, which cannot be read back through the + // COM bindings, and the text form carries all of them. + nrptPolicyInstanceKeyword = "instance of DnsClientPolicyConfiguration" + + nrptPolicyTimeout = 15 * time.Second +) + +// COM initialization results that leave the calling thread usable: S_FALSE for +// a thread this process already initialized, RPC_E_CHANGED_MODE for one that +// belongs to another apartment. +const ( + sFalse = 0x00000001 + rpcEChangedMode = 0x80010106 +) + +// nrptQueryInFlight admits one read of the policy table at a time. A provider +// that stops answering keeps its goroutine and the OS thread that goroutine +// pinned, so a later bundle reports that instead of pinning another one. +var nrptQueryInFlight = make(chan struct{}, 1) + +// nrptPolicyEntry is one namespace of the effective policy table, holding the +// values of an embedded DnsClientPolicyConfiguration instance in the order the +// provider reported them. +type nrptPolicyEntry struct { + namespace string + values []registryValue +} + +// registryValue is a name and its rendered value, shared by the registry and +// policy table readers so both anonymize by value name the same way. +type registryValue struct { + name string + value string +} + +// effectiveNRPTPolicies reads the effective NRPT table. The call is bounded +// because a WMI provider can block indefinitely and a debug bundle must not. +func effectiveNRPTPolicies() ([]nrptPolicyEntry, error) { + type result struct { + text string + err error + } + + select { + case nrptQueryInFlight <- struct{}{}: + default: + return nil, errors.New("an earlier read of the policy table has not returned") + } + + done := make(chan result, 1) + go func() { + // the slot is released here rather than by the caller, so a read that + // outlives the timeout holds it until the provider answers + defer func() { <-nrptQueryInFlight }() + + text, err := nrptPolicyTableText() + done <- result{text: text, err: err} + }() + + select { + case res := <-done: + if res.err != nil { + return nil, res.err + } + return parseNRPTPolicyTable(res.text), nil + case <-time.After(nrptPolicyTimeout): + return nil, errors.New("read of the policy table timed out") + } +} + +// nrptPolicyTableText calls the policy table method and returns the MOF text of +// its out parameters. +func nrptPolicyTableText() (text string, err error) { + // COM is per thread, and the collection is short lived, so the thread is + // pinned for the duration rather than initialized for the process. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + defer func() { + // The COM call chain is dynamically typed, so a provider that answers + // with an unexpected shape must not take the daemon down with it. + if r := recover(); r != nil { + err = fmt.Errorf("read NRPT policy table: %v", r) + } + }() + + owns, err := coInitialize() + if err != nil { + return "", err + } + if owns { + defer ole.CoUninitialize() + } + + locator, err := oleutil.CreateObject("WbemScripting.SWbemLocator") + if err != nil { + return "", fmt.Errorf("create WMI locator: %w", err) + } + defer locator.Release() + + dispatch, err := locator.QueryInterface(ole.IID_IDispatch) + if err != nil { + return "", fmt.Errorf("query WMI locator interface: %w", err) + } + defer dispatch.Release() + + service, err := dispatchCall(dispatch, "ConnectServer", nil, nrptPolicyNamespace) + if err != nil { + return "", fmt.Errorf("connect to %s: %w", nrptPolicyNamespace, err) + } + defer service.Release() + + inParams, err := spawnMethodInParams(service) + if err != nil { + return "", err + } + defer inParams.Release() + + // The effective table is the merge of the local and the group policy + // store, which is what the resolver answers from. + if _, err := oleutil.PutProperty(inParams, "Effective", true); err != nil { + return "", fmt.Errorf("set Effective parameter: %w", err) + } + + outParams, err := dispatchCall(service, "ExecMethod", nrptPolicyClass, nrptPolicyMethod, inParams) + if err != nil { + return "", fmt.Errorf("call %s.%s: %w", nrptPolicyClass, nrptPolicyMethod, err) + } + defer outParams.Release() + + textVariant, err := oleutil.CallMethod(outParams, "GetObjectText_") + if err != nil { + return "", fmt.Errorf("render policy table: %w", err) + } + defer func() { + if err := textVariant.Clear(); err != nil { + log.Debugf("clear policy table variant: %v", err) + } + }() + + return textVariant.ToString(), nil +} + +// spawnMethodInParams builds the in parameters instance the method needs. The +// provider rejects the call without one, even when every parameter is optional. +func spawnMethodInParams(service *ole.IDispatch) (*ole.IDispatch, error) { + class, err := dispatchCall(service, "Get", nrptPolicyClass) + if err != nil { + return nil, fmt.Errorf("get class %s: %w", nrptPolicyClass, err) + } + defer class.Release() + + methods, err := dispatchProperty(class, "Methods_") + if err != nil { + return nil, fmt.Errorf("get class methods: %w", err) + } + defer methods.Release() + + method, err := dispatchCall(methods, "Item", nrptPolicyMethod) + if err != nil { + return nil, fmt.Errorf("get method %s: %w", nrptPolicyMethod, err) + } + defer method.Release() + + params, err := dispatchProperty(method, "InParameters") + if err != nil { + return nil, fmt.Errorf("get method parameters: %w", err) + } + defer params.Release() + + inParams, err := dispatchCall(params, "SpawnInstance_") + if err != nil { + return nil, fmt.Errorf("spawn parameter instance: %w", err) + } + + return inParams, nil +} + +// parseNRPTPolicyTable pulls the embedded instances out of the MOF text. Each +// instance is a namespace of the table, with one name and value per line. +func parseNRPTPolicyTable(text string) []nrptPolicyEntry { + var entries []nrptPolicyEntry + var current *nrptPolicyEntry + + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(line), ";")) + + switch { + case strings.HasPrefix(line, nrptPolicyInstanceKeyword): + entries = append(entries, nrptPolicyEntry{}) + current = &entries[len(entries)-1] + continue + case strings.HasPrefix(line, "}"): + // closes an instance, and the array with the last one, so the + // class level parameters that follow are not read as values + current = nil + continue + case current == nil, line == "{": + continue + } + + name, value, ok := strings.Cut(line, " = ") + if !ok { + continue + } + + value = unquoteMOFValue(value) + if name == "Namespace" { + current.namespace = value + continue + } + + current.values = append(current.values, registryValue{name: name, value: value}) + } + + return entries +} + +// unquoteMOFValue renders a MOF scalar or array as plain text: "a" becomes a, +// and {"a", "b"} becomes a, b. +func unquoteMOFValue(value string) string { + value = strings.TrimSpace(value) + + if inner, ok := strings.CutPrefix(value, "{"); ok { + value = strings.TrimSuffix(inner, "}") + + entries := strings.Split(value, ",") + for i, entry := range entries { + entries[i] = strings.Trim(strings.TrimSpace(entry), `"`) + } + return strings.Join(entries, ", ") + } + + return strings.Trim(value, `"`) +} + +// coInitialize prepares the calling thread for COM and reports whether this +// call owns the initialization, which decides whether it may be balanced with +// CoUninitialize. S_FALSE took a reference on a thread this process had already +// initialized and so has to be released, while RPC_E_CHANGED_MODE took none: +// the thread belongs to another apartment, which is usable but is not ours to +// uninitialize. +func coInitialize() (bool, error) { + err := ole.CoInitializeEx(0, ole.COINIT_MULTITHREADED) + if err == nil { + return true, nil + } + + var oleErr *ole.OleError + if errors.As(err, &oleErr) { + switch oleErr.Code() { + case sFalse: + return true, nil + case rpcEChangedMode: + return false, nil + } + } + + return false, fmt.Errorf("initialize COM: %w", err) +} + +// dispatchCall calls a COM method that returns an object. +func dispatchCall(dispatch *ole.IDispatch, method string, params ...any) (*ole.IDispatch, error) { + variant, err := oleutil.CallMethod(dispatch, method, params...) + if err != nil { + return nil, err + } + + object := variant.ToIDispatch() + if object == nil { + return nil, fmt.Errorf("%s returned no object", method) + } + + return object, nil +} + +// dispatchProperty reads a COM property that holds an object. +func dispatchProperty(dispatch *ole.IDispatch, property string) (*ole.IDispatch, error) { + variant, err := oleutil.GetProperty(dispatch, property) + if err != nil { + return nil, err + } + + object := variant.ToIDispatch() + if object == nil { + return nil, fmt.Errorf("property %s holds no object", property) + } + + return object, nil +} diff --git a/client/internal/debug/wgshow.go b/client/internal/debug/wgshow.go index 1e8a8a6cc..ee24902e6 100644 --- a/client/internal/debug/wgshow.go +++ b/client/internal/debug/wgshow.go @@ -35,14 +35,14 @@ func (g *BundleGenerator) toWGShowFormat(s *configurer.Stats) string { var sb strings.Builder sb.WriteString(fmt.Sprintf("interface: %s\n", s.DeviceName)) - sb.WriteString(fmt.Sprintf(" public key: %s\n", s.PublicKey)) + sb.WriteString(fmt.Sprintf(" public key: %s\n", g.anonymizer.AnonymizeWGKey(s.PublicKey))) sb.WriteString(fmt.Sprintf(" listen port: %d\n", s.ListenPort)) if s.FWMark != 0 { sb.WriteString(fmt.Sprintf(" fwmark: %#x\n", s.FWMark)) } for _, peer := range s.Peers { - sb.WriteString(fmt.Sprintf("\npeer: %s\n", peer.PublicKey)) + sb.WriteString(fmt.Sprintf("\npeer: %s\n", g.anonymizer.AnonymizeWGKey(peer.PublicKey))) if peer.Endpoint.IP != nil { if g.anonymize { anonEndpoint := g.anonymizer.AnonymizeUDPAddr(peer.Endpoint) @@ -54,7 +54,11 @@ func (g *BundleGenerator) toWGShowFormat(s *configurer.Stats) string { if len(peer.AllowedIPs) > 0 { var ipStrings []string for _, ipnet := range peer.AllowedIPs { - ipStrings = append(ipStrings, ipnet.String()) + ipStr := ipnet.String() + if g.anonymize { + ipStr = g.anonymizer.AnonymizeIPString(ipStr) + } + ipStrings = append(ipStrings, ipStr) } sb.WriteString(fmt.Sprintf(" allowed ips: %s\n", strings.Join(ipStrings, ", "))) } diff --git a/client/internal/dns/host_darwin.go b/client/internal/dns/host_darwin.go index 0f4eb6bf8..81029752e 100644 --- a/client/internal/dns/host_darwin.go +++ b/client/internal/dns/host_darwin.go @@ -267,18 +267,38 @@ func (s *systemConfigurator) getSystemDNSSettings() (SystemDNSSettings, error) { return SystemDNSSettings{}, fmt.Errorf("sending the command: %w", err) } - var dnsSettings SystemDNSSettings + dnsSettings, serverAddresses, err := parseSystemDNSSettings(b) + if err != nil { + return dnsSettings, err + } + + s.mu.Lock() + s.origNameservers = serverAddresses + s.mu.Unlock() + + return dnsSettings, nil +} + +// parseSystemDNSSettings parses the output of `scutil show State:/Network/Service//DNS`. +// Lines that don't match the expected "index : value" shape are skipped: hosts with unusual +// network services (e.g. orphaned hardware ports) can produce entries without a value. +func parseSystemDNSSettings(out []byte) (SystemDNSSettings, []netip.Addr, error) { + // port is not exposed by scutil, default to 53 + dnsSettings := SystemDNSSettings{ServerPort: DefaultPort} var serverAddresses []netip.Addr inSearchDomainsArray := false inServerAddressesArray := false - scanner := bufio.NewScanner(bytes.NewReader(b)) + scanner := bufio.NewScanner(bytes.NewReader(out)) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) switch { case strings.HasPrefix(line, "DomainName :"): - domainName := strings.TrimSpace(strings.Split(line, ":")[1]) - dnsSettings.Domains = append(dnsSettings.Domains, domainName) + domainName := strings.TrimSpace(strings.TrimPrefix(line, "DomainName :")) + if domainName != "" { + dnsSettings.Domains = append(dnsSettings.Domains, domainName) + } + continue case line == "SearchDomains : {": inSearchDomainsArray = true continue @@ -288,36 +308,45 @@ func (s *systemConfigurator) getSystemDNSSettings() (SystemDNSSettings, error) { case line == "}": inSearchDomainsArray = false inServerAddressesArray = false + continue + } + + if !inSearchDomainsArray && !inServerAddressesArray { + continue + } + + parts := strings.SplitN(line, " : ", 2) + if len(parts) != 2 { + log.Debugf("skipping unexpected scutil DNS line %q", line) + continue + } + value := strings.TrimSpace(parts[1]) + if value == "" { + continue } if inSearchDomainsArray { - searchDomain := strings.Split(line, " : ")[1] - dnsSettings.Domains = append(dnsSettings.Domains, searchDomain) - } else if inServerAddressesArray { - address := strings.Split(line, " : ")[1] - if ip, err := netip.ParseAddr(address); err == nil && !ip.IsUnspecified() { - ip = ip.Unmap() - serverAddresses = append(serverAddresses, ip) - // Prefer the first IPv4 server as ServerIP since our DNS listener is IPv4. - if !dnsSettings.ServerIP.IsValid() && ip.Is4() { - dnsSettings.ServerIP = ip - } - } + dnsSettings.Domains = append(dnsSettings.Domains, value) + continue + } + + ip, err := netip.ParseAddr(value) + if err != nil || ip.IsUnspecified() { + continue + } + ip = ip.Unmap() + serverAddresses = append(serverAddresses, ip) + // Prefer the first IPv4 server as ServerIP since our DNS listener is IPv4. + if !dnsSettings.ServerIP.IsValid() && ip.Is4() { + dnsSettings.ServerIP = ip } } if err := scanner.Err(); err != nil { - return dnsSettings, err + return dnsSettings, serverAddresses, err } - // default to 53 port - dnsSettings.ServerPort = DefaultPort - - s.mu.Lock() - s.origNameservers = serverAddresses - s.mu.Unlock() - - return dnsSettings, nil + return dnsSettings, serverAddresses, nil } func (s *systemConfigurator) getOriginalNameservers() []netip.Addr { @@ -435,11 +464,15 @@ func (s *systemConfigurator) getPrimaryService() (string, string, error) { router := "" for scanner.Scan() { text := scanner.Text() + parts := strings.SplitN(text, ":", 2) + if len(parts) != 2 { + continue + } if strings.Contains(text, "PrimaryService") { - primaryService = strings.TrimSpace(strings.Split(text, ":")[1]) + primaryService = strings.TrimSpace(parts[1]) } if strings.Contains(text, "Router") { - router = strings.TrimSpace(strings.Split(text, ":")[1]) + router = strings.TrimSpace(parts[1]) } } if err := scanner.Err(); err != nil && err != io.EOF { diff --git a/client/internal/dns/host_darwin_test.go b/client/internal/dns/host_darwin_test.go index 94d020c39..bee691c71 100644 --- a/client/internal/dns/host_darwin_test.go +++ b/client/internal/dns/host_darwin_test.go @@ -328,6 +328,120 @@ func removeTestDNSKey(key string) error { return err } +func TestParseSystemDNSSettings(t *testing.T) { + tests := []struct { + name string + output string + expectedDomains []string + expectedServers []netip.Addr + expectedIP netip.Addr + }{ + { + name: "well_formed", + output: ` { + DomainName : example.com + SearchDomains : { + 0 : example.com + 1 : corp.example.com + } + ServerAddresses : { + 0 : 192.168.1.1 + 1 : fd00::53 + } +} +`, + expectedDomains: []string{"example.com", "example.com", "corp.example.com"}, + expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1"), netip.MustParseAddr("fd00::53")}, + expectedIP: netip.MustParseAddr("192.168.1.1"), + }, + { + // entries without a value after the separator used to panic with + // "index out of range [1] with length 1" + name: "malformed_array_entries_skipped", + output: ` { + SearchDomains : { + 0 : + (null) + + 1 : corp.example.com + } + ServerAddresses : { + 0 : + 1 : 192.168.1.1 + } +} +`, + expectedDomains: []string{"corp.example.com"}, + expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1")}, + expectedIP: netip.MustParseAddr("192.168.1.1"), + }, + { + name: "domain_name_without_value_skipped", + output: ` { + DomainName : + ServerAddresses : { + 0 : 192.168.1.1 + } +} +`, + expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1")}, + expectedIP: netip.MustParseAddr("192.168.1.1"), + }, + { + name: "ipv6_first_prefers_ipv4_server_ip", + output: ` { + ServerAddresses : { + 0 : fd00::53 + 1 : 192.168.1.1 + } +} +`, + expectedServers: []netip.Addr{netip.MustParseAddr("fd00::53"), netip.MustParseAddr("192.168.1.1")}, + expectedIP: netip.MustParseAddr("192.168.1.1"), + }, + { + name: "invalid_and_unspecified_addresses_skipped", + output: ` { + ServerAddresses : { + 0 : (null) + 1 : 0.0.0.0 + 2 : 192.168.1.1 + } +} +`, + expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1")}, + expectedIP: netip.MustParseAddr("192.168.1.1"), + }, + { + name: "v4_mapped_address_unmapped", + output: ` { + ServerAddresses : { + 0 : ::ffff:192.168.1.1 + } +} +`, + expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1")}, + expectedIP: netip.MustParseAddr("192.168.1.1"), + }, + { + name: "empty_output", + output: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + settings, servers, err := parseSystemDNSSettings([]byte(tc.output)) + require.NoError(t, err, "parsing should not fail") + + assert.Equal(t, tc.expectedDomains, settings.Domains, "domains should match") + assert.Equal(t, tc.expectedServers, servers, "server addresses should match") + assert.Equal(t, tc.expectedIP, settings.ServerIP, "server IP should match") + assert.Equal(t, DefaultPort, settings.ServerPort, "server port should default to 53") + }) + } +} + func TestGetOriginalNameservers(t *testing.T) { configurator := &systemConfigurator{ createdKeys: make(map[string]struct{}), diff --git a/client/internal/dns/host_windows.go b/client/internal/dns/host_windows.go index 4f6ece532..2852dddb9 100644 --- a/client/internal/dns/host_windows.go +++ b/client/internal/dns/host_windows.go @@ -31,10 +31,30 @@ var ( dnsFlushResolverCacheFn = dnsapi.NewProc("DnsFlushResolverCache") ) +// Registry locations of the host DNS configuration this package programs, +// exported so a diagnostic reader reports the same locations that are written. const ( - dnsPolicyConfigMatchPath = `SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig\NetBird-Match` - gpoDnsPolicyRoot = `SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig` - gpoDnsPolicyConfigMatchPath = gpoDnsPolicyRoot + `\NetBird-Match` + // NRPTKeyPrefix starts the name of every NRPT rule key this client creates. + // Older versions used different layouts under the same prefix: a single + // unsuffixed key, then one key per domain, now one key per batch of domains. + NRPTKeyPrefix = "NetBird-Match" + + // DNSPolicyConfigRoot holds the NRPT rules of the local policy store. + DNSPolicyConfigRoot = `SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig` + + // GPODNSPolicyConfigRoot holds the NRPT rules of the group policy store, + // which takes precedence over the local one when it is present. + GPODNSPolicyConfigRoot = `SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig` + + // InterfaceConfigPath and InterfaceConfigPathV6 hold the per-interface DNS + // settings, keyed by interface GUID, in separate hives per address family. + InterfaceConfigPath = `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces` + InterfaceConfigPathV6 = `SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces` +) + +const ( + dnsPolicyConfigMatchPath = DNSPolicyConfigRoot + `\` + NRPTKeyPrefix + gpoDnsPolicyConfigMatchPath = GPODNSPolicyConfigRoot + `\` + NRPTKeyPrefix dnsPolicyConfigVersionKey = "Version" dnsPolicyConfigVersionValue = 2 @@ -45,8 +65,6 @@ const ( nrptMaxDomainsPerRule = 50 - interfaceConfigPath = `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces` - interfaceConfigPathV6 = `SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces` interfaceConfigNameServerKey = "NameServer" interfaceConfigDhcpNameSrvKey = "DhcpNameServer" interfaceConfigSearchListKey = "SearchList" @@ -73,7 +91,6 @@ type registryConfigurator struct { guid string routingAll bool gpo bool - nrptEntryCount int origNameservers []netip.Addr } @@ -84,7 +101,7 @@ func newHostManager(wgInterface WGIface) (*registryConfigurator, error) { } var useGPO bool - k, err := registry.OpenKey(registry.LOCAL_MACHINE, gpoDnsPolicyRoot, registry.QUERY_VALUE) + k, err := registry.OpenKey(registry.LOCAL_MACHINE, GPODNSPolicyConfigRoot, registry.QUERY_VALUE) if err != nil { log.Debugf("failed to open GPO DNS policy root: %v", err) } else { @@ -123,7 +140,7 @@ func (r *registryConfigurator) captureOriginalNameservers() ([]netip.Addr, error seen := make(map[netip.Addr]struct{}) var out []netip.Addr var merr *multierror.Error - for _, root := range []string{interfaceConfigPath, interfaceConfigPathV6} { + for _, root := range []string{InterfaceConfigPath, InterfaceConfigPathV6} { addrs, err := r.captureFromTcpipRoot(root) if err != nil { merr = multierror.Append(merr, fmt.Errorf("%s: %w", root, err)) @@ -306,14 +323,9 @@ func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager } if len(matchDomains) != 0 { - count, err := r.addDNSMatchPolicy(matchDomains, config.ServerIP) - // Update count even on error to ensure cleanup covers partially created rules - r.nrptEntryCount = count - if err != nil { + if err := r.addDNSMatchPolicy(matchDomains, config.ServerIP); err != nil { return fmt.Errorf("add dns match policy: %w", err) } - } else { - r.nrptEntryCount = 0 } r.updateState(stateManager) @@ -329,9 +341,8 @@ func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager func (r *registryConfigurator) updateState(stateManager *statemanager.Manager) { if err := stateManager.UpdateState(&ShutdownState{ - Guid: r.guid, - GPO: r.gpo, - NRPTEntryCount: r.nrptEntryCount, + Guid: r.guid, + GPO: r.gpo, }); err != nil { log.Errorf("failed to update shutdown state: %s", err) } @@ -346,7 +357,7 @@ func (r *registryConfigurator) addDNSSetupForAll(ip netip.Addr) error { return nil } -func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr) (int, error) { +func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr) error { // if the gpo key is present, we need to put our DNS settings there, otherwise our config might be ignored // see https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-gpnrpt/8cc31cb9-20cb-4140-9e85-3e08703b4745 @@ -363,19 +374,17 @@ func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr gpoPath := fmt.Sprintf("%s-%d", gpoDnsPolicyConfigMatchPath, ruleIndex) if err := r.configureDNSPolicy(localPath, batchDomains, ip); err != nil { - return ruleIndex, fmt.Errorf("configure DNS Local policy for rule %d: %w", ruleIndex, err) + return fmt.Errorf("configure DNS Local policy for rule %d: %w", ruleIndex, err) } - // Increment immediately so the caller's cleanup path knows about this rule - ruleIndex++ - if r.gpo { if err := r.configureDNSPolicy(gpoPath, batchDomains, ip); err != nil { - return ruleIndex, fmt.Errorf("configure gpo DNS policy for rule %d: %w", ruleIndex-1, err) + return fmt.Errorf("configure gpo DNS policy for rule %d: %w", ruleIndex, err) } } - log.Debugf("added NRPT rule %d with %d domains", ruleIndex-1, len(batchDomains)) + log.Debugf("added NRPT rule %d with %d domains", ruleIndex, len(batchDomains)) + ruleIndex++ } if r.gpo { @@ -385,7 +394,7 @@ func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr } log.Infof("added %d NRPT rules for %d domains", ruleIndex, len(domains)) - return ruleIndex, nil + return nil } func (r *registryConfigurator) configureDNSPolicy(policyPath string, domains []string, ip netip.Addr) error { @@ -496,7 +505,7 @@ func (r *registryConfigurator) deleteInterfaceRegistryKeyProperty(propertyKey st } func (r *registryConfigurator) getInterfaceRegistryKey() (registry.Key, error) { - regKeyPath := interfaceConfigPath + "\\" + r.guid + regKeyPath := InterfaceConfigPath + "\\" + r.guid regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, regKeyPath, registry.SET_VALUE) if err != nil { return regKey, fmt.Errorf("open HKEY_LOCAL_MACHINE\\%s: %w", regKeyPath, err) @@ -518,28 +527,28 @@ func (r *registryConfigurator) restoreHostDNS() error { return nil } +// removeDNSMatchPolicies deletes every NRPT rule this client may have created, +// from the local and the GPO policy store. The rules are found by enumerating +// the registry, the only authoritative record of what was written. Cleanup must +// not depend on a rule count: the in-memory one is scoped to a single +// registryConfigurator and the persisted one is deleted on every clean +// disconnect, and a rule left behind keeps resolving names over an interface +// that is gone, until reboot discards the volatile key. func (r *registryConfigurator) removeDNSMatchPolicies() error { var merr *multierror.Error - // Try to remove the base entries (for backward compatibility) - if err := removeRegistryKeyFromDNSPolicyConfig(dnsPolicyConfigMatchPath); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove local base entry: %w", err)) - } - - if err := removeRegistryKeyFromDNSPolicyConfig(gpoDnsPolicyConfigMatchPath); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove GPO base entry: %w", err)) - } - - for i := 0; i < r.nrptEntryCount; i++ { - localPath := fmt.Sprintf("%s-%d", dnsPolicyConfigMatchPath, i) - gpoPath := fmt.Sprintf("%s-%d", gpoDnsPolicyConfigMatchPath, i) - - if err := removeRegistryKeyFromDNSPolicyConfig(localPath); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove local entry %d: %w", i, err)) + for _, root := range []string{DNSPolicyConfigRoot, GPODNSPolicyConfigRoot} { + names, err := listNRPTRuleKeys(root) + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("list rule keys under %s: %w", root, err)) + continue } - if err := removeRegistryKeyFromDNSPolicyConfig(gpoPath); err != nil { - merr = multierror.Append(merr, fmt.Errorf("remove GPO entry %d: %w", i, err)) + for _, name := range names { + path := root + `\` + name + if err := removeRegistryKeyFromDNSPolicyConfig(path); err != nil { + merr = multierror.Append(merr, fmt.Errorf("remove entry %s: %w", path, err)) + } } } @@ -554,6 +563,39 @@ func (r *registryConfigurator) restoreUncleanShutdownDNS() error { return r.restoreHostDNS() } +// listNRPTRuleKeys returns the names of our NRPT rule keys under a policy store +// root. An absent root holds nothing to clean up, which is the normal state of +// the GPO store on a machine without DNS Client policy. +func listNRPTRuleKeys(root string) ([]string, error) { + k, err := registry.OpenKey(registry.LOCAL_MACHINE, root, registry.ENUMERATE_SUB_KEYS) + switch { + case errors.Is(err, registry.ErrNotExist), errors.Is(err, syscall.ERROR_PATH_NOT_FOUND): + // the GPO store is absent on a machine without DNS client policy + log.Debugf("HKEY_LOCAL_MACHINE\\%s does not exist", root) + return nil, nil + case err != nil: + // any other failure has to reach the caller: reporting no rules would + // report a successful cleanup while leaving the rules in place + return nil, fmt.Errorf("open HKEY_LOCAL_MACHINE\\%s: %w", root, err) + } + defer closer(k) + + names, err := k.ReadSubKeyNames(-1) + if err != nil { + return nil, fmt.Errorf("read subkey names: %w", err) + } + + var ruleKeys []string + for _, name := range names { + // registry key names are case insensitive + if strings.HasPrefix(strings.ToLower(name), strings.ToLower(NRPTKeyPrefix)) { + ruleKeys = append(ruleKeys, name) + } + } + + return ruleKeys, nil +} + func removeRegistryKeyFromDNSPolicyConfig(regKeyPath string) error { k, err := registry.OpenKey(registry.LOCAL_MACHINE, regKeyPath, registry.QUERY_VALUE) if err != nil { diff --git a/client/internal/dns/host_windows_test.go b/client/internal/dns/host_windows_test.go index 3cd2b1bd5..861613c95 100644 --- a/client/internal/dns/host_windows_test.go +++ b/client/internal/dns/host_windows_test.go @@ -25,7 +25,7 @@ func TestNRPTEntriesCleanupOnConfigChange(t *testing.T) { // Create a test interface registry key so updateSearchDomains doesn't fail testGUID := "{12345678-1234-1234-1234-123456789ABC}" - interfacePath := `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces\` + testGUID + interfacePath := InterfaceConfigPath + `\` + testGUID testKey, _, err := registry.CreateKey(registry.LOCAL_MACHINE, interfacePath, registry.SET_VALUE) require.NoError(t, err, "Should create test interface registry key") testKey.Close() @@ -56,7 +56,7 @@ func TestNRPTEntriesCleanupOnConfigChange(t *testing.T) { require.NoError(t, err) // Verify 3 NRPT rules exist - assert.Equal(t, 3, cfg.nrptEntryCount, "Should create 3 NRPT rules for 125 domains") + assert.Equal(t, 3, countNRPTRuleKeys(t), "Should create 3 NRPT rules for 125 domains") for i := 0; i < 3; i++ { exists, err := registryKeyExists(fmt.Sprintf("%s-%d", dnsPolicyConfigMatchPath, i)) require.NoError(t, err) @@ -81,7 +81,7 @@ func TestNRPTEntriesCleanupOnConfigChange(t *testing.T) { require.NoError(t, err) // Verify first 2 NRPT rules exist - assert.Equal(t, 2, cfg.nrptEntryCount, "Should create 2 NRPT rules for 75 domains") + assert.Equal(t, 2, countNRPTRuleKeys(t), "Should create 2 NRPT rules for 75 domains") for i := 0; i < 2; i++ { exists, err := registryKeyExists(fmt.Sprintf("%s-%d", dnsPolicyConfigMatchPath, i)) require.NoError(t, err) @@ -106,9 +106,65 @@ func registryKeyExists(path string) (bool, error) { return true, nil } +// TestNRPTCleanupWithoutRuleCount verifies that rules written by a previous run +// are removed by a configurator that has no record of how many there are: an +// unclean exit loses the in-memory count and a clean disconnect deletes the +// persisted one, so cleanup cannot depend on either. +func TestNRPTCleanupWithoutRuleCount(t *testing.T) { + if testing.Short() { + t.Skip("skipping registry integration test in short mode") + } + + defer cleanupRegistryKeys(t) + cleanupRegistryKeys(t) + + testIP := netip.MustParseAddr("100.64.0.1") + + // 75 domains produce two indexed rules, as the current layout does + domains := make([]string, 75) + for i := range domains { + domains[i] = fmt.Sprintf(".domain%d.com", i+1) + } + + previousRun := ®istryConfigurator{} + require.NoError(t, previousRun.addDNSMatchPolicy(domains, testIP)) + + // the unsuffixed key an older version would have written + require.NoError(t, previousRun.configureDNSPolicy(dnsPolicyConfigMatchPath, []string{".legacy.example.com"}, testIP)) + + // a policy owned by someone else, which cleanup must not touch + foreignPath := DNSPolicyConfigRoot + `\DnsPolicyConfigTestForeign` + foreignKey, _, err := registry.CreateKey(registry.LOCAL_MACHINE, foreignPath, registry.SET_VALUE) + require.NoError(t, err, "Should create foreign policy key") + foreignKey.Close() + defer func() { + _ = registry.DeleteKey(registry.LOCAL_MACHINE, foreignPath) + }() + + require.Equal(t, 3, countNRPTRuleKeys(t), "Should have two indexed rules and the legacy one") + + // a configurator that never applied a DNS config, as one built after a + // restart or from a shutdown state without a count is + freshRun := ®istryConfigurator{} + require.NoError(t, freshRun.removeDNSMatchPolicies()) + + assert.Equal(t, 0, countNRPTRuleKeys(t), "Should remove every rule left by the previous run") + + exists, err := registryKeyExists(foreignPath) + require.NoError(t, err) + assert.True(t, exists, "Should not remove a policy that is not ours") +} + +func countNRPTRuleKeys(t *testing.T) int { + t.Helper() + + names, err := listNRPTRuleKeys(DNSPolicyConfigRoot) + require.NoError(t, err, "Should list NRPT rule keys") + return len(names) +} + func cleanupRegistryKeys(*testing.T) { - // Clean up more entries to account for batching tests with many domains - cfg := ®istryConfigurator{nrptEntryCount: 20} + cfg := ®istryConfigurator{} _ = cfg.removeDNSMatchPolicies() } @@ -125,7 +181,7 @@ func TestNRPTDomainBatching(t *testing.T) { // Create a test interface registry key so updateSearchDomains doesn't fail testGUID := "{12345678-1234-1234-1234-123456789ABC}" - interfacePath := `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces\` + testGUID + interfacePath := InterfaceConfigPath + `\` + testGUID testKey, _, err := registry.CreateKey(registry.LOCAL_MACHINE, interfacePath, registry.SET_VALUE) require.NoError(t, err, "Should create test interface registry key") testKey.Close() @@ -193,7 +249,7 @@ func TestNRPTDomainBatching(t *testing.T) { require.NoError(t, err) // Verify that exactly expectedRuleCount rules were created - assert.Equal(t, tc.expectedRuleCount, cfg.nrptEntryCount, + assert.Equal(t, tc.expectedRuleCount, countNRPTRuleKeys(t), "Should create %d NRPT rules for %d domains", tc.expectedRuleCount, tc.domainCount) // Verify all expected rules exist diff --git a/client/internal/dns/unclean_shutdown_windows.go b/client/internal/dns/unclean_shutdown_windows.go index 24a9eca50..ab0b2cc63 100644 --- a/client/internal/dns/unclean_shutdown_windows.go +++ b/client/internal/dns/unclean_shutdown_windows.go @@ -5,9 +5,8 @@ import ( ) type ShutdownState struct { - Guid string - GPO bool - NRPTEntryCount int + Guid string + GPO bool } func (s *ShutdownState) Name() string { @@ -16,9 +15,8 @@ func (s *ShutdownState) Name() string { func (s *ShutdownState) Cleanup() error { manager := ®istryConfigurator{ - guid: s.Guid, - gpo: s.GPO, - nrptEntryCount: s.NRPTEntryCount, + guid: s.Guid, + gpo: s.GPO, } if err := manager.restoreUncleanShutdownDNS(); err != nil { diff --git a/client/internal/ebpf/ebpf/manager_linux.go b/client/internal/ebpf/ebpf/manager_linux.go index 7520a6387..64a3e5b54 100644 --- a/client/internal/ebpf/ebpf/manager_linux.go +++ b/client/internal/ebpf/ebpf/manager_linux.go @@ -2,17 +2,21 @@ package ebpf import ( _ "embed" + "fmt" "net" "sync" "github.com/cilium/ebpf/link" "github.com/cilium/ebpf/rlimit" log "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" "github.com/netbirdio/netbird/client/internal/ebpf/manager" ) const ( + xdpProgName = "nb_xdp_prog" + mapKeyFeatures uint32 = 0 featureFlagWGProxy = 0b00000001 @@ -68,21 +72,50 @@ func (tf *GeneralManager) loadXdp() error { return err } - // load pre-compiled programs into the kernel. - err = loadBpfObjects(&tf.bpfObjs, nil) + // lo has no native XDP, so the program runs in generic mode. Unless it + // declares multi-buffer support the kernel must linearize every non-linear + // skb before running it. Loopback packets are up to 64 KB, so that is a + // contiguous GFP_ATOMIC allocation per packet, and when it fails the packet + // is dropped before the program runs, stalling local TCP connections. + // Multi-buffer XDP in generic mode requires kernel 6.3, so fall back to a + // plain attach when the kernel rejects it. + err = tf.attachXdp(iFace.Index, true) + if err == nil { + return nil + } + log.Debugf("failed to attach multi-buffer xdp program, retrying without it: %s", err) + + return tf.attachXdp(iFace.Index, false) +} + +func (tf *GeneralManager) attachXdp(iFaceIndex int, multiBuffer bool) error { + spec, err := loadBpf() if err != nil { - return err + return fmt.Errorf("load bpf spec: %w", err) + } + + if multiBuffer { + prog, ok := spec.Programs[xdpProgName] + if !ok { + return fmt.Errorf("program %s not found in bpf spec", xdpProgName) + } + prog.Flags |= unix.BPF_F_XDP_HAS_FRAGS + } + + if err := spec.LoadAndAssign(&tf.bpfObjs, nil); err != nil { + return fmt.Errorf("load bpf objects: %w", err) } tf.link, err = link.AttachXDP(link.XDPOptions{ Program: tf.bpfObjs.NbXdpProg, - Interface: iFace.Index, + Interface: iFaceIndex, }) - if err != nil { - _ = tf.bpfObjs.Close() + if closeErr := tf.bpfObjs.Close(); closeErr != nil { + log.Debugf("failed to close bpf objects after xdp attach error: %s", closeErr) + } tf.link = nil - return err + return fmt.Errorf("attach xdp: %w", err) } return nil } diff --git a/client/internal/engine.go b/client/internal/engine.go index f4f47992f..d92c360f2 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -23,6 +23,7 @@ import ( "golang.zx2c4.com/wireguard/tun/netstack" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "github.com/netbirdio/netbird/client/anonymize" nberrors "github.com/netbirdio/netbird/client/errors" "github.com/netbirdio/netbird/client/firewall" "github.com/netbirdio/netbird/client/firewall/firewalld" @@ -1385,6 +1386,7 @@ func (e *Engine) handleBundle(params *mgmProto.BundleParameters) (*mgmProto.JobR bundleJobParams := debug.BundleConfig{ Anonymize: params.Anonymize, + AnonymizeLevel: anonymize.ParseLevel(params.AnonymizeLevel), IncludeSystemInfo: true, LogFileCount: uint32(params.LogFileCount), } diff --git a/client/internal/metrics/connection_type.go b/client/internal/metrics/connection_type.go index a3406a6b8..d393e5112 100644 --- a/client/internal/metrics/connection_type.go +++ b/client/internal/metrics/connection_type.go @@ -4,11 +4,17 @@ package metrics type ConnectionType string const ( - // ConnectionTypeICE represents a direct peer-to-peer connection using ICE - ConnectionTypeICE ConnectionType = "ice" + // ConnectionTypeICEP2P represents a direct peer-to-peer connection using ICE + ConnectionTypeICEP2P ConnectionType = "ice_p2p" + + // ConnectionTypeICETurn represents an ICE connection through a TURN server + ConnectionTypeICETurn ConnectionType = "ice_turn" // ConnectionTypeRelay represents a relayed connection ConnectionTypeRelay ConnectionType = "relay" + + // ConnectionTypeUnknown represents a connection with no active transport. It is not pushed. + ConnectionTypeUnknown ConnectionType = "unknown" ) // String returns the string representation of the connection type diff --git a/client/internal/metrics/influxdb_test.go b/client/internal/metrics/influxdb_test.go index b964e31a3..6a226fe2f 100644 --- a/client/internal/metrics/influxdb_test.go +++ b/client/internal/metrics/influxdb_test.go @@ -28,7 +28,7 @@ func TestInfluxDBMetrics_RecordAndExport(t *testing.T) { WgHandshakeSuccess: time.Now().Add(-1 * time.Second), } - m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICE, false, ts) + m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICEP2P, false, ts) var buf bytes.Buffer err := m.Export(&buf) @@ -60,7 +60,7 @@ func TestInfluxDBMetrics_ExportDeterministicFieldOrder(t *testing.T) { // Record multiple times and verify consistent field order for i := 0; i < 10; i++ { - m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICE, false, ts) + m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICEP2P, false, ts) } var buf bytes.Buffer diff --git a/client/internal/metrics/infra/README.md b/client/internal/metrics/infra/README.md index 7941a30cf..0a69404df 100644 --- a/client/internal/metrics/infra/README.md +++ b/client/internal/metrics/infra/README.md @@ -56,14 +56,33 @@ Measurement: `netbird_peer_connection` Tags: - `deployment_type`: "cloud" | "selfhosted" | "unknown" -- `connection_type`: "ice" | "relay" +- `connection_type`: "ice_p2p" | "ice_turn" | "relay" (see below) - `attempt_type`: "initial" | "reconnection" - `version`: NetBird version string - `os`: Operating system (linux, darwin, windows, android, ios, etc.) - `arch`: CPU architecture (amd64, arm64, etc.) +- `peer_id`: anonymised peer identifier (truncated SHA-256 of the WireGuard public key) +- `connection_pair_id`: deterministic identifier for the peer pair, identical on both sides **Note:** `SignalingReceived` is set when the first offer or answer arrives from the remote peer (in both initial and reconnection paths). It excludes the potentially unbounded wait for the remote peer to come online. +#### `connection_type` values + +Derived from the connection priority (`conntype.ConnPriority`) by `metricsConnType` in `client/internal/peer/conn.go`: + +| Value | Priority | Traffic is | +|-------|----------|------------| +| `ice_p2p` | `ICEP2P` | direct peer-to-peer | +| `ice_turn` | `ICETurn` | relayed, through a TURN server | +| `relay` | `Relay` | relayed, through a NetBird relay | +| `unknown` | `None` or unrecognised | no active transport — **the sample is not pushed** | + +**Direct traffic is `ice_p2p` only.** `ice_turn` is relayed despite being negotiated by ICE, matching `Conn.isRelayed`. + +`None` means no transport is active: not established yet, or reset after a relay drop or a peer-state reset. Such a sample cannot be attributed to a transport, so `recordConnectionMetrics` drops it instead of pushing it — `unknown` therefore never appears in the bucket. Connection counts are counts of connections whose transport was known at sampling time. + +**Samples recorded before 0.77 used a single `ice` value** which covered `ICEP2P`, `ICETurn` *and* `None`, so historical `ice` samples overstate direct connections by an unknown amount and must not be compared with `ice_p2p`. + ### Sync Duration Measurement: `netbird_sync` diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 09a4e8b02..f3235ec7f 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -307,6 +307,8 @@ func (conn *Conn) Close(signalToRemote bool) { if conn.wgWatcherCancel != nil { conn.wgWatcherCancel() + conn.wgWatcher = nil + conn.wgWatcherCancel = nil } conn.workerRelay.CloseConn() if conn.workerICE != nil { @@ -959,12 +961,9 @@ func (conn *Conn) recordConnectionMetrics() { priority := conn.currentConnPriority conn.mu.Unlock() - var connType metrics.ConnectionType - switch priority { - case conntype.Relay: - connType = metrics.ConnectionTypeRelay - default: - connType = metrics.ConnectionTypeICE + connType := metricsConnType(priority) + if connType == metrics.ConnectionTypeUnknown { + return } // Record metrics with timestamps - duration calculation happens in metrics package @@ -1065,3 +1064,16 @@ func boolToConnStatus(connected bool) guard.ConnStatus { } return guard.ConnStatusDisconnected } + +func metricsConnType(priority conntype.ConnPriority) metrics.ConnectionType { + switch priority { + case conntype.Relay: + return metrics.ConnectionTypeRelay + case conntype.ICETurn: + return metrics.ConnectionTypeICETurn + case conntype.ICEP2P: + return metrics.ConnectionTypeICEP2P + default: + return metrics.ConnectionTypeUnknown + } +} diff --git a/client/internal/peer/conn_test.go b/client/internal/peer/conn_test.go index 49979ea83..b709d5e40 100644 --- a/client/internal/peer/conn_test.go +++ b/client/internal/peer/conn_test.go @@ -11,6 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/internal/metrics" + "github.com/netbirdio/netbird/client/internal/peer/conntype" "github.com/netbirdio/netbird/client/internal/peer/dispatcher" "github.com/netbirdio/netbird/client/internal/peer/guard" "github.com/netbirdio/netbird/client/internal/peer/ice" @@ -386,3 +388,33 @@ func TestConn_onWGDisconnected_NoEscalationWithoutRosenpass(t *testing.T) { } assert.Empty(t, disconnected, "escalation must be limited to rosenpass connections") } + +func TestMetricsConnType(t *testing.T) { + tests := []struct { + name string + priority conntype.ConnPriority + expected metrics.ConnectionType + }{ + {"relay", conntype.Relay, metrics.ConnectionTypeRelay}, + {"ice over turn is relayed, not p2p", conntype.ICETurn, metrics.ConnectionTypeICETurn}, + {"direct p2p", conntype.ICEP2P, metrics.ConnectionTypeICEP2P}, + {"unset priority is unknown, not p2p", conntype.None, metrics.ConnectionTypeUnknown}, + {"unrecognised priority is unknown", conntype.ConnPriority(99), metrics.ConnectionTypeUnknown}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, metricsConnType(tc.priority)) + }) + } +} + +func TestMetricsConnType_RelayedMatchesIsRelayed(t *testing.T) { + for _, priority := range []conntype.ConnPriority{conntype.None, conntype.Relay, conntype.ICETurn, conntype.ICEP2P} { + conn := &Conn{currentConnPriority: priority} + tag := metricsConnType(priority) + relayedTag := tag == metrics.ConnectionTypeRelay || tag == metrics.ConnectionTypeICETurn + assert.Equal(t, conn.isRelayed(), relayedTag, + "priority %s: isRelayed and the %q metric tag must agree", priority, tag) + } +} diff --git a/client/internal/profilemanager/state.go b/client/internal/profilemanager/state.go index fcd1c384c..ddb5dd056 100644 --- a/client/internal/profilemanager/state.go +++ b/client/internal/profilemanager/state.go @@ -87,9 +87,10 @@ func (pm *ProfileManager) SetActiveProfileState(state *ProfileState) error { // RemoveProfileState deletes the per-profile state file (which holds the // account email used for the SSO login hint and the UI display). Called after -// a successful logout so a logged-out profile no longer shows a stale account -// email. The state file only stores the email, so deleting it is equivalent to -// clearing it; the next SSO login recreates it. A missing file is not an error. +// profile removal; logout keeps the file so the next login can pass the email +// as the login_hint. The state file only stores the email, so deleting it is +// equivalent to clearing it; the next SSO login recreates it. A missing file +// is not an error. func (pm *ProfileManager) RemoveProfileState(profileName string) error { configDir, err := getConfigDir() if err != nil { diff --git a/client/internal/routemanager/ipfwdstate/ipfwdstate.go b/client/internal/routemanager/ipfwdstate/ipfwdstate.go index 2be1c2ae7..3d571e16b 100644 --- a/client/internal/routemanager/ipfwdstate/ipfwdstate.go +++ b/client/internal/routemanager/ipfwdstate/ipfwdstate.go @@ -2,54 +2,183 @@ package ipfwdstate import ( "fmt" + "sync" log "github.com/sirupsen/logrus" "github.com/netbirdio/netbird/client/internal/routemanager/systemops" ) -// IPForwardingState is a struct that keeps track of the IP forwarding state. -// todo: read initial state of the IP forwarding from the system and reset the state based on it. -// todo: separate v4/v6 forwarding state, since the sysctls are independent -// (net.ipv4.ip_forward vs net.ipv6.conf.all.forwarding). Currently the nftables -// manager shares one instance between both routers, which works only because -// EnableIPForwarding enables both sysctls in a single call. +// IPForwardingState tracks v4 and v6 IP-forwarding sysctl enables with +// independent refcounts so a v4-only routing setup doesn't flip v6 sysctls. type IPForwardingState struct { - enabledCounter int + mu sync.Mutex + + v4Count int + v6Count int + + // routingV4/routingV6 track whether the routing path currently holds a + // reference, so repeated EnableRouting calls (one per network-map update) + // hold at most one reference per family and an unpaired DisableRouting + // can't release references held by DNAT rules. + routingV4 bool + routingV6 bool + + wgIfaceName string + v6Saved map[string]int } -func NewIPForwardingState() *IPForwardingState { - return &IPForwardingState{} +// NewIPForwardingState returns a state tracker for the IP-forwarding sysctls. +// wgIfaceName is excluded from the per-interface accept_ra handling. +func NewIPForwardingState(wgIfaceName string) *IPForwardingState { + return &IPForwardingState{wgIfaceName: wgIfaceName} } -func (f *IPForwardingState) RequestForwarding() error { - if f.enabledCounter != 0 { - f.enabledCounter++ +// Counts returns the current v4 and v6 refcounts. Intended for diagnostics +// and tests. +func (f *IPForwardingState) Counts() (v4, v6 int) { + f.mu.Lock() + defer f.mu.Unlock() + return f.v4Count, f.v6Count +} + +// RequestRouting takes the forwarding references for the routing path. It is +// idempotent: while routing already holds a reference, further calls don't +// increment the refcounts, and a v4-only request releases a previously held v6 +// reference. A v6 sysctl failure is logged and not returned so it can't take +// down v4 routing (the sysctl may be unwritable, e.g. read-only /proc/sys or +// IPv6 disabled on the kernel command line); v6 is retried on the next call. +func (f *IPForwardingState) RequestRouting(v6 bool) error { + f.mu.Lock() + defer f.mu.Unlock() + + if !f.routingV4 { + if err := f.requestV4(); err != nil { + return err + } + f.routingV4 = true + } + + if !v6 { + if !f.routingV6 { + return nil + } + f.routingV6 = false + return f.releaseV6() + } + + if f.routingV6 { return nil } - - if err := systemops.EnableIPForwarding(); err != nil { - return fmt.Errorf("failed to enable IP forwarding with sysctl: %w", err) + if err := f.requestV6(); err != nil { + log.Warnf("enable IPv6 forwarding for routing: %v", err) + return nil } - f.enabledCounter = 1 - log.Info("IP forwarding enabled") - + f.routingV6 = true return nil } -func (f *IPForwardingState) ReleaseForwarding() error { - if f.enabledCounter == 0 { - return nil +// ReleaseRouting releases the references RequestRouting holds. Calls without a +// held reference are no-ops. +func (f *IPForwardingState) ReleaseRouting() error { + f.mu.Lock() + defer f.mu.Unlock() + + if f.routingV4 { + f.routingV4 = false + f.releaseV4() } - - if f.enabledCounter > 1 { - f.enabledCounter-- - return nil + if f.routingV6 { + f.routingV6 = false + return f.releaseV6() } - - // if failed to disable IP forwarding we anyway decrement the counter - f.enabledCounter = 0 - - // todo call systemops.DisableIPForwarding() + return nil +} + +// RequestForwarding enables the family's forwarding sysctl on first request. +func (f *IPForwardingState) RequestForwarding(v6 bool) error { + f.mu.Lock() + defer f.mu.Unlock() + + if v6 { + return f.requestV6() + } + return f.requestV4() +} + +// ReleaseForwarding decrements the family counter. The last v6 release restores +// what enable captured. v4 stays on: net.ipv4.ip_forward is co-owned by other +// tooling (docker, k8s, libvirt). +func (f *IPForwardingState) ReleaseForwarding(v6 bool) error { + f.mu.Lock() + defer f.mu.Unlock() + + if v6 { + return f.releaseV6() + } + f.releaseV4() + return nil +} + +func (f *IPForwardingState) requestV4() error { + if f.v4Count == 0 { + if err := systemops.EnableV4IPForwarding(); err != nil { + return fmt.Errorf("enable IPv4 forwarding: %w", err) + } + log.Info("IPv4 forwarding enabled") + } + f.v4Count++ + return nil +} + +func (f *IPForwardingState) releaseV4() { + if f.v4Count > 0 { + f.v4Count-- + } +} + +func (f *IPForwardingState) requestV6() error { + if f.v6Count == 0 { + saved, err := systemops.EnableV6IPForwarding(f.wgIfaceName) + if err != nil { + if rerr := systemops.DisableV6IPForwarding(saved); rerr != nil { + log.Warnf("rollback partial v6 sysctls: %v", rerr) + } + return fmt.Errorf("enable IPv6 forwarding: %w", err) + } + // A failed restore on a previous release keeps its saved values; those + // are the true originals, so keep them over what this enable captured. + if f.v6Saved == nil { + f.v6Saved = saved + } else { + for k, v := range saved { + if _, ok := f.v6Saved[k]; !ok { + f.v6Saved[k] = v + } + } + } + log.Info("IPv6 forwarding enabled") + } + f.v6Count++ + return nil +} + +func (f *IPForwardingState) releaseV6() error { + if f.v6Count == 0 { + return nil + } + f.v6Count-- + if f.v6Count > 0 { + return nil + } + + // Keep the saved values on failure so a later release or enable/release + // cycle can still restore them; re-restoring an already-restored key is a + // no-op since the sysctl already holds the desired value. + if err := systemops.DisableV6IPForwarding(f.v6Saved); err != nil { + return fmt.Errorf("disable IPv6 forwarding: %w", err) + } + f.v6Saved = nil + log.Info("IPv6 forwarding disabled") return nil } diff --git a/client/internal/routemanager/ipfwdstate/ipfwdstate_privileged_linux_test.go b/client/internal/routemanager/ipfwdstate/ipfwdstate_privileged_linux_test.go new file mode 100644 index 000000000..b4615ff02 --- /dev/null +++ b/client/internal/routemanager/ipfwdstate/ipfwdstate_privileged_linux_test.go @@ -0,0 +1,39 @@ +//go:build privileged + +package ipfwdstate + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRequestRoutingV6ToV4Transition verifies that a v4-only routing request +// releases a previously held routing-owned v6 reference without touching +// references held by DNAT rules. +func TestRequestRoutingV6ToV4Transition(t *testing.T) { + f := NewIPForwardingState("wt-fwd-test") + + require.NoError(t, f.RequestRouting(true), "request routing with v6") + v4, v6 := f.Counts() + assert.Equal(t, 1, v4, "v4 reference held") + assert.Equal(t, 1, v6, "v6 reference held") + + require.NoError(t, f.RequestRouting(false), "request routing v4-only") + v4, v6 = f.Counts() + assert.Equal(t, 1, v4, "v4 reference kept") + assert.Equal(t, 0, v6, "routing-owned v6 reference released") + + // A DNAT-held reference survives a v4-only routing request. + require.NoError(t, f.RequestForwarding(true), "dnat v6 reference") + require.NoError(t, f.RequestRouting(false), "repeat v4-only request") + _, v6 = f.Counts() + assert.Equal(t, 1, v6, "dnat-held v6 reference survives") + require.NoError(t, f.ReleaseForwarding(true), "release dnat v6 reference") + + require.NoError(t, f.ReleaseRouting(), "release routing") + v4, v6 = f.Counts() + assert.Equal(t, 0, v4, "all v4 references released") + assert.Equal(t, 0, v6, "all v6 references released") +} diff --git a/client/internal/routemanager/sysctl/sysctl_linux.go b/client/internal/routemanager/sysctl/sysctl_linux.go index 46b7c9fb7..bb131c691 100644 --- a/client/internal/routemanager/sysctl/sysctl_linux.go +++ b/client/internal/routemanager/sysctl/sysctl_linux.go @@ -58,11 +58,7 @@ func Setup(wgIface iface) (map[string]int, error) { continue } - // Escape '%' and '.' so they survive the dot-to-slash conversion in Set() - safeName := strings.ReplaceAll(intf.Name, "%", percentEscape) - safeName = strings.ReplaceAll(safeName, ".", dotEscape) - - i := fmt.Sprintf(rpFilterInterfacePath, safeName) + i := fmt.Sprintf(rpFilterInterfacePath, EscapeInterfaceName(intf.Name)) oldVal, err := Set(i, 2, true) if err != nil { result = multierror.Append(result, err) @@ -74,6 +70,13 @@ func Setup(wgIface iface) (map[string]int, error) { return keys, nberrors.FormatErrorOrNil(result) } +// EscapeInterfaceName escapes '%' and '.' in an interface name (e.g. VLANs +// like eth0.100) so the name survives the dot-to-slash conversion in Set. +func EscapeInterfaceName(name string) string { + safe := strings.ReplaceAll(name, "%", percentEscape) + return strings.ReplaceAll(safe, ".", dotEscape) +} + // Set sets a sysctl configuration, if onlyIfOne is true it will only set the new value if it's set to 1 func Set(key string, desiredValue int, onlyIfOne bool) (int, error) { path := strings.ReplaceAll(key, ".", "/") diff --git a/client/internal/routemanager/systemops/routeselection_windows_test.go b/client/internal/routemanager/systemops/routeselection_windows_test.go new file mode 100644 index 000000000..108338dd9 --- /dev/null +++ b/client/internal/routemanager/systemops/routeselection_windows_test.go @@ -0,0 +1,82 @@ +//go:build windows + +package systemops + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSortRouteCandidates(t *testing.T) { + tests := []struct { + name string + candidates []candidateRoute + wantOrder []uint32 + }{ + { + name: "longest prefix wins over metrics", + candidates: []candidateRoute{ + {interfaceIndex: 1, prefixLength: 0, routeMetric: 0, interfaceMetric: 5}, + {interfaceIndex: 2, prefixLength: 24, routeMetric: 100, interfaceMetric: 50}, + }, + wantOrder: []uint32{2, 1}, + }, + { + // Windows ranks equal-length prefixes by route metric + interface metric, + // so a higher route metric on a low metric interface can still win. + name: "combined metric beats route metric alone", + candidates: []candidateRoute{ + {interfaceIndex: 8, prefixLength: 0, routeMetric: 0, interfaceMetric: 100}, + {interfaceIndex: 5, prefixLength: 0, routeMetric: 10, interfaceMetric: 5}, + }, + wantOrder: []uint32{5, 8}, + }, + { + name: "lower combined metric wins", + candidates: []candidateRoute{ + {interfaceIndex: 5, prefixLength: 0, routeMetric: 300, interfaceMetric: 5}, + {interfaceIndex: 8, prefixLength: 0, routeMetric: 0, interfaceMetric: 100}, + }, + wantOrder: []uint32{8, 5}, + }, + { + name: "equal combined metric falls back to route metric", + candidates: []candidateRoute{ + {interfaceIndex: 1, prefixLength: 0, routeMetric: 20, interfaceMetric: 10}, + {interfaceIndex: 2, prefixLength: 0, routeMetric: 5, interfaceMetric: 25}, + }, + wantOrder: []uint32{2, 1}, + }, + { + // The metrics are uint32 on the Windows side, so the sum must not wrap. + name: "combined metric beyond the uint32 range", + candidates: []candidateRoute{ + {interfaceIndex: 1, prefixLength: 0, routeMetric: math.MaxUint32, interfaceMetric: 5}, + {interfaceIndex: 2, prefixLength: 0, routeMetric: math.MaxUint32 - 10, interfaceMetric: 5}, + }, + wantOrder: []uint32{2, 1}, + }, + { + name: "unknown interface metric ranks on route metric only", + candidates: []candidateRoute{ + {interfaceIndex: 1, prefixLength: 0, routeMetric: 30, interfaceMetric: -1}, + {interfaceIndex: 2, prefixLength: 0, routeMetric: 5, interfaceMetric: 10}, + }, + wantOrder: []uint32{2, 1}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sortRouteCandidates(tt.candidates) + + got := make([]uint32, 0, len(tt.candidates)) + for _, c := range tt.candidates { + got = append(got, c.interfaceIndex) + } + assert.Equal(t, tt.wantOrder, got) + }) + } +} diff --git a/client/internal/routemanager/systemops/systemops_android.go b/client/internal/routemanager/systemops/systemops_android.go index 7cb8dae93..97b4ed8ec 100644 --- a/client/internal/routemanager/systemops/systemops_android.go +++ b/client/internal/routemanager/systemops/systemops_android.go @@ -32,8 +32,17 @@ func (r *SysOps) removeFromRouteTable(netip.Prefix, Nexthop) error { return nil } -func EnableIPForwarding() error { - log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS) +func EnableV4IPForwarding() error { + log.Infof("Enable IPv4 forwarding is not implemented on %s", runtime.GOOS) + return nil +} + +func EnableV6IPForwarding(string) (map[string]int, error) { + log.Infof("Enable IPv6 forwarding is not implemented on %s", runtime.GOOS) + return map[string]int{}, nil +} + +func DisableV6IPForwarding(map[string]int) error { return nil } diff --git a/client/internal/routemanager/systemops/systemops_ios.go b/client/internal/routemanager/systemops/systemops_ios.go index 99a363371..0cccd4962 100644 --- a/client/internal/routemanager/systemops/systemops_ios.go +++ b/client/internal/routemanager/systemops/systemops_ios.go @@ -58,8 +58,17 @@ func (r *SysOps) removeFromRouteTable(netip.Prefix, Nexthop) error { return nil } -func EnableIPForwarding() error { - log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS) +func EnableV4IPForwarding() error { + log.Infof("Enable IPv4 forwarding is not implemented on %s", runtime.GOOS) + return nil +} + +func EnableV6IPForwarding(string) (map[string]int, error) { + log.Infof("Enable IPv6 forwarding is not implemented on %s", runtime.GOOS) + return map[string]int{}, nil +} + +func DisableV6IPForwarding(map[string]int) error { return nil } diff --git a/client/internal/routemanager/systemops/systemops_linux.go b/client/internal/routemanager/systemops/systemops_linux.go index 8c6b7d9a9..7d608d886 100644 --- a/client/internal/routemanager/systemops/systemops_linux.go +++ b/client/internal/routemanager/systemops/systemops_linux.go @@ -763,13 +763,10 @@ func flushRoutes(tableID, family int) error { return nberrors.FormatErrorOrNil(result) } -func EnableIPForwarding() error { +func EnableV4IPForwarding() error { if _, err := sysctl.Set(ipv4ForwardingPath, 1, false); err != nil { return err } - if _, err := sysctl.Set(ipv6ForwardingPath, 1, false); err != nil { - log.Warnf("failed to enable IPv6 forwarding: %v", err) - } return nil } diff --git a/client/internal/routemanager/systemops/systemops_nonlinux.go b/client/internal/routemanager/systemops/systemops_nonlinux.go index 016a62ebd..837ac0cd2 100644 --- a/client/internal/routemanager/systemops/systemops_nonlinux.go +++ b/client/internal/routemanager/systemops/systemops_nonlinux.go @@ -43,8 +43,17 @@ func (r *SysOps) RemoveVPNRoute(prefix netip.Prefix, intf *net.Interface) error return r.genericRemoveVPNRoute(prefix, intf) } -func EnableIPForwarding() error { - log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS) +func EnableV4IPForwarding() error { + log.Infof("Enable IPv4 forwarding is not implemented on %s", runtime.GOOS) + return nil +} + +func EnableV6IPForwarding(string) (map[string]int, error) { + log.Infof("Enable IPv6 forwarding is not implemented on %s", runtime.GOOS) + return map[string]int{}, nil +} + +func DisableV6IPForwarding(map[string]int) error { return nil } diff --git a/client/internal/routemanager/systemops/systemops_windows.go b/client/internal/routemanager/systemops/systemops_windows.go index 7bce6af80..47d556cf6 100644 --- a/client/internal/routemanager/systemops/systemops_windows.go +++ b/client/internal/routemanager/systemops/systemops_windows.go @@ -882,26 +882,40 @@ func getInterfaceMetric(interfaceIndex uint32, family int16) int { return int(ipInterfaceRow.Metric) } -// sortRouteCandidates sorts route candidates by priority: prefix length -> route metric -> interface metric +// sortRouteCandidates sorts route candidates by priority: prefix length -> combined metric -> route metric. +// Windows prefers the longest matching prefix and, among prefixes of the same length, the lowest metric, see +// https://learn.microsoft.com/en-us/windows-hardware/customize/desktop/unattend/microsoft-windows-tcpip-interfaces-interface-routes-route-metric func sortRouteCandidates(candidates []candidateRoute) { sort.Slice(candidates, func(i, j int) bool { if candidates[i].prefixLength != candidates[j].prefixLength { return candidates[i].prefixLength > candidates[j].prefixLength } - if candidates[i].routeMetric != candidates[j].routeMetric { - return candidates[i].routeMetric < candidates[j].routeMetric + mi, mj := combinedMetric(candidates[i]), combinedMetric(candidates[j]) + if mi != mj { + return mi < mj } - return candidates[i].interfaceMetric < candidates[j].interfaceMetric + return candidates[i].routeMetric < candidates[j].routeMetric }) } +// combinedMetric returns the effective metric Windows uses to rank routes with an equal prefix length: +// the sum of the route metric and the metric of the interface the route is on, see +// https://learn.microsoft.com/en-us/windows-server/networking/technologies/network-subsystem/net-sub-interface-metric +// An unknown interface metric contributes nothing. +func combinedMetric(candidate candidateRoute) uint64 { + if candidate.interfaceMetric < 0 { + return uint64(candidate.routeMetric) + } + return uint64(candidate.routeMetric) + uint64(candidate.interfaceMetric) +} + // GetBestInterface finds the best interface for reaching a destination, // excluding the VPN interface to avoid routing loops. // // Route selection priority: // 1. Longest prefix match (most specific route) -// 2. Lowest route metric -// 3. Lowest interface metric +// 2. Lowest combined metric (route metric + interface metric) +// 3. Lowest route metric. func GetBestInterface(dest netip.Addr, vpnIntf string) (*net.Interface, error) { var skipInterfaceIndex int if vpnIntf != "" { @@ -925,7 +939,6 @@ func GetBestInterface(dest netip.Addr, vpnIntf string) (*net.Interface, error) { return nil, fmt.Errorf("no route to %s", dest) } - // Sort routes: prefix length -> route metric -> interface metric sortRouteCandidates(candidates) for _, candidate := range candidates { diff --git a/client/internal/routemanager/systemops/v6forwarding_linux.go b/client/internal/routemanager/systemops/v6forwarding_linux.go new file mode 100644 index 000000000..c1e0d4588 --- /dev/null +++ b/client/internal/routemanager/systemops/v6forwarding_linux.go @@ -0,0 +1,92 @@ +//go:build !android + +package systemops + +import ( + "fmt" + "net" + "os" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + + nberrors "github.com/netbirdio/netbird/client/errors" + "github.com/netbirdio/netbird/client/internal/routemanager/sysctl" +) + +const ( + // 1 (default) accepts RAs only while forwarding is off; 2 keeps RA + // acceptance on regardless, so RA-installed host defaults survive our + // v6 forwarding flip. + acceptRAInterfacePath = "net.ipv6.conf.%s.accept_ra" + acceptRADefaultPath = "net.ipv6.conf.default.accept_ra" + acceptRAProcPathFormat = "/proc/sys/net/ipv6/conf/%s/accept_ra" +) + +// EnableV6IPForwarding bumps accept_ra=2 on host v6 interfaces before flipping +// forwarding=1, so RA-installed host defaults survive. Returns the prior values +// of sysctls we actually changed; entries already at the target are omitted. +func EnableV6IPForwarding(wgIfaceName string) (map[string]int, error) { + saved := map[string]int{} + bumpAcceptRA(saved, wgIfaceName) + + oldVal, err := sysctl.Set(ipv6ForwardingPath, 1, false) + if err != nil { + return saved, err + } + if oldVal != 1 { + saved[ipv6ForwardingPath] = oldVal + } + return saved, nil +} + +// DisableV6IPForwarding restores what EnableV6IPForwarding captured. +func DisableV6IPForwarding(saved map[string]int) error { + var result *multierror.Error + for key, value := range saved { + if _, err := sysctl.Set(key, value, false); err != nil { + result = multierror.Append(result, fmt.Errorf("restore %s: %w", key, err)) + } + } + return nberrors.FormatErrorOrNil(result) +} + +func bumpAcceptRA(saved map[string]int, wgIfaceName string) { + // Also bump conf.default so interfaces created while forwarding is on + // (hotplug, new Wi-Fi/dock) inherit accept_ra=2 and keep accepting RAs. + bumpAcceptRAKey(saved, acceptRADefaultPath) + + interfaces, err := net.Interfaces() + if err != nil { + log.Warnf("list interfaces for accept_ra: %v", err) + return + } + for _, intf := range interfaces { + if intf.Name == "lo" || intf.Name == wgIfaceName { + continue + } + bumpAcceptRAForInterface(saved, intf.Name) + } +} + +func bumpAcceptRAForInterface(saved map[string]int, name string) { + // Build procfs path from name, not the dotted key: VLAN names like eth0.100. + if _, err := os.Stat(fmt.Sprintf(acceptRAProcPathFormat, name)); err != nil { + return + } + bumpAcceptRAKey(saved, fmt.Sprintf(acceptRAInterfacePath, sysctl.EscapeInterfaceName(name))) +} + +func bumpAcceptRAKey(saved map[string]int, key string) { + // onlyIfOne=true: leave admin overrides (0, 2) alone. + oldVal, err := sysctl.Set(key, 2, true) + if err != nil { + log.Warnf("bump %s: %v", key, err) + return + } + // With onlyIfOne, a write only happened when the old value was 1; values + // left untouched (0, 2) must not be recorded for restore. + if oldVal == 1 { + saved[key] = oldVal + } +} diff --git a/client/internal/routemanager/systemops/v6route_linux_test.go b/client/internal/routemanager/systemops/v6route_linux_test.go index 449d4cbd2..d8c0012d1 100644 --- a/client/internal/routemanager/systemops/v6route_linux_test.go +++ b/client/internal/routemanager/systemops/v6route_linux_test.go @@ -5,6 +5,7 @@ package systemops import ( "errors" "net" + "net/netip" "syscall" "testing" @@ -29,6 +30,7 @@ func ensureIPv6DefaultRoute(t *testing.T) { } if err := netlink.RouteAdd(route); err != nil { if errors.Is(err, syscall.EEXIST) { + requireUsableIPv6Nexthop(t) return } t.Skipf("install IPv6 fallback default route: %v", err) @@ -38,4 +40,36 @@ func ensureIPv6DefaultRoute(t *testing.T) { t.Logf("delete IPv6 fallback default route: %v", err) } }) + + requireUsableIPv6Nexthop(t) +} + +// requireUsableIPv6Nexthop skips the test unless the resolved IPv6 default +// nexthop can actually carry a route. Installing the default route succeeding +// does not imply the kernel accepts it as a nexthop for a concrete prefix. +func requireUsableIPv6Nexthop(t *testing.T) { + t.Helper() + + nexthop, err := GetNextHop(netip.IPv6Unspecified()) + if err != nil { + t.Skipf("resolve IPv6 default nexthop: %v", err) + } + + probe := &netlink.Route{ + Scope: netlink.SCOPE_UNIVERSE, + Table: syscall.RT_TABLE_MAIN, + Family: netlink.FAMILY_V6, + Dst: &net.IPNet{IP: net.ParseIP("100::64"), Mask: net.CIDRMask(128, 128)}, + } + require.NoError(t, addNextHop(nexthop, probe), "build IPv6 probe route") + + switch err := netlink.RouteAdd(probe); { + case err == nil: + if err := netlink.RouteDel(probe); err != nil && !errors.Is(err, syscall.ESRCH) { + t.Logf("delete IPv6 probe route: %v", err) + } + case errors.Is(err, syscall.EEXIST): + default: + t.Skipf("IPv6 nexthop %s unusable for route installation: %v", nexthop, err) + } } diff --git a/client/ios/NetBirdSDK/client.go b/client/ios/NetBirdSDK/client.go index 2d5460d03..5df6f92f1 100644 --- a/client/ios/NetBirdSDK/client.go +++ b/client/ios/NetBirdSDK/client.go @@ -14,6 +14,7 @@ import ( log "github.com/sirupsen/logrus" + nbAnonymize "github.com/netbirdio/netbird/client/anonymize" "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/auth" "github.com/netbirdio/netbird/client/internal/debug" @@ -28,6 +29,13 @@ import ( types "github.com/netbirdio/netbird/upload-server/types" ) +// AnonymizeLevelDefault and AnonymizeLevelStrict are the accepted +// anonymizeLevel values for DebugBundle. +const ( + AnonymizeLevelDefault = nbAnonymize.LevelDefaultString + AnonymizeLevelStrict = nbAnonymize.LevelStrictString +) + // ConnectionListener export internal Listener for mobile type ConnectionListener interface { peer.Listener @@ -200,8 +208,10 @@ func (c *Client) Stop() { // DebugBundle generates a debug bundle, uploads it and returns the upload key. // It works with or without a running engine: when the engine is up it reuses // the live config, sync response and client metrics; otherwise it loads the -// config from disk (or the preloaded tvOS config). -func (c *Client) DebugBundle(anonymize bool) (string, error) { +// config from disk (or the preloaded tvOS config). anonymizeLevel is "default" +// or "strict"; strict also anonymizes internal IP ranges, peer names, and +// WireGuard public keys, and implies anonymize. +func (c *Client) DebugBundle(anonymize bool, anonymizeLevel string) (string, error) { cfg, cc := c.stateSnapshot() // If the engine hasn't been started, load config so we can reach management. @@ -251,6 +261,7 @@ func (c *Client) DebugBundle(anonymize bool) (string, error) { deps, debug.BundleConfig{ Anonymize: anonymize, + AnonymizeLevel: nbAnonymize.ParseLevel(anonymizeLevel), IncludeSystemInfo: true, }, ) diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index d4deeb8ec..b438a310a 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -2781,6 +2781,11 @@ type DebugBundleRequest struct { // untrusted TLS certificate. Restricted to privileged callers; for // self-hosted upload servers. UploadInsecure bool `protobuf:"varint,7,opt,name=uploadInsecure,proto3" json:"uploadInsecure,omitempty"` + // anonymizeLevel selects how much the anonymizer redacts: "default" + // (or empty) keeps internal IP ranges, "strict" also anonymizes them. + // Unknown values are treated as "strict". Only meaningful with anonymize; + // "strict" implies it. + AnonymizeLevel string `protobuf:"bytes,8,opt,name=anonymizeLevel,proto3" json:"anonymizeLevel,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2857,6 +2862,13 @@ func (x *DebugBundleRequest) GetUploadInsecure() bool { return false } +func (x *DebugBundleRequest) GetAnonymizeLevel() string { + if x != nil { + return x.AnonymizeLevel + } + return "" +} + type DebugBundleResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` @@ -5616,9 +5628,13 @@ func (x *GetPeerSSHHostKeyResponse) GetFound() bool { type RequestJWTAuthRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // hint for OIDC login_hint parameter (typically email address) - Hint *string `protobuf:"bytes,1,opt,name=hint,proto3,oneof" json:"hint,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Hint *string `protobuf:"bytes,1,opt,name=hint,proto3,oneof" json:"hint,omitempty"` + // hasGraphicalSession tells the daemon that the caller has a graphical session, + // which decides whether PKCE or the device code flow is preferred. The daemon + // cannot detect this itself: it does not inherit the session environment. + HasGraphicalSession bool `protobuf:"varint,2,opt,name=hasGraphicalSession,proto3" json:"hasGraphicalSession,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RequestJWTAuthRequest) Reset() { @@ -5658,6 +5674,13 @@ func (x *RequestJWTAuthRequest) GetHint() string { return "" } +func (x *RequestJWTAuthRequest) GetHasGraphicalSession() bool { + if x != nil { + return x.HasGraphicalSession + } + return false +} + // RequestJWTAuthResponse contains authentication flow information type RequestJWTAuthResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -5882,9 +5905,13 @@ type RequestExtendAuthSessionRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // Optional OIDC login_hint (typically the user's email) to pre-fill the // IdP login form. - Hint *string `protobuf:"bytes,1,opt,name=hint,proto3,oneof" json:"hint,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Hint *string `protobuf:"bytes,1,opt,name=hint,proto3,oneof" json:"hint,omitempty"` + // hasGraphicalSession tells the daemon that the caller has a graphical session, + // which decides whether PKCE or the device code flow is preferred. The daemon + // cannot detect this itself: it does not inherit the session environment. + HasGraphicalSession bool `protobuf:"varint,2,opt,name=hasGraphicalSession,proto3" json:"hasGraphicalSession,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RequestExtendAuthSessionRequest) Reset() { @@ -5924,6 +5951,13 @@ func (x *RequestExtendAuthSessionRequest) GetHint() string { return "" } +func (x *RequestExtendAuthSessionRequest) GetHasGraphicalSession() bool { + if x != nil { + return x.HasGraphicalSession + } + return false +} + // RequestExtendAuthSessionResponse carries the verification URI the UI // should open in a browser. The daemon retains the flow state and resolves // it via WaitExtendAuthSession. @@ -7253,7 +7287,7 @@ const file_daemon_proto_rawDesc = "" + "\x12translatedHostname\x18\x04 \x01(\tR\x12translatedHostname\x128\n" + "\x0etranslatedPort\x18\x05 \x01(\v2\x10.daemon.PortInfoR\x0etranslatedPort\"G\n" + "\x17ForwardingRulesResponse\x12,\n" + - "\x05rules\x18\x01 \x03(\v2\x16.daemon.ForwardingRuleR\x05rules\"\xdc\x01\n" + + "\x05rules\x18\x01 \x03(\v2\x16.daemon.ForwardingRuleR\x05rules\"\x84\x02\n" + "\x12DebugBundleRequest\x12\x1c\n" + "\tanonymize\x18\x01 \x01(\bR\tanonymize\x12\x1e\n" + "\n" + @@ -7264,7 +7298,8 @@ const file_daemon_proto_rawDesc = "" + "\n" + "cliVersion\x18\x06 \x01(\tR\n" + "cliVersion\x12&\n" + - "\x0euploadInsecure\x18\a \x01(\bR\x0euploadInsecure\"}\n" + + "\x0euploadInsecure\x18\a \x01(\bR\x0euploadInsecure\x12&\n" + + "\x0eanonymizeLevel\x18\b \x01(\tR\x0eanonymizeLevel\"}\n" + "\x13DebugBundleResponse\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12 \n" + "\vuploadedKey\x18\x02 \x01(\tR\vuploadedKey\x120\n" + @@ -7490,9 +7525,10 @@ const file_daemon_proto_rawDesc = "" + "sshHostKey\x12\x16\n" + "\x06peerIP\x18\x02 \x01(\tR\x06peerIP\x12\x1a\n" + "\bpeerFQDN\x18\x03 \x01(\tR\bpeerFQDN\x12\x14\n" + - "\x05found\x18\x04 \x01(\bR\x05found\"9\n" + + "\x05found\x18\x04 \x01(\bR\x05found\"k\n" + "\x15RequestJWTAuthRequest\x12\x17\n" + - "\x04hint\x18\x01 \x01(\tH\x00R\x04hint\x88\x01\x01B\a\n" + + "\x04hint\x18\x01 \x01(\tH\x00R\x04hint\x88\x01\x01\x120\n" + + "\x13hasGraphicalSession\x18\x02 \x01(\bR\x13hasGraphicalSessionB\a\n" + "\x05_hint\"\x9a\x02\n" + "\x16RequestJWTAuthResponse\x12(\n" + "\x0fverificationURI\x18\x01 \x01(\tR\x0fverificationURI\x128\n" + @@ -7512,9 +7548,10 @@ const file_daemon_proto_rawDesc = "" + "\x14WaitJWTTokenResponse\x12\x14\n" + "\x05token\x18\x01 \x01(\tR\x05token\x12\x1c\n" + "\ttokenType\x18\x02 \x01(\tR\ttokenType\x12\x1c\n" + - "\texpiresIn\x18\x03 \x01(\x03R\texpiresIn\"C\n" + + "\texpiresIn\x18\x03 \x01(\x03R\texpiresIn\"u\n" + "\x1fRequestExtendAuthSessionRequest\x12\x17\n" + - "\x04hint\x18\x01 \x01(\tH\x00R\x04hint\x88\x01\x01B\a\n" + + "\x04hint\x18\x01 \x01(\tH\x00R\x04hint\x88\x01\x01\x120\n" + + "\x13hasGraphicalSession\x18\x02 \x01(\bR\x13hasGraphicalSessionB\a\n" + "\x05_hint\"\xe0\x01\n" + " RequestExtendAuthSessionResponse\x12(\n" + "\x0fverificationURI\x18\x01 \x01(\tR\x0fverificationURI\x128\n" + diff --git a/client/proto/daemon.proto b/client/proto/daemon.proto index 3c31156ec..a3e3f4500 100644 --- a/client/proto/daemon.proto +++ b/client/proto/daemon.proto @@ -540,6 +540,11 @@ message DebugBundleRequest { // untrusted TLS certificate. Restricted to privileged callers; for // self-hosted upload servers. bool uploadInsecure = 7; + // anonymizeLevel selects how much the anonymizer redacts: "default" + // (or empty) keeps internal IP ranges, "strict" also anonymizes them. + // Unknown values are treated as "strict". Only meaningful with anonymize; + // "strict" implies it. + string anonymizeLevel = 8; } message DebugBundleResponse { @@ -889,6 +894,10 @@ message GetPeerSSHHostKeyResponse { message RequestJWTAuthRequest { // hint for OIDC login_hint parameter (typically email address) optional string hint = 1; + // hasGraphicalSession tells the daemon that the caller has a graphical session, + // which decides whether PKCE or the device code flow is preferred. The daemon + // cannot detect this itself: it does not inherit the session environment. + bool hasGraphicalSession = 2; } // RequestJWTAuthResponse contains authentication flow information @@ -932,6 +941,10 @@ message RequestExtendAuthSessionRequest { // Optional OIDC login_hint (typically the user's email) to pre-fill the // IdP login form. optional string hint = 1; + // hasGraphicalSession tells the daemon that the caller has a graphical session, + // which decides whether PKCE or the device code flow is preferred. The daemon + // cannot detect this itself: it does not inherit the session environment. + bool hasGraphicalSession = 2; } // RequestExtendAuthSessionResponse carries the verification URI the UI diff --git a/client/server/debug.go b/client/server/debug.go index 60a401b0e..8f4a506b4 100644 --- a/client/server/debug.go +++ b/client/server/debug.go @@ -16,6 +16,7 @@ import ( "google.golang.org/grpc/codes" gstatus "google.golang.org/grpc/status" + "github.com/netbirdio/netbird/client/anonymize" "github.com/netbirdio/netbird/client/internal/debug" "github.com/netbirdio/netbird/client/internal/ipcauth" "github.com/netbirdio/netbird/client/proto" @@ -122,6 +123,7 @@ func (s *Server) generateDebugBundle(req *proto.DebugBundleRequest, uiOpener deb }, debug.BundleConfig{ Anonymize: req.GetAnonymize(), + AnonymizeLevel: anonymize.ParseLevel(req.GetAnonymizeLevel()), IncludeSystemInfo: req.GetSystemInfo(), LogFileCount: req.GetLogFileCount(), }, diff --git a/client/server/server.go b/client/server/server.go index 01778b8e0..f33e19075 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -1723,8 +1723,8 @@ func (s *Server) RequestJWTAuth( hint = profilemanager.GetLoginHint() } - isDesktop := isUnixRunningDesktop() - oAuthFlow, err := auth.NewOAuthFlow(ctx, config, isDesktop, false, hint) + // the daemon has no graphical session of its own, only the caller can answer this + oAuthFlow, err := auth.NewOAuthFlow(ctx, config, msg.GetHasGraphicalSession(), false, hint) if err != nil { return nil, gstatus.Errorf(codes.Internal, "failed to create OAuth flow: %v", err) } @@ -1827,8 +1827,8 @@ func (s *Server) RequestExtendAuthSession( hint = profilemanager.GetLoginHint() } - isDesktop := isUnixRunningDesktop() - oAuthFlow, err := auth.NewOAuthFlow(ctx, config, isDesktop, false, hint) + // the daemon has no graphical session of its own, only the caller can answer this + oAuthFlow, err := auth.NewOAuthFlow(ctx, config, msg.GetHasGraphicalSession(), false, hint) if err != nil { return nil, gstatus.Errorf(codes.Internal, "failed to create OAuth flow: %v", err) } @@ -2000,13 +2000,6 @@ func (s *Server) ExposeService(req *proto.ExposeServiceRequest, srv proto.Daemon return nil } -func isUnixRunningDesktop() bool { - if runtime.GOOS != "linux" && runtime.GOOS != "freebsd" { - return false - } - return os.Getenv("DESKTOP_SESSION") != "" || os.Getenv("XDG_CURRENT_DESKTOP") != "" -} - func (s *Server) runProbes(ctx context.Context, waitForProbeResult bool) { if s.connectClient == nil { return diff --git a/client/ssh/common.go b/client/ssh/common.go index 92e647b7d..3f4f3e9d1 100644 --- a/client/ssh/common.go +++ b/client/ssh/common.go @@ -13,6 +13,7 @@ import ( "golang.org/x/crypto/ssh" "github.com/netbirdio/netbird/client/proto" + "github.com/netbirdio/netbird/util" ) const ( @@ -92,7 +93,8 @@ func printAuthInstructions(stderr io.Writer, authResponse *proto.RequestJWTAuthR // RequestJWTToken requests or retrieves a JWT token for SSH authentication func RequestJWTToken(ctx context.Context, client proto.DaemonServiceClient, stdout, stderr io.Writer, useCache bool, hint string, openBrowser func(string) error) (string, error) { - req := &proto.RequestJWTAuthRequest{} + // the ssh client runs in the user's session, the daemon does not: tell it what we can see + req := &proto.RequestJWTAuthRequest{HasGraphicalSession: util.HasGraphicalSession()} if hint != "" { req.Hint = &hint } @@ -193,4 +195,3 @@ func buildAddressList(hostname string, remote net.Addr) []string { } return addresses } - diff --git a/client/ssh/server/command_execution_windows.go b/client/ssh/server/command_execution_windows.go index e1ba777f6..feb8daa26 100644 --- a/client/ssh/server/command_execution_windows.go +++ b/client/ssh/server/command_execution_windows.go @@ -243,7 +243,7 @@ func (s *Server) setUserEnvironmentVariables(envMap map[string]string, userProfi // prepareCommandEnv prepares environment variables for command execution on Windows func (s *Server) prepareCommandEnv(logger *log.Entry, localUser *user.User, session ssh.Session) []string { - username, domain := s.parseUsername(localUser.Username) + username, domain := parseUsername(localUser.Username) userEnv, err := s.getUserEnvironment(logger, username, domain) if err != nil { log.Debugf("failed to get user environment for %s\\%s, using fallback: %v", domain, username, err) @@ -383,7 +383,7 @@ func (s *Server) executeCommandWithPty(logger *log.Entry, session ssh.Session, _ return false } - username, domain := s.parseUsername(localUser.Username) + username, domain := parseUsername(localUser.Username) shell := getUserShell(localUser.Uid) req := PtyExecutionRequest{ diff --git a/client/ssh/server/port_forwarding.go b/client/ssh/server/port_forwarding.go index a47fdb48a..81d3b9173 100644 --- a/client/ssh/server/port_forwarding.go +++ b/client/ssh/server/port_forwarding.go @@ -133,7 +133,12 @@ func (s *Server) checkPrivilegedPortAccess(forwardType string, port uint32, resu return nil } - if result.User != nil && isPrivilegedUsername(result.User.Username) { + // Only uid 0 may bind below the threshold, which is the kernel's own rule and + // is asked directly rather than through isPrivilegedOrUnknown: that helper + // reports an account it cannot evaluate as privileged, which is safe for a + // refusal and unsafe for a grant such as this one. Windows has returned + // above, so Uid here is a Unix uid and never a SID. + if result.User != nil && result.User.Uid == "0" { return nil } diff --git a/client/ssh/server/privileges_other.go b/client/ssh/server/privileges_other.go new file mode 100644 index 000000000..89440dea8 --- /dev/null +++ b/client/ssh/server/privileges_other.go @@ -0,0 +1,16 @@ +//go:build !windows + +package server + +// isProcessElevated is only meaningful on Windows; other platforms use the +// effective UID check in isCurrentProcessPrivileged. +func isProcessElevated() bool { + return false +} + +// isWindowsAccountPrivilegedOrUnknown is only reachable on Windows. Report +// privileged on other platforms so a caller refusing privileged accounts fails +// closed. +func isWindowsAccountPrivilegedOrUnknown(string) bool { + return true +} diff --git a/client/ssh/server/privileges_windows.go b/client/ssh/server/privileges_windows.go new file mode 100644 index 000000000..41ea00fd7 --- /dev/null +++ b/client/ssh/server/privileges_windows.go @@ -0,0 +1,228 @@ +//go:build windows + +package server + +import ( + "fmt" + "strings" + "unsafe" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" +) + +var ( + netapi32 = windows.NewLazySystemDLL("netapi32.dll") + procNetUserGetLocalGroups = netapi32.NewProc("NetUserGetLocalGroups") +) + +const ( + // lgIncludeIndirect makes NetUserGetLocalGroups also return local groups + // the user belongs to through a global group. + lgIncludeIndirect = 0x1 + maxPreferredLength = 0xFFFFFFFF +) + +// localGroupUsersInfo0 mirrors LOCALGROUP_USERS_INFO_0. +type localGroupUsersInfo0 struct { + name *uint16 +} + +// isProcessElevated reports whether the current process token is elevated +// (TokenElevation): true for elevated administrators, the built-in +// Administrator, administrators with UAC disabled, and SYSTEM; false for +// standard users and administrators running with a UAC-filtered token. +func isProcessElevated() bool { + return windows.GetCurrentProcessToken().IsElevated() +} + +// isWindowsAccountPrivilegedOrUnknown reports whether the account is privileged +// on this machine: a well-known service account, a built-in Administrator +// (RID 500), or a member of the local Administrators group, directly or through +// nested groups. +// +// An account whose privilege cannot be determined counts as privileged, which +// is why the name says "or unknown". That is fail-closed for a caller that +// refuses privileged accounts, and fail-open for a caller that grants something +// to them, so only the former may use this. +func isWindowsAccountPrivilegedOrUnknown(username string) bool { + sid, _, _, err := windows.LookupSID("", username) + if err != nil { + log.Warnf("privilege check: SID lookup for %q failed, treating as privileged: %v", username, err) + return true + } + + if isPrivilegedUserSID(sid) { + return true + } + + member, err := isLocalAdminsMember(username) + if err != nil { + log.Warnf("privilege check: cannot determine Administrators membership for %q, treating as privileged: %v", username, err) + return true + } + return member +} + +// isPrivilegedUserSID reports whether the SID itself identifies a privileged +// principal, without consulting group membership. +func isPrivilegedUserSID(sid *windows.SID) bool { + wellKnown := []windows.WELL_KNOWN_SID_TYPE{ + windows.WinLocalSystemSid, + windows.WinLocalServiceSid, + windows.WinNetworkServiceSid, + windows.WinBuiltinAdministratorsSid, + } + for _, sidType := range wellKnown { + if sid.IsWellKnown(sidType) { + return true + } + } + return isBuiltinAdministratorSID(sid) +} + +// isBuiltinAdministratorSID reports whether the SID is a machine or domain +// built-in Administrator account (S-1-5-21-...-500). RID 500 is reserved for +// that account; it can be renamed but cannot be removed from the +// Administrators group. +func isBuiltinAdministratorSID(sid *windows.SID) bool { + if sid.IdentifierAuthority() != windows.SECURITY_NT_AUTHORITY { + return false + } + count := sid.SubAuthorityCount() + if count < 2 || sid.SubAuthority(0) != 21 { + return false + } + return sid.SubAuthority(uint32(count-1)) == 500 +} + +// isLocalAdminsMember reports whether the account is a member of the local +// Administrators group. +// +// Local accounts are checked against the local SAM, which is authoritative for +// them and, unlike a token, cannot under-report: UAC filters the tokens of +// local administrators, and a filtered token carries Administrators as +// deny-only, which a membership check on the token would read as "not a +// member". Domain accounts are exempt from that filtering, so for them an S4U +// token is preferred because its group list is LSA's transitive expansion and +// therefore covers nested and universal groups plus the machine's own local +// groups. NetUserGetLocalGroups expands only one global-group hop but needs no +// logon, so it serves as the fallback when no token can be obtained. +func isLocalAdminsMember(username string) (bool, error) { + adminSid, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + if err != nil { + return false, fmt.Errorf("create Administrators SID: %w", err) + } + + account, domain := parseUsername(username) + if NewPrivilegeDropper().isLocalUser(domain) { + return localGroupsContainSID(account, adminSid) + } + + member, s4uErr := s4uTokenIsMember(account, domain, adminSid) + if s4uErr == nil { + return member, nil + } + log.Debugf("privilege check: S4U membership check for %q failed, falling back to local group enumeration: %v", username, s4uErr) + + member, err = localGroupsContainSID(buildUserCpn(account, domain), adminSid) + if err != nil { + return false, fmt.Errorf("S4U check: %w; local group enumeration: %w", s4uErr, err) + } + return member, nil +} + +// s4uTokenIsMember obtains an S4U token for the account and checks whether the +// given SID is enabled in it. +func s4uTokenIsMember(account, domain string, sid *windows.SID) (bool, error) { + token, err := generateS4UUserToken(log.NewEntry(log.StandardLogger()), account, domain) + if err != nil { + return false, err + } + defer func() { + if err := windows.CloseHandle(token); err != nil { + log.Debugf("close S4U token: %v", err) + } + }() + return windows.Token(token).IsMember(sid) +} + +// localGroupsContainSID reports whether the wanted group is among the local +// groups the account belongs to, directly or through a global group. +// +// The wanted SID is resolved to its group name once and compared against the +// enumerated names. Well-known SIDs resolve from a static table, so that lookup +// needs no domain controller, and it keeps the comparison correct for a renamed +// or localized group because both sides then carry the new name. Resolving each +// enumerated name back to a SID instead would add a lookup per group that can +// block until it times out while a domain controller is unreachable, and cannot +// change the outcome: the names enumerated here are local groups of this +// machine, whose names are unique, so a name match identifies the group. +// +// A failure to resolve the wanted SID is returned rather than reported as +// "not a member", so a privilege check built on this fails closed. +func localGroupsContainSID(username string, want *windows.SID) (bool, error) { + wantName, _, _, err := want.LookupAccount("") + if err != nil { + return false, fmt.Errorf("resolve group SID %s to a name: %w", want, err) + } + + groups, err := netUserGetLocalGroups(username) + if err != nil { + return false, err + } + + for _, group := range groups { + if strings.EqualFold(group, wantName) { + return true, nil + } + } + return false, nil +} + +// netUserGetLocalGroups returns the names of the local groups the account is a +// member of, including indirect membership through global groups. +func netUserGetLocalGroups(username string) ([]string, error) { + name16, err := windows.UTF16PtrFromString(username) + if err != nil { + return nil, fmt.Errorf("convert username: %w", err) + } + + var buf *byte + var entriesRead, totalEntries uint32 + status, _, _ := procNetUserGetLocalGroups.Call( + 0, // local server + uintptr(unsafe.Pointer(name16)), + 0, // level 0: LOCALGROUP_USERS_INFO_0 + lgIncludeIndirect, + uintptr(unsafe.Pointer(&buf)), + maxPreferredLength, + uintptr(unsafe.Pointer(&entriesRead)), + uintptr(unsafe.Pointer(&totalEntries)), + ) + if status != 0 { + return nil, fmt.Errorf("NetUserGetLocalGroups for %q: status %d", username, status) + } + if buf == nil { + return nil, nil + } + defer func() { + if err := windows.NetApiBufferFree(buf); err != nil { + log.Debugf("free NetApi buffer: %v", err) + } + }() + + // MAX_PREFERRED_LENGTH makes the API allocate as much as it needs, so a + // short read is not expected. Report it rather than silently returning a + // subset of the account's groups. + if entriesRead != totalEntries { + return nil, fmt.Errorf("NetUserGetLocalGroups for %q returned %d of %d groups", username, entriesRead, totalEntries) + } + + entries := unsafe.Slice((*localGroupUsersInfo0)(unsafe.Pointer(buf)), entriesRead) + groups := make([]string, 0, entriesRead) + for _, entry := range entries { + groups = append(groups, windows.UTF16PtrToString(entry.name)) + } + return groups, nil +} diff --git a/client/ssh/server/privileges_windows_test.go b/client/ssh/server/privileges_windows_test.go new file mode 100644 index 000000000..983fccdf7 --- /dev/null +++ b/client/ssh/server/privileges_windows_test.go @@ -0,0 +1,293 @@ +//go:build windows + +package server + +import ( + "os/user" + "testing" + "unsafe" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" +) + +// filterNormalAccount limits NetUserEnum to normal user accounts. +const filterNormalAccount = 0x2 + +// TOKEN_ELEVATION_TYPE values. +const ( + tokenElevationTypeDefault = 1 + tokenElevationTypeFull = 2 + tokenElevationTypeLimited = 3 +) + +// tokenElevationType reads TokenElevationType from a token. +func tokenElevationType(token windows.Token) (uint32, error) { + var elevationType, returnedLen uint32 + err := windows.GetTokenInformation(token, windows.TokenElevationType, + (*byte)(unsafe.Pointer(&elevationType)), uint32(unsafe.Sizeof(elevationType)), &returnedLen) + if err != nil { + return 0, err + } + return elevationType, nil +} + +// userInfo0 mirrors USER_INFO_0. +type userInfo0 struct { + name *uint16 +} + +func mustParseSID(t *testing.T, s string) *windows.SID { + t.Helper() + sid, err := windows.StringToSid(s) + require.NoError(t, err, "parse SID %s", s) + return sid +} + +// localAccountNames returns the names of the local user accounts. +func localAccountNames(t *testing.T) []string { + t.Helper() + + var buf *byte + var entriesRead, totalEntries, resume uint32 + err := windows.NetUserEnum(nil, 0, filterNormalAccount, &buf, maxPreferredLength, + &entriesRead, &totalEntries, &resume) + require.NoError(t, err, "enumerate local users") + t.Cleanup(func() { + require.NoError(t, windows.NetApiBufferFree(buf), "free NetApi buffer") + }) + + entries := unsafe.Slice((*userInfo0)(unsafe.Pointer(buf)), entriesRead) + names := make([]string, 0, entriesRead) + for _, entry := range entries { + names = append(names, windows.UTF16PtrToString(entry.name)) + } + return names +} + +// localAccountNameByRID returns the name of the local account carrying the +// given RID. Accounts such as Administrator and Guest can be renamed and are +// localized, so tests must not name them literally. +func localAccountNameByRID(t *testing.T, rid uint32) string { + t.Helper() + + for _, name := range localAccountNames(t) { + sid, _, _, err := windows.LookupSID("", name) + if err != nil { + continue + } + if sid.IdentifierAuthority() != windows.SECURITY_NT_AUTHORITY { + continue + } + count := sid.SubAuthorityCount() + if count < 2 || sid.SubAuthority(0) != 21 { + continue + } + if sid.SubAuthority(uint32(count-1)) == rid { + return name + } + } + + t.Fatalf("no local account with RID %d", rid) + return "" +} + +// wellKnownAccountName resolves a well-known SID to the qualified account name +// the local system uses for it, which is localized. +func wellKnownAccountName(t *testing.T, sidType windows.WELL_KNOWN_SID_TYPE) string { + t.Helper() + + sid, err := windows.CreateWellKnownSid(sidType) + require.NoError(t, err, "create well-known SID") + name, domain, _, err := sid.LookupAccount("") + require.NoError(t, err, "resolve %s to an account name", sid) + if domain == "" { + return name + } + return domain + `\` + name +} + +func TestIsBuiltinAdministratorSID(t *testing.T) { + tests := []struct { + name string + sid string + want bool + }{ + {"machine_administrator", "S-1-5-21-1111111111-2222222222-3333333333-500", true}, + {"domain_administrator", "S-1-5-21-3390233681-4087452608-412898826-500", true}, + {"regular_user", "S-1-5-21-1111111111-2222222222-3333333333-1001", false}, + {"guest_account", "S-1-5-21-1111111111-2222222222-3333333333-501", false}, + {"domain_admins_group", "S-1-5-21-1111111111-2222222222-3333333333-512", false}, + {"system", "S-1-5-18", false}, + {"administrators_group", "S-1-5-32-544", false}, + {"non_nt_authority", "S-1-1-0", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isBuiltinAdministratorSID(mustParseSID(t, tt.sid)) + assert.Equal(t, tt.want, result, "RID 500 detection for %s", tt.sid) + }) + } +} + +func TestIsPrivilegedUserSID(t *testing.T) { + tests := []struct { + name string + sid string + want bool + }{ + {"local_system", "S-1-5-18", true}, + {"local_service", "S-1-5-19", true}, + {"network_service", "S-1-5-20", true}, + {"administrators_group", "S-1-5-32-544", true}, + {"builtin_administrator", "S-1-5-21-1111111111-2222222222-3333333333-500", true}, + {"regular_user", "S-1-5-21-1111111111-2222222222-3333333333-1001", false}, + {"users_group", "S-1-5-32-545", false}, + {"everyone", "S-1-1-0", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isPrivilegedUserSID(mustParseSID(t, tt.sid)) + assert.Equal(t, tt.want, result, "SID privilege classification for %s", tt.sid) + }) + } +} + +func TestIsWindowsAccountPrivilegedOrUnknown(t *testing.T) { + tests := []struct { + name string + username string + want bool + }{ + {"system", wellKnownAccountName(t, windows.WinLocalSystemSid), true}, + {"local_service", wellKnownAccountName(t, windows.WinLocalServiceSid), true}, + {"network_service", wellKnownAccountName(t, windows.WinNetworkServiceSid), true}, + {"administrators_group", wellKnownAccountName(t, windows.WinBuiltinAdministratorsSid), true}, + // The built-in Administrator (RID 500) and Guest (RID 501) accounts + // exist on every Windows installation, though they may be disabled. + {"builtin_administrator", localAccountNameByRID(t, 500), true}, + {"guest", localAccountNameByRID(t, 501), false}, + // Unresolvable accounts fail closed. + {"nonexistent_user", "netbird-no-such-user", true}, + {"empty_username", "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isWindowsAccountPrivilegedOrUnknown(tt.username) + assert.Equal(t, tt.want, result, "account privilege classification for %q", tt.username) + }) + } +} + +func TestIsProcessElevated(t *testing.T) { + elevated := isProcessElevated() + + // TokenElevationType is a second, independent view of the same token: + // Full means elevated and Limited means a filtered administrator, while + // Default covers both a standard user and an administrator with no linked + // token (UAC off, the built-in Administrator, SYSTEM), so it implies nothing. + elevationType, err := tokenElevationType(windows.GetCurrentProcessToken()) + require.NoError(t, err, "read token elevation type") + + adminSid, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + require.NoError(t, err, "create Administrators SID") + + // Token(0) makes CheckTokenMembership evaluate the caller's own token. It + // counts only enabled SIDs, so a filtered administrator reports false here. + member, err := windows.Token(0).IsMember(adminSid) + require.NoError(t, err, "check own Administrators membership") + + t.Logf("elevated=%v elevationType=%d memberOfAdministrators=%v", elevated, elevationType, member) + + switch elevationType { + case tokenElevationTypeFull: + assert.True(t, elevated, "a token of elevation type Full must report elevated") + case tokenElevationTypeLimited: + assert.False(t, elevated, "a filtered administrator token must not report elevated") + } + + // Administrators enabled in the token means the token wields administrative + // rights, which is what elevation reports. + if member { + assert.True(t, elevated, "token with enabled Administrators membership must report elevated") + } +} + +// TestS4UMembershipAgreesWithLocalGroups exercises the S4U token path used +// for domain accounts. S4U logons need the TCB privilege, so the test runs +// only as SYSTEM (which is how CI executes the suite). For local accounts the +// token's Administrators membership must agree with the SAM enumeration. +func TestS4UMembershipAgreesWithLocalGroups(t *testing.T) { + system, err := windows.CreateWellKnownSid(windows.WinLocalSystemSid) + require.NoError(t, err, "create SYSTEM SID") + current, err := user.Current() + require.NoError(t, err, "get current user") + if current.Uid != system.String() { + t.Skipf("S4U logon requires SYSTEM (running as %s)", current.Username) + } + + adminSid, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + require.NoError(t, err, "create Administrators SID") + + checked := 0 + for _, name := range localAccountNames(t) { + viaToken, err := s4uTokenIsMember(name, ".", adminSid) + if err != nil { + // Disabled or logon-restricted accounts cannot get an S4U logon. + t.Logf("skipping %s: %v", name, err) + continue + } + viaSAM, err := localGroupsContainSID(name, adminSid) + require.NoError(t, err, "enumerate local groups for %s", name) + + assert.Equal(t, viaSAM, viaToken, "S4U token and SAM enumeration must agree on Administrators membership for %s", name) + checked++ + } + // Ineligible accounts are skipped, so without this the test could report + // success while comparing nothing at all. + require.Positive(t, checked, "no local account completed an S4U logon, so nothing was compared") + t.Logf("checked %d local accounts via S4U", checked) +} + +// TestLocalGroupsContainSID_Administrator checks the positive case against the +// built-in Administrator, a member of Administrators on every installation. +func TestLocalGroupsContainSID_Administrator(t *testing.T) { + adminSid, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + require.NoError(t, err, "create Administrators SID") + + administrator := localAccountNameByRID(t, 500) + member, err := localGroupsContainSID(administrator, adminSid) + require.NoError(t, err, "enumerate local groups for %s", administrator) + assert.True(t, member, "%s is a member of the Administrators group", administrator) +} + +// TestLocalGroupsContainSID_UnresolvableGroupFailsClosed covers a wanted SID +// that resolves to no group: the error must surface rather than being reported +// as "not a member", so the privilege check treats the account as privileged. +func TestLocalGroupsContainSID_UnresolvableGroupFailsClosed(t *testing.T) { + unknown := mustParseSID(t, "S-1-5-21-1111111111-2222222222-3333333333-4444") + + _, err := localGroupsContainSID(localAccountNameByRID(t, 500), unknown) + require.Error(t, err, "must report an error when the wanted group cannot be identified") +} + +func TestLocalGroupsContainSID_Guest(t *testing.T) { + guestsSid, err := windows.CreateWellKnownSid(windows.WinBuiltinGuestsSid) + require.NoError(t, err, "create Guests SID") + adminsSid, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + require.NoError(t, err, "create Administrators SID") + + guest := localAccountNameByRID(t, 501) + + inGuests, err := localGroupsContainSID(guest, guestsSid) + require.NoError(t, err, "enumerate local groups for %s", guest) + assert.True(t, inGuests, "%s is a member of the Guests group", guest) + + inAdmins, err := localGroupsContainSID(guest, adminsSid) + require.NoError(t, err, "enumerate local groups for %s", guest) + assert.False(t, inAdmins, "%s is not a member of the Administrators group", guest) +} diff --git a/client/ssh/server/server_config_test.go b/client/ssh/server/server_config_test.go index f70e29963..983bd7a43 100644 --- a/client/ssh/server/server_config_test.go +++ b/client/ssh/server/server_config_test.go @@ -239,6 +239,7 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { forwardType string port uint32 username string + uid string expectError bool errorMsg string skipOnWindows bool @@ -248,6 +249,7 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { forwardType: "remote", port: 80, username: "testuser", + uid: "1000", expectError: true, errorMsg: "cannot bind to privileged port", skipOnWindows: true, @@ -257,6 +259,7 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { forwardType: "tcpip-forward", port: 443, username: "testuser", + uid: "1000", expectError: true, errorMsg: "cannot bind to privileged port", skipOnWindows: true, @@ -266,6 +269,7 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { forwardType: "remote", port: 8080, username: "testuser", + uid: "1000", expectError: false, }, { @@ -273,6 +277,7 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { forwardType: "remote", port: 0, username: "testuser", + uid: "1000", expectError: false, }, { @@ -280,13 +285,35 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { forwardType: "remote", port: 22, username: "root", + uid: "0", expectError: false, }, + { + // Only uid 0 is privileged, whatever the account is called. + name: "uid 0 under another name may bind a privileged port", + forwardType: "remote", + port: 22, + username: "toor", + uid: "0", + expectError: false, + skipOnWindows: true, + }, + { + name: "account named root without uid 0 may not", + forwardType: "remote", + port: 22, + username: "root", + uid: "1000", + expectError: true, + errorMsg: "cannot bind to privileged port", + skipOnWindows: true, + }, { name: "local forward privileged port allowed for non-root", forwardType: "local", port: 80, username: "testuser", + uid: "1000", expectError: false, }, } @@ -299,7 +326,7 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { result := PrivilegeCheckResult{ Allowed: true, - User: &user.User{Username: tt.username}, + User: &user.User{Username: tt.username, Uid: tt.uid}, } err := server.checkPrivilegedPortAccess(tt.forwardType, tt.port, result) @@ -420,6 +447,13 @@ func TestServer_PortConflictHandling(t *testing.T) { func TestServer_IsPrivilegedUser(t *testing.T) { + // Windows classification depends on account SIDs and group membership, and + // the accounts involved carry localized, renameable names. It is covered by + // TestIsWindowsAccountPrivileged, which resolves them from well-known SIDs. + if runtime.GOOS == "windows" { + t.Skip("covered by TestIsWindowsAccountPrivileged") + } + tests := []struct { username string expected bool @@ -440,44 +474,16 @@ func TestServer_IsPrivilegedUser(t *testing.T) { expected: false, description: "empty username should not be privileged", }, - } - - // Add Windows-specific tests - if runtime.GOOS == "windows" { - tests = append(tests, []struct { - username string - expected bool - description string - }{ - { - username: "Administrator", - expected: true, - description: "Administrator should be considered privileged on Windows", - }, - { - username: "administrator", - expected: true, - description: "administrator should be considered privileged on Windows (case insensitive)", - }, - }...) - } else { - // On non-Windows systems, Administrator should not be privileged - tests = append(tests, []struct { - username string - expected bool - description string - }{ - { - username: "Administrator", - expected: false, - description: "Administrator should not be privileged on non-Windows systems", - }, - }...) + { + username: "Administrator", + expected: false, + description: "Administrator should not be privileged on non-Windows systems", + }, } for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { - result := isPrivilegedUsername(tt.username) + result := isPrivilegedOrUnknown(tt.username) assert.Equal(t, tt.expected, result, tt.description) }) } diff --git a/client/ssh/server/sftp_windows.go b/client/ssh/server/sftp_windows.go index dc532b9e7..25cd17298 100644 --- a/client/ssh/server/sftp_windows.go +++ b/client/ssh/server/sftp_windows.go @@ -17,7 +17,7 @@ import ( // createSftpCommand creates a Windows SFTP command with user switching. // The caller must close the returned token handle after starting the process. func (s *Server) createSftpCommand(targetUser *user.User, sess ssh.Session) (*exec.Cmd, windows.Token, error) { - username, domain := s.parseUsername(targetUser.Username) + username, domain := parseUsername(targetUser.Username) netbirdPath, err := os.Executable() if err != nil { diff --git a/client/ssh/server/user_utils.go b/client/ssh/server/user_utils.go index bc2aa2d7d..6c8142b30 100644 --- a/client/ssh/server/user_utils.go +++ b/client/ssh/server/user_utils.go @@ -16,11 +16,6 @@ var ( ErrPrivilegedUserSwitch = errors.New("cannot switch to privileged user - current user lacks required privileges") ) -// isPlatformUnix returns true for Unix-like platforms (Linux, macOS, etc.) -func isPlatformUnix() bool { - return getCurrentOS() != "windows" -} - // Dependency injection variables for testing - allows mocking dynamic runtime checks var ( getCurrentUser = currentUserWithGetent @@ -29,6 +24,9 @@ var ( getIsProcessPrivileged = isCurrentProcessPrivileged getEuid = os.Geteuid + + getProcessElevated = isProcessElevated + getWindowsAccountPrivilegedOrUnknown = isWindowsAccountPrivilegedOrUnknown ) const ( @@ -65,6 +63,13 @@ type PrivilegeCheckResult struct { RequiresUserSwitching bool } +// privilegeCheckContext holds all context needed for privilege checking +type privilegeCheckContext struct { + currentUser *user.User + currentUserPrivileged bool + allowRoot bool +} + // CheckPrivileges performs comprehensive privilege checking for all SSH features. // This is the single source of truth for privilege decisions across the SSH server. func (s *Server) CheckPrivileges(req PrivilegeCheckRequest) PrivilegeCheckResult { @@ -75,7 +80,7 @@ func (s *Server) CheckPrivileges(req PrivilegeCheckRequest) PrivilegeCheckResult // Handle empty username case - but still check root access controls if req.RequestedUsername == "" { - if isPrivilegedUsername(context.currentUser.Username) && !context.allowRoot { + if isPrivilegedOrUnknown(context.currentUser.Username) && !context.allowRoot { return PrivilegeCheckResult{ Allowed: false, Error: &PrivilegedUserError{Username: context.currentUser.Username}, @@ -135,7 +140,7 @@ func (s *Server) checkUserRequest(ctx *privilegeCheckContext, req PrivilegeCheck needsUserSwitching := !isSameResolvedUser(resolvedUser, ctx.currentUser) - if isPrivilegedUsername(resolvedUser.Username) && !ctx.allowRoot { + if isPrivilegedOrUnknown(resolvedUser.Username) && !ctx.allowRoot { return PrivilegeCheckResult{ Allowed: false, Error: &PrivilegedUserError{Username: resolvedUser.Username}, @@ -175,6 +180,42 @@ func (s *Server) resolveRequestedUser(requestedUsername string) (*user.User, err return u, nil } +// SetAllowRootLogin configures root login access +func (s *Server) SetAllowRootLogin(allow bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.allowRootLogin = allow +} + +// userNameLookup performs user lookup with root login permission check +func (s *Server) userNameLookup(username string) (*user.User, error) { + result, err := s.userPrivilegeCheck(username) + if err != nil { + return nil, err + } + return result.User, nil +} + +// userPrivilegeCheck performs user lookup with full privilege check result +func (s *Server) userPrivilegeCheck(username string) (PrivilegeCheckResult, error) { + result := s.CheckPrivileges(PrivilegeCheckRequest{ + RequestedUsername: username, + FeatureSupportsUserSwitch: true, + FeatureName: FeatureSSHLogin, + }) + + if !result.Allowed { + return result, result.Error + } + + return result, nil +} + +// isPlatformUnix returns true for Unix-like platforms (Linux, macOS, etc.) +func isPlatformUnix() bool { + return getCurrentOS() != "windows" +} + // isSameResolvedUser compares two resolved user identities func isSameResolvedUser(user1, user2 *user.User) bool { if user1 == nil || user2 == nil { @@ -183,13 +224,6 @@ func isSameResolvedUser(user1, user2 *user.User) bool { return user1.Uid == user2.Uid } -// privilegeCheckContext holds all context needed for privilege checking -type privilegeCheckContext struct { - currentUser *user.User - currentUserPrivileged bool - allowRoot bool -} - // isSameUser checks if two usernames refer to the same user // SECURITY: This function must be conservative - it should only return true // when we're certain both usernames refer to the exact same user identity @@ -253,159 +287,30 @@ func isWindowsSameUser(requestedUsername, currentUsername string) bool { return strings.EqualFold(reqDomain, curDomain) } -// SetAllowRootLogin configures root login access -func (s *Server) SetAllowRootLogin(allow bool) { - s.mu.Lock() - defer s.mu.Unlock() - s.allowRootLogin = allow -} - -// userNameLookup performs user lookup with root login permission check -func (s *Server) userNameLookup(username string) (*user.User, error) { - result := s.CheckPrivileges(PrivilegeCheckRequest{ - RequestedUsername: username, - FeatureSupportsUserSwitch: true, - FeatureName: FeatureSSHLogin, - }) - - if !result.Allowed { - return nil, result.Error - } - - return result.User, nil -} - -// userPrivilegeCheck performs user lookup with full privilege check result -func (s *Server) userPrivilegeCheck(username string) (PrivilegeCheckResult, error) { - result := s.CheckPrivileges(PrivilegeCheckRequest{ - RequestedUsername: username, - FeatureSupportsUserSwitch: true, - FeatureName: FeatureSSHLogin, - }) - - if !result.Allowed { - return result, result.Error - } - - return result, nil -} - -// isPrivilegedUsername checks if the given username represents a privileged user across platforms. -// On Unix: root -// On Windows: Administrator, SYSTEM (case-insensitive) -// Handles domain-qualified usernames like "DOMAIN\Administrator" or "user@domain.com" -func isPrivilegedUsername(username string) bool { +// isPrivilegedOrUnknown reports whether the given username represents a +// privileged user, or on Windows an account whose privilege could not be +// determined. +// On Unix: root. +// On Windows: well-known service accounts, built-in Administrator accounts, +// and members of the local Administrators group; handles domain-qualified +// usernames like "DOMAIN\user" or "user@domain.com". An account that cannot be +// resolved or evaluated is reported as privileged. +// +// Use this to refuse privileged accounts, never to grant them anything: the +// undetermined case is safe for a refusal and unsafe for a grant. +func isPrivilegedOrUnknown(username string) bool { if getCurrentOS() != "windows" { return username == "root" } - - bareUsername := username - // Handle Windows domain format: DOMAIN\username - if idx := strings.LastIndex(username, `\`); idx != -1 { - bareUsername = username[idx+1:] - } - // Handle email-style format: username@domain.com - if idx := strings.Index(bareUsername, "@"); idx != -1 { - bareUsername = bareUsername[:idx] - } - - return isWindowsPrivilegedUser(bareUsername) -} - -// isWindowsPrivilegedUser checks if a bare username (domain already stripped) represents a Windows privileged account -func isWindowsPrivilegedUser(bareUsername string) bool { - // common privileged usernames (case insensitive) - privilegedNames := []string{ - "administrator", - "admin", - "root", - "system", - "localsystem", - "networkservice", - "localservice", - } - - usernameLower := strings.ToLower(bareUsername) - for _, privilegedName := range privilegedNames { - if usernameLower == privilegedName { - return true - } - } - - // computer accounts (ending with $) are not privileged by themselves - // They only gain privileges through group membership or specific SIDs - - if targetUser, err := lookupUser(bareUsername); err == nil { - return isWindowsPrivilegedSID(targetUser.Uid) - } - - return false -} - -// isWindowsPrivilegedSID checks if a Windows SID represents a privileged account -func isWindowsPrivilegedSID(sid string) bool { - privilegedSIDs := []string{ - "S-1-5-18", // Local System (SYSTEM) - "S-1-5-19", // Local Service (NT AUTHORITY\LOCAL SERVICE) - "S-1-5-20", // Network Service (NT AUTHORITY\NETWORK SERVICE) - "S-1-5-32-544", // Administrators group (BUILTIN\Administrators) - "S-1-5-500", // Built-in Administrator account (local machine RID 500) - } - - for _, privilegedSID := range privilegedSIDs { - if sid == privilegedSID { - return true - } - } - - // Check for domain administrator accounts (RID 500 in any domain) - // Format: S-1-5-21-domain-domain-domain-500 - // This is reliable as RID 500 is reserved for the domain Administrator account - if strings.HasPrefix(sid, "S-1-5-21-") && strings.HasSuffix(sid, "-500") { - return true - } - - // Check for other well-known privileged RIDs in domain contexts - // RID 512 = Domain Admins group, RID 516 = Domain Controllers group - if strings.HasPrefix(sid, "S-1-5-21-") { - if strings.HasSuffix(sid, "-512") || // Domain Admins group - strings.HasSuffix(sid, "-516") || // Domain Controllers group - strings.HasSuffix(sid, "-519") { // Enterprise Admins group - return true - } - } - - return false + return getWindowsAccountPrivilegedOrUnknown(username) } // isCurrentProcessPrivileged checks if the current process is running with elevated privileges. // On Unix systems, this means running as root (UID 0). -// On Windows, this means running as Administrator or SYSTEM. +// On Windows, this means the process token is elevated (administrators, SYSTEM). func isCurrentProcessPrivileged() bool { if getCurrentOS() == "windows" { - return isWindowsElevated() + return getProcessElevated() } return getEuid() == 0 } - -// isWindowsElevated checks if the current process is running with elevated privileges on Windows -func isWindowsElevated() bool { - currentUser, err := getCurrentUser() - if err != nil { - log.Errorf("failed to get current user for privilege check, assuming non-privileged: %v", err) - return false - } - - if isWindowsPrivilegedSID(currentUser.Uid) { - log.Debugf("Windows user switching supported: running as privileged SID %s", currentUser.Uid) - return true - } - - if isPrivilegedUsername(currentUser.Username) { - log.Debugf("Windows user switching supported: running as privileged username %s", currentUser.Username) - return true - } - - log.Debugf("Windows user switching not supported: not running as privileged user (current: %s)", currentUser.Uid) - return false -} diff --git a/client/ssh/server/user_utils_test.go b/client/ssh/server/user_utils_test.go index 637dc10d0..2fa9b68ee 100644 --- a/client/ssh/server/user_utils_test.go +++ b/client/ssh/server/user_utils_test.go @@ -4,6 +4,7 @@ import ( "errors" "os/user" "runtime" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -27,8 +28,8 @@ func setupTestDependencies(currentUser *user.User, currentUserErr error, os stri originalLookupUser := lookupUser originalGetCurrentOS := getCurrentOS originalGetEuid := getEuid - - // Reset caches to ensure clean test state + originalGetProcessElevated := getProcessElevated + originalGetWindowsAccountPrivilegedOrUnknown := getWindowsAccountPrivilegedOrUnknown // Set test values - inject platform dependencies getCurrentUser = func() (*user.User, error) { @@ -53,16 +54,31 @@ func setupTestDependencies(currentUser *user.User, currentUserErr error, os stri return euid } - // Mock privilege detection based on the test user - getIsProcessPrivileged = func() bool { + // Simulate the Windows token elevation check based on the fixture user: + // the built-in Administrator (RID 500) and SYSTEM run elevated. + getProcessElevated = func() bool { if currentUser == nil { return false } - // Check both username and SID for Windows systems - if os == "windows" && isWindowsPrivilegedSID(currentUser.Uid) { + return currentUser.Uid == "S-1-5-18" || strings.HasSuffix(currentUser.Uid, "-500") + } + + // Simulate the Windows account classifier for the fixture accounts. + // "root" does not exist on Windows; the real classifier fails closed on + // unresolvable accounts, so it counts as privileged here too. + getWindowsAccountPrivilegedOrUnknown = func(username string) bool { + bare := username + if idx := strings.LastIndex(bare, `\`); idx != -1 { + bare = bare[idx+1:] + } + if idx := strings.Index(bare, "@"); idx != -1 { + bare = bare[:idx] + } + switch strings.ToLower(bare) { + case "administrator", "system", "root": return true } - return isPrivilegedUsername(currentUser.Username) + return false } // Return cleanup function @@ -71,10 +87,8 @@ func setupTestDependencies(currentUser *user.User, currentUserErr error, os stri lookupUser = originalLookupUser getCurrentOS = originalGetCurrentOS getEuid = originalGetEuid - - getIsProcessPrivileged = isCurrentProcessPrivileged - - // Reset caches after test + getProcessElevated = originalGetProcessElevated + getWindowsAccountPrivilegedOrUnknown = originalGetWindowsAccountPrivilegedOrUnknown } } @@ -421,6 +435,9 @@ func TestUsedFallback_MeansNoPrivilegeDropping(t *testing.T) { } func TestPrivilegedUsernameDetection(t *testing.T) { + // Windows classification is syscall-backed (SID resolution, group + // membership) and is covered by privileges_windows_test.go; here only the + // Unix logic and the platform dispatch are exercised. tests := []struct { name string username string @@ -432,25 +449,9 @@ func TestPrivilegedUsernameDetection(t *testing.T) { {"unix_regular_user", "alice", "linux", false}, {"unix_root_capital", "Root", "linux", false}, // Case-sensitive - // Windows tests + // Windows dispatch to the (mocked) account classifier {"windows_administrator", "Administrator", "windows", true}, - {"windows_system", "SYSTEM", "windows", true}, - {"windows_admin", "admin", "windows", true}, - {"windows_admin_lowercase", "administrator", "windows", true}, // Case-insensitive - {"windows_domain_admin", "DOMAIN\\Administrator", "windows", true}, - {"windows_email_admin", "admin@domain.com", "windows", true}, {"windows_regular_user", "alice", "windows", false}, - {"windows_domain_user", "DOMAIN\\alice", "windows", false}, - {"windows_localsystem", "localsystem", "windows", true}, - {"windows_networkservice", "networkservice", "windows", true}, - {"windows_localservice", "localservice", "windows", true}, - - // Computer accounts (these depend on current user context in real implementation) - {"windows_computer_account", "WIN2K19-C2$", "windows", false}, // Computer account by itself not privileged - {"windows_domain_computer", "DOMAIN\\COMPUTER$", "windows", false}, // Domain computer account - - // Cross-platform - {"root_on_windows", "root", "windows", true}, // Root should be privileged everywhere } for _, tt := range tests { @@ -459,50 +460,8 @@ func TestPrivilegedUsernameDetection(t *testing.T) { cleanup := setupTestDependencies(nil, nil, tt.platform, 1000, nil, nil) defer cleanup() - result := isPrivilegedUsername(tt.username) - assert.Equal(t, tt.privileged, result) - }) - } -} - -func TestWindowsPrivilegedSIDDetection(t *testing.T) { - tests := []struct { - name string - sid string - privileged bool - description string - }{ - // Well-known system accounts - {"system_account", "S-1-5-18", true, "Local System (SYSTEM)"}, - {"local_service", "S-1-5-19", true, "Local Service"}, - {"network_service", "S-1-5-20", true, "Network Service"}, - {"administrators_group", "S-1-5-32-544", true, "Administrators group"}, - {"builtin_administrator", "S-1-5-500", true, "Built-in Administrator"}, - - // Domain accounts - {"domain_administrator", "S-1-5-21-1234567890-1234567890-1234567890-500", true, "Domain Administrator (RID 500)"}, - {"domain_admins_group", "S-1-5-21-1234567890-1234567890-1234567890-512", true, "Domain Admins group"}, - {"domain_controllers_group", "S-1-5-21-1234567890-1234567890-1234567890-516", true, "Domain Controllers group"}, - {"enterprise_admins_group", "S-1-5-21-1234567890-1234567890-1234567890-519", true, "Enterprise Admins group"}, - - // Regular users - {"regular_user", "S-1-5-21-1234567890-1234567890-1234567890-1001", false, "Regular domain user"}, - {"another_regular_user", "S-1-5-21-1234567890-1234567890-1234567890-1234", false, "Another regular user"}, - {"local_user", "S-1-5-21-1234567890-1234567890-1234567890-1000", false, "Local regular user"}, - - // Groups that are not privileged - {"domain_users", "S-1-5-21-1234567890-1234567890-1234567890-513", false, "Domain Users group"}, - {"power_users", "S-1-5-32-547", false, "Power Users group"}, - - // Invalid SIDs - {"malformed_sid", "S-1-5-invalid", false, "Malformed SID"}, - {"empty_sid", "", false, "Empty SID"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := isWindowsPrivilegedSID(tt.sid) - assert.Equal(t, tt.privileged, result, "Failed for %s: %s", tt.description, tt.sid) + result := isPrivilegedOrUnknown(tt.username) + assert.Equal(t, tt.privileged, result, "privilege classification for %s on %s", tt.username, tt.platform) }) } } diff --git a/client/ssh/server/userswitching_windows.go b/client/ssh/server/userswitching_windows.go index 260e1301e..9e8cd5b30 100644 --- a/client/ssh/server/userswitching_windows.go +++ b/client/ssh/server/userswitching_windows.go @@ -91,7 +91,7 @@ func validateUsernameFormat(username string) error { func (s *Server) createExecutorCommand(logger *log.Entry, session ssh.Session, localUser *user.User, hasPty bool) (*exec.Cmd, func(), error) { logger.Debugf("creating Windows executor command for user %s (Pty: %v)", localUser.Username, hasPty) - username, _ := s.parseUsername(localUser.Username) + username, _ := parseUsername(localUser.Username) if err := validateUsername(username); err != nil { return nil, nil, fmt.Errorf("invalid username %q: %w", username, err) } @@ -102,7 +102,7 @@ func (s *Server) createExecutorCommand(logger *log.Entry, session ssh.Session, l // createUserSwitchCommand creates a command with Windows user switching. // Returns the command and a cleanup function that must be called after starting the process. func (s *Server) createUserSwitchCommand(logger *log.Entry, session ssh.Session, localUser *user.User) (*exec.Cmd, func(), error) { - username, domain := s.parseUsername(localUser.Username) + username, domain := parseUsername(localUser.Username) shell := getUserShell(localUser.Uid) @@ -138,7 +138,7 @@ func (s *Server) createUserSwitchCommand(logger *log.Entry, session ssh.Session, } // parseUsername extracts username and domain from a Windows username -func (s *Server) parseUsername(fullUsername string) (username, domain string) { +func parseUsername(fullUsername string) (username, domain string) { // Handle DOMAIN\username format if idx := strings.LastIndex(fullUsername, `\`); idx != -1 { domain = fullUsername[:idx] diff --git a/client/status/status.go b/client/status/status.go index e8276d0fa..1c204cdb1 100644 --- a/client/status/status.go +++ b/client/status/status.go @@ -46,7 +46,10 @@ func ParseDaemonStatus(s string) DaemonStatus { // ConvertOptions holds parameters for ConvertToStatusOutputOverview. type ConvertOptions struct { - Anonymize bool + Anonymize bool + // AnonymizeLevel selects how much the anonymizer redacts. Only + // meaningful when Anonymize is set. + AnonymizeLevel anonymize.Level DaemonVersion string DaemonStatus DaemonStatus StatusFilter string @@ -217,6 +220,7 @@ func ConvertToStatusOutputOverview(pbFullStatus *proto.FullStatus, opts ConvertO if opts.Anonymize { anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(opts.AnonymizeLevel) anonymizeOverview(anonymizer, &overview) } @@ -976,6 +980,7 @@ func timeAgo(t time.Time) string { func anonymizePeerDetail(a *anonymize.Anonymizer, peer *PeerStateDetailOutput) { peer.FQDN = a.AnonymizeDomain(peer.FQDN) + peer.PubKey = a.AnonymizeWGKey(peer.PubKey) if localIP, port, err := net.SplitHostPort(peer.IceCandidateEndpoint.Local); err == nil { peer.IceCandidateEndpoint.Local = fmt.Sprintf("%s:%s", a.AnonymizeIPString(localIP), port) } @@ -1007,6 +1012,7 @@ func anonymizeOverview(a *anonymize.Anonymizer, overview *OutputOverview) { overview.SignalState.URL = a.AnonymizeURI(overview.SignalState.URL) overview.SignalState.Error = a.AnonymizeString(overview.SignalState.Error) + overview.PubKey = a.AnonymizeWGKey(overview.PubKey) overview.IP = a.AnonymizeIPString(overview.IP) overview.IPv6 = a.AnonymizeIPString(overview.IPv6) for i, detail := range overview.Relays.Details { diff --git a/client/ui/authsession/service.go b/client/ui/authsession/service.go index 28efe7cfd..9c094c2a1 100644 --- a/client/ui/authsession/service.go +++ b/client/ui/authsession/service.go @@ -6,9 +6,11 @@ import ( "context" "time" + log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" gstatus "google.golang.org/grpc/status" + "github.com/netbirdio/netbird/client/internal/profilemanager" "github.com/netbirdio/netbird/client/proto" ) @@ -58,10 +60,21 @@ func (s *Session) RequestExtend(ctx context.Context, p ExtendStartParams) (Exten return ExtendStartResult{}, err } - req := &proto.RequestExtendAuthSessionRequest{} - if p.Hint != "" { - h := p.Hint - req.Hint = &h + // a request from the UI implies a graphical session, which the daemon cannot detect itself + req := &proto.RequestExtendAuthSessionRequest{HasGraphicalSession: true} + hint := p.Hint + if hint == "" { + pm := profilemanager.NewProfileManager() + if active, perr := pm.GetActiveProfile(); perr != nil { + log.Debugf("failed to get active profile for login hint: %v", perr) + } else if state, serr := pm.GetProfileState(active.ID); serr != nil { + log.Debugf("failed to get profile state for login hint: %v", serr) + } else { + hint = state.Email + } + } + if hint != "" { + req.Hint = &hint } resp, err := cli.RequestExtendAuthSession(ctx, req) diff --git a/client/ui/frontend/package.json b/client/ui/frontend/package.json index 3131b36cd..dcef99ad3 100644 --- a/client/ui/frontend/package.json +++ b/client/ui/frontend/package.json @@ -15,7 +15,8 @@ "lint": "eslint \"src/**/*.{ts,tsx}\"", "lint:fix": "eslint \"src/**/*.{ts,tsx}\" --fix", "check": "pnpm lint && pnpm typecheck && pnpm format:check", - "check:fix": "pnpm lint:fix && pnpm format && pnpm typecheck" + "check:fix": "pnpm lint:fix && pnpm format && pnpm typecheck", + "i18n:check": "node ../i18n/check-translations.mjs" }, "dependencies": { "@radix-ui/react-dialog": "^1.1.15", diff --git a/client/ui/frontend/src/contexts/DebugBundleContext.tsx b/client/ui/frontend/src/contexts/DebugBundleContext.tsx index a0a131fbf..5f2ed9041 100644 --- a/client/ui/frontend/src/contexts/DebugBundleContext.tsx +++ b/client/ui/frontend/src/contexts/DebugBundleContext.tsx @@ -71,10 +71,12 @@ type BundleOptions = { hasWindow: boolean; totalSec: number; uploadUrl: string; - anonymize: boolean; + anonymizeLevel: AnonymizeLevel; systemInfo: boolean; }; +export type AnonymizeLevel = "none" | "default" | "strict"; + const startCaptureBestEffort = async (totalSec: number, pcap: CaptureState) => { try { // Mirror the CLI's safety margin: window + 30s, server caps at 10m. @@ -187,7 +189,10 @@ const runBundleFlow = async ( if (opts.uploadUrl) setStage({ kind: "uploading" }); const result = await DebugSvc.Bundle({ - anonymize: opts.anonymize, + anonymize: opts.anonymizeLevel !== "none", + // The daemon only knows "default" and "strict"; "none" is expressed + // through the anonymize flag being off. + anonymizeLevel: opts.anonymizeLevel === "strict" ? "strict" : "default", systemInfo: opts.systemInfo, uploadUrl: opts.uploadUrl, logFileCount, @@ -198,7 +203,7 @@ const runBundleFlow = async ( }; const useDebugBundle = () => { - const [anonymize, setAnonymize] = useState(false); + const [anonymizeLevel, setAnonymizeLevel] = useState("none"); const [systemInfo, setSystemInfo] = useState(true); const [upload, setUpload] = useState(true); const [trace, setTrace] = useState(true); @@ -240,7 +245,7 @@ const useDebugBundle = () => { hasWindow: capture && totalSec > 0, totalSec, uploadUrl: upload ? NETBIRD_UPLOAD_URL : "", - anonymize, + anonymizeLevel, systemInfo, }; @@ -272,8 +277,8 @@ const useDebugBundle = () => { }; return { - anonymize, - setAnonymize, + anonymizeLevel, + setAnonymizeLevel, systemInfo, setSystemInfo, upload, diff --git a/client/ui/frontend/src/modules/settings/SettingsTroubleshooting.tsx b/client/ui/frontend/src/modules/settings/SettingsTroubleshooting.tsx index 991937719..8b1774ed6 100644 --- a/client/ui/frontend/src/modules/settings/SettingsTroubleshooting.tsx +++ b/client/ui/frontend/src/modules/settings/SettingsTroubleshooting.tsx @@ -1,6 +1,6 @@ import { useId, type ReactNode } from "react"; import { Trans, useTranslation } from "react-i18next"; -import { CircleCheckBig, FolderOpen, Loader2 } from "lucide-react"; +import { ChevronDown, CircleCheckBig, FolderOpen, Info, Loader2 } from "lucide-react"; import { Browser } from "@wailsio/runtime"; import { Debug as DebugSvc } from "@bindings/services"; import type { DebugBundleResult } from "@bindings/services/models.js"; @@ -8,13 +8,22 @@ import { Button } from "@/components/buttons/Button"; import { DialogActions } from "@/components/dialog/DialogActions"; import { DialogDescription } from "@/components/dialog/DialogDescription"; import { DialogHeading } from "@/components/dialog/DialogHeading"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuRadioGroup, + DropdownMenuRadioItem, + DropdownMenuTrigger, +} from "@/components/DropdownMenu"; import FancyToggleSwitch from "@/components/switches/FancyToggleSwitch"; import HelpText from "@/components/typography/HelpText.tsx"; import { Input } from "@/components/inputs/Input"; import { Label } from "@/components/typography/Label"; import { SquareIcon } from "@/components/SquareIcon"; +import { Tooltip } from "@/components/Tooltip"; +import { cn } from "@/lib/cn"; import { formatRemaining } from "@/lib/formatters"; -import type { DebugStage } from "@/contexts/DebugBundleContext"; +import type { AnonymizeLevel, DebugStage } from "@/contexts/DebugBundleContext"; import { useDebugBundleContext } from "@/contexts/DebugBundleContext"; import { SectionGroup, SettingsBottomBar } from "@/modules/settings/SettingsSection.tsx"; @@ -24,8 +33,8 @@ export function SettingsTroubleshooting() { const { t } = useTranslation(); const durationId = useId(); const { - anonymize, - setAnonymize, + anonymizeLevel, + setAnonymizeLevel, systemInfo, setSystemInfo, upload, @@ -55,12 +64,71 @@ export function SettingsTroubleshooting() { return ( - +
+
+
+ } + > + + + + + + {t("settings.troubleshooting.anonymize.help")} + +
+
+ + + + + + setAnonymizeLevel(v as AnonymizeLevel)} + > + + {t("settings.troubleshooting.anonymize.none")} + + + {t("settings.troubleshooting.anonymize.default")} + + + {t("settings.troubleshooting.anonymize.strict")} + + + + +
+ .** Join the project, pick your language, and translate in the editor. Each string carries a context note (the `description` from the source file) telling you what it is and where it shows up, and the project's glossary, style guide, and QA checks mirror this document. -> 💡 **The one habit that matters most:** read each key's `description` before translating it. Labels are terse and ambiguous on their own; the `description` tells you what the string is, where it shows up, what to keep verbatim, and what it actually means. +> 💡 **The one habit that matters most:** read each string's context before translating it. Labels are terse and ambiguous on their own; the context tells you what the string is, where it shows up, what to keep verbatim, and what it actually means. + +--- + +## How contributions flow + +```text +i18n/locales/en/common.json ──sync──▶ Crowdin ──service PR──▶ i18n/locales//common.json +``` + +- `i18n/locales/en/common.json` is the source of truth. New and changed strings sync to Crowdin automatically (see `crowdin.yml` in the repository root). +- Crowdin opens and updates a service pull request with the translated bundles, keeping the source's file shape and key order. Keys nobody has translated yet are left out of the export; the app falls back to English for them at runtime. Maintainers review and merge that PR. +- Don't hand-edit `i18n/locales//common.json` in your own PRs: the next sync would conflict with or overwrite your changes. Translate on Crowdin instead. +- Missing your language? Request it on the Crowdin project page or in a [GitHub discussion](https://github.com/netbirdio/netbird/discussions). When a language first ships, a maintainer adds its row to `i18n/locales/_index.json` with `code`, `displayName` (the native name), and `englishName`, which puts it in the app's language picker. + +**Prefer translating with an AI agent?** That still works: drive it with *"Read `i18n/TRANSLATING.md` and translate the UI to Russian"* as before, but deliver the result to Crowdin instead of a pull request. Download your language's file from the Crowdin editor, let the agent translate it, and upload it back (the editor's offline translation flow). Crowdin runs its QA checks on upload, and the next service PR carries the strings into the repo. --- @@ -30,25 +45,6 @@ A **business zero-trust VPN** — an encrypted **overlay mesh** between a compan --- -## The files - -``` -i18n/locales/_index.json shipped-language list -i18n/locales/en/common.json source of truth — message + description -i18n/locales//common.json a target — message only -``` - -Chrome-extension JSON, each key → `{ "message", "description" }`. You translate the **`message`**. - -| ✅ Do | ❌ Don't | -|---|---| -| Keep **every key** from `en`, in the same order | Translate, rename, reorder, drop, or add keys (they're identifiers; the set grows over time) | -| Put **only `message`** in target bundles | Copy `description` into a target bundle | -| Give every key a non-empty `message` | Leave keys missing or empty | -| Save valid UTF-8 JSON, no BOM | Add trailing commas or break the JSON | - ---- - ## Hard rules — get these exactly right These are the usual ways a translation *breaks the app*, not just reads oddly. @@ -58,7 +54,7 @@ These are the usual ways a translation *breaks the app*, not just reads oddly. | Copy `{placeholders}` verbatim — `{version}`, `{count}`, `{name}`… | Translate the word inside the braces (`{verbleibend}` breaks it) | | Reposition a placeholder so the sentence flows | Drop or duplicate a placeholder | | Preserve every `\n`, leading/trailing space, and trailing `...` | Trim "invisible" spaces or the `...` (they're load-bearing) | -| Keep `®` in WireGuard® and quotes around `{name}` | Strip punctuation the description flags | +| Keep `®` in WireGuard® and quotes around `{name}` | Strip punctuation the context flags | **Plurals:** the app has only a *one / other* split — the singular key fires only when `count == 1`; the `{count}` key covers everything else (0, 2, 5, 100…). Languages with more than two forms (ru, pl, uk) can't be fully correct here — use the form that fits the widest range (Russian genitive plural: `минут` / `часов` / `дней`). Don't invent extra keys or cram multiple forms into one string. When no single form fits every value — a unit label after a number field, say — reach for a number-agnostic form (an abbreviation, or wording that reads the same for 1 and 100) instead of forcing a plural the *one / other* split can't supply. @@ -78,13 +74,15 @@ When a brand sits beside a common noun, keep its exact spelling but join them th > **Use the word that language's IT users actually say.** Translate when a natural, common term exists; keep the English term *only* when the literal translation would be awkward or no one in that field really uses it. -Apply each term **consistently** — same English term → same translation everywhere — and keep a term once you've settled it. Whether a term stays English or takes a native word is **language-dependent**: a technical loanword (e.g. *Daemon*, *Handshake*) often stays, an everyday word (e.g. *Latency*, *Public key*) usually localizes, and some (*Exit Node*, *Peer*) go either way depending on the language. Decide per term with the rule above — a foreign origin alone is no reason to keep English. **Your main reference is the existing bundles:** match how a term was already rendered for your language rather than re-deciding it. +Apply each term **consistently** — same English term → same translation everywhere — and keep a term once you've settled it. Whether a term stays English or takes a native word is **language-dependent**: a technical loanword (e.g. *Daemon*, *Handshake*) often stays, an everyday word (e.g. *Latency*, *Public key*) usually localizes, and some (*Exit Node*, *Peer*) go either way depending on the language. Decide per term with the rule above — a foreign origin alone is no reason to keep English. **Your main reference is the existing translation:** match how a term was already rendered for your language rather than re-deciding it. Two checks before you commit a term: - **Prefer established localized wording.** If a widely used tool in this space (for example WireGuard) ships your language, its wording for a shared term such as *handshake* is what users already expect — look at the translated app, not just English docs. For generic UI verbs and formal address, follow your OS vendor's style guide (Microsoft / Apple / Google). - **Watch for false friends.** A literal translation can collide with a *different* established term in your field — confirm your word doesn't already mean something else in this domain before using it. +These tiers are mirrored in the Crowdin project glossary, so the editor highlights them inline. When you settle a new Tier C term for your language, add its translation to the glossary entry so it sticks for everyone who comes after you. + --- ## Style @@ -98,7 +96,7 @@ Two checks before you commit a term: Where it reads naturally, aim to keep each string **roughly the same length** as the English — the UI is tight and over-long strings can wrap or truncate. It's a soft preference, not a rule: if your language simply needs more words, use them. -A few habits that keep a bundle reading like one product rather than a word-for-word port: +A few habits that keep a translation reading like one product rather than a word-for-word port: - **Translate meaning, not words.** Render what a string *does*. An idiom or an awkward source phrase should become natural in your language, not a literal calque. - **Keep one voice within a family.** Sibling strings — the connection states, every settings *help* caption, every "… Failed" title — should share a grammatical form. If one member sounds wrong in that form, re-voice the whole family rather than leave one odd sibling. @@ -107,27 +105,26 @@ A few habits that keep a bundle reading like one product rather than a word-for- --- -## Procedure +## Reviewing a language -**New language** — read `en/common.json` *with* descriptions → settle your Tier C terms → write `i18n/locales//common.json` (same keys and order as `en`, `message` only, placeholders & brands preserved) → add a row to `_index.json` (`{"code","displayName"` = native name`,"englishName"}`) → run the QA list. Use the locale-code style the existing entries use (e.g. `fr`, `pt`, `zh-CN`). +**On Crowdin:** proofread in the editor — context, glossary highlights, and QA flags sit inline next to each string. -**Review (de / hu / …)** — read source and target side by side; for each key check glossary conformance (e.g. de `Exit-Node` → `Exit Node`, hu `Kilépő csomópont` → `Exit Node`), placeholder/`\n` integrity, consistency, tone, and that the meaning matches the English `description`. Fix in place, then report what you changed (especially term standardizations) so a native speaker can sanity-check. +**In the repo** — e.g. driving an AI agent with *"Read `i18n/TRANSLATING.md` and review the existing German translation"* — read source and target side by side; for each key check glossary conformance (e.g. de `Exit-Node` → `Exit Node`, hu `Kilépő csomópont` → `Exit Node`), placeholder/`\n` integrity, consistency, tone, and that the meaning matches the English `description`. Report what you found, and apply the fixes **on Crowdin** — direct edits to the locale files are overwritten by the next sync. --- ## QA before you finish -- [ ] Valid JSON · **every `en` key** present, same order · **no `description`** fields - [ ] Every `{placeholder}`, `\n`, and intentional space preserved · `...` / `… Failed` / `{name}` quotes kept -- [ ] Tier A/B left intact · Tier C applied consistently (and matching the existing bundle for your language) +- [ ] Tier A/B left intact · Tier C applied consistently (and matching the existing translation for your language) - [ ] Buttons & tray short · locale punctuation and capitalization applied -- [ ] New language added to `_index.json` +- [ ] Crowdin QA flags resolved (variables, glossary terms, punctuation) - [ ] **Tested in the running app** ↓ --- ## Test it in the app -A bundle can pass every check above and still read wrong on screen. **Run the app, switch to your language, and click through the real surfaces** — tray menu, main window, every Settings tab, the dialogs. Watch for text overflow or truncation, labels that are technically right but wrong *for what the control does*, leaked placeholders, and terms that drift between screens. +A translation can pass every check above and still read wrong on screen. **Run the app, switch to your language, and click through the real surfaces** — tray menu, main window, every Settings tab, the dialogs. Watch for text overflow or truncation, labels that are technically right but wrong *for what the control does*, leaked placeholders, and terms that drift between screens. How to run the app and switch language: see the project README. Can't run it (e.g. a headless agent)? Say so in your summary — don't silently skip this step. diff --git a/client/ui/i18n/check-translations.mjs b/client/ui/i18n/check-translations.mjs new file mode 100644 index 000000000..bd076e0e0 --- /dev/null +++ b/client/ui/i18n/check-translations.mjs @@ -0,0 +1,104 @@ +#!/usr/bin/env node +// Validates that every shipped translation bundle carries exactly the same set +// of keys as the English source of truth. English (en) defines the keys; every +// other locale declared in _index.json must match it 1:1: +// +// - no missing keys — a missing key silently falls back to English at runtime +// (see i18n bundle fallback), so the gap never surfaces to users or CI +// without this check; +// - no orphaned keys — keys left behind after an English key is renamed or +// removed are dead weight and a sign the locale is drifting. +// +// Pure Node, no dependencies, so it runs without installing the frontend +// toolchain. +// +// Local: node client/ui/i18n/check-translations.mjs (or: pnpm i18n:check) +// CI: .github/workflows/ui-translations.yml + +import { readdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const SOURCE = "en"; +const localesDir = join(dirname(fileURLToPath(import.meta.url)), "locales"); +const isCI = Boolean(process.env.GITHUB_ACTIONS); + +function readJSON(path) { + return JSON.parse(readFileSync(path, "utf8")); +} + +function keysOf(langCode) { + return Object.keys(readJSON(join(localesDir, langCode, "common.json"))); +} + +// Emit a GitHub Actions annotation so failures render inline on the PR diff. +function annotate(file, message) { + if (isCI) console.log(`::error file=${file}::${message}`); +} + +const index = readJSON(join(localesDir, "_index.json")); +const declared = index.languages.map((l) => l.code); + +if (!declared.includes(SOURCE)) { + console.error(`FATAL: source language "${SOURCE}" is not declared in _index.json`); + process.exit(1); +} + +const sourceKeys = keysOf(SOURCE); +const sourceSet = new Set(sourceKeys); +console.log(`Source of truth: ${SOURCE}/common.json — ${sourceKeys.length} keys\n`); + +let failed = false; + +for (const code of declared) { + if (code === SOURCE) continue; + const file = `client/ui/i18n/locales/${code}/common.json`; + + let keys; + try { + keys = keysOf(code); + } catch (e) { + failed = true; + const msg = `bundle is declared in _index.json but common.json is missing or invalid (${e.message})`; + console.error(`✗ ${code}: ${msg}`); + annotate("client/ui/i18n/locales/_index.json", `${code}: ${msg}`); + continue; + } + + const set = new Set(keys); + const missing = sourceKeys.filter((k) => !set.has(k)); + const extra = keys.filter((k) => !sourceSet.has(k)); + + if (missing.length === 0 && extra.length === 0) { + console.log(`✓ ${code}: ${keys.length} keys`); + continue; + } + + failed = true; + console.error(`✗ ${code}: ${keys.length} keys (expected ${sourceKeys.length})`); + if (missing.length) { + console.error(` missing ${missing.length}: ${missing.join(", ")}`); + annotate(file, `Missing ${missing.length} key(s) present in ${SOURCE}: ${missing.join(", ")}`); + } + if (extra.length) { + console.error(` extra ${extra.length}: ${extra.join(", ")}`); + annotate(file, `Has ${extra.length} key(s) not present in ${SOURCE}: ${extra.join(", ")}`); + } +} + +// Locale directories present on disk but not declared in _index.json are never +// loaded by the app — surface them so dead translation files don't rot silently. +const onDisk = readdirSync(localesDir, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => e.name); +const undeclared = onDisk.filter((d) => !declared.includes(d)); +if (undeclared.length) { + console.warn(`\n⚠ locale directories not declared in _index.json (not shipped): ${undeclared.join(", ")}`); +} + +console.log(); +if (failed) { + console.error("Translation check FAILED — every locale must match the English key set."); + process.exit(1); +} +console.log("Translation check passed — all locales match the English key set."); diff --git a/client/ui/i18n/locales/en/common.json b/client/ui/i18n/locales/en/common.json index 9769e772f..694444497 100644 --- a/client/ui/i18n/locales/en/common.json +++ b/client/ui/i18n/locales/en/common.json @@ -1013,11 +1013,27 @@ }, "settings.troubleshooting.anonymize.label": { "message": "Anonymize Sensitive Information", - "description": "Toggle label: anonymize sensitive information in the bundle." + "description": "Label for the anonymization level dropdown (None, Default, Strict)." }, "settings.troubleshooting.anonymize.help": { - "message": "Hides public IP addresses and non-NetBird domains from logs.", - "description": "Helper text for anonymizing logs (hides public IPs and non-NetBird domains)." + "message": "Hides IP addresses, domains, and other sensitive values.", + "description": "Helper text under the anonymization dropdown. The level details live in the info tooltip." + }, + "settings.troubleshooting.anonymize.info": { + "message": "Default keeps internal IPv4 addresses and peer names readable for support. Strict additionally anonymizes private (RFC 1918), CGNAT, and link-local IP addresses, peer names, and WireGuard public keys. Recurring values map to the same placeholder, so peers stay distinguishable. Use Strict when sharing the bundle outside your organization.", + "description": "Info tooltip explaining the anonymization levels. 'RFC 1918', 'CGNAT', 'link-local', and 'WireGuard' are technical terms — keep them." + }, + "settings.troubleshooting.anonymize.none": { + "message": "None", + "description": "Dropdown option: no anonymization." + }, + "settings.troubleshooting.anonymize.default": { + "message": "Default", + "description": "Dropdown option: default anonymization level." + }, + "settings.troubleshooting.anonymize.strict": { + "message": "Strict", + "description": "Dropdown option: strict anonymization level." }, "settings.troubleshooting.systemInfo.label": { "message": "Include System Information", diff --git a/client/ui/i18n/locales/ja/common.json b/client/ui/i18n/locales/ja/common.json index 10cf7598d..ec69de9a5 100644 --- a/client/ui/i18n/locales/ja/common.json +++ b/client/ui/i18n/locales/ja/common.json @@ -1312,6 +1312,9 @@ "daemon.outdated.description": { "message": "このアプリを使用するには NetBird サービスを更新してください。" }, + "daemon.outdated.download": { + "message": "最新版をダウンロード" + }, "error.jwt_clock_skew": { "message": "サインインに失敗しました: このデバイスの時計がサーバーと同期していません。システムの時計を同期してからもう一度お試しください。" }, diff --git a/client/ui/services/connection.go b/client/ui/services/connection.go index 1069f8754..f78ce4c0f 100644 --- a/client/ui/services/connection.go +++ b/client/ui/services/connection.go @@ -108,10 +108,11 @@ func (s *Connection) Login(ctx context.Context, p LoginParams) (LoginResult, err } req := &proto.LoginRequest{ - ManagementUrl: p.ManagementURL, - SetupKey: p.SetupKey, - Hostname: p.Hostname, - IsUnixDesktopClient: runtime.GOOS == "linux", + ManagementUrl: p.ManagementURL, + SetupKey: p.SetupKey, + Hostname: p.Hostname, + // a login driven by the UI always has a graphical session available + IsUnixDesktopClient: true, } if profileName != "" { req.ProfileName = ptrStr(profileName) @@ -122,8 +123,16 @@ func (s *Connection) Login(ctx context.Context, p LoginParams) (LoginResult, err if p.PreSharedKey != "" { req.OptionalPreSharedKey = ptrStr(p.PreSharedKey) } - if p.Hint != "" { - req.Hint = ptrStr(p.Hint) + hint := p.Hint + if hint == "" && profileID != "" { + if state, serr := profilemanager.NewProfileManager().GetProfileState(profilemanager.ID(profileID)); serr == nil { + hint = state.Email + } else { + log.Debugf("failed to get profile state for login hint: %v", serr) + } + } + if hint != "" { + req.Hint = ptrStr(hint) } resp, err := cli.Login(ctx, req) @@ -227,16 +236,6 @@ func (s *Connection) Logout(ctx context.Context, p LogoutParams) error { return s.classifyDaemonError(err) } - // The daemon runs as root and can't reach the user-owned per-profile state - // file holding the account email (see Profiles.List), so clear the stale - // email here; the next SSO login recreates it. - if p.ProfileName != "" { - if err := profilemanager.NewProfileManager().RemoveProfileState(p.ProfileName); err != nil { - // Non-fatal: the logout itself succeeded. - log.Warnf("failed to remove profile state for %s: %v", p.ProfileName, err) - } - } - return nil } @@ -260,7 +259,7 @@ func (s *Connection) waitSSOLogin(ctx context.Context, p WaitSSOParams) (string, // Persist the account email the same way the CLI does after its own // WaitSSOLogin: the daemon returns it but cannot store it, since it runs as - // root and the per-profile state file is user-owned (see Logout below). + // root and the per-profile state file is user-owned (see Profiles.List). // Without this the profile has no email, so Profiles.List shows no account // and later logins and session extends go out without a login_hint — // leaving the IdP to guess which account was meant. diff --git a/client/ui/services/debug.go b/client/ui/services/debug.go index 034086747..d1f6555a8 100644 --- a/client/ui/services/debug.go +++ b/client/ui/services/debug.go @@ -15,10 +15,13 @@ import ( ) type DebugBundleParams struct { - Anonymize bool `json:"anonymize"` - SystemInfo bool `json:"systemInfo"` - UploadURL string `json:"uploadUrl"` - LogFileCount uint32 `json:"logFileCount"` + Anonymize bool `json:"anonymize"` + // AnonymizeLevel is "default" or "strict"; strict also anonymizes + // private IP ranges, peer names, and WireGuard public keys. + AnonymizeLevel string `json:"anonymizeLevel"` + SystemInfo bool `json:"systemInfo"` + UploadURL string `json:"uploadUrl"` + LogFileCount uint32 `json:"logFileCount"` } // DebugBundleResult: Path is set for local-only bundles, UploadedKey on upload @@ -48,11 +51,12 @@ func (s *Debug) Bundle(ctx context.Context, p DebugBundleParams) (DebugBundleRes return DebugBundleResult{}, err } resp, err := cli.DebugBundle(ctx, &proto.DebugBundleRequest{ - Anonymize: p.Anonymize, - SystemInfo: p.SystemInfo, - UploadURL: p.UploadURL, - LogFileCount: p.LogFileCount, - CliVersion: version.NetbirdVersion(), + Anonymize: p.Anonymize, + AnonymizeLevel: p.AnonymizeLevel, + SystemInfo: p.SystemInfo, + UploadURL: p.UploadURL, + LogFileCount: p.LogFileCount, + CliVersion: version.NetbirdVersion(), }) if err != nil { return DebugBundleResult{}, err diff --git a/client/ui/services/profile.go b/client/ui/services/profile.go index 5a9a0e68d..e76ab3db6 100644 --- a/client/ui/services/profile.go +++ b/client/ui/services/profile.go @@ -162,8 +162,9 @@ func (s *Profiles) Remove(ctx context.Context, p ProfileRef) error { } // The daemon deletes what it owns but runs as root, so it leaves the - // user-owned state file holding the account email behind (same split as - // Connection.Logout). Legacy profiles are keyed by name rather than by a + // user-owned state file holding the account email behind. Logout keeps the + // email on purpose so later logins can pass it as the login_hint; profile + // removal is what deletes it. Legacy profiles are keyed by name rather than by a // generated ID, so a recreated profile of the same name would inherit the // deleted one's email and offer it as the login_hint. // diff --git a/crowdin.yml b/crowdin.yml new file mode 100644 index 000000000..efdbceb8d --- /dev/null +++ b/crowdin.yml @@ -0,0 +1,11 @@ +skip_untranslated_strings: true +skip_untranslated_files: true +import_eq_suggestions: true + +files: + - source: /client/ui/i18n/locales/en/common.json + translation: /client/ui/i18n/locales/%two_letters_code%/common.json + type: chrome + languages_mapping: + two_letters_code: + zh-CN: zh-CN diff --git a/docs/agent-networks/01-end-to-end-flows.md b/docs/agent-networks/01-end-to-end-flows.md index b8891001b..0de6b4c33 100644 --- a/docs/agent-networks/01-end-to-end-flows.md +++ b/docs/agent-networks/01-end-to-end-flows.md @@ -115,7 +115,7 @@ sequenceDiagram Resp->>Resp: parse usage tokens, completion Note over Resp: capture_completion gates raw
completion capture Resp->>Cost: tokens - Cost->>Cost: lookup pricing.yaml + compute cost + Cost->>Cost: lookup rates from config-delivered
pricing table + compute cost Cost->>Rec: tokens + cost Rec->>MgmtGrpc: RecordLLMUsage(provider, model, prompt_t, completion_t, cost, groups, user) Rec-->>Log: emit access-log entry
(if EnableLogCollection) diff --git a/docs/agent-networks/modules/21-management-agentnetwork.md b/docs/agent-networks/modules/21-management-agentnetwork.md index cc74206e9..f91c369f7 100644 --- a/docs/agent-networks/modules/21-management-agentnetwork.md +++ b/docs/agent-networks/modules/21-management-agentnetwork.md @@ -15,6 +15,10 @@ Inside the package: `manager.go` is the CRUD + permissions-gated facade; `synthe | ---- | ---- | | `agentnetwork/manager.go` | Manager interface + CRUD + permission gates + bootstrap-settings + reconcile trigger | | `agentnetwork/synthesizer.go` | Settings/policy → wire-format synthesis; sole writer of the proxy middleware chain | +| `agentnetwork/synthesizer_pricing.go` | `buildCostMeterConfigJSON` — default table + per-provider prices → `cost_meter` config | +| `agentnetwork/pricing/defaults.go` | Default pricing table derived from the catalog + supplementals; `DefaultTable`, `LookupDefault`, wire `Entry` | +| `agentnetwork/pricing/override.go` | `LoadFile`/`StartReloader` for `AgentNetwork.PricingDefaultsFile` (mtime poll, merge over compiled-in base) | +| `agentnetwork/pricing/{exampleyaml,gen}.go` | Generates `defaults_llm_pricing.example.yaml` from the compiled-in table (golden-tested) | | `agentnetwork/policyselect.go` | Per-request policy attribution + account-budget ceiling (min-wins) | | `agentnetwork/reconcile.go` | Per-account synth diff vs in-memory cache → Create/Update/Delete | | `agentnetwork/catalog/catalog.go` | Static provider catalogue (auth headers, identity-injection shapes) | @@ -48,6 +52,8 @@ flowchart TD I --> J[indexProviderGroups: providerID -> sorted source groups] J --> K[buildRouterConfigJSON drops orphan providers] J --> L[buildIdentityInjectConfigJSON per catalog entry] + J --> K2[buildCostMeterConfigJSON: default table + per-provider prices] + K2 --> P H --> M[mergeGuardrails: union allowlist, OR redact] M --> N[applyAccountCollectionControls account toggle = SOLE capture control] N --> O[marshalGuardrailConfig] @@ -60,6 +66,84 @@ flowchart TD R --> T[accountManager.UpdateAccountPeers — fans synth ACLs into network map] ``` +### LLM pricing (management is the sole authority) + +**The proxy carries no price list.** Management synthesizes the entire pricing +table and ships it inside `cost_meter`'s `ConfigJSON`, so a price change reaches +the proxies as an ordinary mapping push — the chain rebuild installs a fresh +table and there is nothing to reload on the proxy side. + +```mermaid +flowchart TD + A[catalog.All — PricingSurfaces x Models] --> B[buildDefaultTable + supplementalDefaults] + B --> C{AgentNetwork.PricingDefaultsFile} + C -- absent --> D[compiled-in table serves] + C -- loaded --> E[LoadFile: merge file entries WHOLE over compiled base] + E --> F[mergedTable atomic.Pointer] + D --> G[DefaultTable] + F --> G + G --> H[buildCostMeterConfigJSON — pricing.defaults] + I[types.Provider.Models operator prices] --> J[normalizePricingModelID
bedrock ARN/region/version, vertex @version] + J --> K[materializeEntry: default entry as base,
operator input/output verbatim,
cache pointers only when non-nil] + K --> L[pricing.providers keyed by provider record ID] + H --> M[cost_meter ConfigJSON] + L --> M + G --> N[GET /catalog — applyDefaultPricing prefills dashboard rows] + O[StartReloader: mtime poll every ReloadInterval 1m] --> E +``` + +**Two tiers, resolved per request on the proxy** (`synthesizer_pricing.go:22-35`): + +- `pricing.defaults` — surface (`openai`/`anthropic`/`bedrock`) → normalized model + id → rates. The **full** default table ships to every account: it is small + (~10 KB) and it is what keeps gateway-style providers (which enumerate no + models, so they claim every model) priced. +- `pricing.providers` — provider **record** id → normalized model id → rates, + matched against the `llm.resolved_provider_id` the router stamps. Entries are + **fully materialized here**, at synth time: `materializeEntry` starts from the + default entry for that model so cache rates the operator didn't state are + inherited, overlays operator `input`/`output` verbatim (**including an explicit + 0**, which prices a self-hosted or internal endpoint as free rather than + silently reverting to list price), and overlays cache-rate **pointers only when + non-nil** — `nil` means "inherit the default", an explicit `0` means "no + discount, bill this bucket at the input rate". The proxy therefore does two map + lookups and no merging. + +Same orphan rule as the router: a provider no enabled policy authorises is +unreachable, so its prices aren't shipped. Model ids are normalized with the +**same** functions the request parser uses (`NormalizeBedrockModel` / +`NormalizeVertexModel`), which is what makes the per-record lookup key compare +equal to the `llm.model` the proxy meters. Post-normalization duplicates resolve +first-occurrence-wins, matching the routing dedup order. + +**`AgentNetwork.PricingDefaultsFile`** (`config.go:190-207`) lets an operator +replace default rates without a rebuild. Schema is `surface → model → rates` +(`input_per_1k`, `output_per_1k`, and optional `cached_input_per_1k` / +`cache_read_per_1k` / `cache_creation_per_1k`). Semantics: + +- A **relative** path resolves against ``, so a bare filename lands + alongside the store. Empty config probes `/defaults_llm_pricing.yaml`. +- An **explicitly configured** path is *required to load*: a typo or malformed + file fails startup, because the operator believes those rates are live. The + conventional probe is optional — an absent file just serves compiled-in + defaults, and the path stays watched in case it appears later. +- File entries **replace** the compiled-in entry for the same (surface, model) + **whole** — they are not field-merged, so an entry must repeat the cache rates + it wants to keep. Everything the file doesn't mention keeps built-in rates. +- Unknown YAML fields are rejected (`KnownFields(true)`) and every rate must be + finite and non-negative — the same constraints the HTTP API enforces on + operator per-provider prices. +- Reload is an mtime poll (`ReloadInterval`, 1 min) and is **lenient at runtime**: + a parse error keeps the previous table, a deleted file reverts to compiled-in + defaults. A mid-edit save can never take pricing down. + +The live table feeds **both** consumers, which is what keeps them consistent: the +synthesizer (what proxies actually bill with) and `GET /api/agent-network/catalog` +via `applyDefaultPricing` (what the dashboard's model-row prices prefill with). +`defaults_llm_pricing.example.yaml` is generated from the compiled-in table +(`go generate ./management/internals/modules/agentnetwork/pricing`) and +golden-tested, so operators start from a file matching the built-in rates exactly. + ### Budget rule resolution (min-wins, group+user bound) ```mermaid @@ -124,7 +208,7 @@ At request time the path is independent: the proxy calls `SelectPolicyForRequest | on_request | 3 | `llm_identity_inject` | `{"providers":[{provider_id, header_pair?, json_metadata?, extra_headers?}]}` | **true** | | on_request | 4 | `llm_guardrail` | `{"provider_allowlists"?: {providerID: []model}, "prompt_capture":{enabled,redact_pii}}` | – | | on_response | 5 | `llm_limit_record` | `{}` (runs LAST at runtime) | – | - | on_response | 6 | `cost_meter` | `{}` | – | + | on_response | 6 | `cost_meter` | `{"pricing":{"defaults":{surface:{model:rates}},"providers"?:{providerRecordID:{model:rates}}}}` — rates are `{input_per_1k, output_per_1k, cached_input_per_1k?, cache_read_per_1k?, cache_creation_per_1k?}` | – | | on_response | 7 | `llm_response_parser` | `{"capture_completion": , "redact_pii"?: true}` | – | - **Synthesized service shape** (`synthesizer.go:739`): `Mode=HTTP`, `Private=true`, `Domain=.`, `AccessGroups=unionSourceGroups(enabledPolicies)`, one `TargetTypeCluster` target with `Host=noop.invalid:443` (router rewrites per request), `Options.{DirectUpstream,AgentNetwork}=true`, `DisableAccessLog=!settings.EnableLogCollection`, `CaptureMax{Req,Resp}Bytes=1<<20`, `CaptureContentTypes=["application/json","text/event-stream"]`. @@ -139,6 +223,12 @@ At request time the path is independent: the proxy calls `SelectPolicyForRequest - **Orphan providers (no enabled policy authorises them) NEVER reach the router** (`synthesizer.go:351-357`); skipped from `identity_inject` for symmetry. - **Provider creation refuses empty `api_key`** (`manager.go:175`); **deletion refuses while any policy still references it** (`manager.go:265-273`). - **Session keypair stability across provider edits** (`manager.go:226-228`) — server-managed, copied through every `UpdateProvider`, never API-surfaced. +- **Management is the sole pricing authority.** The proxy has no embedded price list, so an account whose `cost_meter` config carries no `pricing` block bills **nothing** (`cost.skipped=unknown_model`, $0) rather than falling back to stale built-ins. The top-level `pricing` wrapper is also the feature-detection signal in both directions: an old proxy ignores it as an unknown field, and a new proxy reads its absence as "old management". +- **Per-provider prices are materialized at synth time, not merged on the proxy** (`synthesizer_pricing.go:114-131`). A per-record entry is always complete, so the proxy's lookup is per-record-then-defaults with no field-level fallback between tiers. +- **An explicit operator price of `0` prices the model as free** — it must not be treated as "unset" and reverted to list price (`synthesizer_pricing.go:49-54`). Only *cache*-rate fields distinguish unset from zero, via `*float64`. +- **Pricing model ids are normalized with the same functions the request parser uses** (`normalizePricingModelID`). If the two ever diverge, per-record prices silently stop matching and every request falls through to surface defaults. +- **The default table's coverage is structural, not curated.** It is derived from the catalog via each provider's `PricingSurfaces`; `TestDefaultTable_CoversEveryCatalogModel` fails on an unpriced catalog model and `TestDefaultTable_NoConflictingContributions` fails if two providers contribute the same (surface, model) at different rates. +- **A pricing-defaults file failure is fatal only at startup, and only for an explicitly configured path.** Runtime reload failures keep the previous table; a deleted file reverts to compiled-in defaults (`pricing/override.go:62-81, 113-148`). ## Things to scrutinize @@ -176,10 +266,12 @@ At request time the path is independent: the proxy calls `SelectPolicyForRequest - **Capture-pointer semantics (restated):** non-agent-network callers see no field → legacy nil-default emit, identical to pre-PR. Agent-network targets always carry an explicit `capture_*` value. - **`TestSynthesizeServices_HappyPath` was updated:** request-parser config moved from `{}` to `{"capture_prompt":false}` (`synthesizer_test.go:174`). External snapshot tests against synth output need updating. - **`MergedGuardrails` retains zeroed `TokenLimits`/`Budget`/`Retention`** even though `Policy.Limits` carries the real values now; `llm_limit_check` is the authoritative enforcement. Comment at `synthesizer.go:940-948` calls this out. +- **`cost_meter`'s `pricing` block is version-skew-safe in both directions.** A proxy predating config-delivered pricing ignores the field as unknown JSON (it previously priced from its own embedded table, so it keeps billing — at its own rates, which is the skew to be aware of during a rolling upgrade). A current proxy paired with old management sees no `pricing` block, logs one warning at chain-build time, and records `cost.skipped=unknown_model` — token counting and cap enforcement are unaffected, only the USD annotation goes to $0. ### Performance - **`SynthesizeServices` runs on every controller tick / mutation reconcile.** Cost: 4 store reads + optional per-provider keypair backfill. Sort + index + merge are O(N log N) / O(P × G); dominant cost is JSON marshalling. No nested loops escape these dimensions. +- **The full default pricing table is marshalled into every account's `cost_meter` config on every synth** (~10 KB serialized). This is a deliberate trade: it keeps gateway-style providers priced for every catalog model, and it is the largest single contributor to the synth JSON. `DefaultTable()` itself is a pointer load (or a `sync.Once`-built map) — the cost is the marshal, not the build. - **`reconcile.diffMappings` is O(N + M)** with N=M=1 per account today — effectively constant. - **`SynthesizeServicesForCluster`** (`synthesizer.go:71`) walks every account on a cluster; per-account failures are **swallowed** (`synthesizer.go:91-93`) so a single misconfigured account doesn't drop the cluster. Runs per proxy reconnect. @@ -188,6 +280,7 @@ At request time the path is independent: the proxy calls `SelectPolicyForRequest - **Activity codes:** `AgentNetwork{Provider,Policy,Guardrail,BudgetRule}{Created,Updated,Deleted}`; `AgentNetworkSettingsUpdated` with `log_collection/prompt_collection/redact_pii` payload (`manager.go:567-571`). **No activity code for `SelectPolicyForRequest` denies** — surfaced via proxy access log only (likely intentional given volume). - **Deny codes** namespaced: `llm_policy.{token,budget}_cap_exceeded`, `llm_account.{token,budget}_cap_exceeded` (`policyselect.go:18-26`). - **Reconcile failures are logged at warn and swallowed** (`reconcile.go:42-44`). Persistent synth failures (e.g. unknown catalog id) silently keep the proxy out of sync — consider a manager-level synth-health surface if this becomes a support burden. +- **Pricing-file lifecycle logs at info** (load, reload, revert-to-built-ins) and **at warn** for a runtime reload failure; the mtime check itself is `Debugf`. There is no metric on reload failures, so an operator who breaks the file mid-flight keeps billing at the previous table with only a log line to show it (`pricing/override.go:113-148`). ## Test coverage @@ -198,6 +291,9 @@ At request time the path is independent: the proxy calls `SelectPolicyForRequest | `synthesizer_guardrail_realstore_test.go` | `PromptCaptureAccountIsSoleControl`; `PromptCaptureFlowsWhenAccountOptsIn`; `AccountRedactWithoutGuardrailRedact`; `NoGuardrail_CaptureOff`. | | `synthesizer_log_collection_realstore_test.go` | `LogCollection{Off_SuppressesAccessLog,On_PermitsAccessLog}` — verifies `DisableAccessLog` propagation through `ToProtoMapping`. | | `synthesizer_parser_redact_realstore_test.go` | **Capture-pointer regression suite:** `ParserConfigsCarryRedactPii`; `ParserConfigsSuppressCaptureWhenLogCollectionOnly` (log=on/prompt=off ⇒ both capture flags false); `ParserConfigsOmitRedactPiiWhenOff`. | +| `synthesizer_pricing_test.go` | `BuildCostMeterConfig_{BedrockModelNormalization,CacheRateNilVsZero,OrphanAndGatewayProviders}` — the per-record tier's three load-bearing rules: keys normalized like the parser's, `nil` cache pointer inherits vs explicit `0` bills at input rate, and orphan / gateway (empty `Models`) providers ship no per-record entry. | +| `pricing/defaults_test.go` | `DefaultTable_{CoversEveryCatalogModel,NoConflictingContributions,AllRatesFiniteNonNegative,PinnedRates}`; `LookupDefault_SurfaceOrder`. Catalog-derived coverage + rate sanity are structural, not curated. | +| `pricing/override_test.go` | `LoadFile_{MergesOverCompiledDefaults,MissingPath,RejectsInvalid}`; `Reload_LifeCycle` (mtime detect, parse error keeps previous, delete reverts to built-ins); `ExampleYAML_InSyncWithBuiltins` golden. | | `policyselect_test.go` | Mock-store: `NoApplicablePolicies`; `AllowWithLowestGroupAttribution`; `LargerPoolWinsAcrossUsageLevels`; `StaysOnLargerPoolAfterPartialDrain`; `FallsThroughToSmallerPoolWhenLargerExhausted`; `TiebreakBy{LargerGroupPool,CreatedAt}`; `DeniesWhenAllExhausted`; `UncappedPolicyAlwaysWinsAgainstCapped`; `DisabledPolicyIgnored`; `StoreErrorPropagates`; `RejectsEmptyAccount`; `SharesGroupCounterAcrossPolicies`; `AntiFallThroughOnLowestGroup`; `BudgetOnlyExhaustionDenies`; `BudgetTighterThanTokenWins`. | | `policyselect_realstore_test.go` | Real-sqlite regression guard: `NoApplicablePolicies`; `AllowAndLowestGroupAttribution`; `LargerPoolWins_FallsThroughWhenExhausted`; `BudgetCapDenies`; `GroupCounterSharedAcrossPolicies`; `DisabledPolicyIgnored`. | | `policyselect_account_realstore_test.go` | Account budget rules: `AccountCeilingBindsEvenWithUncappedPolicy` (min-wins); `AccountGroupCeiling`; `AccountTargetUsersBindsOnlyThatUser`; `AccountRuleRecordsToOwnWindow`. | diff --git a/docs/agent-networks/modules/31-proxy-middleware-builtin.md b/docs/agent-networks/modules/31-proxy-middleware-builtin.md index efe1bc4ce..ad56feb77 100644 --- a/docs/agent-networks/modules/31-proxy-middleware-builtin.md +++ b/docs/agent-networks/modules/31-proxy-middleware-builtin.md @@ -5,7 +5,7 @@ LLM request. The two highest-blast-radius areas are the **capture-pointer semantics** and the **limit_check ⇒ limit_record** record-once invariant. Sibling module: [32-proxy-llm-parsers.md](./32-proxy-llm-parsers.md) — the SDK -adapters + pricing catalog this chain delegates to. +adapters + pricing table and cost formula this chain delegates to. --- @@ -34,7 +34,7 @@ rewrites. | `llm_identity_inject` | OnRequest | `llm.{resolved_provider_id,authorising_groups}`, `Input.{UserEmail,UserID,UserGroups,UserGroupNames}` | none | header strip/inject + optional body rewrite | | `llm_guardrail` | OnRequest | `llm.{model,request_prompt_raw}` | `llm_policy.{decision,reason}`, `llm.request_prompt` | none (model allowlist deny) | | `llm_response_parser` | OnResponse | `llm.provider`, `Input.{RespHeaders,RespBody,Status}` | `llm.{input,output,total,cached_input,cache_creation}_tokens`, `llm.response_completion` | none | -| `cost_meter` | OnResponse | `llm.{provider,model}`, token buckets | `cost.usd_total` or `cost.skipped` | pricing lookup | +| `cost_meter` | OnResponse | `llm.{provider,model,resolved_provider_id}`, token buckets | `cost.usd_{input,cached_input,cache_creation,output,total,cache}` or `cost.skipped` | none (in-memory pricing lookup) | | `llm_limit_record` | OnResponse | `llm.{attribution_group_id,attribution_window_seconds,input_tokens,output_tokens}`, `cost.usd_total` | none | gRPC `RecordLLMUsage` | [all_test.go:26–40](../../../proxy/internal/middleware/builtin/all_test.go) @@ -44,7 +44,7 @@ locks the ID set; adding or removing one is a conscious extension. | File | LOC | Notes | |---|---:|---| -| `builtin.go` | 86 | Registry + `FactoryContext` (ctx, data dir, meter, logger, mgmt client) | +| `builtin.go` | 90 | Registry + `FactoryContext` (ctx, meter, logger, mgmt client) | | `all_test.go` | 41 | Locks the 8-ID registry surface | | `agentnetwork_chain_integration_test.go` | 319 | Live sqlite + real gRPC bufconn; gate→recorder wire path | | `llm_request_parser/*` | 162 / 66 / 356 | Provider detection, body parse, prompt extraction with capture-pointer gating | @@ -53,7 +53,7 @@ locks the ID set; adding or removing one is a conscious extension. | `llm_identity_inject/*` | 440 / 108 / 666 | HeaderPair (LiteLLM) + JSONMetadata (Portkey) + ExtraHeaders | | `llm_guardrail/*` | 176 / 82 / 75 / 219 / 217 | Model allowlist + optional prompt capture with PII redaction | | `llm_response_parser/*` | 258 / 222 / 43 / 433 / 169 / 111 | Buffered + SSE accumulation; AWS event-stream accumulator (`streaming_bedrock.go`) for Bedrock; capture-pointer gates completion emit | -| `cost_meter/*` | 181 / 84 / 439 | Token → USD via `proxy/internal/llm/pricing` | +| `cost_meter/*` | 236 / 98 / 586 | Token → USD via `proxy/internal/llm/pricing`; both pricing tiers arrive in the middleware config | | `llm_limit_record/*` | 144 / 35 / 191 | Post-flight `RecordLLMUsage` (5s, debug-on-error) | ## Per-middleware @@ -168,12 +168,46 @@ token schema. ### cost_meter -Reads `llm.provider` + `llm.model` + token buckets, looks up per-1k rate via -`pricing.Loader`, emits `cost.usd_total` or a closed-set `cost.skipped` -reason (`missing_provider/model/tokens`, `unparseable_tokens`, `zero_tokens`, -`unknown_model`). Loader's hot-reload goroutine is bound to proxy-lifetime -context via `startReloader`. **Key invariant:** provider-shape switch lives -in `pricing.Table.Cost` (sibling doc) — `cost_meter` stays provider-agnostic. +Reads `llm.provider` + `llm.model` + token buckets, looks up the per-1k rates, +and emits the full `cost.usd_*` breakdown (four per-bucket values plus the +`_total` and `_cache` aggregates) or a closed-set `cost.skipped` reason +(`missing_provider/model/tokens`, `unparseable_tokens`, `zero_tokens`, +`unknown_model`). + +**Management owns pricing.** The proxy carries no embedded price list: the whole +table arrives in this middleware's `ConfigJSON` as +`{pricing: {defaults, providers}}`, synthesized by management from the catalog +plus the operator's stored per-provider prices +([factory.go:13–34](../../../proxy/internal/middleware/builtin/cost_meter/factory.go)). +Both tiers are validated by `pricing.NewTable` / `pricing.NewEntries` at +construction, so a non-finite or negative rate fails the chain build. A price +change is an ordinary mapping push — the chain rebuild yields a fresh instance +over a fresh immutable table, so there is no data dir, no pricing file, no +reload goroutine, and nothing to invalidate. + +**Two-tier lookup** +([middleware.go:165–183](../../../proxy/internal/middleware/builtin/cost_meter/middleware.go)): + +1. **Per-provider-record** — the operator's stored price for the route that + actually served the request, keyed by the `llm.resolved_provider_id` that + `llm_router` stamped on the allow path, then by normalized model id. Entries + arrive fully materialized (management folds default cache rates in at synth + time), so there is no merging here. Absent metadata — no router in the chain + — skips this tier. +2. **Surface defaults** — the catalog-derived table keyed by `llm.provider` + (`openai`/`anthropic`/`bedrock`). This is also what prices gateway-style + providers, which enumerate no models and therefore get no per-record entry. + +**Backward compatibility:** a config with no `pricing` block means management +predates config-delivered pricing. The factory logs one warning at build time +and the instance records `cost.skipped=unknown_model` ($0) for every request +rather than falling back to a stale built-in price list +([factory.go:55–60](../../../proxy/internal/middleware/builtin/cost_meter/factory.go)). + +**Key invariant:** the provider-shape switch lives in `pricing.EntryCosts` +(sibling doc) and is selected by the **surface**, not by which tier the entry +came from — `cost_meter` stays provider-agnostic, and a per-record override on +an Anthropic route still bills its cache buckets additively. ### llm_limit_record @@ -246,12 +280,14 @@ no mocks. Tests: `TestChain_AllowPath_StampsAttributionAndRecordsCounter` | `llm_identity_inject` | `{providers: [{provider_id, header_pair?|json_metadata?, extra_headers?}]}` | | `llm_guardrail` | `{provider_allowlists: {providerID: []string}, prompt_capture: {enabled, redact_pii}}` — allowlist keyed by resolved provider id; a provider absent from the map is unrestricted (fail-closed backstop; authoritative per-policy/group check is management's `CheckLLMPolicyLimits`) | | `llm_response_parser` | `{redact_pii?, capture_completion?: *bool}` | -| `cost_meter` | `{pricing_path?}` (basename inside data-dir; defaults `pricing.yaml`) | +| `cost_meter` | `{pricing: {defaults: {surface: {model: rates}}, providers: {providerRecordID: {model: rates}}}}` — rates are `{input_per_1k, output_per_1k, cached_input_per_1k?, cache_read_per_1k?, cache_creation_per_1k?}`. A missing `pricing` key means "management predates config-delivered pricing": every request records `cost.skipped=unknown_model` | | `llm_limit_record` | `{}` — same pattern as `llm_limit_check` | All factories accept empty / null / `{}` / whitespace as zero-value config; only structurally invalid JSON is rejected so misconfig surfaces at chain -build time. +build time. `cost_meter` adds a semantic check on top of that: a `pricing` +block carrying a negative or non-finite rate fails the build too, rather than +mispricing live traffic. ## Invariants @@ -320,10 +356,11 @@ non-object `metadata` field — header path still attributes, but body-level tag-budget enforcement doesn't run for that request. -**Concurrency.** `cost_meter` shares a `pricing.Loader` via -`atomic.Pointer[Table]`; readers always see a consistent table. Every -middleware is a stateless value receiver. Integration test uses real bufconn -gRPC — race detector is the meaningful bar. +**Concurrency.** `cost_meter`'s two pricing tables are built once from the +middleware config and never mutated, so the lookup path needs no lock or atomic +swap — a price change replaces the whole instance. Every middleware is +otherwise a stateless value receiver. Integration test uses real bufconn gRPC — +race detector is the meaningful bar. **Perf.** Hot path is `lookupKV` linear scan over <10 KVs; `cost_meter.Cost` is O(1); SSE accumulation is single-pass. No map allocation per call. @@ -349,13 +386,13 @@ counter accuracy. | `llm_guardrail/redact_test.go` | 15 | Email, SSN, phone (E.164 + NA), bearer, IPv4; fixture-driven | | `llm_response_parser/middleware_test.go` | 18 | Buffered OAI+Anthro, capture-pointer, redact, truncation | | `llm_response_parser/streaming_test.go` | 7 | OAI usage frame, Anthro message_delta, truncated body best-effort | -| `cost_meter/middleware_test.go` | 17 | Each skip reason, provider-shape, pricing loader integration | +| `cost_meter/middleware_test.go` | 22 | Each skip reason, provider-shape formulas, config-delivered defaults, per-record-beats-defaults + miss-falls-back, per-record uses surface formula, nil-pricing skips everything, invalid-rate rejection | | `llm_limit_record/middleware_test.go` | 7 | Skip-on-no-signal, skip-on-missing-attribution, RPC failure swallowed | ## Cross-references - Sibling: [32-proxy-llm-parsers.md](./32-proxy-llm-parsers.md) — SDK adapters - + SSE framer + pricing loader. + + SSE framer + pricing table and cost formula. - Path-routed providers (Vertex AI + Bedrock), `keyfile::` credential, GCP token minting, `/bedrock` prefix: [50-path-routed-providers.md](./50-path-routed-providers.md). diff --git a/docs/agent-networks/modules/32-proxy-llm-parsers.md b/docs/agent-networks/modules/32-proxy-llm-parsers.md index 0376bc988..52faeaac1 100644 --- a/docs/agent-networks/modules/32-proxy-llm-parsers.md +++ b/docs/agent-networks/modules/32-proxy-llm-parsers.md @@ -9,7 +9,7 @@ pricing table's per-provider cost formula is the highest-leverage place a small bug would silently mis-bill operators. Sibling module: [31-proxy-middleware-builtin.md](./31-proxy-middleware-builtin.md) -— the 8 middlewares that consume this package's parsers + pricing loader. +— the 8 middlewares that consume this package's parsers + pricing table. --- @@ -24,8 +24,9 @@ proxy-framework dependencies: - `openai.go` / `anthropic.go` / `bedrock.go` — per-provider `Parser` impls. - `sse.go` — SSE scanner (`Scanner`, `Event`, `NewScanner`). - `errors.go` — sentinels callers branch on with `errors.Is`. -- `pricing/` — embedded-default + hot-reload override table with - symlink-safe Unix loader (build-tagged stub elsewhere). +- `pricing/` — immutable pricing table + the per-surface cost formula. The + rates themselves come from management inside `cost_meter`'s middleware + config; this package holds no price list and reads no files. - `fixtures/` — captured request/response/stream bodies the tests replay. The package carries zero proxy-framework dependencies so the same parsers can @@ -47,12 +48,9 @@ be reused later by a WASM adapter | `sse_test.go` | 175 | 12 tests; fixture replay + multiline + size limits | | `parser_test.go` | 53 | `Parsers()`, `DetectParser`, provider enum values | | `errors.go` | 31 | 6 sentinels: `Err{Unknown,Unsupported}Provider/Model`, `Err{NotLLM,Malformed}Response`, `ErrStreamingUnsupported`, `ErrMalformedRequest` | -| `pricing/pricing.go` | 421 | `Loader`, `Table`, `Entry`; embedded defaults + atomic swap + mtime reload | -| `pricing/pricing_unix.go` | 69 | `O_NOFOLLOW` + fstat-from-FD + 1 MiB cap | -| `pricing/pricing_other.go` | 21 | Stub returning "not supported on this platform" | -| `pricing/pricing_test.go` | 432 | 21 tests — symlink rejection, reload race, path traversal, oversize | -| `pricing/defaults_pricing.yaml` | 85 | go:embed source of truth | -| `fixtures/*` | 21–59 | OAI chat/responses/stream + Anthro messages/stream + pricing starter | +| `pricing/pricing.go` | 234 | `Table`, `Entry`, `EntryJSON`, `Costs`; `NewTable`/`NewEntries` validation + `EntryCosts` formula. No I/O, no reload, no embedded rates | +| `pricing/pricing_test.go` | 177 | 10 tests — provider-shape formulas, cached clamp, rate fallback, nil-safety, rate validation | +| `fixtures/*` | 21–59 | OAI chat/responses/stream + Anthro messages/stream | ## Request body → parser dispatch @@ -188,9 +186,11 @@ response leg, covering both Bedrock body shapes: `totalTokens`). `firstNonZero` folds the two naming conventions into one `Usage`; when Converse omits `totalTokens` the parser sums the buckets. -`ProviderName()` returns `"bedrock"` — its own `defaults_pricing.yaml` block, -keyed by the **normalised** model id (region prefix + version suffix stripped by -the request parser). `ParseResponse` returns `ErrStreamingUnsupported` for an +`ProviderName()` returns `"bedrock"` — its own pricing surface in the table +management ships, keyed by the **normalised** model id (region prefix + version +suffix stripped by the request parser; management normalises its keys the same +way at synth time so the two compare equal). `ParseResponse` returns +`ErrStreamingUnsupported` for an AWS binary event-stream content-type (`application/vnd.amazon.eventstream`, `isAWSEventStream`) so the caller routes to the streaming accumulator instead. @@ -205,11 +205,34 @@ response body. Streaming accumulators live in the middleware package ([llm_response_parser/streaming.go](../../../proxy/internal/middleware/builtin/llm_response_parser/streaming.go)) but use `llm.NewScanner` so the framing contract stays here. -### Pricing catalog +### Pricing table -`Table.Cost` -([pricing.go:129–174](../../../proxy/internal/llm/pricing/pricing.go)) -is the cost formula — most security-relevant math in this module: +**Management is the sole pricing authority.** The proxy carries no embedded +price list and reads no pricing file: the whole table arrives inside +`cost_meter`'s `ConfigJSON` on the ordinary mapping push, and a price change +is just another push — the chain rebuild constructs a fresh `Table`, so there +is nothing to reload +([pricing.go:1–7](../../../proxy/internal/llm/pricing/pricing.go)). The +management side of the contract (catalog defaults, the operator's stored +per-provider prices, and `AgentNetwork.PricingDefaultsFile`) is covered in the +management-side module guide; `cost_meter`'s wire shape is in +[31-proxy-middleware-builtin.md](./31-proxy-middleware-builtin.md). + +`EntryJSON` +([pricing.go:36–45](../../../proxy/internal/llm/pricing/pricing.go)) is the +management→proxy contract — five USD-per-1k rates under `input_per_1k`, +`output_per_1k`, `cached_input_per_1k`, `cache_read_per_1k`, +`cache_creation_per_1k`. Management's `pricing.Entry` marshals the identical +names, and `EntryJSON`/`Entry` are field-identical so `NewEntries` converts by +direct struct conversion rather than field-by-field copying (a new rate can't +be silently dropped in transit). + +`EntryCosts` +([pricing.go:183–234](../../../proxy/internal/llm/pricing/pricing.go)) +is the cost formula — most security-relevant math in this module. The +**surface** (the `llm.provider` value the request parser stamped) selects the +formula, never the tier the entry came from: a per-provider-record override on +an Anthropic route still bills its cache buckets additively. | Provider | Formula | |---|---| @@ -218,7 +241,7 @@ is the cost formula — most security-relevant math in this module: | default | `inTokens × InputPer1K + outTokens × OutputPer1K` | `bedrock` shares the Anthropic additive-cache formula -([pricing.go:172-174](../../../proxy/internal/llm/pricing/pricing.go)): +([pricing.go:214–229](../../../proxy/internal/llm/pricing/pricing.go)): Anthropic-on-Bedrock reports the same additive cache buckets, while non-Anthropic Bedrock models (Nova, Llama) simply report zero in those buckets so cost reduces to `input + output`. @@ -226,15 +249,12 @@ to `input + output`. Each per-bucket rate falls back to `InputPer1K` when zero — operators opt in to discounts by setting the field. -`Loader` -([pricing.go:212–268](../../../proxy/internal/llm/pricing/pricing.go)) -overlays an optional `pricing.yaml` from data-dir on top of the go:embed -defaults. Atomic pointer swap means readers never observe a partial update. -The mtime-poll reloader (30s default cadence) keeps the previous table on -parse failure so cost annotation never goes blank during a botched edit. - -`defaults_pricing.yaml` is the source of truth for built-in pricing. -Operator overrides only carry the entries they want to change. +`Costs` +([pricing.go:143–163](../../../proxy/internal/llm/pricing/pricing.go)) is the +per-request split. The four per-bucket fields are the base; `TotalUSD` and +`CacheUSD` are **derived** in `newCosts` so the aggregates can never drift from +the breakdown. `InputUSD` is always the non-cached input bucket on both +provider shapes, so input and cached-input never double-count. ## Public contracts @@ -264,29 +284,38 @@ Order matters: `DetectFromURL` ties resolve by registration order. `ProviderBedrock = 3`. Numeric values are persisted in nothing today but treat them as wire-stable — new providers must take fresh numbers. -**`Pricing` lookup** -([pricing.go:129](../../../proxy/internal/llm/pricing/pricing.go)): +**`Pricing` construction + lookup** +([pricing.go:60–130](../../../proxy/internal/llm/pricing/pricing.go)): ```go +func NewEntries(raw map[string]map[string]EntryJSON) (map[string]map[string]Entry, error) +func NewTable(raw map[string]map[string]EntryJSON) (*Table, error) + +func (t *Table) Lookup(provider, model string) (Entry, bool) func (t *Table) Cost(provider, model string, inTokens, outTokens, cachedInput, cacheCreation int64) (float64, bool) +func (t *Table) Costs(provider, model string, inTokens, outTokens, cachedInput, cacheCreation int64) (Costs, bool) +func EntryCosts(entry Entry, surface string, inTokens, outTokens, cachedInput, cacheCreation int64) Costs ``` -Nil-safe: `t.Cost` on a nil receiver returns `(0, false)` -([pricing.go:130–132](../../../proxy/internal/llm/pricing/pricing.go)). -`ok=false` means provider or model is absent from the loaded table; the caller -emits `cost.skipped=unknown_model`. +`NewTable` is the surface-keyed defaults table; `NewEntries` returns the raw +two-level map `cost_meter` uses for the per-provider-record tier (it looks up an +`Entry` directly and calls `EntryCosts`, so it needs no `Table` wrapper). Both +reject any non-finite or negative rate, so a corrupt config fails the chain +build rather than mispricing silently. Nil input yields an empty, +never-matching table. + +Nil-safe: `t.Cost`/`t.Lookup` on a nil receiver returns `ok=false` +([pricing.go:96–99](../../../proxy/internal/llm/pricing/pricing.go)). +`ok=false` means the surface or model is absent from the table management sent; +the caller emits `cost.skipped=unknown_model`. ## Invariants -1. **Cross-platform pricing build.** `pricing_unix.go` carries the only - functional `loadPricing` (uses `syscall.O_NOFOLLOW` and `f.Stat()` on an - open descriptor — both Unix-only). `pricing_other.go` is a build-tag - fallback that returns `"not supported on this platform"` - ([pricing_other.go:14–16](../../../proxy/internal/llm/pricing/pricing_other.go)). - The proxy is Linux-only in production today; a Windows port needs an - equivalent path-as-handle implementation. Reviewers building on Windows - should expect this surface to return an error at startup if an override - file is configured. +1. **The pricing package is pure and platform-independent.** No file I/O, no + `//go:embed`, no goroutines, no build tags — the rates arrive as config, so + there is nothing platform-specific left to port. Anything reintroducing a + read-from-disk path here re-splits pricing authority between management and + the proxy, which is exactly what this design removed. 2. **SSE scanner handles partial chunks.** A buffered prefix that doesn't end in `\n\n` still yields its accumulated event before `io.EOF` @@ -298,38 +327,45 @@ emits `cost.skipped=unknown_model`. usage rather than aborting ([streaming.go:68–73, 144–150](../../../proxy/internal/middleware/builtin/llm_response_parser/streaming.go)). -3. **`defaults_pricing.yaml` is the source of truth.** Compiled into the - binary via `//go:embed` - ([pricing.go:29–30](../../../proxy/internal/llm/pricing/pricing.go)). - `DefaultTable()` parses once and panics on parse failure - ([pricing.go:42–49](../../../proxy/internal/llm/pricing/pricing.go)) - — by design: a broken embedded YAML must not ship to production. +3. **Management is the only source of rates.** `Table` has no constructor that + invents prices: the only way in is `NewTable`/`NewEntries` over the wire map + management sent. A missing or empty `pricing` block therefore means *no + prices at all* (`cost_meter` records `cost.skipped=unknown_model`, $0) — + never a stale built-in fallback that would silently bill list price. -4. **Loader path validation.** `resolveMiddlewareDataPath` - ([pricing.go:370–394](../../../proxy/internal/llm/pricing/pricing.go)) - rejects absolute paths, traversal segments, and basenames that fail - `basenameRegex = ^[a-zA-Z0-9._-]+$`. The resolved path must remain - inside `baseDir` even after `filepath.Clean`. Tests: - `TestNewLoader_PathValidation`, `TestNewLoader_PathValidation_Extended`, - `TestNewLoader_SymlinkOutsideBaseDirRejected`, `TestNewLoader_SymlinkRejected`. +4. **Tables are immutable once built.** `Table.entries` is written only in + `NewEntries` and never mutated afterwards, and `cost_meter`'s `perRecord` + map is likewise build-time-only + ([pricing.go:47–52](../../../proxy/internal/llm/pricing/pricing.go)). This + is what makes the no-reload design safe: a price change arrives as a mapping + push that builds a new middleware instance over a new table, so concurrent + readers can't observe a half-updated price list and no atomic swap or lock + is needed on the hot path. -5. **Unix loader symlink safety.** `O_NOFOLLOW` on open, `f.Stat()` on the - open descriptor (never re-stat by path), `info.Mode().IsRegular()` check, - `io.LimitReader(f, maxPricingBytes+1)` with a final size assertion - ([pricing_unix.go:25–57](../../../proxy/internal/llm/pricing/pricing_unix.go)). - A mid-read symlink swap is detected because the fstat is on the original - fd. Test: `TestNewLoader_RejectsOversizedFile_FixesM4`. +5. **Rate validation happens at chain-build time, not per request.** + `NewEntries` rejects negative, NaN, and ±Inf rates field by field + ([pricing.go:60–83](../../../proxy/internal/llm/pricing/pricing.go)), naming + the offending surface/model/field in the error. Management enforces the same + constraints at its API boundary and in its YAML parser, so this is + defense-in-depth — but it means a corrupt push fails loudly at build instead + of producing negative costs on live traffic. Test: + `TestNewTable_ValidatesRates`. -6. **`yaml.NewDecoder(...).KnownFields(true)`** - ([pricing.go:397–398](../../../proxy/internal/llm/pricing/pricing.go)) - rejects YAML files that carry fields not in the schema. A typo in an - operator override file fails loud instead of silently zeroing rates. +6. **New rates must be added to `Entry`, `EntryJSON`, *and* management's + `pricing.Entry` together.** `NewEntries` converts by direct struct + conversion `Entry(e)` + ([pricing.go:76–78](../../../proxy/internal/llm/pricing/pricing.go)), which + only compiles while the two structs stay field-identical — so the proxy half + is compiler-enforced. The management half is not: a rate added there but not + here unmarshals into nothing and prices that bucket at `InputPer1K`. ## Things to scrutinise -**Correctness.** Verify OpenAI cached-prompt clamp at -[pricing.go:147–149](../../../proxy/internal/llm/pricing/pricing.go) -short-circuits before subtraction. `Anthropic.TotalTokens` sums all four +**Correctness.** Verify the OpenAI cached-prompt clamp at +[pricing.go:203–206](../../../proxy/internal/llm/pricing/pricing.go) +short-circuits before subtraction. Negative token counts are clamped to zero up +front ([pricing.go:186–197](../../../proxy/internal/llm/pricing/pricing.go)) so +no formula can yield a negative cost. `Anthropic.TotalTokens` sums all four buckets (in + out + cache_read + cache_creation) — downstream dashboards need to know this differs from `input + output`. `OpenAIParser.ExtractPrompt` falls through `messages → input → prompt`; a @@ -338,22 +374,27 @@ noting). **Security.** `Scanner.maxLine = 1 MiB`; a 2 MiB single-line `data:` event errors from `Scanner.Next` and both accumulators stop with partial usage. -Pricing file 1 MiB cap is orders of magnitude larger than realistic. Confirm -new schema additions are mirrored in both `pricingFile` and `Entry`; -`KnownFields(true)` will reject silently-typo'd operator overrides -otherwise. +Pricing is no longer file-backed, so the loader's path-traversal / symlink / +oversize surface is gone entirely — the config channel (an authenticated +mapping push from management) is now the only way rates enter the proxy, and +`NewEntries` is the validation boundary on it. A new rate added to management's +`pricing.Entry` but not to `EntryJSON` here is the remaining silent-mispricing +path (see invariant 6). -**Concurrency.** `Loader.table` is `atomic.Pointer[Table]`; readers never -block or see a torn table. `Loader.Reload` is one goroutine, cancelled via -context (`TestLoader_ReloadBackgroundLoopCancellation`). `DefaultTable()` -uses `sync.Once`. Per-call `Scanner` instances mean no shared state across -concurrent response-parser calls. +**Concurrency.** Nothing in this package is shared mutable state: tables are +built once and never written again, so `cost_meter`'s hot path is lock-free by +construction rather than by atomic swap. Per-call `Scanner` instances mean no +shared state across concurrent response-parser calls. -**Perf.** `Table.Cost` is two map lookups + multiplications, O(1). -`Scanner.Next` is one `ReadString('\n')` per line. Pricing reload poll 30s. +**Perf.** `Table.Cost` is two map lookups + multiplications, O(1); the +per-provider-record tier adds at most one more lookup. `Scanner.Next` is one +`ReadString('\n')` per line. No background goroutines and no per-request +allocation of pricing state. -**Observability.** Reload failures count via `metric.Int64Counter` keyed -`plugin`; warning log rate-limited at 5 min so a broken file doesn't flood. +**Observability.** A config carrying no `pricing` block logs one warning at +chain-build time (`cost_meter` factory) and then records +`cost.skipped=unknown_model` per request, so an old-management deployment is +visible in both logs and the access log rather than quietly reporting $0. Parser errors return sentinels — middleware uses `errors.Is` to map to the right `cost.skipped` reason. @@ -365,7 +406,7 @@ right `cost.skipped` reason. | `openai_test.go` | 11 | Chat Completions + Responses API + legacy `prompt`; cached-tokens subset for both naming conventions; fixture replays | | `anthropic_test.go` | 7 | Messages + legacy `/v1/complete`; streaming REJECTED on `ParseResponse` (must use scanner); fixture replays | | `sse_test.go` | 12 | Fixture replay both providers; multiline `data:`; CRLF; comment skip; trailing-event-without-blank-line; oversize rejection | -| `pricing/pricing_test.go` | 21 | Provider-shape switch; cached-rate fallback; cached-clamp; symlink rejection (target outside basedir + symlink to file); path validation matrix; oversize rejection; reload-keeps-previous-on-parse-error; mtime change detection; goroutine cancellation | +| `pricing/pricing_test.go` | 10 | Provider-shape switch (surface selects the formula); cached-rate + cache-read/creation fallback to `InputPer1K`; cached-clamp; negative-token clamp; nil-receiver safety; rate validation (negative / NaN / Inf rejected); nil + empty table | **Fixtures** ([proxy/internal/llm/fixtures/](../../../proxy/internal/llm/fixtures/)): `openai_chat_completion.json` (chat.completions with usage), @@ -373,14 +414,15 @@ right `cost.skipped` reason. `openai_stream.txt` (3 deltas + usage + `[DONE]`), `anthropic_messages.json` (Messages API non-streaming), `anthropic_stream.txt` (full 7-event sequence: message_start → -content_block_{start,delta×2,stop} → message_delta (usage) → message_stop), -`pricing.yaml` (realistic-pricing starter for operator overrides). +content_block_{start,delta×2,stop} → message_delta (usage) → message_stop). +No pricing fixture: the table is config-delivered, so pricing tests construct +it in-process from a wire-shape map. ## Cross-references - Sibling: [31-proxy-middleware-builtin.md](./31-proxy-middleware-builtin.md) — the chain that calls `llm.Parsers()`, `llm.ParserByName`, - `llm.NewScanner`, `pricing.NewLoader`. + `llm.NewScanner`, `pricing.NewTable` / `pricing.NewEntries`. - Path-routed providers (Vertex AI + Bedrock), credential syntax, and the Bedrock AWS event-stream accumulator: [50-path-routed-providers.md](./50-path-routed-providers.md). diff --git a/docs/agent-networks/modules/33-proxy-runtime.md b/docs/agent-networks/modules/33-proxy-runtime.md index f553473f8..54046b614 100644 --- a/docs/agent-networks/modules/33-proxy-runtime.md +++ b/docs/agent-networks/modules/33-proxy-runtime.md @@ -1,7 +1,7 @@ # proxy/runtime — translate + serve + log > **Risk level:** High — every config push from management is translated here, and the chain runs on every HTTP request to a synth target. -> **Backward-compat impact:** Additive at the wire (`PathTargetOptions.middlewares`, `agent_network`, `disable_access_log`, capture caps) and on the proxy `Server` struct (`MiddlewareDataDir`, `MiddlewareCaptureBudgetBytes`). Non-agent-network targets stay on the no-middleware fast path. +> **Backward-compat impact:** Additive at the wire (`PathTargetOptions.middlewares`, `agent_network`, `disable_access_log`, capture caps) and on the proxy `Server` struct (`MiddlewareCaptureBudgetBytes`). Non-agent-network targets stay on the no-middleware fast path. Middleware config is entirely wire-delivered — no proxy-side data dir is involved, including for LLM pricing, which management ships inside `cost_meter`'s config. ## Module boundary @@ -114,8 +114,7 @@ At **request time** the access-log middleware stamps `CapturedData`; the auth ch ## Public contracts touched -- `proxy.Server.MiddlewareDataDir` (string) — base dir for file-backed middleware config (server.go:238-241). -- `proxy.Server.MiddlewareCaptureBudgetBytes` (int64) — process-wide capture cap; defaults to 256 MiB (server.go:248-250). +- `proxy.Server.MiddlewareCaptureBudgetBytes` (int64) — process-wide capture cap; defaults to 256 MiB (server.go:249-253). There is no `MiddlewareDataDir`: no built-in middleware reads config from disk, so `builtin.FactoryContext` carries only the proxy-lifetime context, meter, logger, and management client. - `proxy/internal/proxy.WithMiddlewareManager(*middleware.Manager) Option` — new option on `NewReverseProxy`; nil keeps the fast path (reverseproxy.go:48-56). - `proxy/internal/proxy.PathTarget` adds `Middlewares`, `CaptureConfig`, `AgentNetwork`, `DisableAccessLog` (servicemapping.go:27-51), all zero-default. - `proxy/internal/proxy.CapturedData` adds `agentNetwork`, `suppressAccessLog`, `userGroupNames` behind `sync.RWMutex`; slices deep-copied (context.go:47-66, 183-258). diff --git a/docs/agent-networks/modules/50-path-routed-providers.md b/docs/agent-networks/modules/50-path-routed-providers.md index b7cda3a97..08c976c5f 100644 --- a/docs/agent-networks/modules/50-path-routed-providers.md +++ b/docs/agent-networks/modules/50-path-routed-providers.md @@ -87,9 +87,9 @@ strips the `@version` suffix from the model, and maps the publisher to a parser surface via `vertexPublisherVendor`: - `anthropic` → `llm.provider="anthropic"` → metered through the Anthropic - parser, priced under the **`anthropic`** block in `defaults_pricing.yaml` - (the parser emits the standard Anthropic provider label, so Vertex Claude - reuses first-party Anthropic prices). + parser, priced under the **`anthropic`** surface of the pricing table + management ships (the parser emits the standard Anthropic provider label, so + Vertex Claude reuses first-party Anthropic prices). - `openai` → `llm.provider="openai"` (reserved; not in the catalog lineup today). - anything else (notably `google` / Gemini) → empty vendor → **no parser**. @@ -104,8 +104,9 @@ is omitted from the catalog. > Caveat: cross-region inference profiles in `eu` / `apac` carry a ~10% price > premium that the base per-token rates do **not** model — cost annotations for -> those regions read low. Operators who need exact regional billing override -> the affected entries in `pricing.yaml`. +> those regions read low. Operators who need exact regional billing set the +> affected models' prices on the provider record, or replace the default entries +> via management's `AgentNetwork.PricingDefaultsFile`. ## AWS Bedrock (`bedrock_api`) @@ -211,15 +212,19 @@ so a model-listing call can't be rewritten onto an upstream that would 404 it. ## Catalog ↔ pricing cross-check Catalog prices and context windows are cross-checked against LiteLLM's -`model_prices_and_context_window.json`. The proxy's embedded -`defaults_pricing.yaml` covers **every metered first-party model** the catalog -enumerates — guarded by -`TestDefaultTable_FirstPartyModelCoverage` -([pricing/defaults_coverage_test.go](../../../proxy/internal/llm/pricing/defaults_coverage_test.go)), -which fails if a catalog model has no embedded price. Bedrock entries are keyed -by the **normalised** id the request parser emits (region prefix + version -suffix stripped). Vertex Claude carries no Bedrock-style prefix, so it prices -straight off the `anthropic` block. +`model_prices_and_context_window.json`. The **catalog is the source of default +prices**: management's `pricing.DefaultTable` folds every catalog provider's +models into the surfaces that provider declares (`PricingSurfaces`), so coverage +is structural rather than maintained in a parallel file +([pricing/defaults.go](../../../management/internals/modules/agentnetwork/pricing/defaults.go)). +`TestDefaultTable_CoversEveryCatalogModel` fails if a catalog model ends up +unpriced, and `TestDefaultTable_NoConflictingContributions` fails if two +providers contribute the same (surface, model) at different rates. Bedrock +entries are keyed by the **normalised** id the request parser emits (region +prefix + version suffix stripped) — management applies the same normalisation to +per-provider prices at synth time, so the two keys compare equal. Vertex Claude +carries no Bedrock-style prefix, so it prices straight off the `anthropic` +surface. ## Things to scrutinise @@ -232,16 +237,17 @@ operator-misconfigured Vertex provider and unmetered Gemini traffic; verify publishers). **Correctness.** `normalizeBedrockModel` is the join between the wire id and the -pricing key — a model that normalises to something not in `defaults_pricing.yaml` -meters at `cost.skipped=unknown_model` rather than failing the request. The +pricing key — a model that normalises to something absent from the shipped +pricing table meters at `cost.skipped=unknown_model` rather than failing the +request. The `/bedrock` prefix strip must run on both the parser side (so the model is extracted) and the router side (so the upstream path is native); a regression in either silently breaks the other. **Metering caveats.** eu/apac cross-region Bedrock + Vertex profiles carry a -~10% premium not modelled by base pricing — flagged in both the catalog comment -and `defaults_pricing.yaml`. Operators needing exact regional billing override -the relevant entries. +~10% premium not modelled by base pricing — flagged in the catalog comment. +Operators needing exact regional billing set per-provider prices on the model +rows (or replace the default entries via `AgentNetwork.PricingDefaultsFile`). ## Cross-references diff --git a/e2e/agentnetwork/chat_test.go b/e2e/agentnetwork/chat_test.go index ed94623fe..792f67c89 100644 --- a/e2e/agentnetwork/chat_test.go +++ b/e2e/agentnetwork/chat_test.go @@ -438,14 +438,10 @@ func TestProvidersMatrix(t *testing.T) { // Create every provider, all enabled, each with a unique model string so the // proxy's connect-time snapshot carries them all and model→provider routing // is unambiguous (provider toggles after connect don't reconcile to the - // proxy, so we enable everything up front). The first create bootstraps the - // cluster. + // proxy, so we enable everything up front). ids := make([]string, 0, len(matrix)) - for i, pc := range matrix { + for _, pc := range matrix { req := providerRequest(pc) - if i == 0 { - req.BootstrapCluster = ptr(harness.AgentNetworkCluster) - } prov, perr := srv.CreateProvider(ctx, req) require.NoError(t, perr, "create provider %s", pc.name) ids = append(ids, prov.Id) diff --git a/e2e/agentnetwork/custom_pricing_test.go b/e2e/agentnetwork/custom_pricing_test.go index 33788a16a..e3750258f 100644 --- a/e2e/agentnetwork/custom_pricing_test.go +++ b/e2e/agentnetwork/custom_pricing_test.go @@ -82,13 +82,12 @@ func provisionPricedProvider(t *testing.T, ctx context.Context, name string, mod // need NOT be in the catalog — the operator names it and prices it here. dummyKey := "sk-price-e2e" prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: name, - ProviderId: "openai_api", - UpstreamUrl: vllm.URL, - ApiKey: &dummyKey, - Enabled: ptr(true), - BootstrapCluster: ptr(harness.AgentNetworkCluster), - Models: &models, + Name: name, + ProviderId: "openai_api", + UpstreamUrl: vllm.URL, + ApiKey: &dummyKey, + Enabled: ptr(true), + Models: &models, }) require.NoError(t, err, "create provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) diff --git a/e2e/agentnetwork/guardrail_block_test.go b/e2e/agentnetwork/guardrail_block_test.go index c4b22ae25..302b5107e 100644 --- a/e2e/agentnetwork/guardrail_block_test.go +++ b/e2e/agentnetwork/guardrail_block_test.go @@ -113,15 +113,14 @@ func runPathRoutedGuardrailCase(t *testing.T, tc pathRoutedGuardrailCase) { // Catch-all provider (no models) so the router forwards any model; a static // bearer key means the router injects a static auth header instead of minting - // a GCP token. Bootstraps the cluster if it isn't already. + // a GCP token. staticKey := "static-e2e-token" prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: tc.name, - ProviderId: tc.catalogID, - UpstreamUrl: vllm.URL, - ApiKey: &staticKey, - Enabled: ptr(true), - BootstrapCluster: ptr(harness.AgentNetworkCluster), + Name: tc.name, + ProviderId: tc.catalogID, + UpstreamUrl: vllm.URL, + ApiKey: &staticKey, + Enabled: ptr(true), }) require.NoError(t, err, "create %s provider", tc.name) t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) diff --git a/e2e/agentnetwork/guardrail_groupswitch_test.go b/e2e/agentnetwork/guardrail_groupswitch_test.go index 5f2dce52e..7108bb587 100644 --- a/e2e/agentnetwork/guardrail_groupswitch_test.go +++ b/e2e/agentnetwork/guardrail_groupswitch_test.go @@ -73,7 +73,6 @@ func TestGuardrailGroupSwitchTakesEffectAfterTTL(t *testing.T) { {Id: modelA, InputPer1k: 0.001, OutputPer1k: 0.001}, {Id: modelB, InputPer1k: 0.001, OutputPer1k: 0.001}, }, - BootstrapCluster: ptr(harness.AgentNetworkCluster), }) require.NoError(t, err, "create provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) diff --git a/e2e/agentnetwork/guardrail_multipolicy_test.go b/e2e/agentnetwork/guardrail_multipolicy_test.go index 1d664fb58..2bb56b3bc 100644 --- a/e2e/agentnetwork/guardrail_multipolicy_test.go +++ b/e2e/agentnetwork/guardrail_multipolicy_test.go @@ -61,15 +61,14 @@ func TestGuardrailMultiPolicyModelAllowlist(t *testing.T) { } // pRestricted declares the two guardrailed models so routing is deterministic - // (model -> provider). Created first, so it carries the bootstrap cluster. + // (model -> provider). pRestricted, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: "restricted", - ProviderId: "openai_api", - UpstreamUrl: vllm.URL, - ApiKey: &staticKey, - Enabled: ptr(true), - Models: models(modelSelected, modelOther), - BootstrapCluster: ptr(harness.AgentNetworkCluster), + Name: "restricted", + ProviderId: "openai_api", + UpstreamUrl: vllm.URL, + ApiKey: &staticKey, + Enabled: ptr(true), + Models: models(modelSelected, modelOther), }) require.NoError(t, err, "create restricted provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), pRestricted.Id) }) diff --git a/e2e/agentnetwork/guardrail_pergroup_providers_test.go b/e2e/agentnetwork/guardrail_pergroup_providers_test.go index eddae65c3..0e98330c4 100644 --- a/e2e/agentnetwork/guardrail_pergroup_providers_test.go +++ b/e2e/agentnetwork/guardrail_pergroup_providers_test.go @@ -115,7 +115,7 @@ func TestGuardrailPerGroupAllowlist_AllProviders(t *testing.T) { staticKey := "static-e2e-token" enabled := true - for i, c := range cases { + for _, c := range cases { req := api.AgentNetworkProviderRequest{ Name: "e2e-pergroup-" + c.name, ProviderId: c.catalogID, @@ -124,9 +124,6 @@ func TestGuardrailPerGroupAllowlist_AllProviders(t *testing.T) { Enabled: ptr(true), Models: c.models, } - if i == 0 { - req.BootstrapCluster = ptr(harness.AgentNetworkCluster) - } prov, perr := srv.CreateProvider(ctx, req) require.NoError(t, perr, "create provider %s", c.name) c.providerID = prov.Id @@ -283,13 +280,12 @@ func TestGuardrailMultiGroupUser(t *testing.T) { // P1 — union scenario: two restricting policies, one per group. p1, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: "e2e-mg-union", - ProviderId: "openai_api", - UpstreamUrl: vllm.URL, - ApiKey: &staticKey, - Enabled: ptr(true), - Models: priced(unionA, unionB, unionC), - BootstrapCluster: ptr(harness.AgentNetworkCluster), + Name: "e2e-mg-union", + ProviderId: "openai_api", + UpstreamUrl: vllm.URL, + ApiKey: &staticKey, + Enabled: ptr(true), + Models: priced(unionA, unionB, unionC), }) require.NoError(t, err, "create union provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), p1.Id) }) diff --git a/e2e/agentnetwork/guardrail_test.go b/e2e/agentnetwork/guardrail_test.go index 6a2487a88..ddc95d62d 100644 --- a/e2e/agentnetwork/guardrail_test.go +++ b/e2e/agentnetwork/guardrail_test.go @@ -115,14 +115,11 @@ func TestModelAllowlistEnforced(t *testing.T) { }) require.NoError(t, err, "mint setup key") - // Providers with their configured (allowed) models; the first bootstraps the cluster. + // Providers with their configured (allowed) models ids := make([]string, 0, len(providers)) allowed := make([]string, 0, len(providers)) - for i, pc := range providers { + for _, pc := range providers { req := providerRequest(pc) - if i == 0 { - req.BootstrapCluster = ptr(harness.AgentNetworkCluster) - } prov, perr := srv.CreateProvider(ctx, req) require.NoError(t, perr, "create provider %s", pc.name) id := prov.Id diff --git a/e2e/agentnetwork/main_test.go b/e2e/agentnetwork/main_test.go index 17c5e00be..cc366b3fb 100644 --- a/e2e/agentnetwork/main_test.go +++ b/e2e/agentnetwork/main_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/netbirdio/netbird/e2e/harness" + "github.com/netbirdio/netbird/shared/management/http/api" ) // srv is the shared combined server for the package, ready (PAT-authenticated) @@ -42,5 +43,14 @@ func run(m *testing.M) int { return 1 } + // Bootstrap the account's agent-network endpoint once for the package: + // providers no longer have settings side effects, and every data-plane + // test expects the shared account pinned to the combined proxy cluster. + cluster := harness.AgentNetworkCluster + if _, err := srv.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ProxyAddress: &cluster}); err != nil { + fmt.Fprintf(os.Stderr, "e2e: bootstrap agent-network endpoint: %v\n", err) + return 1 + } + return m.Run() } diff --git a/e2e/agentnetwork/management_test.go b/e2e/agentnetwork/management_test.go index 6962f9796..9e3176d68 100644 --- a/e2e/agentnetwork/management_test.go +++ b/e2e/agentnetwork/management_test.go @@ -21,11 +21,10 @@ func ptr[T any](v T) *T { return &v } func newProvider(t *testing.T, ctx context.Context, name string) api.AgentNetworkProvider { t.Helper() prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: name, - ProviderId: "openai_api", - UpstreamUrl: "https://api.openai.com", - ApiKey: ptr("sk-dummy-e2e-key"), - BootstrapCluster: ptr("eu.proxy.netbird.test"), + Name: name, + ProviderId: "openai_api", + UpstreamUrl: "https://api.openai.com", + ApiKey: ptr("sk-dummy-e2e-key"), }) require.NoError(t, err, "create provider %q", name) t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) @@ -57,17 +56,11 @@ func TestProviderLifecycle(t *testing.T) { }} } - for i, pc := range cases { - i, pc := i, pc + for _, pc := range cases { + pc := pc t.Run(pc.name, func(t *testing.T) { req := providerRequest(pc) req.Name = "lc-" + pc.name - // Bootstrap the cluster on the first create in case the matrix has - // not run (e.g. no provider keys → settings not yet bootstrapped). - if i == 0 { - req.BootstrapCluster = ptr(harness.AgentNetworkCluster) - } - prov, err := srv.CreateProvider(ctx, req) require.NoError(t, err, "create %s provider", pc.name) t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) @@ -137,45 +130,65 @@ func TestProviderValidation(t *testing.T) { requireClientError(t, err) } -// TestSettingsRoundTrip flips the collection toggles and confirms cluster / -// subdomain stay immutable, then restores the original state. +// TestSettingsRoundTrip flips the collection toggles and confirms the +// endpoint and proxy address stay immutable, then restores the original +// state. A second bootstrap attempt must be rejected as a conflict. func TestSettingsRoundTrip(t *testing.T) { ctx := context.Background() - // Settings are bootstrapped on first provider create. - newProvider(t, ctx, "Settings Bootstrap") - + // The package's TestMain bootstrapped the shared account's endpoint. before, err := srv.GetSettings(ctx) require.NoError(t, err, "get settings") - require.NotEmpty(t, before.Cluster, "settings must carry an assigned cluster") + require.NotEmpty(t, before.Endpoint, "settings must carry the bootstrapped endpoint") + require.NotEmpty(t, before.ProxyAddress, "settings must carry the bootstrapped proxy address") + + require.NotNil(t, before.AccessLogRetentionDays, "bootstrapped settings must carry a retention") + beforeRetention := *before.AccessLogRetentionDays flipped, err := srv.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ + Endpoint: before.Endpoint, + ProxyAddress: before.ProxyAddress, EnableLogCollection: !before.EnableLogCollection, EnablePromptCollection: !before.EnablePromptCollection, RedactPii: !before.RedactPii, + AccessLogRetentionDays: beforeRetention, }) require.NoError(t, err, "update settings") assert.Equal(t, !before.EnableLogCollection, flipped.EnableLogCollection, "log collection toggle must flip") assert.Equal(t, !before.EnablePromptCollection, flipped.EnablePromptCollection, "prompt collection toggle must flip") - assert.Equal(t, before.Cluster, flipped.Cluster, "cluster must be immutable across updates") - assert.Equal(t, before.Subdomain, flipped.Subdomain, "subdomain must be immutable across updates") + require.NotNil(t, flipped.AccessLogRetentionDays) + assert.Equal(t, beforeRetention, *flipped.AccessLogRetentionDays, + "retention sent unchanged must round-trip, not reset to the zero value") + assert.Equal(t, before.Endpoint, flipped.Endpoint, "endpoint must be immutable across updates") + assert.Equal(t, before.ProxyAddress, flipped.ProxyAddress, "proxy address must be immutable across updates") - // A cluster different from the pinned one must be rejected; echoing the - // pinned one back is valid. + // The account is already bootstrapped: a second bootstrap is a conflict, + // whatever shape it asks for. + _, err = srv.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + Endpoint: ptr("attacker.cluster.invalid"), + }) + requireClientError(t, err) + + // The identity fields ride along on the PUT as a required echo: a request + // carrying a different endpoint is rejected without applying anything. _, err = srv.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - Cluster: ptr("attacker.cluster.invalid"), + Endpoint: "other.cluster.invalid", + ProxyAddress: before.ProxyAddress, EnableLogCollection: before.EnableLogCollection, EnablePromptCollection: before.EnablePromptCollection, RedactPii: before.RedactPii, + AccessLogRetentionDays: beforeRetention, }) requireClientError(t, err) // Restore the original toggles. _, err = srv.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - Cluster: ptr(before.Cluster), + Endpoint: before.Endpoint, + ProxyAddress: before.ProxyAddress, EnableLogCollection: before.EnableLogCollection, EnablePromptCollection: before.EnablePromptCollection, RedactPii: before.RedactPii, + AccessLogRetentionDays: beforeRetention, }) require.NoError(t, err, "restore settings") } diff --git a/e2e/agentnetwork/settings_bootstrap_test.go b/e2e/agentnetwork/settings_bootstrap_test.go index ea56f7064..806ce82f4 100644 --- a/e2e/agentnetwork/settings_bootstrap_test.go +++ b/e2e/agentnetwork/settings_bootstrap_test.go @@ -4,6 +4,7 @@ package agentnetwork import ( "context" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -14,7 +15,8 @@ import ( ) // harnessStartFresh boots a dedicated combined server with its own fresh -// account and registers its teardown on t. +// account and registers its teardown on t. Unlike the shared srv, the fresh +// account has NOT had its agent-network endpoint bootstrapped. func harnessStartFresh(ctx context.Context, t *testing.T) (*harness.Combined, error) { t.Helper() fresh, err := harness.StartCombined(ctx) @@ -28,16 +30,16 @@ func harnessStartFresh(ctx context.Context, t *testing.T) (*harness.Combined, er return fresh, nil } -// TestSettingsBootstrapViaPut covers the settings-first bootstrap path on an +// TestSettingsBootstrapViaPost covers the explicit bootstrap contract on an // account that has never been bootstrapped: the GET reads as the defaults -// with an empty cluster/subdomain/endpoint, a PUT without a cluster has -// nothing to pin and fails, and a PUT carrying a cluster creates the row and -// pins it immutably. The shared srv cannot provide that starting state (any -// provider-creating test bootstraps it, and test order is deliberately not -// relied on), so this boots a dedicated combined server — the image is -// already built and cached by TestMain's StartCombined, so the extra cost is -// one container start. -func TestSettingsBootstrapViaPut(t *testing.T) { +// with an empty endpoint/proxy_address, a PUT has no row to update and fails, +// and a POST creates the row and assigns the immutable endpoint — labeled +// beneath a proxy address here, with the toggle overrides from the same +// request applied. The shared srv cannot provide that starting state +// (TestMain bootstraps it), so this boots a dedicated combined server — the +// image is already built and cached by TestMain's StartCombined, so the extra +// cost is one container start. +func TestSettingsBootstrapViaPost(t *testing.T) { ctx := context.Background() fresh, err := harnessStartFresh(ctx, t) @@ -47,32 +49,35 @@ func TestSettingsBootstrapViaPut(t *testing.T) { // as an error and not as a null body. before, err := fresh.GetSettings(ctx) require.NoError(t, err, "get settings on a fresh account must succeed") - assert.Empty(t, before.Cluster, "cluster must be empty before bootstrap") - assert.Empty(t, before.Subdomain, "subdomain must be empty before bootstrap") - assert.Empty(t, before.Endpoint, "endpoint must be empty before bootstrap, not a bare dot") + assert.Empty(t, before.Endpoint, "endpoint must be empty before bootstrap") + assert.Empty(t, before.ProxyAddress, "proxy address must be empty before bootstrap") + assert.False(t, before.Dedicated, "an unbootstrapped account has no serving shape") assert.True(t, before.EnableLogCollection, "defaults must show log collection on, matching bootstrap") assert.False(t, before.EnablePromptCollection, "defaults must show prompt collection off") - // A PUT without a cluster has nothing to pin the account to. + // A PUT has no row to update yet — bootstrap is the explicit POST. _, err = fresh.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - EnableLogCollection: true, + EnableLogCollection: true, + AccessLogRetentionDays: 30, }) requireClientError(t, err) - // A PUT carrying a cluster bootstraps the account and applies the - // mutable fields from the same request. Every toggle is set away from - // its bootstrap default so each assertion can actually fail. + // A POST with a proxy address bootstraps a labeled endpoint and applies + // the toggles from the same request. Every toggle is set away from its + // bootstrap default so each assertion can actually fail. const cluster = "e2e.bootstrap.netbird.selfhosted" - bootstrapped, err := fresh.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - Cluster: ptr(cluster), - EnableLogCollection: false, - EnablePromptCollection: true, - RedactPii: true, + bootstrapped, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + ProxyAddress: ptr(cluster), + EnableLogCollection: ptr(false), + EnablePromptCollection: ptr(true), + RedactPii: ptr(true), }) - require.NoError(t, err, "bootstrap settings via PUT must succeed") - assert.Equal(t, cluster, bootstrapped.Cluster, "cluster must be pinned from the request") - require.NotEmpty(t, bootstrapped.Subdomain, "subdomain must be assigned at bootstrap") - assert.Equal(t, bootstrapped.Subdomain+"."+cluster, bootstrapped.Endpoint, "endpoint must combine subdomain and cluster") + require.NoError(t, err, "bootstrap settings via POST must succeed") + assert.Equal(t, cluster, bootstrapped.ProxyAddress, "proxy address must be pinned from the request") + require.NotEmpty(t, bootstrapped.Endpoint, "endpoint must be assigned at bootstrap") + assert.True(t, strings.HasSuffix(bootstrapped.Endpoint, "."+cluster), + "labeled endpoint must hang one label beneath the proxy address: %s", bootstrapped.Endpoint) + assert.False(t, bootstrapped.Dedicated, "a labeled pin is not dedicated") assert.False(t, bootstrapped.EnableLogCollection, "log collection from the bootstrap request must override the default") assert.True(t, bootstrapped.EnablePromptCollection, "prompt collection from the bootstrap request must apply") assert.True(t, bootstrapped.RedactPii, "redact toggle from the bootstrap request must apply") @@ -85,30 +90,90 @@ func TestSettingsBootstrapViaPut(t *testing.T) { assert.Equal(t, bootstrapped.EnablePromptCollection, after.EnablePromptCollection, "prompt collection must persist") assert.Equal(t, bootstrapped.RedactPii, after.RedactPii, "redact toggle must persist") - // Once bootstrapped, later updates may omit the cluster entirely. + // Once bootstrapped, PUT updates the toggles. The identity fields ride + // along as a required echo of the assigned values; a matching echo is + // accepted and never written. persisted, err := fresh.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ + Endpoint: bootstrapped.Endpoint, + ProxyAddress: bootstrapped.ProxyAddress, EnableLogCollection: true, EnablePromptCollection: false, RedactPii: true, + AccessLogRetentionDays: 21, }) - require.NoError(t, err, "post-bootstrap update without cluster must succeed") - assert.Equal(t, cluster, persisted.Cluster, "omitted cluster must keep the pinned value") + require.NoError(t, err, "post-bootstrap update must succeed") + require.NotNil(t, persisted.AccessLogRetentionDays) + assert.Equal(t, 21, *persisted.AccessLogRetentionDays, "retention from the update must apply") + assert.Equal(t, bootstrapped.Endpoint, persisted.Endpoint, "endpoint must survive updates untouched") + assert.Equal(t, cluster, persisted.ProxyAddress, "proxy address must survive updates untouched") assert.True(t, persisted.EnableLogCollection, "post-bootstrap toggle must apply") assert.False(t, persisted.EnablePromptCollection, "post-bootstrap toggle must apply") - // The cluster is immutable: a different value is rejected rather than - // silently ignored, and the rejected update must not disturb anything. + // The endpoint is immutable: a PUT carrying a different endpoint is + // rejected, and a second bootstrap is rejected as a conflict. Neither + // rejected write may disturb anything. _, err = fresh.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - Cluster: ptr("other.cluster.invalid"), - EnableLogCollection: false, + Endpoint: "other.cluster.invalid", + ProxyAddress: persisted.ProxyAddress, + EnableLogCollection: persisted.EnableLogCollection, + EnablePromptCollection: persisted.EnablePromptCollection, + RedactPii: persisted.RedactPii, + AccessLogRetentionDays: 21, + }) + requireClientError(t, err) + + _, err = fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + Endpoint: ptr("other.cluster.invalid"), }) requireClientError(t, err) final, err := fresh.GetSettings(ctx) - require.NoError(t, err, "get settings after the rejected cluster change must succeed") - assert.Equal(t, persisted.Cluster, final.Cluster, "rejected update must not change the cluster") - assert.Equal(t, persisted.Endpoint, final.Endpoint, "rejected update must not change the endpoint") - assert.Equal(t, persisted.EnableLogCollection, final.EnableLogCollection, "rejected update must not apply its toggles") - assert.Equal(t, persisted.EnablePromptCollection, final.EnablePromptCollection, "rejected update must not apply its toggles") - assert.Equal(t, persisted.RedactPii, final.RedactPii, "rejected update must not apply its toggles") + require.NoError(t, err, "get settings after the rejected bootstrap must succeed") + assert.Equal(t, persisted.Endpoint, final.Endpoint, "rejected bootstrap must not change the endpoint") + assert.Equal(t, persisted.ProxyAddress, final.ProxyAddress, "rejected bootstrap must not change the proxy address") + assert.Equal(t, persisted.EnableLogCollection, final.EnableLogCollection, "rejected bootstrap must not apply its toggles") + assert.Equal(t, persisted.EnablePromptCollection, final.EnablePromptCollection, "rejected bootstrap must not apply its toggles") + assert.Equal(t, persisted.RedactPii, final.RedactPii, "rejected bootstrap must not apply its toggles") +} + +// TestSettingsBootstrapSelfAddressed covers the dedicated shape end to end: +// a POST carrying an endpoint claims the hostname verbatim, the proxy address +// equals it, and the pin reads as dedicated — the address-first flow a +// self-hosted operator uses before deploying the proxy that will declare it. +// The tail covers the recovery path the guarded DELETE exists for: with no +// providers and no proxy at the address, the claim can be released and a +// fresh bootstrap succeeds — the fix for a typo'd immutable endpoint. +func TestSettingsBootstrapSelfAddressed(t *testing.T) { + ctx := context.Background() + + fresh, err := harnessStartFresh(ctx, t) + require.NoError(t, err, "start dedicated combined server") + + created, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + Endpoint: ptr("gw.e2e.netbird.selfhosted"), + }) + require.NoError(t, err, "self-addressed bootstrap must succeed") + assert.Equal(t, "gw.e2e.netbird.selfhosted", created.Endpoint, "endpoint must be claimed verbatim") + assert.Equal(t, created.Endpoint, created.ProxyAddress, "self-addressed: proxy address is the endpoint") + assert.True(t, created.Dedicated, "a self-addressed pin is dedicated") + + // No providers exist and no proxy declares the address, so both delete + // guards are clear: the delete releases the claim and the account reads + // as unbootstrapped defaults again. + require.NoError(t, fresh.DeleteSettings(ctx), "guarded delete with both guards clear must succeed") + + after, err := fresh.GetSettings(ctx) + require.NoError(t, err, "get settings after delete must succeed") + assert.Empty(t, after.Endpoint, "a deleted account must read as unbootstrapped") + + // A second delete has nothing to remove. + requireClientError(t, fresh.DeleteSettings(ctx)) + + // Re-creating is a fresh bootstrap — the released hostname is free to be + // claimed again, or a different one chosen. + recreated, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + Endpoint: ptr("gw2.e2e.netbird.selfhosted"), + }) + require.NoError(t, err, "bootstrap after delete must succeed") + assert.Equal(t, "gw2.e2e.netbird.selfhosted", recreated.Endpoint, "the fresh bootstrap claims the new hostname") } diff --git a/e2e/agentnetwork/skiptls_test.go b/e2e/agentnetwork/skiptls_test.go index 1f57605e3..85af8c604 100644 --- a/e2e/agentnetwork/skiptls_test.go +++ b/e2e/agentnetwork/skiptls_test.go @@ -66,9 +66,7 @@ func TestProviderSkipTLSVerification(t *testing.T) { } } - // First create bootstraps the account cluster. insecureReq := newReq("skip-tls", insecureModel, true) - insecureReq.BootstrapCluster = ptr(harness.AgentNetworkCluster) insecureProv, err := srv.CreateProvider(ctx, insecureReq) require.NoError(t, err, "create skip-tls provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), insecureProv.Id) }) diff --git a/e2e/agentnetwork/vllm_test.go b/e2e/agentnetwork/vllm_test.go index cd598f1ed..6f5117b2f 100644 --- a/e2e/agentnetwork/vllm_test.go +++ b/e2e/agentnetwork/vllm_test.go @@ -57,12 +57,11 @@ func TestVLLMProvider(t *testing.T) { // is enumerated so the router dispatches this model string to this provider. dummyKey := "sk-vllm-e2e" prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: "vllm", - ProviderId: "vllm", - UpstreamUrl: vllm.URL, - ApiKey: &dummyKey, - Enabled: ptr(true), - BootstrapCluster: ptr(harness.AgentNetworkCluster), + Name: "vllm", + ProviderId: "vllm", + UpstreamUrl: vllm.URL, + ApiKey: &dummyKey, + Enabled: ptr(true), Models: &[]api.AgentNetworkProviderModel{ {Id: harness.VLLMModel, InputPer1k: 0.001, OutputPer1k: 0.002}, }, diff --git a/e2e/harness/Dockerfile.client b/e2e/harness/Dockerfile.client index 114577d60..74a3ec245 100644 --- a/e2e/harness/Dockerfile.client +++ b/e2e/harness/Dockerfile.client @@ -20,5 +20,9 @@ ENV NETBIRD_BIN="/usr/local/bin/netbird" \ NB_ENABLE_CAPTURE="false" \ NB_ENTRYPOINT_SERVICE_TIMEOUT="30" ENTRYPOINT [ "/usr/local/bin/netbird-entrypoint.sh" ] -COPY client/netbird-entrypoint.sh /usr/local/bin/netbird-entrypoint.sh +# --chmod because the build context is not always a git checkout. A suite in +# another module builds from this module's extracted copy in the module cache, +# where every file is 0444 — the cache drops the executable bit git records — and +# a bare COPY then produces an entrypoint the runtime cannot exec. +COPY --chmod=0755 client/netbird-entrypoint.sh /usr/local/bin/netbird-entrypoint.sh COPY --from=builder /out/netbird /usr/local/bin/netbird diff --git a/e2e/harness/agentnetwork.go b/e2e/harness/agentnetwork.go index 078e697af..e85475dff 100644 --- a/e2e/harness/agentnetwork.go +++ b/e2e/harness/agentnetwork.go @@ -126,17 +126,33 @@ func (c *Combined) DeleteGuardrail(ctx context.Context, id string) error { return anDelete(ctx, c, "/api/agent-network/guardrails/"+id) } -// GetSettings returns the account's agent-network settings row. It exists only -// after the first provider create bootstraps it. +// CreateSettings bootstraps the account's agent-network settings row, +// assigning the immutable endpoint. Exactly one of req.ProxyAddress (labeled +// endpoint beneath that cluster) and req.Endpoint (self-addressed dedicated +// endpoint) must be set; a second bootstrap returns a conflict. +func (c *Combined) CreateSettings(ctx context.Context, req api.AgentNetworkSettingsCreateRequest) (api.AgentNetworkSettings, error) { + return anRequest[api.AgentNetworkSettings](ctx, c, http.MethodPost, "/api/agent-network/settings", req) +} + +// GetSettings returns the account's agent-network settings row. Before the +// CreateSettings bootstrap it reads as the defaults with an empty endpoint. func (c *Combined) GetSettings(ctx context.Context) (api.AgentNetworkSettings, error) { return anRequest[api.AgentNetworkSettings](ctx, c, http.MethodGet, "/api/agent-network/settings", nil) } -// UpdateSettings applies the mutable collection toggles. +// UpdateSettings applies the mutable collection toggles. The request must +// echo the assigned endpoint and proxy address unchanged — the server rejects +// a PUT that tries to change them. func (c *Combined) UpdateSettings(ctx context.Context, req api.AgentNetworkSettingsRequest) (api.AgentNetworkSettings, error) { return anRequest[api.AgentNetworkSettings](ctx, c, http.MethodPut, "/api/agent-network/settings", req) } +// DeleteSettings removes the account's settings row, releasing the endpoint. +// Refused while providers exist or a proxy is actively serving the endpoint. +func (c *Combined) DeleteSettings(ctx context.Context) error { + return anDelete(ctx, c, "/api/agent-network/settings") +} + // ListConsumption returns the account's consumption rows (possibly empty). func (c *Combined) ListConsumption(ctx context.Context) ([]api.AgentNetworkConsumption, error) { return anRequest[[]api.AgentNetworkConsumption](ctx, c, http.MethodGet, "/api/agent-network/consumption", nil) diff --git a/e2e/harness/client.go b/e2e/harness/client.go index f53d0ea64..0d7f016a6 100644 --- a/e2e/harness/client.go +++ b/e2e/harness/client.go @@ -32,12 +32,36 @@ type Client struct { container testcontainers.Container } +// clientOptions is what the ClientOption values assemble. +type clientOptions struct { + name string +} + +// ClientOption adjusts how StartClient runs the agent. +type ClientOption func(*clientOptions) + +// WithClientName names the agent, which sets both its network alias and its +// container hostname. The hostname matters beyond addressing: the agent reports +// it to management at registration, so it is the name the peer appears under in +// the API. +// +// Required to run more than one agent against the same server — the default name +// is shared, and two containers cannot hold the same alias on one network. +func WithClientName(name string) ClientOption { + return func(o *clientOptions) { o.name = name } +} + // StartClient builds the client image and runs it on the combined server's // network, joining via the given setup key. The image entrypoint brings the // daemon up automatically; callers wait for connectivity with WaitConnected / // WaitProxyPeer. -func StartClient(ctx context.Context, c *Combined, setupKey string) (*Client, error) { - root, err := repoRoot() +func StartClient(ctx context.Context, c *Combined, setupKey string, opts ...ClientOption) (*Client, error) { + o := clientOptions{name: clientAlias} + for _, opt := range opts { + opt(&o) + } + + root, err := repoRoot(ctx) if err != nil { return nil, err } @@ -47,9 +71,13 @@ func StartClient(ctx context.Context, c *Combined, setupKey string) (*Client, er } req := testcontainers.ContainerRequest{ - Image: clientImage, + Image: clientImage, + // The agent reports the container's hostname to management, so this is + // the name the peer is addressable by in the API as well as on the + // network. The entrypoint takes no hostname flag of its own. + Hostname: o.name, Networks: []string{c.network.Name}, - NetworkAliases: map[string][]string{c.network.Name: {clientAlias}}, + NetworkAliases: map[string][]string{c.network.Name: {o.name}}, Env: map[string]string{ "NB_MANAGEMENT_URL": combinedExposedURL, "NB_SETUP_KEY": setupKey, diff --git a/e2e/harness/combined.go b/e2e/harness/combined.go index b2f0d89d2..e03f9f256 100644 --- a/e2e/harness/combined.go +++ b/e2e/harness/combined.go @@ -61,11 +61,68 @@ type Combined struct { workDir string } +// combinedOptions is what the CombinedOption values assemble. +type combinedOptions struct { + geolocation bool + env map[string]string +} + +// CombinedOption adjusts how StartCombined boots the server. The defaults suit a +// suite that only drives the API; the options exist for the ones that need more +// of the product than that. +type CombinedOption func(*combinedOptions) + +// WithGeolocation leaves the GeoLite database download enabled. It is off by +// default because the download adds startup latency that most suites get nothing +// for. A suite asserting on location-based posture checks needs it: management +// evaluates those rules against the database, and without it the rule fails +// instead of passing without having been checked. +func WithGeolocation() CombinedOption { + return func(o *combinedOptions) { o.geolocation = true } +} + +// WithServerEnv adds environment variables to the combined container, overriding +// the defaults on a key collision. For settings this harness does not model +// directly, so a suite needing one does not have to fork the harness to get it. +func WithServerEnv(env map[string]string) CombinedOption { + return func(o *combinedOptions) { + if o.env == nil { + o.env = map[string]string{} + } + for k, v := range env { + o.env[k] = v + } + } +} + +// combinedEnv is the combined container's environment: setup-PAT enabled so the +// caller can mint an admin token through /api/setup, geolocation off unless the +// suite asked for it, and whatever the suite added on top. +func combinedEnv(o combinedOptions) map[string]string { + env := map[string]string{ + "NB_SETUP_PAT_ENABLED": "true", + } + if !o.geolocation { + // Skip the GeoLite DB download — it blocks startup and agent-network + // ingest doesn't use geolocation. + env["NB_DISABLE_GEOLOCATION"] = "true" + } + for k, v := range o.env { + env[k] = v + } + return env +} + // StartCombined builds the combined server from its multistage Dockerfile and // boots it with setup-PAT enabled on a fresh shared network, returning once the // API is serving. The caller still owns minting the admin PAT via Bootstrap. -func StartCombined(ctx context.Context) (*Combined, error) { - root, err := repoRoot() +func StartCombined(ctx context.Context, opts ...CombinedOption) (*Combined, error) { + var o combinedOptions + for _, opt := range opts { + opt(&o) + } + + root, err := repoRoot(ctx) if err != nil { return nil, err } @@ -88,7 +145,7 @@ func StartCombined(ctx context.Context) (*Combined, error) { return nil, fmt.Errorf("create work dir: %w", err) } - cfg := fmt.Sprintf(combinedConfigYAML, combinedExposedURL, containerIssuer) + cfg := fmt.Sprintf(combinedConfigYAML, combinedExposedURL, !o.geolocation, containerIssuer) if err := os.WriteFile(filepath.Join(workDir, "config.yaml"), []byte(cfg), 0o644); err != nil { //nolint:gosec // non-secret config, bind-mounted and read by the container _ = net.Remove(ctx) return nil, fmt.Errorf("write combined config: %w", err) @@ -112,13 +169,8 @@ func StartCombined(ctx context.Context) (*Combined, error) { ExposedPorts: []string{combinedHTTPPort}, Networks: []string{net.Name}, NetworkAliases: map[string][]string{net.Name: {combinedAlias}}, - Env: map[string]string{ - "NB_SETUP_PAT_ENABLED": "true", - // Skip the GeoLite DB download — it blocks startup and agent-network - // ingest doesn't use geolocation. - "NB_DISABLE_GEOLOCATION": "true", - }, - Cmd: []string{"--config", "/nb/config.yaml"}, + Env: combinedEnv(o), + Cmd: []string{"--config", "/nb/config.yaml"}, HostConfigModifier: func(hc *container.HostConfig) { hc.Binds = append(hc.Binds, workDir+":/nb") }, diff --git a/e2e/harness/config.go b/e2e/harness/config.go index 71b3656c5..f0952b18c 100644 --- a/e2e/harness/config.go +++ b/e2e/harness/config.go @@ -15,6 +15,11 @@ package harness // server is required to load it — a broken path or malformed file fails startup // rather than silently falling back to the compiled-in rates, and TestMain then // fails with the container logs. +// +// disableGeoliteUpdate is a parameter rather than a fixed true because a suite +// that exercises geolocation needs the database: management can only evaluate a +// location rule with GeoLite loaded, and a rule it cannot evaluate fails rather +// than passing vacuously. See WithGeolocation. const combinedConfigYAML = `server: listenAddress: ":8080" exposedAddress: "%s" @@ -25,7 +30,7 @@ const combinedConfigYAML = `server: authSecret: "e2e-relay-secret" dataDir: "/nb/data" disableAnonymousMetrics: true - disableGeoliteUpdate: true + disableGeoliteUpdate: %t auth: issuer: "%s" store: diff --git a/e2e/harness/options_test.go b/e2e/harness/options_test.go new file mode 100644 index 000000000..8a5557a83 --- /dev/null +++ b/e2e/harness/options_test.go @@ -0,0 +1,161 @@ +//go:build e2e + +package harness + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The options exist so a suite can ask for a deployment this harness would not +// otherwise give it. What they configure is a container environment and a config +// file, both assembled before anything is started, so they are checkable without +// Docker — which is the point: a wiring mistake here would otherwise only show up +// as a puzzling failure minutes into a container run. + +func TestCombinedEnvGeolocation(t *testing.T) { + var off combinedOptions + assert.Equal(t, "true", combinedEnv(off)["NB_DISABLE_GEOLOCATION"], + "geolocation should be off by default") + + var on combinedOptions + WithGeolocation()(&on) + assert.NotContains(t, combinedEnv(on), "NB_DISABLE_GEOLOCATION", + "WithGeolocation must leave NB_DISABLE_GEOLOCATION unset, so the server downloads the database") + assert.Equal(t, "true", combinedEnv(on)["NB_SETUP_PAT_ENABLED"], + "the setup PAT must stay enabled whatever else is configured; Bootstrap depends on it") +} + +// The config file carries the same decision as the environment variable, and the +// server needs both to agree: disableGeoliteUpdate suppresses the download even +// when geolocation itself is enabled. +func TestCombinedConfigGeolocation(t *testing.T) { + for _, tc := range []struct { + name string + opts []CombinedOption + want string + }{ + {name: "default", want: "disableGeoliteUpdate: true"}, + {name: "with geolocation", opts: []CombinedOption{WithGeolocation()}, want: "disableGeoliteUpdate: false"}, + } { + t.Run(tc.name, func(t *testing.T) { + var o combinedOptions + for _, opt := range tc.opts { + opt(&o) + } + cfg := fmt.Sprintf(combinedConfigYAML, combinedExposedURL, !o.geolocation, containerIssuer) + assert.Contains(t, cfg, tc.want, "geolocation not rendered as expected") + // The issuer is the last verb; a mis-ordered argument list would put + // the boolean here instead and the server would fail to start. + assert.Contains(t, cfg, `issuer: "`+containerIssuer+`"`, "issuer not rendered") + }) + } +} + +func TestWithServerEnvOverrides(t *testing.T) { + var o combinedOptions + WithServerEnv(map[string]string{"NB_LOG_LEVEL": "debug"})(&o) + WithServerEnv(map[string]string{"NB_SETUP_PAT_ENABLED": "false"})(&o) + + env := combinedEnv(o) + assert.Equal(t, "debug", env["NB_LOG_LEVEL"], "added variable missing") + assert.Equal(t, "false", env["NB_SETUP_PAT_ENABLED"], "a suite must be able to override a default") +} + +// Two agents on one network cannot share an alias, so the name has to reach both +// the alias and the hostname. The hostname is the one management records, so it is +// also what the peer is addressable by through the API. +func TestWithClientName(t *testing.T) { + o := clientOptions{name: clientAlias} + require.Equal(t, "client", o.name, "unexpected default client name") + + WithClientName("peer2")(&o) + assert.Equal(t, "peer2", o.name, "WithClientName did not take") +} + +// repoRoot has to recognise this module rather than merely finding a go.mod, or a +// suite in another module gets its own root and a build context without the +// component Dockerfiles in it. +func TestIsModule(t *testing.T) { + dir := t.TempDir() + + other := filepath.Join(dir, "go.mod") + require.NoError(t, os.WriteFile(other, []byte("module example.com/other\n\ngo 1.25\n"), 0o600)) + assert.False(t, isModule(other, modulePath), "another module's go.mod must not be taken for this repo") + + ours := filepath.Join(dir, "ours.mod") + require.NoError(t, os.WriteFile(ours, []byte("// a comment\n\nmodule "+modulePath+"\n\ngo 1.25\n"), 0o600)) + assert.True(t, isModule(ours, modulePath), "this repo's go.mod was not recognised") + + assert.False(t, isModule(filepath.Join(dir, "absent.mod"), modulePath), + "a missing go.mod must not report a match") +} + +// Running from inside the repo, repoRoot finds it by walking up — the module +// lookup is only the fallback, and this asserts the walk still wins so an in-repo +// run never depends on the module cache. +func TestRepoRootFindsThisRepo(t *testing.T) { + root, err := repoRoot(context.Background()) + require.NoError(t, err) + assert.True(t, isModule(filepath.Join(root, "go.mod"), modulePath), + "repoRoot returned %s, which is not this module", root) + + for _, f := range []string{combinedDockerfile, clientDockerfile} { + _, err := os.Stat(filepath.Join(root, f)) + assert.NoError(t, err, "%s is not present under the reported root %s", f, root) + } +} + +// A caller that vendors its dependencies puts the go command in automatic vendor +// mode, where `go list -m -f {{.Dir}}` succeeds and reports an EMPTY directory: +// vendor/ holds packages, not module source. Without -mod=readonly the lookup +// would come back empty and the harness would report a missing module for a +// dependency that is present. +func TestModuleDirResolvesUnderVendorMode(t *testing.T) { + if _, err := exec.LookPath("go"); err != nil { + t.Skip("no go tool on PATH") + } + ctx := context.Background() + + base := t.TempDir() + dep := filepath.Join(base, "dep") + main := filepath.Join(base, "main") + require.NoError(t, os.MkdirAll(dep, 0o750)) + require.NoError(t, os.MkdirAll(main, 0o750)) + + // A local replacement rather than a real dependency, so this needs no network. + require.NoError(t, os.WriteFile(filepath.Join(dep, "go.mod"), + []byte("module example.com/dep\n\ngo 1.25\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dep, "dep.go"), + []byte("package dep\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(main, "go.mod"), + []byte("module example.com/main\n\ngo 1.25\n\nrequire example.com/dep v0.0.0\n\nreplace example.com/dep v0.0.0 => ../dep\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(main, "main.go"), + []byte("package main\n\nimport _ \"example.com/dep\"\n\nfunc main() {}\n"), 0o600)) + + t.Chdir(main) + vendor := exec.CommandContext(ctx, "go", "mod", "vendor") + out, err := vendor.CombinedOutput() + require.NoError(t, err, "go mod vendor: %s", out) + + dir, err := moduleDir(ctx, "example.com/dep") + require.NoError(t, err, "the module must still resolve with a vendor directory present") + assert.Equal(t, dep, dir, "resolved the wrong directory") +} + +// A cancelled context has to stop the lookup rather than leaving the caller +// waiting on a subprocess it has already given up on. +func TestModuleDirHonoursContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := moduleDir(ctx, modulePath) + assert.ErrorIs(t, err, context.Canceled, "a cancelled context must stop the lookup") +} diff --git a/e2e/harness/paths.go b/e2e/harness/paths.go index d7df6bbfa..569c32efc 100644 --- a/e2e/harness/paths.go +++ b/e2e/harness/paths.go @@ -3,27 +3,82 @@ package harness import ( + "context" "fmt" "os" + "os/exec" "path/filepath" + "strings" ) -// repoRoot walks up from the working directory to the module root (the -// directory holding go.mod), so the Docker build context is correct no matter -// which package the test runs from. -func repoRoot() (string, error) { +// modulePath is this module, used both to recognise the repo when walking up +// from the working directory and to locate it when the suite lives elsewhere. +const modulePath = "github.com/netbirdio/netbird" + +// repoRoot returns the directory the component Dockerfiles are built from. +// +// Walking up from the working directory finds it for any test inside this repo, +// no matter which package it runs from. A suite in another module gets a +// different answer that way — its own module root, where combined/Dockerfile +// does not exist — so the ancestor has to be this module and not merely some +// module. When it is not, the build context is the extracted module directory of +// whichever version that suite depends on, which is the right one: the server it +// tests against is then built from the same revision as the client library it +// was compiled with. +func repoRoot(ctx context.Context) (string, error) { dir, err := os.Getwd() if err != nil { return "", err } for { - if _, statErr := os.Stat(filepath.Join(dir, "go.mod")); statErr == nil { + if isModule(filepath.Join(dir, "go.mod"), modulePath) { return dir, nil } parent := filepath.Dir(dir) if parent == dir { - return "", fmt.Errorf("go.mod not found above %s", dir) + break } dir = parent } + return moduleDir(ctx, modulePath) +} + +// isModule reports whether the go.mod at path declares the given module. +func isModule(path, want string) bool { + b, err := os.ReadFile(path) + if err != nil { + return false + } + for _, line := range strings.Split(string(b), "\n") { + if rest, ok := strings.CutPrefix(strings.TrimSpace(line), "module "); ok { + return strings.TrimSpace(rest) == want + } + } + return false +} + +// moduleDir asks the go tool where a module's source is, which for a dependent +// module is its extracted copy in the module cache. The cache is read-only, and +// a Docker build context is only ever read. +// +// -mod=readonly is required rather than cosmetic. A caller that vendors its +// dependencies puts the go command in automatic vendor mode, where this lookup +// succeeds with an EMPTY directory — vendor/ holds packages, not module source, +// so there is nothing to report. Asking in readonly mode resolves against the +// module graph instead, which answers for both a cached module and a local +// replacement, and neither writes to go.mod. +func moduleDir(ctx context.Context, module string) (string, error) { + cmd := exec.CommandContext(ctx, "go", "list", "-mod=readonly", "-m", "-f", "{{.Dir}}", module) + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("locate %s: %w", module, err) + } + dir := strings.TrimSpace(string(out)) + if dir == "" { + return "", fmt.Errorf("locate %s: the go tool reported no directory; run `go mod download %s`", module, module) + } + if _, err := os.Stat(dir); err != nil { + return "", fmt.Errorf("locate %s: %w", module, err) + } + return dir, nil } diff --git a/e2e/harness/proxy.go b/e2e/harness/proxy.go index 85f3518d4..3d709b439 100644 --- a/e2e/harness/proxy.go +++ b/e2e/harness/proxy.go @@ -43,7 +43,7 @@ type Proxy struct { // or override any NB_PROXY_* var (e.g. NB_PROXY_TUNNEL_CACHE_TTL for tests that // need a short authorization-cache window). func StartProxy(ctx context.Context, c *Combined, proxyToken string, envOverrides ...map[string]string) (*Proxy, error) { - root, err := repoRoot() + root, err := repoRoot(ctx) if err != nil { return nil, err } diff --git a/funding.json b/funding.json index 6b509a992..34ee9fe46 100644 --- a/funding.json +++ b/funding.json @@ -6,7 +6,7 @@ "name": "NetBird GmbH", "email": "hello@netbird.io", "phone": "", - "description": "NetBird GmbH is a Berlin-based software company specializing in the development of open-source network security solutions. Network security is utterly complex and expensive, accessible only to companies with multi-million dollar IT budgets. In contrast, there are millions of companies left behind. Our mission is to create an advanced network and cybersecurity platform that is both easy-to-use and affordable for teams of all sizes and budgets. By leveraging the open-source strategy and technological advancements, NetBird aims to set the industry standard for connecting and securing IT infrastructure.", + "description": "NetBird GmbH is a Berlin-based software company specializing in the development of open source network security solutions. Network security is utterly complex and expensive, accessible only to companies with multi-million dollar IT budgets. In contrast, there are millions of companies left behind. Our mission is to create an advanced network and cybersecurity platform that is both easy-to-use and affordable for teams of all sizes and budgets. By leveraging the open source strategy and technological advancements, NetBird aims to set the industry standard for connecting and securing IT infrastructure.", "webpageUrl": { "url": "https://github.com/netbirdio" } @@ -15,7 +15,7 @@ { "guid": "netbird", "name": "NetBird", - "description": "NetBird is a configuration-free peer-to-peer private network and a centralized access control system combined in a single open-source platform. It makes it easy to create secure WireGuard-based private networks for your organization or home.", + "description": "NetBird is a configuration-free peer-to-peer private network and a centralized access control system combined in a single open source platform. It makes it easy to create secure WireGuard-based private networks for your organization or home.", "webpageUrl": { "url": "https://github.com/netbirdio/netbird" }, @@ -59,7 +59,7 @@ "guid": "support-yearly", "status": "active", "name": "Support Open Source Development and Maintenance - Yearly", - "description": "This will help us partially cover the yearly cost of maintaining the open-source NetBird project.", + "description": "This will help us partially cover the yearly cost of maintaining the open source NetBird project.", "amount": 100000, "currency": "USD", "frequency": "yearly", @@ -72,7 +72,7 @@ "guid": "support-one-time-year", "status": "active", "name": "Support Open Source Development and Maintenance - One Year", - "description": "This will help us partially cover the yearly cost of maintaining the open-source NetBird project.", + "description": "This will help us partially cover the yearly cost of maintaining the open source NetBird project.", "amount": 100000, "currency": "USD", "frequency": "one-time", @@ -85,7 +85,7 @@ "guid": "support-one-time-monthly", "status": "active", "name": "Support Open Source Development and Maintenance - Monthly", - "description": "This will help us partially cover the monthly cost of maintaining the open-source NetBird project.", + "description": "This will help us partially cover the monthly cost of maintaining the open source NetBird project.", "amount": 10000, "currency": "USD", "frequency": "monthly", @@ -98,7 +98,7 @@ "guid": "support-monthly", "status": "active", "name": "Support Open Source Development and Maintenance - One Month", - "description": "This will help us partially cover the monthly cost of maintaining the open-source NetBird project.", + "description": "This will help us partially cover the monthly cost of maintaining the open source NetBird project.", "amount": 10000, "currency": "USD", "frequency": "monthly", diff --git a/infrastructure_files/getting-started.sh b/infrastructure_files/getting-started.sh index 4f2c1d82e..0fc5b23c5 100755 --- a/infrastructure_files/getting-started.sh +++ b/infrastructure_files/getting-started.sh @@ -111,6 +111,59 @@ check_nb_domain() { return 0 } +# Non-interactive configuration +# ------------------------------ +# Every prompt below can be pre-answered with an environment variable, so the +# script runs unattended (cloud-init, CI, Terraform, curl | bash). resolve() +# is the single place that decides env var vs prompt vs default; the read_* +# helpers stay pure prompts. +# +# Supported env vars: +# NETBIRD_DOMAIN domain/FQDN (required) +# NETBIRD_LETSENCRYPT_EMAIL ACME email (required for built-in Traefik) +# NETBIRD_AGENT_NETWORK true enables the agent-network preset +# NETBIRD_REVERSE_PROXY_TYPE 0-5 (default 0 = built-in Traefik) +# NETBIRD_ENABLE_PROXY true/false (default false) +# NETBIRD_ENABLE_CROWDSEC true/false (default false) +# NETBIRD_TRAEFIK_EXTERNAL_NETWORK external-Traefik network (type 1) +# NETBIRD_TRAEFIK_ENTRYPOINT external-Traefik entrypoint (type 1, default websecure) +# NETBIRD_TRAEFIK_CERTRESOLVER external-Traefik cert resolver (type 1) +# NETBIRD_BIND_LOCALHOST_ONLY true/false (default true, types 2-5) +# NETBIRD_EXTERNAL_PROXY_NETWORK docker network to join (types 2-4) +# NETBIRD_NON_INTERACTIVE true forces unattended mode even with a TTY + +# tty_available succeeds only when we may prompt: never when the operator has +# set NETBIRD_NON_INTERACTIVE=true, otherwise only when /dev/tty can actually +# be opened. A PTY can be attached in automation (CI runners, some +# provisioners), so the env override is the authoritative signal and the +# /dev/tty probe is the fallback. /dev/tty is a world-rw device node even with +# no terminal, so a permission test ([ -r ]) is not enough - we must open it. +tty_available() { + [[ "${NETBIRD_NON_INTERACTIVE:-}" == "true" ]] && return 1 + { true < /dev/tty; } 2>/dev/null +} + +# resolve ENV_VAR_NAME DEFAULT PROMPT_FN [prompt args...] +# env var set and non-empty -> its value +# interactive -> PROMPT_FN "$@" (prompt behavior unchanged) +# otherwise -> DEFAULT, or abort when DEFAULT is "required" +resolve() { + local env_name="$1" default="$2" prompt_fn="$3" + shift 3 + local env_value="${!env_name:-}" + if [[ -n "$env_value" ]]; then + echo "$env_value" + elif tty_available; then + "$prompt_fn" "$@" + elif [[ "$default" == "required" ]]; then + echo "$env_name is required for a non-interactive install." > /dev/stderr + exit 1 + else + echo "$default" + fi + return 0 +} + read_nb_domain() { READ_NETBIRD_DOMAIN="" echo -n "Enter the domain you want to use for NetBird (e.g. netbird.my-domain.com): " > /dev/stderr @@ -383,7 +436,14 @@ initialize_default_values() { } configure_domain() { + # Domain is validated (not a free-form value), so it keeps its own guard + # rather than going through resolve(): a valid NETBIRD_DOMAIN is used as-is, + # otherwise we prompt, or abort when there is no terminal to prompt on. if ! check_nb_domain "$NETBIRD_DOMAIN"; then + if ! tty_available; then + echo "NETBIRD_DOMAIN is required for a non-interactive install." > /dev/stderr + exit 1 + fi NETBIRD_DOMAIN=$(read_nb_domain) fi @@ -411,11 +471,7 @@ apply_agent_network_preset() { ENABLE_PROXY="true" ENABLE_CROWDSEC="false" - if [[ -n "${NETBIRD_LETSENCRYPT_EMAIL}" ]]; then - TRAEFIK_ACME_EMAIL="${NETBIRD_LETSENCRYPT_EMAIL}" - else - TRAEFIK_ACME_EMAIL=$(read_traefik_acme_email) - fi + TRAEFIK_ACME_EMAIL=$(resolve NETBIRD_LETSENCRYPT_EMAIL required read_traefik_acme_email) echo "" > /dev/stderr echo "Agent-network preset enabled (NETBIRD_AGENT_NETWORK=true):" > /dev/stderr @@ -437,35 +493,35 @@ configure_reverse_proxy() { return 0 fi - # Prompt for reverse proxy type - REVERSE_PROXY_TYPE=$(read_reverse_proxy_type) + # Reverse proxy type (env NETBIRD_REVERSE_PROXY_TYPE, else prompt, else 0) + REVERSE_PROXY_TYPE=$(resolve NETBIRD_REVERSE_PROXY_TYPE 0 read_reverse_proxy_type) # Handle built-in Traefik prompts (option 0) if [[ "$REVERSE_PROXY_TYPE" == "0" ]]; then - TRAEFIK_ACME_EMAIL=$(read_traefik_acme_email) - ENABLE_PROXY=$(read_enable_proxy) + TRAEFIK_ACME_EMAIL=$(resolve NETBIRD_LETSENCRYPT_EMAIL required read_traefik_acme_email) + ENABLE_PROXY=$(resolve NETBIRD_ENABLE_PROXY false read_enable_proxy) if [[ "$ENABLE_PROXY" == "true" ]]; then - ENABLE_CROWDSEC=$(read_enable_crowdsec) + ENABLE_CROWDSEC=$(resolve NETBIRD_ENABLE_CROWDSEC false read_enable_crowdsec) fi fi # Handle external Traefik-specific prompts (option 1) if [[ "$REVERSE_PROXY_TYPE" == "1" ]]; then - TRAEFIK_EXTERNAL_NETWORK=$(read_traefik_network) - TRAEFIK_ENTRYPOINT=$(read_traefik_entrypoint) - TRAEFIK_CERTRESOLVER=$(read_traefik_certresolver) + TRAEFIK_EXTERNAL_NETWORK=$(resolve NETBIRD_TRAEFIK_EXTERNAL_NETWORK "" read_traefik_network) + TRAEFIK_ENTRYPOINT=$(resolve NETBIRD_TRAEFIK_ENTRYPOINT websecure read_traefik_entrypoint) + TRAEFIK_CERTRESOLVER=$(resolve NETBIRD_TRAEFIK_CERTRESOLVER "" read_traefik_certresolver) fi # Handle port binding for external proxy options (2-5) if [[ "$REVERSE_PROXY_TYPE" -ge 2 ]]; then - BIND_LOCALHOST_ONLY=$(read_port_binding_preference) + BIND_LOCALHOST_ONLY=$(resolve NETBIRD_BIND_LOCALHOST_ONLY true read_port_binding_preference) fi # Handle Docker network prompts for external proxies (options 2-4) case "$REVERSE_PROXY_TYPE" in - 2) EXTERNAL_PROXY_NETWORK=$(read_proxy_docker_network "Nginx") ;; - 3) EXTERNAL_PROXY_NETWORK=$(read_proxy_docker_network "Nginx Proxy Manager") ;; - 4) EXTERNAL_PROXY_NETWORK=$(read_proxy_docker_network "Caddy") ;; + 2) EXTERNAL_PROXY_NETWORK=$(resolve NETBIRD_EXTERNAL_PROXY_NETWORK "" read_proxy_docker_network "Nginx") ;; + 3) EXTERNAL_PROXY_NETWORK=$(resolve NETBIRD_EXTERNAL_PROXY_NETWORK "" read_proxy_docker_network "Nginx Proxy Manager") ;; + 4) EXTERNAL_PROXY_NETWORK=$(resolve NETBIRD_EXTERNAL_PROXY_NETWORK "" read_proxy_docker_network "Caddy") ;; *) ;; # No network prompt for other options esac return 0 @@ -643,8 +699,13 @@ start_services_and_show_instructions() { print_post_setup_instructions echo "" - echo -n "Press Enter when your reverse proxy is configured (or Ctrl+C to exit)... " - read -r < /dev/tty + if tty_available; then + echo -n "Press Enter when your reverse proxy is configured (or Ctrl+C to exit)... " + read -r < /dev/tty + else + echo "Non-interactive mode: starting NetBird containers now. Finish configuring" + echo "your reverse proxy using the instructions above so it can reach them." + fi echo -e "$MSG_STARTING_SERVICES" $DOCKER_COMPOSE_COMMAND up -d diff --git a/infrastructure_files/migrate-to-enterprise.sh b/infrastructure_files/migrate-to-enterprise.sh index 8e1fcf521..e2713c902 100755 --- a/infrastructure_files/migrate-to-enterprise.sh +++ b/infrastructure_files/migrate-to-enterprise.sh @@ -173,11 +173,11 @@ EOF # --------------------------------------------------------------------------- detect_combined_service() { - yq eval '.services | to_entries | map(select(.value.image | test("^netbirdio/netbird-server"))) | .[0].key // ""' "$COMPOSE_FILE" + yq eval '.services | to_entries | map(select(.value.image | test("^(ghcr\\.io/)?netbirdio/netbird-server([:@]|$)"))) | .[0].key // ""' "$COMPOSE_FILE" } detect_dashboard_service() { - yq eval '.services | to_entries | map(select(.value.image | test("^netbirdio/dashboard"))) | .[0].key // ""' "$COMPOSE_FILE" + yq eval '.services | to_entries | map(select(.value.image | test("^(ghcr\\.io/)?netbirdio/dashboard([:@]|$)"))) | .[0].key // ""' "$COMPOSE_FILE" } detect_config_yaml_host_path() { @@ -221,9 +221,6 @@ render_override() { # Remove this file (and config.yaml.enterprise if present) to revert. services: - ${DASHBOARD_SERVICE}: - image: \${NETBIRD_DASHBOARD_IMAGE:-ghcr.io/netbirdio/dashboard-cloud:latest} - ${COMBINED_SERVICE}: image: \${NETBIRD_SERVER_IMAGE:-ghcr.io/netbirdio/netbird-server-cloud:latest} environment: @@ -661,12 +658,12 @@ init_migration() { COMPOSE_NETWORK=$(detect_compose_network) if [[ -z "$COMBINED_SERVICE" ]]; then - echo "Could not find a service running netbirdio/netbird-server* in $COMPOSE_FILE." > /dev/stderr + echo "Could not find a service running netbirdio/netbird-server or ghcr.io/netbirdio/netbird-server in $COMPOSE_FILE." > /dev/stderr echo "This script targets the community combined-server deployment." > /dev/stderr exit 1 fi if [[ -z "$DASHBOARD_SERVICE" ]]; then - echo "Could not find a service running netbirdio/dashboard* in $COMPOSE_FILE." > /dev/stderr + echo "Could not find a service running netbirdio/dashboard or ghcr.io/netbirdio/dashboard in $COMPOSE_FILE." > /dev/stderr exit 1 fi if [[ -z "$CONFIG_YAML_HOST" ]]; then diff --git a/infrastructure_files/observability/grafana/dashboards/management-enterprise.json b/infrastructure_files/observability/grafana/dashboards/management-enterprise.json new file mode 100644 index 000000000..481050a02 --- /dev/null +++ b/infrastructure_files/observability/grafana/dashboards/management-enterprise.json @@ -0,0 +1,8857 @@ +{ + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "builtIn": true, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "query": { + "datasource": { + "name": "-- Grafana --" + }, + "group": "grafana", + "kind": "DataQuery", + "spec": {}, + "version": "v0" + } + } + }, + { + "kind": "AnnotationQuery", + "spec": { + "enable": true, + "hide": false, + "iconColor": "red", + "name": "Deployments", + "query": { + "datasource": { + "name": "grafana" + }, + "group": "datasource", + "kind": "DataQuery", + "spec": { + "expr": "", + "interval": "", + "refId": "Anno", + "tags": [], + "type": "tags" + }, + "version": "v0" + } + } + } + ], + "cursorSync": "Crosshair", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, sum by(le) (rate(management_grpc_updatechannel_queue_length_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Connected grpc streams", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p99 of the update-channel queue length: how many network map updates are waiting to be delivered to a peer. The channel buffer is 100 messages - approaching it means updates start getting dropped and peers fall behind.", + "id": 1, + "links": [], + "title": "UpdateChannel Queue length max", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-10": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_total_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\", type=\"read\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HTTP read duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of read API requests (GET/OPTIONS), from the total request duration histogram tagged type=read.", + "id": 10, + "links": [], + "title": "Read HTTP API Requests p95 Duration (GET/OPTIONS)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-11": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_updatechannel_close_one_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "CloseOne", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(rate(management_updatechannel_send_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "Send", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(rate(management_updatechannel_create_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "Create", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_updatechannel_get_all_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GetAll", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_updatechannel_haschannel_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HasChannel", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_updatechannel_close_multiple_channels_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "CloseMultiple", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "F" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate per minute of update-channel operations: close one, send update, create, get all connected peers, has-channel and close multiple. Together they show how much channel churn an instance is handling.", + "id": 11, + "links": [], + "title": "Update Channel operations", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 69, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-12": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_updatechannel_create_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\", closed=\"true\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_updatechannel_create_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "__expr__" + }, + "group": "__expr__", + "kind": "DataQuery", + "spec": { + "expression": "100-$A/$B*100", + "type": "math" + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Tracks how often a new peer update channel replaces a channel that was still open (closed=true), i.e. peers reconnecting before management noticed the previous stream was gone. The plotted expression is 100 - recreated/total, so the line drops as recreations rise.", + "id": 12, + "links": [], + "title": "Percentage of Recreated channels", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "displayName", + "value": "Recreation" + } + ] + } + ] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-13": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_close_one_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "CloseOne", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_send_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "Send", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_create_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "Create", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_get_all_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GetAll", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_haschannel_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "CloseMultiple", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_close_multiple_channels_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HasChannel", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "F" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of each update-channel operation, in microseconds. These are in-memory operations behind a lock, so growth here means lock contention rather than I/O.", + "id": 13, + "links": [], + "title": "Update Channel methods p95 Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-14": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum(management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Connected grpc streams", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Total number of peers currently holding an open gRPC Sync stream across all management instances (management.grpc.connected.streams). A sudden drop means peers were disconnected and will come back as a Login/Sync reconnect storm.", + "id": 14, + "links": [], + "title": "Connected peers", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "red", + "value": 0 + }, + { + "color": "#EAB839", + "value": 8000 + }, + { + "color": "green", + "value": 11000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": true, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-15": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "sum by(instance) (management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"})", + "instant": false, + "legendFormat": "Total", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Open peer Sync streams over time, in total per instance (A) and as raw series (B). Steps down mark instance restarts or LB rebalancing; the ramp back up is the reconnect wave.", + "id": 15, + "links": [], + "title": "Connected peers historical", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-16": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_sync_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Sync duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of the Sync gRPC request - the time management needs to authenticate the peer, build its first network map and open the update channel. Primary latency SLI for peer connects.", + "id": 16, + "links": [], + "title": "gRPC Sync Request p95 Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-17": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_login_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Login duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of the Login gRPC request (peer authentication plus initial configuration and relay credentials). Rises when the IdP, the store or account locking is slow.", + "id": 17, + "links": [], + "title": "gRPC Login Request p95 Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-18": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_account_get_peer_network_map_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken to build the network map returned to a single peer, in milliseconds.", + "id": 18, + "links": [], + "title": "GetPeerNetworkMap Latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-19": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(le) (increase(management_account_network_map_object_count_objects_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the number of objects (peers, routes, firewall rules, DNS entries and so on) in the network maps sent to clients. Large maps make both calculation and serialization more expensive.", + "id": 19, + "links": [], + "title": "NetworkMap Objects", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 0, + "reverse": false, + "unit": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "login", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "sync", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (rate(management_grpc_key_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "get key", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of the three peer-facing gRPC calls: Login (authenticate, return initial config and relay credentials), Sync (open the update channel and receive network map updates) and GetServerKey (fetch the server WireGuard public key). Elevated Login/Sync rates usually mean peers are reconnecting in a loop.", + "id": 2, + "links": [], + "title": "gRPC Requests", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 51, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": true, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": {}, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-20": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_updatechannel_tosyncresponse_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken to convert a calculated network map into the gRPC SyncResponse sent to the peer, in microseconds.", + "id": 20, + "links": [], + "title": "To Sync Response", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "µs" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-23": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(le) (increase(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of per-account p95 Sync durations - each sample is one account's p95, flushed every 60s. Shows whether slow Syncs are spread across the fleet or concentrated in a few accounts.", + "id": 23, + "links": [], + "title": "GRPC Sync Latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.6, + "fill": "dark-red", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-24": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_grpc_login_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of Login gRPC request durations across all peers. The right tail is what a peer experiences on startup; samples beyond 7s are what the high-latency counters in the Debugging row track.", + "id": 24, + "links": [], + "title": "GRPC Login Latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-25": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by(environment) (rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Total", + "range": true + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Login requests, per instance (A) and in total (B). Login is the most expensive peer call (IdP plus store writes); a sustained high rate means clients are restarting or failing to hold their Sync stream.", + "id": 25, + "links": [], + "title": "gRPC Login Request rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-26": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by(environment) (rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Total", + "range": true + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Sync requests, per instance (A) and in total (B). Every Sync opens or re-opens a peer update channel, so a spike here is a reconnect storm and predicts CPU spent on network map calculation.", + "id": 26, + "links": [], + "title": "gRPC Sync Request rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-27": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_grpc_updatechannel_queue_length_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of update-channel queue lengths across all peers. Mass moving to the right shows a growing backlog of undelivered network map updates.", + "id": 27, + "links": [], + "title": "Update Channel heat map", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Greens", + "steps": 73 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-29": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(endpoint) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval])*60) > 10", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{endpoint}} > 10", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Total", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Successful (2xx) GET requests per minute broken down by endpoint, limited to endpoints above 10 rpm. Shows which read endpoints dashboard and API clients hit hardest.", + "id": 29, + "links": [], + "title": "HTTP Read request counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-30": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(endpoint) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60) > 1", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{endpoint}} > 1", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Total", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Successful (2xx) non-GET/OPTIONS requests per minute by endpoint, limited to endpoints above 1 rpm. Writes take account locks and fan out network map updates, so read this together with the Network Map row.", + "id": 30, + "links": [], + "title": "HTTP Write request counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-31": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_store_persistence_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of how long it takes to save or delete an account in the store (management.store.persistence.duration.ms). Widening buckets point at database write pressure.", + "id": 31, + "links": [], + "title": "Store Persistence latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.6, + "fill": "dark-red", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Reds", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-32": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_store_transaction_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of store transaction execution time. Nearly every API and gRPC call runs through a transaction, so this is the first place to look when latency rises everywhere at once.", + "id": 32, + "links": [], + "title": "Store Transaction latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.6, + "fill": "dark-red", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Reds", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-33": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_account_peer_meta_update_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{endpoint}} > 10", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate per minute of peers reporting changed metadata (OS, version, hostname, addresses). Each change persists the peer and can trigger an account peers update.", + "id": 33, + "links": [], + "title": "Peer meta updates counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-34": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_grpc_sync_request_blocked_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Sync request rate", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Sync requests rejected because the peer is blocked (login expired, peer disabled or account restricted). A rising line means clients retrying without valid access.", + "id": 34, + "links": [], + "title": "gRPC Sync Request Blocked rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-35": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_grpc_login_request_blocked_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Login request rate", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Login requests rejected because the peer is blocked. Persistent volume here is usually expired peer logins retrying in a loop.", + "id": 35, + "links": [], + "title": "gRPC Login Request Blocked rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-36": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_account_update_account_peers_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken by an account peers update: preparing the data and pushing a fresh network map to every peer in the account, in milliseconds. Scales with account size.", + "id": 36, + "links": [], + "title": "UpdateAccountPeers Latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-37": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_updatechannel_calc_networkmap_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken to calculate one peer's network map, in milliseconds. This is the dominant CPU cost when many peers reconnect at once.", + "id": 37, + "links": [], + "title": "Network Map Calculation", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-39": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_updatechannel_calc_posturechecks_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken to evaluate a peer's posture checks while building its network map, in microseconds.", + "id": 39, + "links": [], + "title": "Posture Check Calculation", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "µs" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-45": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "sort_desc(\n topk(\n 10,\n sum by (account_id) (\n increase(management_grpc_login_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n )\n )\n)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [ + { + "group": "organize", + "kind": "Transformation", + "spec": { + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Time": "" + } + } + } + } + ] + } + }, + "description": "Top 10 accounts by number of Login requests that exceeded the 7s high-latency threshold.", + "id": 45, + "links": [], + "title": "High Login Latency Ranking (login count per account)", + "vizConfig": { + "group": "table", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "footer": { + "reducers": [] + }, + "inspect": false, + "tooltip": { + "placement": "auto" + }, + "wrapHeaderText": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Time" + } + ] + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-46": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "sort_desc(\n topk(\n 10,\n sum by (account_id) (\n increase(management_grpc_sync_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n )\n )\n)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [ + { + "group": "organize", + "kind": "Transformation", + "spec": { + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Time": "" + } + } + } + } + ] + } + }, + "description": "Top 10 accounts by number of Sync requests that exceeded the 7s high-latency threshold. Use it to find which tenant is behind a latency spike.", + "id": 46, + "links": [], + "title": "High Sync Latency Ranking (sync count per account)", + "vizConfig": { + "group": "table", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "footer": { + "reducers": [] + }, + "inspect": false, + "tooltip": { + "placement": "auto" + }, + "wrapHeaderText": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Time" + } + ] + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-47": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "count(\n sum by (account_id) (\n increase(management_grpc_sync_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n ) > 1\n)", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of distinct accounts with more than one slow (>7s) Sync in the interval - tells you whether a latency spike is fleet-wide or limited to a few tenants.", + "id": 47, + "links": [], + "title": "Accounts with high Sync latency", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-48": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "count(\n sum by (account_id) (\n increase(management_grpc_login_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n ) > 1\n)", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of distinct accounts with more than one slow (>7s) Login in the interval.", + "id": 48, + "links": [], + "title": "Accounts with high Login latency", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-50": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${logs_datasource}" + }, + "group": "loki", + "kind": "DataQuery", + "spec": { + "direction": "backward", + "editorMode": "code", + "expr": "sort_desc(\ntopk(\n 20,\n sum by (peerID) (\n count_over_time(\n {job=\"$log_job\"} |= \"Sync took\" | regexp `peerID:\\s(?P[^,\\]]+)` [$__range]\n )\n )\n)\n)", + "queryType": "instant" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [ + { + "group": "organize", + "kind": "Transformation", + "spec": { + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value #A": "number of times ", + "peerID": "peerPubKey" + } + } + } + } + ] + } + }, + "description": "Top 20 peers by number of Sync log lines in the selected range, parsed out of the management logs to find individual peers stuck in a reconnect loop. Pick the log stream with the `log stream` variable; the query matches the `Sync took` debug line, so management must run at debug level for this panel to have data.", + "id": 50, + "links": [], + "title": "Highest Syncs by peer key", + "vizConfig": { + "group": "table", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "footer": { + "reducers": [] + }, + "inspect": false, + "tooltip": { + "placement": "auto" + }, + "wrapHeaderText": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "labels" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "peerPubKey" + }, + "properties": [ + { + "id": "custom.width", + "value": 455 + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "number of times " + } + ] + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-52": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "wsproxy_active_connections{job=~\"$job\", instance=~\"$instance\"}", + "legendFormat": "{{__name__}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Currently open WebSocket proxy connections. The wsproxy fronts gRPC for browser-based clients, so this is the number of connected web clients.", + "id": 52, + "links": [], + "title": "Proxy Active Connections", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-53": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "delta(wsproxy_bytes_transferred_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])", + "legendFormat": "{{direction}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Bytes transferred through the WebSocket proxy per interval, split by direction.", + "id": 53, + "links": [], + "title": "Proxy Traffic by Direction", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-54": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "increase(wsproxy_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{error_type}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "WebSocket proxy errors grouped by error type - dial failures, read/write errors and protocol issues on browser client connections.", + "id": 54, + "links": [], + "title": "Errors by Type", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-65": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "exemplar": false, + "expr": "management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "legendFormat": "{{host}}", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Open peer Sync streams per management instance. Use it to confirm the load balancer spreads streams evenly - a skewed instance carries proportionally more update-channel and network map work.", + "id": 65, + "links": [], + "title": "Connected Peers per Server", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-66": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(host) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval]) * 60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Successful GET requests per minute grouped by management instance - read load distribution across the fleet.", + "id": 66, + "links": [], + "title": "HTTP Read request counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-67": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(host) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Successful write requests per minute grouped by management instance - write load distribution across the fleet.", + "id": 67, + "links": [], + "title": "HTTP Write request counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_total_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\", type=\"write\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HTTP write duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of write API requests (PUT/POST/DELETE), tagged type=write. Writes acquire account locks and trigger peer updates, so they are normally slower than reads.", + "id": 7, + "links": [], + "title": "Write HTTP API Requests p95 Duration (PUT/POST/DELETE)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-74": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_pat_usage_distribution_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of personal access token usage counts, sampled once per minute per token. Mass in the high buckets means a single PAT is hammering the API.", + "id": 74, + "links": [], + "title": "PAT usage counter", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-76": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "gnatsd_connz_subscriptions{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "{{host}}", + "range": false, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Current subscriptions held by each client connected to NATS (connz). Signal instances subscribe per connected peer, so this tracks how peers are spread across signal nodes.", + "id": 76, + "links": [], + "title": "Subscriptions per node", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-77": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "gnatsd_connz_subscriptions{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Subscriptions per connected NATS client over time. Steps mark signal instances restarting and re-subscribing their peers.", + "id": 77, + "links": [], + "title": "Subscriptions per node", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-78": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "gnatsd_connz_num_connections{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of client connections on each NATS node - effectively how many signal instances are attached to that node.", + "id": 78, + "links": [], + "title": "Signal nodes connected per NATs", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-79": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(environment) (rate(gnatsd_connz_out_msgs{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of messages NATS delivers out to its clients: the peer signalling throughput flowing through the cluster.", + "id": 79, + "links": [], + "title": "Messages per Second", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HTTP request duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of incoming REST API requests by endpoint and method (management.http.request.duration.ms). Covers dashboard and API clients only - peer traffic is on the gRPC panels.", + "id": 8, + "links": [], + "title": "HTTP API Requests p95 Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-80": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(environment) (rate(gnatsd_connz_out_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of bytes NATS delivers out to its clients - signalling bandwidth, useful next to the message rate to spot unusually large payloads.", + "id": 80, + "links": [], + "title": "Data per Second", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-81": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(environment) (increase(gnatsd_varz_slow_consumers{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Increase of the NATS slow-consumer counter. Anything above zero means a signal node could not read fast enough and NATS dropped its connection, so peers lose signalling until it reconnects.", + "id": 81, + "links": [], + "title": "Slow consumers detected", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-82": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "gnatsd_varz_subscriptions{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Total subscriptions registered on each NATS server (varz). Grows roughly with the number of connected peers.", + "id": 82, + "links": [], + "title": "Total number of NATS subscriptions", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-83": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{instance}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Resident memory of the scraped processes. process_resident_memory_bytes is exposed by every Go/Prometheus process, so narrow the job and instance variables to the NATS nodes to read this panel as NATS memory.", + "id": 83, + "links": [], + "title": "NATS memory consumption", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-84": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "idelta(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[2m])", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Per-account Sync p95 samples as a heatmap (2m deltas). Use it to spot a handful of large or slow accounts pulling the overall Sync latency up.", + "id": 84, + "links": [], + "title": "GRPC Sync Latency By Account", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.6, + "fill": "dark-red", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-85": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "idelta(management_grpc_login_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[2m])", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Per-account Login p95 samples as a heatmap (2m deltas). Highlights accounts whose logins are much slower than the fleet average.", + "id": 85, + "links": [], + "title": "GRPC Login Latency By Account", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-86": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Sync duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 of the per-account Sync p95 distribution - the latency seen by the worst-served accounts rather than by the average request. Diverges from the global p95 when only a few accounts are hurting.", + "id": 86, + "links": [], + "title": "gRPC Sync Request p95 Duration (by Account)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-87": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_login_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Login duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 of the per-account Login p95 distribution - login latency for the worst-served accounts rather than the average request.", + "id": 87, + "links": [], + "title": "gRPC Login Request p95 Duration (by Account)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-88": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{environment}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Account peers updates triggered per minute (2m increase halved). Each trigger recalculates and pushes network maps to every peer in the account.", + "id": 88, + "links": [], + "title": "NetworkMap Triggers Total", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 19, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-89": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(resource) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Account peers updates over 2m intervals, grouped by the resource that triggered them - the time series behind the pie chart above.", + "id": 89, + "links": [], + "title": "NetworkMap Triggers By Source", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.9, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_idp_authenticate_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "authenticate (mgtm->auth0)", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_idp_update_user_meta_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "update user meta", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_idp_get_account_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "get account", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Requests per minute that management sends to the configured identity provider: authenticate (service token refresh), user metadata updates and account lookups. IdP throttling shows up here before it shows up as Login latency.", + "id": 9, + "links": [], + "title": "IdP Requests", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 69, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-90": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "sum by(resource) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[1h]))", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Account peers updates over the last hour, grouped by the resource that triggered them (policy, group, peer, route, user, network, ...). Shows which kind of configuration change generates the most fan-out work.", + "id": 90, + "links": [], + "title": "NetworkMap Triggers by source in last hour", + "vizConfig": { + "group": "piechart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#73BF69", + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + } + }, + "overrides": [] + }, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-91": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(resource) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[1h]))", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Network maps computed in the last hour, grouped by the resource whose change triggered the computation.", + "id": 91, + "links": [], + "title": "NetworkMaps by source in last hour", + "vizConfig": { + "group": "piechart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#73BF69", + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + } + }, + "overrides": [] + }, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-92": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{environment}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Network maps computed per minute (2m increase halved). One account peers update fans out into one map per peer, so this - not the trigger count - is the real CPU driver.", + "id": 92, + "links": [], + "title": "NetworkMaps Total", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 19, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-93": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(resource) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{resource}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Network maps computed per minute, grouped by the resource that triggered the computation.", + "id": 93, + "links": [], + "title": "NetworkMaps Total By Source", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.9, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-14" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-65" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-15" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-2" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-23" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-24" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-16" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-17" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-84" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-85" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-86" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-87" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-26" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 48 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-25" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 48 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-34" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 56 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-35" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 56 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-31" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 64 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-32" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 64 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-9" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 72 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-8" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 72 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-29" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 80 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-30" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 80 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-66" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 88 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-67" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 88 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-10" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 96 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-7" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 96 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-33" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 104 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-74" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 104 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-11" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 112 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-13" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 112 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-12" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 120 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-1" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 120 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-27" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 128 + } + } + ] + } + }, + "title": "General" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-76" + }, + "height": 8, + "width": 24, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-77" + }, + "height": 8, + "width": 24, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-78" + }, + "height": 8, + "width": 24, + "x": 0, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-79" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-80" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-81" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-82" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-83" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 40 + } + } + ] + } + }, + "title": "NATS" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-46" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-45" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-47" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-48" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-50" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 16 + } + } + ] + } + }, + "title": "Debugging" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-37" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-20" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-39" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-36" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-18" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-19" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-90" + }, + "height": 19, + "width": 6, + "x": 0, + "y": 48 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-88" + }, + "height": 10, + "width": 17, + "x": 6, + "y": 48 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-89" + }, + "height": 9, + "width": 17, + "x": 6, + "y": 58 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-91" + }, + "height": 19, + "width": 6, + "x": 0, + "y": 67 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-92" + }, + "height": 10, + "width": 17, + "x": 6, + "y": 67 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-93" + }, + "height": 9, + "width": 17, + "x": 6, + "y": 77 + } + } + ] + } + }, + "title": "Network Map" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-52" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-53" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-54" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + } + ] + } + }, + "title": "Browser Client" + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [], + "timeSettings": { + "autoRefresh": "auto", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "fiscalYearStartMonth": 0, + "from": "now-1h", + "hideTimepicker": false, + "timezone": "browser", + "to": "now" + }, + "title": "Management generic", + "variables": [ + { + "kind": "DatasourceVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "description": "Prometheus data source scraping the management metrics.", + "hide": "dontHide", + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "pluginId": "prometheus", + "refresh": "onDashboardLoad", + "regex": "", + "skipUrlSync": false + } + }, + { + "kind": "DatasourceVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "description": "Loki data source holding the management logs.", + "hide": "dontHide", + "includeAll": false, + "label": "Logs data source", + "multi": false, + "name": "logs_datasource", + "options": [], + "pluginId": "loki", + "refresh": "onDashboardLoad", + "regex": "", + "skipUrlSync": false + } + }, + { + "kind": "QueryVariable", + "spec": { + "allValue": ".*", + "allowCustomValue": true, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "definition": "label_values(job)", + "description": "Prometheus scrape job(s) exposing the management metrics.", + "hide": "dontHide", + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [], + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allValue": ".*", + "allowCustomValue": true, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "definition": "label_values(up{job=~\"$job\"},instance)", + "description": "Instance(s) of the selected job(s). Depends on the job variable.", + "hide": "dontHide", + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(up{job=~\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "definition": "label_values(job)", + "description": "Loki stream carrying the management logs (adjust the label if your setup does not use `job`).", + "hide": "dontHide", + "includeAll": false, + "label": "log stream", + "multi": false, + "name": "log_job", + "options": [], + "query": { + "datasource": { + "name": "${logs_datasource}" + }, + "group": "loki", + "kind": "DataQuery", + "spec": { + "label": "job", + "refId": "LokiVariableQueryEditor-VariableQuery", + "type": 1 + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + }, + { + "kind": "CustomVariable", + "spec": { + "allValue": "5m", + "allowCustomValue": true, + "current": { + "text": "60s", + "value": "60s" + }, + "hide": "dontHide", + "includeAll": false, + "label": "interval", + "multi": false, + "name": "interval", + "options": [], + "query": "5m,60s", + "skipUrlSync": false, + "valuesFormat": "csv" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(management_store_query_count_total,method)", + "hide": "hideVariable", + "includeAll": true, + "multi": true, + "name": "method", + "options": [], + "query": { + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(management_store_query_count_total,method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "disabled" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "definition": "label_values(aws_rds_cpuutilization_average,dimension_DBClusterIdentifier)", + "hide": "hideVariable", + "includeAll": true, + "multi": true, + "name": "dimension_DBClusterIdentifier", + "options": [], + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(aws_rds_cpuutilization_average,dimension_DBClusterIdentifier)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "disabled" + } + } + ] +} diff --git a/management/internals/controllers/network_map/controller/controller.go b/management/internals/controllers/network_map/controller/controller.go index 5785004db..356dc9f67 100644 --- a/management/internals/controllers/network_map/controller/controller.go +++ b/management/internals/controllers/network_map/controller/controller.go @@ -176,6 +176,7 @@ func (c *Controller) sendUpdateAccountPeers(ctx context.Context, accountID strin semaphore := make(chan struct{}, 10) c.injectAllProxyPolicies(ctx, account) + account.PrecomputePostureValidation(ctx) dnsCache := &cache.DNSConfigCache{} dnsDomain := c.GetDNSDomain(account.Settings) peersCustomZone := account.GetPeersCustomZone(ctx, dnsDomain) @@ -357,6 +358,7 @@ func (c *Controller) sendUpdateForAffectedPeers(ctx context.Context, accountID s // network map that omitted the synth DNS zone, and the agent kept // resolving against the stale or absent record. c.injectAllProxyPolicies(ctx, account) + account.PrecomputePostureValidation(ctx) dnsCache := &cache.DNSConfigCache{} dnsDomain := c.GetDNSDomain(account.Settings) peersCustomZone := account.GetPeersCustomZone(ctx, dnsDomain) diff --git a/management/internals/modules/agentnetwork/handlers/budget_handler_test.go b/management/internals/modules/agentnetwork/handlers/budget_handler_test.go index 4038761c5..3a7709461 100644 --- a/management/internals/modules/agentnetwork/handlers/budget_handler_test.go +++ b/management/internals/modules/agentnetwork/handlers/budget_handler_test.go @@ -102,8 +102,8 @@ func TestSettingsHandler_GetExposesCollectionToggles(t *testing.T) { require.NoError(t, f.store.SaveAgentNetworkSettings(context.Background(), &agentNetworkTypes.Settings{ AccountID: testAccountID, - Cluster: "eu.proxy.netbird.io", - Subdomain: "violet", + Domain: "violet.eu.proxy.netbird.io", + ProxyAddress: "eu.proxy.netbird.io", EnableLogCollection: true, EnablePromptCollection: true, RedactPii: false, diff --git a/management/internals/modules/agentnetwork/handlers/providers_handler.go b/management/internals/modules/agentnetwork/handlers/providers_handler.go index c05363101..0d8a44ca3 100644 --- a/management/internals/modules/agentnetwork/handlers/providers_handler.go +++ b/management/internals/modules/agentnetwork/handlers/providers_handler.go @@ -155,12 +155,7 @@ func (h *handler) createProvider(w http.ResponseWriter, r *http.Request) { provider := types.NewProvider(userAuth.AccountId) provider.FromAPIRequest(&req) - bootstrapCluster := "" - if req.BootstrapCluster != nil { - bootstrapCluster = *req.BootstrapCluster - } - - created, err := h.manager.CreateProvider(r.Context(), userAuth.UserId, provider, bootstrapCluster) + created, err := h.manager.CreateProvider(r.Context(), userAuth.UserId, provider) if err != nil { util.WriteError(r.Context(), err, w) return diff --git a/management/internals/modules/agentnetwork/handlers/settings_handler.go b/management/internals/modules/agentnetwork/handlers/settings_handler.go index 171750838..0a365f9ce 100644 --- a/management/internals/modules/agentnetwork/handlers/settings_handler.go +++ b/management/internals/modules/agentnetwork/handlers/settings_handler.go @@ -12,13 +12,55 @@ import ( "github.com/netbirdio/netbird/shared/management/http/util" ) -// addSettingsEndpoints registers the Agent Network settings routes. The -// settings row is bootstrapped server-side on first provider create or on the -// first PUT carrying a cluster; GET reads it and PUT applies a partial update -// of the mutable collection toggles (cluster/subdomain stay immutable). +// addSettingsEndpoints registers the Agent Network settings routes. POST +// bootstraps the settings row, assigning the account's immutable endpoint; +// GET reads it (defaults with an empty endpoint before bootstrap); PUT +// carries every field, replacing the mutable collection toggles and rejecting +// any change to the identity fields; DELETE removes the row — guarded so it +// stays a bootstrap-repair operation — releasing the endpoint for a fresh +// bootstrap. func (h *handler) addSettingsEndpoints(router *mux.Router) { router.HandleFunc("/agent-network/settings", h.getSettings).Methods("GET", "OPTIONS") + router.HandleFunc("/agent-network/settings", h.createSettings).Methods("POST", "OPTIONS") router.HandleFunc("/agent-network/settings", h.updateSettings).Methods("PUT", "OPTIONS") + router.HandleFunc("/agent-network/settings", h.deleteSettings).Methods("DELETE", "OPTIONS") +} + +// createSettings bootstraps the account's settings row. Exactly one of +// proxy_address (labeled endpoint; the server allocates the label) and +// endpoint (self-addressed, claimed verbatim) must be provided; optional +// collection toggles ride along with defaults for omitted fields. +func (h *handler) createSettings(w http.ResponseWriter, r *http.Request) { + userAuth, err := nbcontext.GetUserAuthFromContext(r.Context()) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + + var req api.AgentNetworkSettingsCreateRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + util.WriteErrorResponse("couldn't parse JSON request", http.StatusBadRequest, w) + return + } + + settings := types.DefaultSettings(userAuth.AccountId) + settings.FromAPICreateRequest(&req) + + proxyAddress := "" + if req.ProxyAddress != nil { + proxyAddress = *req.ProxyAddress + } + endpoint := "" + if req.Endpoint != nil { + endpoint = *req.Endpoint + } + + created, err := h.manager.CreateSettings(r.Context(), userAuth.UserId, settings, proxyAddress, endpoint) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + util.WriteJSONObject(r.Context(), w, created.ToAPIResponse()) } // updateSettings replaces the mutable settings fields on the account's row. @@ -48,6 +90,24 @@ func (h *handler) updateSettings(w http.ResponseWriter, r *http.Request) { util.WriteJSONObject(r.Context(), w, updated.ToAPIResponse()) } +// deleteSettings removes the account's settings row, releasing the endpoint. +// The manager refuses (412) while providers exist or a proxy is actively +// serving the endpoint; a later POST bootstraps fresh, allocating a new +// endpoint. +func (h *handler) deleteSettings(w http.ResponseWriter, r *http.Request) { + userAuth, err := nbcontext.GetUserAuthFromContext(r.Context()) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + + if err := h.manager.DeleteSettings(r.Context(), userAuth.AccountId, userAuth.UserId); err != nil { + util.WriteError(r.Context(), err, w) + return + } + util.WriteJSONObject(r.Context(), w, util.EmptyObject{}) +} + // getSettings returns the account's agent-network settings. Accounts that // haven't been bootstrapped yet read as the defaults with an empty cluster, // subdomain and endpoint; the manager synthesises that view. diff --git a/management/internals/modules/agentnetwork/handlers/settings_handler_test.go b/management/internals/modules/agentnetwork/handlers/settings_handler_test.go index 636ec5b26..400208e1c 100644 --- a/management/internals/modules/agentnetwork/handlers/settings_handler_test.go +++ b/management/internals/modules/agentnetwork/handlers/settings_handler_test.go @@ -1,20 +1,25 @@ package handlers import ( + "context" "encoding/json" + "fmt" "net/http" + "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + rpproxy "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy" "github.com/netbirdio/netbird/shared/management/http/api" ) // TestSettingsHandler_GetUnbootstrappedReturnsDefaults pins the settings-read // convention shared with the account and DNS settings endpoints: settings // always read as a JSON object. Before bootstrap that object carries the -// defaults with an empty cluster/subdomain/endpoint (the "not bootstrapped" +// defaults with an empty endpoint/proxy_address (the "not bootstrapped" // signal) and no timestamps — never a 404 and never the legacy null body. func TestSettingsHandler_GetUnbootstrappedReturnsDefaults(t *testing.T) { f := newAgentNetworkHandlerFixture(t) @@ -27,9 +32,9 @@ func TestSettingsHandler_GetUnbootstrappedReturnsDefaults(t *testing.T) { var got api.AgentNetworkSettings require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) - assert.Empty(t, got.Cluster, "cluster must be empty until bootstrapped") - assert.Empty(t, got.Subdomain, "subdomain must be empty until bootstrapped") - assert.Empty(t, got.Endpoint, "endpoint must be empty until bootstrapped, not a bare dot") + assert.Empty(t, got.Endpoint, "endpoint must be empty until bootstrapped") + assert.Empty(t, got.ProxyAddress, "proxy address must be empty until bootstrapped") + assert.False(t, got.Dedicated, "an unbootstrapped account has no serving shape") assert.True(t, got.EnableLogCollection, "defaults must show log collection on, matching bootstrap") assert.False(t, got.EnablePromptCollection, "defaults must show prompt collection off") assert.False(t, got.RedactPii, "defaults must show redaction off") @@ -39,62 +44,149 @@ func TestSettingsHandler_GetUnbootstrappedReturnsDefaults(t *testing.T) { assert.Nil(t, got.UpdatedAt, "no timestamps before a row exists") } -// TestSettingsHandler_PutBootstrapsWithCluster covers the settings-first -// bootstrap path: a PUT carrying a cluster on an unbootstrapped account -// creates the row (cluster pinned, subdomain assigned) and applies the -// mutable fields from the same request. -func TestSettingsHandler_PutBootstrapsWithCluster(t *testing.T) { +// TestSettingsHandler_PostBootstrapsLabeled covers the labeled bootstrap +// shape: a POST carrying a proxy_address allocates a label beneath it, so the +// endpoint hangs one label under the shared cluster's address and the pin is +// not dedicated. Toggles riding along apply; omitted ones keep defaults. +func TestSettingsHandler_PostBootstrapsLabeled(t *testing.T) { f := newAgentNetworkHandlerFixture(t) - rec := f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "eu.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": true, "redact_pii": false, "access_log_retention_days": 30}`) - require.Equal(t, http.StatusOK, rec.Code, "bootstrap PUT must succeed: %s", rec.Body.String()) + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "enable_prompt_collection": true, "access_log_retention_days": 14}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) var got api.AgentNetworkSettings require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) - assert.Equal(t, "eu.proxy.netbird.io", got.Cluster, "cluster must be pinned from the request") - assert.NotEmpty(t, got.Subdomain, "subdomain must be assigned at bootstrap") - assert.Equal(t, got.Subdomain+".eu.proxy.netbird.io", got.Endpoint, "endpoint must combine subdomain and cluster") - assert.True(t, got.EnableLogCollection, "toggle from the bootstrap request must apply") + assert.Equal(t, "eu.proxy.netbird.io", got.ProxyAddress, "proxy address must be pinned from the request") + require.NotEmpty(t, got.Endpoint, "endpoint must be allocated at bootstrap") + assert.True(t, strings.HasSuffix(got.Endpoint, ".eu.proxy.netbird.io"), + "labeled endpoint must hang off the proxy address: %s", got.Endpoint) + label := strings.TrimSuffix(got.Endpoint, ".eu.proxy.netbird.io") + assert.NotContains(t, label, ".", "the allocated label must be a single DNS label: %s", label) + assert.False(t, got.Dedicated, "a labeled pin is not dedicated") + assert.True(t, got.EnableLogCollection, "omitted toggle must keep its default") assert.True(t, got.EnablePromptCollection, "toggle from the bootstrap request must apply") require.NotNil(t, got.AccessLogRetentionDays) - assert.Equal(t, 30, *got.AccessLogRetentionDays, "retention from the bootstrap request must apply") + assert.Equal(t, 14, *got.AccessLogRetentionDays, "retention from the bootstrap request must apply") + assert.NotNil(t, got.CreatedAt, "a persisted row carries timestamps") // The row is now readable via GET. rec = f.do(t, http.MethodGet, "/agent-network/settings", "") require.Equal(t, http.StatusOK, rec.Code, "GET after bootstrap must succeed") + var read api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &read)) + assert.Equal(t, got.Endpoint, read.Endpoint, "GET must return the bootstrapped endpoint") } -// TestSettingsHandler_PutWithoutClusterOnUnbootstrapped pins that a PUT -// without a cluster cannot conjure a settings row out of nothing — there is -// no cluster to pin — and surfaces as 404 like the GET. -func TestSettingsHandler_PutWithoutClusterOnUnbootstrapped(t *testing.T) { +// TestSettingsHandler_PostBootstrapsSelfAddressed covers the dedicated shape: +// a POST carrying an endpoint claims the hostname verbatim, the proxy address +// equals it, and the pin reads as dedicated. The claim is legitimate before +// any proxy declares the address (address-first). +func TestSettingsHandler_PostBootstrapsSelfAddressed(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"endpoint": "Brave-Otter.Gateway.Example.com"}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + + var got api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, "brave-otter.gateway.example.com", got.Endpoint, + "endpoint must be claimed verbatim, lowercased") + assert.Equal(t, got.Endpoint, got.ProxyAddress, "self-addressed: the proxy address is the endpoint") + assert.True(t, got.Dedicated, "a self-addressed pin is dedicated") + assert.True(t, got.EnableLogCollection, "omitted toggles must keep their defaults") +} + +// TestSettingsHandler_PostRequiresExactlyOneIdentityField pins the request +// contract: proxy_address and endpoint are mutually exclusive and one is +// required — both or neither is a validation error, not a guess. +func TestSettingsHandler_PostRequiresExactlyOneIdentityField(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", `{}`) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, + "empty POST must be rejected: got %d body=%s", rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "endpoint": "brave-otter.gateway.example.com"}`) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, + "POST with both identity fields must be rejected: got %d body=%s", rec.Code, rec.Body.String()) +} + +// TestSettingsHandler_PostRejectsMalformedHostnames pins per-write input +// validation: shapes canonicalization cannot repair — trailing dots, embedded +// whitespace, empty labels — are rejected with a validation error instead of +// landing in an immutable column. +func TestSettingsHandler_PostRejectsMalformedHostnames(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + for name, body := range map[string]string{ + "trailing dot": `{"endpoint": "gateway.example.com."}`, + "leading dot": `{"endpoint": ".gateway.example.com"}`, + "inner whitespace": `{"endpoint": "gate way.example.com"}`, + "empty label": `{"proxy_address": "eu..proxy.netbird.io"}`, + } { + rec := f.do(t, http.MethodPost, "/agent-network/settings", body) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, + "%s must be rejected: got %d body=%s", name, rec.Code, rec.Body.String()) + } +} + +// TestSettingsHandler_PostConflictsOnSecondBootstrap pins that bootstrap is a +// one-time create: a second POST returns 409 and leaves the row untouched. +func TestSettingsHandler_PostConflictsOnSecondBootstrap(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", `{"proxy_address": "eu.proxy.netbird.io"}`) + require.Equal(t, http.StatusOK, rec.Code, "first bootstrap must succeed: %s", rec.Body.String()) + var first api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &first)) + + rec = f.do(t, http.MethodPost, "/agent-network/settings", `{"proxy_address": "us.proxy.netbird.io"}`) + assert.Equal(t, http.StatusConflict, rec.Code, + "second bootstrap must 409: got %d body=%s", rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodGet, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code) + var got api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, first.Endpoint, got.Endpoint, "the original endpoint must survive the rejected bootstrap") + assert.Equal(t, first.ProxyAddress, got.ProxyAddress, "the original proxy address must survive") +} + +// TestSettingsHandler_PutBeforeBootstrapIs404 pins that a PUT cannot conjure a +// settings row out of nothing — bootstrap is the explicit POST — and the +// error points the caller there. +func TestSettingsHandler_PutBeforeBootstrapIs404(t *testing.T) { f := newAgentNetworkHandlerFixture(t) rec := f.do(t, http.MethodPut, "/agent-network/settings", `{"enable_log_collection": false, "enable_prompt_collection": false, "redact_pii": false}`) assert.Equal(t, http.StatusNotFound, rec.Code, - "cluster-less PUT on an unbootstrapped account must 404: got %d body=%s", rec.Code, rec.Body.String()) - assert.Contains(t, rec.Body.String(), "cluster", - "the error must point the caller at the bootstrap paths: %s", rec.Body.String()) + "PUT on an unbootstrapped account must 404: got %d body=%s", rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), "/api/agent-network/settings", + "the error must point the caller at the bootstrap POST: %s", rec.Body.String()) } // TestSettingsHandler_PutReplacesMutableFields pins the update contract shared -// with the other PUT endpoints: the request replaces every mutable field, so a -// toggle absent from the JSON lands as its zero value rather than being -// preserved. Cluster and subdomain survive untouched. +// with the other PUT endpoints: the request carries every field, replacing the +// mutable ones. The identity fields ride along as a required echo of the +// assigned values — compared, never written — so the endpoint and proxy +// address survive every accepted update. func TestSettingsHandler_PutReplacesMutableFields(t *testing.T) { f := newAgentNetworkHandlerFixture(t) - rec := f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "eu.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": true, "redact_pii": true, "access_log_retention_days": 14}`) - require.Equal(t, http.StatusOK, rec.Code, "bootstrap PUT must succeed: %s", rec.Body.String()) + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "enable_prompt_collection": true, "redact_pii": true, "access_log_retention_days": 14}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) var before api.AgentNetworkSettings require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &before)) - rec = f.do(t, http.MethodPut, "/agent-network/settings", - `{"enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false}`) + rec = f.do(t, http.MethodPut, "/agent-network/settings", fmt.Sprintf( + `{"endpoint": %q, "proxy_address": %q, "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false, "access_log_retention_days": 7}`, + before.Endpoint, before.ProxyAddress)) require.Equal(t, http.StatusOK, rec.Code, "update PUT must succeed: %s", rec.Body.String()) var got api.AgentNetworkSettings @@ -103,35 +195,201 @@ func TestSettingsHandler_PutReplacesMutableFields(t *testing.T) { assert.False(t, got.EnablePromptCollection, "sent toggle must apply") assert.False(t, got.RedactPii, "sent toggle must apply") require.NotNil(t, got.AccessLogRetentionDays) - assert.Equal(t, 0, *got.AccessLogRetentionDays, - "retention absent from the request must land as the zero value — PUT replaces all mutable fields") - assert.Equal(t, before.Cluster, got.Cluster, "cluster must survive updates untouched") - assert.Equal(t, before.Subdomain, got.Subdomain, "subdomain must survive updates untouched") + assert.Equal(t, 7, *got.AccessLogRetentionDays, "sent retention must apply") + assert.Equal(t, before.Endpoint, got.Endpoint, "endpoint must survive updates untouched") + assert.Equal(t, before.ProxyAddress, got.ProxyAddress, "proxy address must survive updates untouched") } -// TestSettingsHandler_PutRejectsClusterChange pins cluster immutability: once -// assigned, a differing cluster is rejected as a validation error instead of -// being silently ignored, so callers never observe a value other than the one -// they sent. Echoing the assigned cluster back stays valid, which lets -// declarative clients send their full desired state idempotently. -func TestSettingsHandler_PutRejectsClusterChange(t *testing.T) { +// TestSettingsHandler_PutRejectsChangedIdentity pins the immutability contract: +// the PUT carries the identity fields like every other field, but they are an +// echo — a request carrying a different endpoint or proxy address is rejected +// as a validation error and the row is left untouched. The comparison is +// lenient about casing (the stored values are normalized lowercase), so a +// client replaying a GET response with different casing is not rejected. +func TestSettingsHandler_PutRejectsChangedIdentity(t *testing.T) { f := newAgentNetworkHandlerFixture(t) - rec := f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "eu.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false}`) - require.Equal(t, http.StatusOK, rec.Code, "bootstrap PUT must succeed: %s", rec.Body.String()) + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "enable_prompt_collection": true}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + var before api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &before)) - rec = f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "us.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false}`) - assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, - "cluster change must be rejected as a validation error: got %d body=%s", rec.Code, rec.Body.String()) + for name, body := range map[string]string{ + "changed endpoint": fmt.Sprintf( + `{"endpoint": "other.gateway.example.com", "proxy_address": %q, "enable_log_collection": false, "enable_prompt_collection": false, "redact_pii": false, "access_log_retention_days": 30}`, + before.ProxyAddress), + "changed proxy_address": fmt.Sprintf( + `{"endpoint": %q, "proxy_address": "us.proxy.netbird.io", "enable_log_collection": false, "enable_prompt_collection": false, "redact_pii": false, "access_log_retention_days": 30}`, + before.Endpoint), + "omitted identity": `{"enable_log_collection": false, "enable_prompt_collection": false, "redact_pii": false, "access_log_retention_days": 30}`, + } { + rec = f.do(t, http.MethodPut, "/agent-network/settings", body) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, + "%s must be rejected: got %d body=%s", name, rec.Code, rec.Body.String()) + } - rec = f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "eu.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": true}`) - require.Equal(t, http.StatusOK, rec.Code, "echoing the assigned cluster must stay valid: %s", rec.Body.String()) + // The rejected updates must not have applied anything — toggles included. + rec = f.do(t, http.MethodGet, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code) + var got api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, before.Endpoint, got.Endpoint, "rejected PUT must not change the endpoint") + assert.Equal(t, before.ProxyAddress, got.ProxyAddress, "rejected PUT must not change the proxy address") + assert.True(t, got.EnablePromptCollection, "rejected PUT must not apply its toggles") + + // An uppercased echo of the assigned values still names the same host and + // must be accepted. + rec = f.do(t, http.MethodPut, "/agent-network/settings", fmt.Sprintf( + `{"endpoint": %q, "proxy_address": %q, "enable_log_collection": true, "enable_prompt_collection": true, "redact_pii": false, "access_log_retention_days": 30}`, + strings.ToUpper(before.Endpoint), strings.ToUpper(before.ProxyAddress))) + assert.Equal(t, http.StatusOK, rec.Code, + "an uppercased identity echo must be accepted: got %d body=%s", rec.Code, rec.Body.String()) +} + +// TestSettingsHandler_PutOmittedRetentionLandsAsZero documents a residual the +// required-ness of access_log_retention_days does not remove. Marking the field +// required changes the generated client type from *int to int, so a generated +// client cannot omit it — but nothing validates OpenAPI required-ness at +// runtime, so a hand-rolled body without the field still decodes as 0, which +// the API documents as "keep indefinitely". +// +// That is the same latitude the three booleans already have, so it is left +// consistent rather than special-cased. This test exists to make the gap +// explicit: if request validation is ever added, this expectation is what +// changes. +func TestSettingsHandler_PutOmittedRetentionLandsAsZero(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "access_log_retention_days": 14}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + var before api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &before)) + + rec = f.do(t, http.MethodPut, "/agent-network/settings", fmt.Sprintf( + `{"endpoint": %q, "proxy_address": %q, "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false}`, + before.Endpoint, before.ProxyAddress)) + require.Equal(t, http.StatusOK, rec.Code, "update PUT must succeed: %s", rec.Body.String()) var got api.AgentNetworkSettings require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) - assert.Equal(t, "eu.proxy.netbird.io", got.Cluster, "cluster must be unchanged") - assert.True(t, got.RedactPii, "toggle sent alongside the echoed cluster must apply") + require.NotNil(t, got.AccessLogRetentionDays) + assert.Equal(t, 0, *got.AccessLogRetentionDays, + "a non-conforming body that omits retention still replaces it with the zero value") +} + +// TestSettingsHandler_DeleteBeforeBootstrapIs404 pins that DELETE on an +// account with no settings row is a 404, mirroring the PUT. +func TestSettingsHandler_DeleteBeforeBootstrapIs404(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodDelete, "/agent-network/settings", "") + assert.Equal(t, http.StatusNotFound, rec.Code, + "DELETE on an unbootstrapped account must 404: got %d body=%s", rec.Code, rec.Body.String()) +} + +// TestSettingsHandler_DeleteBlockedByProviders pins the first delete guard: +// while any provider exists for the account, the delete is refused with 412 +// and the row survives. Providers route through the endpoint — the guard +// keeps DELETE a bootstrap-repair operation rather than a way to abandon a +// configured gateway. +func TestSettingsHandler_DeleteBlockedByProviders(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", `{"proxy_address": "eu.proxy.netbird.io"}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + var before api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &before)) + + f.seedProvider(t, "prov-guard") + + rec = f.do(t, http.MethodDelete, "/agent-network/settings", "") + assert.Equal(t, http.StatusPreconditionFailed, rec.Code, + "delete with a provider present must be refused: got %d body=%s", rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodGet, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code) + var got api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, before.Endpoint, got.Endpoint, "the refused delete must leave the row intact") +} + +// TestSettingsHandler_DeleteBlockedByActiveProxy pins the second delete +// guard: while a proxy is actively serving the endpoint — an active proxy +// row declaring the endpoint hostname as its cluster address, the dedicated +// shape — the delete is refused with 412. A proxy that has disconnected no +// longer blocks: the guard is about a live serving path, not history. +// +// The proxy declares its address with mixed casing on purpose: Connect +// stores the declared address verbatim while the settings row is normalized +// lowercase, and hostnames are case-insensitive, so the guard must match +// across the casing difference rather than be sidestepped by it. +func TestSettingsHandler_DeleteBlockedByActiveProxy(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + const endpoint = "gw.dedicated.example.com" + rec := f.do(t, http.MethodPost, "/agent-network/settings", fmt.Sprintf(`{"endpoint": %q}`, endpoint)) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + + now := time.Now() + accountID := testAccountID + proxyRow := &rpproxy.Proxy{ + ID: "proxy-guard", + SessionID: "sess-1", + ClusterAddress: "GW.Dedicated.Example.Com", + AccountID: &accountID, + LastSeen: now, + ConnectedAt: &now, + Status: rpproxy.StatusConnected, + } + require.NoError(t, f.store.SaveProxy(context.Background(), proxyRow)) + + rec = f.do(t, http.MethodDelete, "/agent-network/settings", "") + assert.Equal(t, http.StatusPreconditionFailed, rec.Code, + "delete with an active proxy at the endpoint must be refused: got %d body=%s", rec.Code, rec.Body.String()) + + // Once the proxy disconnects it no longer serves the endpoint, so the + // delete goes through. + require.NoError(t, f.store.DisconnectProxy(context.Background(), proxyRow.ID, proxyRow.SessionID)) + rec = f.do(t, http.MethodDelete, "/agent-network/settings", "") + assert.Equal(t, http.StatusOK, rec.Code, + "delete after the proxy disconnected must succeed: got %d body=%s", rec.Code, rec.Body.String()) +} + +// TestSettingsHandler_DeleteReleasesEndpointForFreshBootstrap pins the +// full-reset semantic that gives replace-on-change clients (e.g. Terraform's +// RequiresReplace) a real path: with both guards clear the delete succeeds, +// the account reads as the defaults again, and a fresh bootstrap draws a +// fresh label. The released hostname is not reserved — a fresh draw may even +// legitimately re-pick it — so the assertions check the new row's shape, not +// that the label differs. +func TestSettingsHandler_DeleteReleasesEndpointForFreshBootstrap(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "enable_prompt_collection": true}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + + rec = f.do(t, http.MethodDelete, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code, + "delete with both guards clear must succeed: got %d body=%s", rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodGet, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code) + var after api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &after)) + assert.Empty(t, after.Endpoint, "a deleted account must read as unbootstrapped defaults") + assert.False(t, after.EnablePromptCollection, "the deleted row's toggles must not linger") + + rec = f.do(t, http.MethodPost, "/agent-network/settings", `{"proxy_address": "eu.proxy.netbird.io"}`) + require.Equal(t, http.StatusOK, rec.Code, "re-bootstrap after delete must succeed: %s", rec.Body.String()) + var second api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &second)) + require.NotEmpty(t, second.Endpoint, "the fresh bootstrap must allocate an endpoint") + assert.True(t, strings.HasSuffix(second.Endpoint, ".eu.proxy.netbird.io"), + "the fresh endpoint must hang beneath the requested proxy address: %s", second.Endpoint) + assert.False(t, second.EnablePromptCollection, + "the fresh row must carry bootstrap defaults, not the deleted row's toggles") + assert.NotNil(t, second.CreatedAt, "the fresh row is persisted and carries timestamps") } diff --git a/management/internals/modules/agentnetwork/labelgen/adjectives.go b/management/internals/modules/agentnetwork/labelgen/adjectives.go new file mode 100644 index 000000000..5058c2f1c --- /dev/null +++ b/management/internals/modules/agentnetwork/labelgen/adjectives.go @@ -0,0 +1,37 @@ +package labelgen + +// adjectives is the descriptor half of a generated label. It pairs with the +// noun pool in words.go to form `-` labels, and is kept +// separate because words.go is almost entirely nouns — drawing both halves +// from it produced unreadable pairs like "millet-hammock". Entries are +// lowercase ASCII, 4-12 chars, free of hyphens and digits, screened for +// offensive/brand/region-specific terms, and disjoint from the noun pool +// (enforced by TestAdjectives_AreDisjointFromNouns). +var adjectives = []string{ + "able", "active", "adept", "agile", "airy", "alert", "amiable", "ample", + "ancient", "ardent", "artful", "astute", "balmy", "blithe", "bold", "bonny", + "brave", "breezy", "brisk", "bubbly", "buoyant", "bushy", "candid", "canny", + "cheery", "chilly", "chipper", "chunky", "civil", "classic", "clever", "comely", + "compact", "cordial", "cosmic", "courtly", "crafty", "creamy", "crisp", "cuddly", + "curious", "dainty", "dapper", "daring", "dashing", "deft", "dewy", "diligent", + "downy", "dreamy", "dulcet", "durable", "dusky", "eager", "earnest", "earthy", + "easy", "elated", "elegant", "epic", "fabled", "faithful", "fancy", "fearless", + "feisty", "fervent", "fleet", "fluffy", "fond", "frisky", "frosty", "gallant", + "genial", "genteel", "gentle", "giddy", "gilded", "glad", "glassy", "gleaming", + "glossy", "graceful", "grand", "grainy", "hale", "hardy", "hearty", "hefty", + "honest", "hopeful", "humble", "hushed", "immense", "jaunty", "jolly", "jovial", + "joyful", "jubilant", "keen", "kindly", "kindred", "lanky", "leafy", "limber", + "lively", "lofty", "loyal", "lucent", "lucid", "luminous", "lush", "maroon", + "mellow", "merry", "mighty", "mindful", "mirthful", "misty", "modest", "muted", + "nifty", "nimble", "noble", "patient", "peaceful", "pearly", "peppy", "perky", + "petite", "placid", "playful", "pleasant", "plucky", "plush", "polite", "posh", + "prancing", "pristine", "prompt", "proud", "prudent", "quaint", "quick", "quirky", + "radiant", "ready", "regal", "restful", "robust", "rosy", "ruddy", "rugged", + "sandy", "satin", "saucy", "savvy", "sedate", "serene", "shady", "shiny", + "silken", "silky", "sincere", "sleek", "slender", "smart", "smooth", "snappy", + "snug", "soaring", "sparkly", "spiffy", "spirited", "sprightly", "spry", "stalwart", + "stately", "steady", "sterling", "stoic", "stormy", "stout", "sturdy", "sunlit", + "supple", "svelte", "tawny", "tender", "tidy", "timeless", "trusty", "upbeat", + "urbane", "valiant", "vast", "vernal", "vibrant", "vintage", "whimsy", "willing", + "windy", "winsome", "wintry", "witty", "worthy", "zesty", "zippy", +} diff --git a/management/internals/modules/agentnetwork/labelgen/labelgen.go b/management/internals/modules/agentnetwork/labelgen/labelgen.go index b45ff4ea8..549767096 100644 --- a/management/internals/modules/agentnetwork/labelgen/labelgen.go +++ b/management/internals/modules/agentnetwork/labelgen/labelgen.go @@ -64,3 +64,20 @@ func PickUnique(rng *rand.Rand, taken map[string]struct{}, fallbackSuffix string w := pool[rng.Intn(len(pool))] return fmt.Sprintf("%s-%s", w, fallbackSuffix) } + +// PickTuple returns an adjective-noun label such as "brave-otter". It is still +// a single DNS label. +// +// Unlike PickUnique it takes no `taken` set and has no fallback suffix. The +// noun pool holds 857 entries, which is ample per cluster but a hard ceiling +// once labels must be unique across one shared zone; pairing an adjective with +// a noun spans len(adjectives) * 857 instead. Uniqueness is enforced by a +// database constraint and retried by the caller, rather than guessed from a +// pre-read set that a concurrent allocation can invalidate. +func PickTuple(rng *rand.Rand) string { + nouns := uniqueWords() + if len(nouns) == 0 || len(adjectives) == 0 { + return "" + } + return adjectives[rng.Intn(len(adjectives))] + "-" + nouns[rng.Intn(len(nouns))] +} diff --git a/management/internals/modules/agentnetwork/labelgen/labelgen_test.go b/management/internals/modules/agentnetwork/labelgen/labelgen_test.go index f03a3501d..7e12fc133 100644 --- a/management/internals/modules/agentnetwork/labelgen/labelgen_test.go +++ b/management/internals/modules/agentnetwork/labelgen/labelgen_test.go @@ -99,3 +99,82 @@ func TestUniqueWords_DropsDuplicates(t *testing.T) { } assert.GreaterOrEqual(t, len(pool), 500, "Pool must contain at least 500 unique words") } + +// TestPickTuple_ShapeAndPoolMembership locks the wire-visible shape: an +// adjective and a noun, each from its own pool, joined by a single hyphen so +// the result stays one DNS label. +func TestPickTuple_ShapeAndPoolMembership(t *testing.T) { + nouns := uniqueWords() + inNouns := make(map[string]struct{}, len(nouns)) + for _, w := range nouns { + inNouns[w] = struct{}{} + } + inAdjectives := make(map[string]struct{}, len(adjectives)) + for _, a := range adjectives { + inAdjectives[a] = struct{}{} + } + + rng := rand.New(rand.NewSource(7)) + for i := 0; i < 200; i++ { + got := PickTuple(rng) + + parts := strings.Split(got, "-") + require.Len(t, parts, 2, "PickTuple must produce exactly two hyphen-joined words; got %q", got) + + _, adjOK := inAdjectives[parts[0]] + assert.True(t, adjOK, "First half must be an adjective; %q not in adjectives (from %q)", parts[0], got) + _, nounOK := inNouns[parts[1]] + assert.True(t, nounOK, "Second half must be a noun; %q not in words (from %q)", parts[1], got) + + assert.LessOrEqual(t, len(got), 63, "Label must fit a DNS label; got %q (%d chars)", got, len(got)) + } +} + +// TestAdjectives_AreDisjointFromNouns keeps the namespace a clean product and +// prevents nonsense like "azure-azure": a handful of the noun pool's entries +// are adjectival, and any overlap would let the same word land on both sides. +func TestAdjectives_AreDisjointFromNouns(t *testing.T) { + nouns := make(map[string]struct{}, len(uniqueWords())) + for _, w := range uniqueWords() { + nouns[w] = struct{}{} + } + for _, a := range adjectives { + _, clash := nouns[a] + assert.False(t, clash, "Adjective %q also appears in the noun pool; remove it from one list", a) + } +} + +// TestAdjectives_AreDNSSafeAndDeduplicated mirrors the curation contract stated +// in words.go: lowercase ASCII, 4-12 chars, no digits or hyphens, no repeats. +func TestAdjectives_AreDNSSafeAndDeduplicated(t *testing.T) { + seen := make(map[string]struct{}, len(adjectives)) + for _, a := range adjectives { + _, dup := seen[a] + assert.False(t, dup, "Duplicate adjective %q", a) + seen[a] = struct{}{} + + assert.Regexp(t, `^[a-z]{4,12}$`, a, "Adjective %q must be 4-12 lowercase ASCII letters", a) + } + assert.Greater(t, len(adjectives), 150, "Adjective pool too small to give a useful namespace") +} + +// TestPickTuple_DeterministicWithSeededRng documents that generation is a pure +// function of the rng, which is what makes allocation retries reproducible in tests. +func TestPickTuple_DeterministicWithSeededRng(t *testing.T) { + a := PickTuple(rand.New(rand.NewSource(42))) + b := PickTuple(rand.New(rand.NewSource(42))) + assert.Equal(t, a, b, "Same seed must yield the same tuple") +} + +// TestPickTuple_SpansALargeNamespace guards the reason we moved to tuples: a +// single-word pool caps the GLOBAL namespace at 857. Drawing many tuples must +// yield overwhelmingly distinct values. +func TestPickTuple_SpansALargeNamespace(t *testing.T) { + rng := rand.New(rand.NewSource(11)) + seen := make(map[string]struct{}, 2000) + for i := 0; i < 2000; i++ { + seen[PickTuple(rng)] = struct{}{} + } + assert.Greater(t, len(seen), 1900, + "2000 draws should be nearly all distinct across a ~200k namespace; got %d unique", len(seen)) +} diff --git a/management/internals/modules/agentnetwork/manager.go b/management/internals/modules/agentnetwork/manager.go index ba2c06826..379672989 100644 --- a/management/internals/modules/agentnetwork/manager.go +++ b/management/internals/modules/agentnetwork/manager.go @@ -22,7 +22,6 @@ import ( "github.com/netbirdio/netbird/management/server/permissions/modules" "github.com/netbirdio/netbird/management/server/permissions/operations" "github.com/netbirdio/netbird/management/server/store" - "github.com/netbirdio/netbird/shared/management/proto" "github.com/netbirdio/netbird/shared/management/status" ) @@ -48,7 +47,7 @@ func ensureSessionKeys(p *types.Provider) error { type Manager interface { GetAllProviders(ctx context.Context, accountID, userID string) ([]*types.Provider, error) GetProvider(ctx context.Context, accountID, userID, providerID string) (*types.Provider, error) - CreateProvider(ctx context.Context, userID string, provider *types.Provider, bootstrapCluster string) (*types.Provider, error) + CreateProvider(ctx context.Context, userID string, provider *types.Provider) (*types.Provider, error) UpdateProvider(ctx context.Context, userID string, provider *types.Provider) (*types.Provider, error) DeleteProvider(ctx context.Context, accountID, userID, providerID string) error @@ -71,7 +70,9 @@ type Manager interface { DeleteBudgetRule(ctx context.Context, accountID, userID, ruleID string) error GetSettings(ctx context.Context, accountID, userID string) (*types.Settings, error) + CreateSettings(ctx context.Context, userID string, settings *types.Settings, proxyAddress, endpoint string) (*types.Settings, error) UpdateSettings(ctx context.Context, userID string, settings *types.Settings) (*types.Settings, error) + DeleteSettings(ctx context.Context, accountID, userID string) error ListConsumption(ctx context.Context, accountID, userID string) ([]*types.Consumption, error) ListAccessLogs(ctx context.Context, accountID, userID string, filter types.AgentNetworkAccessLogFilter) ([]*types.AgentNetworkAccessLog, int64, error) @@ -123,11 +124,10 @@ type managerImpl struct { proxyController proxy.Controller // reconcileCache holds the last set of synthesised proxy mappings - // per account so reconcile can emit precise Create/Update/Delete - // updates instead of a full re-push on every mutation. Keyed by - // accountID, then by synthesised service ID. + // per account, each paired with the proxy that served it, so a change + // of serving proxy can be diffed without re-deriving it. reconcileMu sync.Mutex - reconcileCache map[string]map[string]*proto.ProxyMapping + reconcileCache map[string]map[string]syntheticMapping // labelRngMu guards labelRng. PickUnique consumes math/rand.Source // state; concurrent provider creates would otherwise race. @@ -151,7 +151,7 @@ func NewManager( accountManager: accountManager, permissionsManager: permissionsManager, proxyController: proxyController, - reconcileCache: make(map[string]map[string]*proto.ProxyMapping), + reconcileCache: make(map[string]map[string]syntheticMapping), labelRng: rand.New(rand.NewSource(time.Now().UnixNano())), } } @@ -170,19 +170,14 @@ func (m *managerImpl) GetProvider(ctx context.Context, accountID, userID, provid return m.store.GetAgentNetworkProviderByID(ctx, store.LockingStrengthNone, accountID, providerID) } -// CreateProvider persists a new provider for the account. bootstrapCluster -// is used only when the per-account agent-network Settings row hasn't -// been created yet; otherwise it is ignored (the cluster is pinned on -// Settings and every provider in the account routes through it). -func (m *managerImpl) CreateProvider(ctx context.Context, userID string, provider *types.Provider, bootstrapCluster string) (*types.Provider, error) { +// CreateProvider persists a new provider for the account. Providers have no +// settings side effects: the account's endpoint is bootstrapped separately and +// explicitly via CreateSettings, and every provider in the account routes +// through it. +func (m *managerImpl) CreateProvider(ctx context.Context, userID string, provider *types.Provider) (*types.Provider, error) { if err := m.requirePermission(ctx, provider.AccountID, userID, modules.AgentNetworkProviders, operations.Create); err != nil { return nil, err } - if strings.TrimSpace(bootstrapCluster) != "" { - if err := m.requireSettingsBootstrapPermission(ctx, provider.AccountID, userID); err != nil { - return nil, err - } - } // An empty api_key would silently produce a synthesised service // that 401s on every upstream request. Surface the misconfiguration @@ -206,16 +201,6 @@ func (m *managerImpl) CreateProvider(ctx context.Context, userID string, provide return nil, fmt.Errorf("save agent network provider: %w", err) } - if strings.TrimSpace(bootstrapCluster) != "" { - if _, err := m.bootstrapSettingsIfNeeded(ctx, m.store, provider.AccountID, bootstrapCluster); err != nil { - // The provider create has already succeeded; logging the - // bootstrap miss matches the plan's PoC behaviour. The synth - // path treats a missing settings row as a no-op, and the next - // provider create retries the bootstrap. - log.WithContext(ctx).Debugf("agent-network bootstrap settings for account %s on cluster %s: %v", provider.AccountID, bootstrapCluster, err) - } - } - m.accountManager.StoreEvent(ctx, userID, provider.ID, provider.AccountID, activity.AgentNetworkProviderCreated, provider.EventMeta()) m.reconcile(ctx, provider.AccountID) @@ -560,52 +545,44 @@ func (m *managerImpl) DeleteBudgetRule(ctx context.Context, accountID, userID, r } // UpdateSettings replaces the mutable account-level settings — the collection -// toggles and retention — on the account's row. When the account has no -// settings row yet, a non-empty settings.Cluster bootstraps one (same path as -// first provider create); without it the update fails with NotFound. On an -// existing row the cluster and subdomain are immutable: a differing -// settings.Cluster is rejected rather than silently ignored so callers never -// observe a value other than what they sent. Because the collection toggles -// change the synthesised service config (prompt-capture gating, access-log -// emission), a reconcile is triggered so the proxy and peer network maps -// converge on the new state. +// toggles and retention — on the account's row. The identity fields (Domain, +// ProxyAddress) are assigned at bootstrap (CreateSettings) and immutable: the +// request carries them, matching the PUT convention of every other endpoint, +// but they are only compared against the stored row — a request carrying +// different values is rejected, and the stored values are never overwritten. +// When the account has no settings row yet the update fails with NotFound. +// Because the collection toggles change the synthesised service config +// (prompt-capture gating, access-log emission), a reconcile is triggered so +// the proxy and peer network maps converge on the new state. func (m *managerImpl) UpdateSettings(ctx context.Context, userID string, settings *types.Settings) (*types.Settings, error) { if err := m.requirePermission(ctx, settings.AccountID, userID, modules.AgentNetworkSettings, operations.Update); err != nil { return nil, err } - requestedCluster := strings.TrimSpace(settings.Cluster) - // The row lock from LockingStrengthUpdate only holds for the duration of - // the surrounding transaction, so the read, the cluster-immutability - // check, and the save must share one — otherwise concurrent PUTs could - // interleave between them. + // the surrounding transaction, so the read and the save must share one — + // otherwise concurrent PUTs could interleave between them. var updated *types.Settings err := m.store.ExecuteInTransaction(ctx, func(tx store.Store) error { existing, err := tx.GetAgentNetworkSettings(ctx, store.LockingStrengthUpdate, settings.AccountID) switch { case err == nil: - if requestedCluster != "" && requestedCluster != existing.Cluster { - return status.Errorf(status.InvalidArgument, "cluster is immutable once assigned (current: %s)", existing.Cluster) - } case isNotFound(err): - if requestedCluster == "" { - return status.Errorf(status.NotFound, "agent network settings have not been bootstrapped yet; pass cluster to bootstrap them, or create a provider with bootstrap_cluster set") - } - // Bootstrapping pins the cluster and subdomain — a settings - // create on top of the update the caller already passed, matching - // the gate on the provider-create bootstrap path. - if err := m.requirePermission(ctx, settings.AccountID, userID, modules.AgentNetworkSettings, operations.Create); err != nil { - return err - } - existing, err = m.bootstrapSettingsIfNeeded(ctx, tx, settings.AccountID, requestedCluster) - if err != nil { - return err - } + return status.Errorf(status.NotFound, "agent network settings have not been bootstrapped yet; POST /api/agent-network/settings to bootstrap them") default: return fmt.Errorf("get agent network settings: %w", err) } + // The identity echo is compared leniently (trimmed, case-insensitive): + // the stored values are normalized lowercase, and a client replaying a + // GET response must never be rejected over casing it didn't choose. + if !hostnamesEquivalent(settings.Domain, existing.Domain) { + return status.Errorf(status.InvalidArgument, "endpoint is immutable: it must match the assigned endpoint %q; delete the settings to release it and bootstrap again", existing.Domain) + } + if !hostnamesEquivalent(settings.ProxyAddress, existing.ProxyAddress) { + return status.Errorf(status.InvalidArgument, "proxy_address is immutable: it must match the assigned proxy address %q; delete the settings to release it and bootstrap again", existing.ProxyAddress) + } + existing.EnableLogCollection = settings.EnableLogCollection existing.EnablePromptCollection = settings.EnablePromptCollection existing.RedactPii = settings.RedactPii @@ -632,6 +609,83 @@ func (m *managerImpl) UpdateSettings(ctx context.Context, userID string, setting return updated, nil } +// hostnamesEquivalent reports whether a caller-supplied hostname names the +// same host as a stored (normalized, lowercase) one: equal after trimming and +// case folding. No structural validation — an arbitrary mismatch and a +// malformed value are both simply "not the assigned value". +func hostnamesEquivalent(supplied, stored string) bool { + return strings.EqualFold(strings.TrimSpace(supplied), stored) +} + +// DeleteSettings removes the account's settings row, releasing the endpoint. +// Two guards make this a bootstrap-repair operation rather than a way to tear +// down a serving gateway, both re-checked under the row lock: +// +// - No Agent Network providers may exist for the account. Providers route +// through the endpoint; delete them first. +// - No proxy may be actively serving the endpoint — that is, no active proxy +// declares the endpoint hostname as its cluster address. This is the +// dedicated (self-addressed) shape's guard: the proxy at the address IS +// this account's gateway. A labeled endpoint hangs beneath a shared +// cluster's address, and with the account's providers already gone the +// shared proxy serves nothing of the account's, so the parent cluster +// being up does not block the delete. +// +// Bootstrapping again after a delete allocates fresh — the released hostname +// is not reserved. That full-reset semantic is what gives clients that model +// immutability as replace-on-change (e.g. Terraform's RequiresReplace) a real +// path: tear down providers, delete, re-create. +func (m *managerImpl) DeleteSettings(ctx context.Context, accountID, userID string) error { + if err := m.requirePermission(ctx, accountID, userID, modules.AgentNetworkSettings, operations.Delete); err != nil { + return err + } + + var deleted *types.Settings + err := m.store.ExecuteInTransaction(ctx, func(tx store.Store) error { + existing, err := tx.GetAgentNetworkSettings(ctx, store.LockingStrengthUpdate, accountID) + switch { + case err == nil: + case isNotFound(err): + return status.Errorf(status.NotFound, "agent network settings have not been bootstrapped yet; there is nothing to delete") + default: + return fmt.Errorf("get agent network settings: %w", err) + } + + providers, err := tx.GetAccountAgentNetworkProviders(ctx, store.LockingStrengthNone, accountID) + if err != nil { + return fmt.Errorf("get agent network providers: %w", err) + } + if len(providers) > 0 { + return status.Errorf(status.PreconditionFailed, "agent network settings cannot be deleted while %d provider(s) exist; delete the providers first", len(providers)) + } + + serving, err := tx.HasActiveProxyAtClusterAddress(ctx, existing.Domain) + if err != nil { + return fmt.Errorf("check for a proxy serving the endpoint: %w", err) + } + if serving { + return status.Errorf(status.PreconditionFailed, "agent network settings cannot be deleted while a proxy is actively serving the endpoint %q", existing.Domain) + } + + if err := tx.DeleteAgentNetworkSettings(ctx, accountID); err != nil { + return fmt.Errorf("delete agent network settings: %w", err) + } + deleted = existing + return nil + }) + if err != nil { + return err + } + + m.accountManager.StoreEvent(ctx, userID, accountID, accountID, activity.AgentNetworkSettingsDeleted, map[string]any{ + "endpoint": deleted.Domain, + "proxy_address": deleted.ProxyAddress, + }) + m.reconcile(ctx, accountID) + + return nil +} + // isNotFound reports whether err is a status.NotFound error. func isNotFound(err error) bool { var sErr *status.Error @@ -678,74 +732,162 @@ func (m *managerImpl) GetSettings(ctx context.Context, accountID, userID string) } } -// requireSettingsBootstrapPermission gates the one-time settings bootstrap a -// first provider create performs. Pinning the account's cluster and subdomain -// is a settings write, so it needs the settings permission on top of the -// provider one. No-op once the settings row exists. -func (m *managerImpl) requireSettingsBootstrapPermission(ctx context.Context, accountID, userID string) error { - _, err := m.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, accountID) - if err == nil { - return nil - } - if !isNotFound(err) { - return fmt.Errorf("get agent network settings: %w", err) - } - return m.requirePermission(ctx, accountID, userID, modules.AgentNetworkSettings, operations.Create) -} +// maxDomainAllocationAttempts bounds the label search when bootstrapping a +// labeled endpoint. Package-level (rather than function-local) so tests can +// assert on the exhaustion path without duplicating the literal. +const maxDomainAllocationAttempts = 10 -// bootstrapSettingsIfNeeded creates the per-account agent-network -// settings row when missing. The cluster comes from the create-time -// hint the dashboard sends (auto-picked from the active cluster list); -// the subdomain is picked from the curated wordlist avoiding -// collisions on the same cluster. Idempotent: if a row already exists -// it is returned untouched and the hint is ignored. st is the store to -// operate on — pass the transaction store when calling from within one. -func (m *managerImpl) bootstrapSettingsIfNeeded(ctx context.Context, st store.Store, accountID, providerCluster string) (*types.Settings, error) { - if accountID == "" { - return nil, fmt.Errorf("bootstrap settings: account id is required") +// CreateSettings bootstraps the per-account settings row, assigning the +// account's immutable endpoint. Exactly one of proxyAddress and endpoint must +// be non-empty: proxyAddress allocates a labeled endpoint one label beneath +// the given cluster address; endpoint claims the given hostname verbatim as a +// self-addressed (dedicated) endpoint — a legitimate claim before any proxy +// declares the address (address-first). settings carries the account ID and +// the initial collection toggles; its identity fields are assigned here. +func (m *managerImpl) CreateSettings(ctx context.Context, userID string, settings *types.Settings, proxyAddress, endpoint string) (*types.Settings, error) { + if settings == nil || settings.AccountID == "" { + return nil, status.Errorf(status.InvalidArgument, "account id is required") } - if strings.TrimSpace(providerCluster) == "" { - return nil, fmt.Errorf("bootstrap settings: provider cluster is required") + if err := m.requirePermission(ctx, settings.AccountID, userID, modules.AgentNetworkSettings, operations.Create); err != nil { + return nil, err } - existing, err := st.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, accountID) - if err == nil { - return existing, nil + hasProxyAddress := strings.TrimSpace(proxyAddress) != "" + hasEndpoint := strings.TrimSpace(endpoint) != "" + if hasProxyAddress == hasEndpoint { + return nil, status.Errorf(status.InvalidArgument, "exactly one of proxy_address and endpoint is required") } - if !isNotFound(err) { + + // Fail fast on an existing row for a clean 409; the insert below stays + // the authority against concurrent bootstraps (the primary key wins). + if _, err := m.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, settings.AccountID); err == nil { + return nil, status.Errorf(status.AlreadyExists, "agent network settings already bootstrapped for account %s", settings.AccountID) + } else if !isNotFound(err) { return nil, fmt.Errorf("get agent network settings: %w", err) } - siblings, err := st.GetAgentNetworkSettingsByCluster(ctx, store.LockingStrengthNone, providerCluster) - if err != nil { - return nil, fmt.Errorf("list agent network settings on cluster: %w", err) - } - taken := make(map[string]struct{}, len(siblings)) - for _, s := range siblings { - taken[s.Subdomain] = struct{}{} - } - - suffix := accountID - if len(suffix) > 4 { - suffix = suffix[:4] - } - - m.labelRngMu.Lock() - subdomain := labelgen.PickUnique(m.labelRng, taken, suffix) - m.labelRngMu.Unlock() - now := time.Now().UTC() - settings := types.DefaultSettings(accountID) - settings.Cluster = providerCluster - settings.Subdomain = subdomain settings.CreatedAt = now settings.UpdatedAt = now - if err := st.SaveAgentNetworkSettings(ctx, settings); err != nil { - return nil, fmt.Errorf("save agent network settings: %w", err) + + var err error + if hasEndpoint { + err = m.bootstrapSelfAddressed(ctx, settings, endpoint) + } else { + err = m.bootstrapLabeled(ctx, settings, proxyAddress) } + if err != nil { + return nil, err + } + + m.accountManager.StoreEvent(ctx, userID, settings.AccountID, settings.AccountID, activity.AgentNetworkSettingsUpdated, map[string]any{ + "bootstrapped": true, + "endpoint": settings.Domain, + "dedicated": settings.Dedicated(), + }) + m.reconcile(ctx, settings.AccountID) + return settings, nil } +// bootstrapSelfAddressed claims the given hostname as the account's endpoint, +// served only by a proxy declaring exactly that address (Domain == +// ProxyAddress). The domain unique index is the arbiter of availability. +func (m *managerImpl) bootstrapSelfAddressed(ctx context.Context, settings *types.Settings, endpoint string) error { + hostname, err := types.NormalizeHostname(endpoint) + if err != nil { + return status.Errorf(status.InvalidArgument, "invalid endpoint: %s", err) + } + + settings.Domain = hostname + settings.ProxyAddress = hostname + if err := m.store.CreateAgentNetworkSettings(ctx, settings); err != nil { + if isUniqueConstraintError(err) { + // The violation is either the account primary key (a concurrent + // bootstrap for the same account won) or the domain index + // (another account holds the hostname). Distinguish by re-read. + if _, getErr := m.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, settings.AccountID); getErr == nil { + return status.Errorf(status.AlreadyExists, "agent network settings already bootstrapped for account %s", settings.AccountID) + } + return status.Errorf(status.AlreadyExists, "endpoint %s is already taken", hostname) + } + return fmt.Errorf("create agent network settings: %w", err) + } + return nil +} + +// bootstrapLabeled allocates a labeled endpoint one label beneath the given +// cluster address: Domain =