From 524b8b97188e4817b0b616ff97241d5604cf041f Mon Sep 17 00:00:00 2001 From: Pascal Fischer <32096965+pascal-fischer@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:03:40 +0200 Subject: [PATCH 01/22] [management] prewarm a posture check cache on network map generation (#7093) --- .../network_map/controller/controller.go | 2 + management/server/account_test.go | 1 + management/server/types/account.go | 3 + management/server/types/account_components.go | 115 ++++++++++++++++-- .../types/account_posture_validation_test.go | 72 +++++++++++ .../server/types/networkmap_benchmark_test.go | 37 ++++++ 6 files changed, 223 insertions(+), 7 deletions(-) create mode 100644 management/server/types/account_posture_validation_test.go diff --git a/management/internals/controllers/network_map/controller/controller.go b/management/internals/controllers/network_map/controller/controller.go index 5785004db..356dc9f67 100644 --- a/management/internals/controllers/network_map/controller/controller.go +++ b/management/internals/controllers/network_map/controller/controller.go @@ -176,6 +176,7 @@ func (c *Controller) sendUpdateAccountPeers(ctx context.Context, accountID strin semaphore := make(chan struct{}, 10) c.injectAllProxyPolicies(ctx, account) + account.PrecomputePostureValidation(ctx) dnsCache := &cache.DNSConfigCache{} dnsDomain := c.GetDNSDomain(account.Settings) peersCustomZone := account.GetPeersCustomZone(ctx, dnsDomain) @@ -357,6 +358,7 @@ func (c *Controller) sendUpdateForAffectedPeers(ctx context.Context, accountID s // network map that omitted the synth DNS zone, and the agent kept // resolving against the stale or absent record. c.injectAllProxyPolicies(ctx, account) + account.PrecomputePostureValidation(ctx) dnsCache := &cache.DNSConfigCache{} dnsDomain := c.GetDNSDomain(account.Settings) peersCustomZone := account.GetPeersCustomZone(ctx, dnsDomain) diff --git a/management/server/account_test.go b/management/server/account_test.go index 3c0bb25da..73126a496 100644 --- a/management/server/account_test.go +++ b/management/server/account_test.go @@ -1757,6 +1757,7 @@ func TestAccount_Copy(t *testing.T) { AccountID: "account1", }, }, + PostureValidation: map[string]map[string]bool{"1": {"1": true}}, } err := hasNilField(account) if err != nil { diff --git a/management/server/types/account.go b/management/server/types/account.go index 474825281..4616fe26b 100644 --- a/management/server/types/account.go +++ b/management/server/types/account.go @@ -91,6 +91,8 @@ type Account struct { Onboarding AccountOnboarding `gorm:"foreignKey:AccountID;references:id;constraint:OnDelete:CASCADE"` ReverseProxyFreeDomainNonce string + + PostureValidation map[string]map[string]bool `gorm:"-"` } // this class is used by gorm only @@ -874,6 +876,7 @@ func (a *Account) Copy() *Account { Services: services, Onboarding: a.Onboarding, Domains: domains, + PostureValidation: a.PostureValidation, } } diff --git a/management/server/types/account_components.go b/management/server/types/account_components.go index 6fc904c0b..624a778fe 100644 --- a/management/server/types/account_components.go +++ b/management/server/types/account_components.go @@ -10,6 +10,8 @@ import ( nbdns "github.com/netbirdio/netbird/dns" "github.com/netbirdio/netbird/management/internals/modules/zones" routerTypes "github.com/netbirdio/netbird/management/server/networks/routers/types" + nbpeer "github.com/netbirdio/netbird/management/server/peer" + "github.com/netbirdio/netbird/management/server/posture" "github.com/netbirdio/netbird/management/server/telemetry" "github.com/netbirdio/netbird/route" ) @@ -506,8 +508,8 @@ func (a *Account) getPeersGroupsPoliciesRoutes( func (a *Account) getPeersFromGroups(ctx context.Context, groups []string, peerID string, sourcePostureChecksIDs []string, validatedPeersMap map[string]struct{}, postureFailedPeers *map[string]map[string]struct{}) ([]string, bool) { peerInGroups := false - filteredPeerIDs := make([]string, 0, len(groups)) - seenPeerIds := make(map[string]struct{}, len(groups)) + var filteredPeerIDs []string + var seenPeerIds map[string]struct{} for _, gid := range groups { group := a.GetGroup(gid) @@ -547,6 +549,17 @@ func (a *Account) getPeersFromGroups(ctx context.Context, groups []string, peerI return filteredPeerIDs, peerInGroups } + if seenPeerIds == nil { + totalGroupPeers := 0 + for _, g := range groups { + if grp := a.GetGroup(g); grp != nil { + totalGroupPeers += len(grp.Peers) + } + } + filteredPeerIDs = make([]string, 0, totalGroupPeers) + seenPeerIds = make(map[string]struct{}, totalGroupPeers) + } + for _, pid := range group.Peers { if _, seen := seenPeerIds[pid]; seen { continue @@ -589,21 +602,109 @@ func (a *Account) validatePostureChecksOnPeerGetFailed(ctx context.Context, sour } for _, postureChecksID := range sourcePostureChecksID { + if valid, cached := a.cachedPostureCheckResult(postureChecksID, peerID); cached { + if !valid { + return false, postureChecksID + } + continue + } + postureChecks := a.GetPostureChecks(postureChecksID) if postureChecks == nil { continue } - for _, check := range postureChecks.GetChecks() { - isValid, _ := check.Check(ctx, *peer) - if !isValid { - return false, postureChecksID - } + if !peerPassesPostureChecks(ctx, postureChecks.GetChecks(), peer) { + return false, postureChecksID } } return true, "" } +// PrecomputePostureValidation evaluates every posture check referenced by an enabled +// policy once against the peers of that policy's source groups and stores the results, +// so the per-peer network map calculations that follow look them up instead of +// re-evaluating checks for every peer pair. It must be called before the account is +// shared across goroutines; lookups not covered by the precomputed results fall back +// to direct evaluation. +func (a *Account) PrecomputePostureValidation(ctx context.Context) { + if len(a.PostureChecks) == 0 { + a.PostureValidation = nil + return + } + + checkPeerIDs := make(map[string]map[string]struct{}) + for _, policy := range a.Policies { + if !policy.Enabled || len(policy.SourcePostureChecks) == 0 { + continue + } + + peerIDs := a.getUniquePeerIDsFromGroupsIDs(ctx, policy.SourceGroups()) + for _, rule := range policy.Rules { + if rule.SourceResource.Type == ResourceTypePeer && rule.SourceResource.ID != "" { + peerIDs = append(peerIDs, rule.SourceResource.ID) + } + } + + for _, postureChecksID := range policy.SourcePostureChecks { + set := checkPeerIDs[postureChecksID] + if set == nil { + set = make(map[string]struct{}, len(peerIDs)) + checkPeerIDs[postureChecksID] = set + } + for _, pid := range peerIDs { + set[pid] = struct{}{} + } + } + } + + results := make(map[string]map[string]bool, len(checkPeerIDs)) + for postureChecksID, peerIDs := range checkPeerIDs { + results[postureChecksID] = a.evaluatePostureChecksForPeers(ctx, postureChecksID, peerIDs) + } + a.PostureValidation = results +} + +func (a *Account) evaluatePostureChecksForPeers(ctx context.Context, postureChecksID string, peerIDs map[string]struct{}) map[string]bool { + postureChecks := a.GetPostureChecks(postureChecksID) + if postureChecks == nil { + return nil + } + + checks := postureChecks.GetChecks() + results := make(map[string]bool, len(peerIDs)) + for peerID := range peerIDs { + peer, ok := a.Peers[peerID] + if !ok || peer == nil { + continue + } + results[peerID] = peerPassesPostureChecks(ctx, checks, peer) + } + return results +} + +func (a *Account) cachedPostureCheckResult(postureChecksID, peerID string) (bool, bool) { + results, ok := a.PostureValidation[postureChecksID] + if !ok { + return false, false + } + if results == nil { + return true, true + } + valid, found := results[peerID] + return valid, found +} + +func peerPassesPostureChecks(ctx context.Context, checks []posture.Check, peer *nbpeer.Peer) bool { + for _, check := range checks { + isValid, _ := check.Check(ctx, *peer) + if !isValid { + return false + } + } + return true +} + func (a *Account) getPostureValidPeersSaveFailed(inputPeers []string, postureChecksIDs []string, validatedPeersMap map[string]struct{}, postureFailedPeers *map[string]map[string]struct{}) []string { var dest []string for _, peerID := range inputPeers { diff --git a/management/server/types/account_posture_validation_test.go b/management/server/types/account_posture_validation_test.go new file mode 100644 index 000000000..69a744bea --- /dev/null +++ b/management/server/types/account_posture_validation_test.go @@ -0,0 +1,72 @@ +package types_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + nbdns "github.com/netbirdio/netbird/dns" + "github.com/netbirdio/netbird/management/server/posture" +) + +func TestPrecomputePostureValidation_MatchesDirectEvaluation(t *testing.T) { + account, validatedPeers := scalableTestAccount(60, 5) + + account.PostureChecks = append(account.PostureChecks, &posture.Checks{ + ID: "posture-check-strict", Name: "Strict version", + Checks: posture.ChecksDefinition{ + NBVersionCheck: &posture.NBVersionCheck{MinVersion: "0.50.0"}, + }, + }) + account.Policies[0].SourcePostureChecks = []string{"posture-check-ver", "posture-check-unknown"} + account.Policies[1].SourcePostureChecks = []string{"posture-check-strict"} + account.Policies[2].SourcePostureChecks = []string{"posture-check-ver"} + account.Policies[2].Enabled = false + + ctx := context.Background() + resourcePolicies := account.GetResourcePoliciesMap() + routers := account.GetResourceRoutersMap() + + type result struct { + peers map[string]struct{} + postureFailedPeers map[string]map[string]struct{} + } + snapshot := func() map[string]result { + results := make(map[string]result, len(account.Peers)) + for peerID := range account.Peers { + components := account.GetPeerNetworkMapComponents(ctx, peerID, nbdns.CustomZone{}, nil, validatedPeers, resourcePolicies, routers, nil) + require.NotNil(t, components) + peerSet := make(map[string]struct{}, len(components.Peers)) + for id := range components.Peers { + peerSet[id] = struct{}{} + } + results[peerID] = result{peers: peerSet, postureFailedPeers: components.PostureFailedPeers} + } + return results + } + + direct := snapshot() + account.PrecomputePostureValidation(ctx) + memoized := snapshot() + + require.Equal(t, len(direct), len(memoized)) + for peerID, want := range direct { + got := memoized[peerID] + assert.Equal(t, want.peers, got.peers, "visible peers changed for %s", peerID) + assert.Equal(t, want.postureFailedPeers, got.postureFailedPeers, "posture failed peers changed for %s", peerID) + } +} + +func TestPrecomputePostureValidation_NoPostureChecks(t *testing.T) { + account, validatedPeers := scalableTestAccount(10, 2) + account.PostureChecks = nil + + ctx := context.Background() + account.PrecomputePostureValidation(ctx) + + components := account.GetPeerNetworkMapComponents(ctx, "peer-0", nbdns.CustomZone{}, nil, validatedPeers, account.GetResourcePoliciesMap(), account.GetResourceRoutersMap(), nil) + require.NotNil(t, components) + assert.NotEmpty(t, components.Peers) +} diff --git a/management/server/types/networkmap_benchmark_test.go b/management/server/types/networkmap_benchmark_test.go index 38272e7b0..118ef9baf 100644 --- a/management/server/types/networkmap_benchmark_test.go +++ b/management/server/types/networkmap_benchmark_test.go @@ -86,6 +86,43 @@ func BenchmarkNetworkMapGeneration_AllPeers(b *testing.B) { b.ReportAllocs() b.ResetTimer() for range b.N { + account.PrecomputePostureValidation(ctx) + for _, peerID := range peerIDs { + _ = account.GetPeerNetworkMapFromComponents(ctx, peerID, nbdns.CustomZone{}, nil, validatedPeers, resourcePolicies, routers, nil, groupIDToUserIDs) + } + } + }) + } +} + +// BenchmarkNetworkMapGeneration_AllPeersPostureChecks benchmarks the UpdateAccountPeers +// hot path with a posture check attached to the account-wide policy, so posture +// validation runs for every source peer of every target peer's map. +func BenchmarkNetworkMapGeneration_AllPeersPostureChecks(b *testing.B) { + skipCIBenchmark(b) + scales := []benchmarkScale{ + {"500peers_20groups", 500, 20}, + {"1000peers_50groups", 1000, 50}, + } + + for _, scale := range scales { + account, validatedPeers := scalableTestAccount(scale.peers, scale.groups) + account.Policies[0].SourcePostureChecks = []string{"posture-check-ver"} + ctx := context.Background() + + peerIDs := make([]string, 0, len(account.Peers)) + for peerID := range account.Peers { + peerIDs = append(peerIDs, peerID) + } + + b.Run("components/"+scale.name, func(b *testing.B) { + resourcePolicies := account.GetResourcePoliciesMap() + routers := account.GetResourceRoutersMap() + groupIDToUserIDs := account.GetActiveGroupUsers() + b.ReportAllocs() + b.ResetTimer() + for range b.N { + account.PrecomputePostureValidation(ctx) for _, peerID := range peerIDs { _ = account.GetPeerNetworkMapFromComponents(ctx, peerID, nbdns.CustomZone{}, nil, validatedPeers, resourcePolicies, routers, nil, groupIDToUserIDs) } From f63fd21e0cb4ff622f544af2f0ede88ce80aacac Mon Sep 17 00:00:00 2001 From: Riccardo Manfrin <3090891+riccardomanfrin@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:01:08 +0200 Subject: [PATCH 02/22] [client] peer: re-arm the WireGuard watcher after a lazy wake (#7091) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Describe your changes The Conn struct is reused across lazy-connection deactivate/activate. Close cancels the WireGuard watcher (via wgWatcherCancel, and ctxCancel also tears down its context) but left conn.wgWatcher pointing at the stopped instance. enableWgWatcherIfNeeded skips while conn.wgWatcher is non-nil, so the next Open never started a fresh watcher: once a lazy connection had idled and woken, the peer ran with no watcher at all — no WireGuard handshake-timeout detection and none of the escalation that depends on it. Clear conn.wgWatcher and conn.wgWatcherCancel in Close so the next Open re-arms a fresh watcher. ## Issue ticket number and link ## Stack ### Checklist - [X] Is it a bug fix - [ ] Is a typo/documentation fix - [ ] Is a feature enhancement - [ ] It is a refactor - [ ] Created tests that fail without the change (if possible) - [ ] I ran and tested this change locally — I did not rely on CI to find out whether it works - [ ] This PR has a single purpose (not a fix + refactor + feature in one) - [ ] This change is a trivial fix, **OR** it links an issue the NetBird team agreed on beforehand. Changes to the public API, gRPC protocols, functionality behavior, CLI / service flags, or new features always need that agreement first. See [CONTRIBUTING.md](https://github.com/netbirdio/netbird/blob/main/CONTRIBUTING.md#ticket-first-pr-second). > By submitting this pull request, you confirm that you have read and agree to the terms of the [Contributor License Agreement](https://github.com/netbirdio/netbird/blob/main/CONTRIBUTOR_LICENSE_AGREEMENT.md). ## Documentation Select exactly one: - [ ] I added/updated documentation for this change - [X] Documentation is **not needed** for this change (explain why) ### Docs PR URL (required if "docs added" is checked) Paste the PR link from https://github.com/netbirdio/docs here: https://github.com/netbirdio/docs/pull/__ ## Summary by CodeRabbit * **Bug Fixes** * Improved connection cleanup by fully releasing WireGuard watcher resources when a connection closes. --- client/internal/peer/conn.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 09a4e8b02..fe68bb15d 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -307,6 +307,8 @@ func (conn *Conn) Close(signalToRemote bool) { if conn.wgWatcherCancel != nil { conn.wgWatcherCancel() + conn.wgWatcher = nil + conn.wgWatcherCancel = nil } conn.workerRelay.CloseConn() if conn.workerICE != nil { From 8632a0d2150d3c42a2e0340662854eda1ac02b9a Mon Sep 17 00:00:00 2001 From: Nicolas Frati Date: Fri, 7 Aug 2026 17:04:58 +0200 Subject: [PATCH 03/22] [infrastructure] Detect community GHCR images during enterprise migration (#7101) The migration wizard detected the community deployment only by the Docker Hub image prefix (netbirdio/netbird-server), so deployments installed from the ghcr.io mirror failed with "Could not find a service running netbirdio/netbird-server*". This broadens the server and dashboard detection to also accept ghcr.io/netbirdio/... images. The regexes are anchored at the tag/digest separator so Enterprise images (netbird-server-cloud, dashboard-cloud) are still rejected, an already-migrated deployment must not be detected as a community one. Error messages updated to mention both forms. --- infrastructure_files/migrate-to-enterprise.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infrastructure_files/migrate-to-enterprise.sh b/infrastructure_files/migrate-to-enterprise.sh index 8e1fcf521..5f69b4a90 100755 --- a/infrastructure_files/migrate-to-enterprise.sh +++ b/infrastructure_files/migrate-to-enterprise.sh @@ -173,11 +173,11 @@ EOF # --------------------------------------------------------------------------- detect_combined_service() { - yq eval '.services | to_entries | map(select(.value.image | test("^netbirdio/netbird-server"))) | .[0].key // ""' "$COMPOSE_FILE" + yq eval '.services | to_entries | map(select(.value.image | test("^(ghcr\\.io/)?netbirdio/netbird-server([:@]|$)"))) | .[0].key // ""' "$COMPOSE_FILE" } detect_dashboard_service() { - yq eval '.services | to_entries | map(select(.value.image | test("^netbirdio/dashboard"))) | .[0].key // ""' "$COMPOSE_FILE" + yq eval '.services | to_entries | map(select(.value.image | test("^(ghcr\\.io/)?netbirdio/dashboard([:@]|$)"))) | .[0].key // ""' "$COMPOSE_FILE" } detect_config_yaml_host_path() { @@ -661,12 +661,12 @@ init_migration() { COMPOSE_NETWORK=$(detect_compose_network) if [[ -z "$COMBINED_SERVICE" ]]; then - echo "Could not find a service running netbirdio/netbird-server* in $COMPOSE_FILE." > /dev/stderr + echo "Could not find a service running netbirdio/netbird-server or ghcr.io/netbirdio/netbird-server in $COMPOSE_FILE." > /dev/stderr echo "This script targets the community combined-server deployment." > /dev/stderr exit 1 fi if [[ -z "$DASHBOARD_SERVICE" ]]; then - echo "Could not find a service running netbirdio/dashboard* in $COMPOSE_FILE." > /dev/stderr + echo "Could not find a service running netbirdio/dashboard or ghcr.io/netbirdio/dashboard in $COMPOSE_FILE." > /dev/stderr exit 1 fi if [[ -z "$CONFIG_YAML_HOST" ]]; then From eb619fc7e39f8ab17936829ce57911c39d300064 Mon Sep 17 00:00:00 2001 From: Riccardo Manfrin <3090891+riccardomanfrin@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:11:05 +0200 Subject: [PATCH 04/22] [client] disambiguate the connection_type metric tag (#7043) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Describe your changes `recordConnectionMetrics` mapped only `conntype.Relay` to `relay` and let a `default` branch record everything else as `ice`. That silently included `ICETurn` — an ICE connection through a TURN server, which [`conn.isRelayed`](https://github.com/netbirdio/netbird/blob/main/client/internal/peer/conn.go#L788-L795) itself counts as relayed — and `None`, the transient state set when the relay drops ([conn.go:632](https://github.com/netbirdio/netbird/blob/main/client/internal/peer/conn.go#L632)) or the peer state is reset ([conn.go:757](https://github.com/netbirdio/netbird/blob/main/client/internal/peer/conn.go#L757)). Both were reported as direct peer-to-peer, so the `ice` share overstated direct connections on every platform. The mapping now lists every priority explicitly and emits `ice_p2p`, `ice_turn`, `relay` or `unknown`. The new values deliberately do not reuse `ice` to avoid ambuguity. ## Issue ticket number and link No public issue. Found while reviewing the first production sample of client metrics: 38% of iOS connection events were tagged `ice` on a platform that forces relay by default, which traced back to the `default` branch at [client/internal/peer/conn.go#L963-L968](https://github.com/netbirdio/netbird/blob/main/client/internal/peer/conn.go#L963-L968). ## Stack ### Checklist - [x] Is it a bug fix - [ ] Is a typo/documentation fix - [ ] Is a feature enhancement - [ ] It is a refactor - [x] Created tests that fail without the change (if possible) > By submitting this pull request, you confirm that you have read and agree to the terms of the [Contributor License Agreement](https://github.com/netbirdio/netbird/blob/main/CONTRIBUTOR_LICENSE_AGREEMENT.md). ## Documentation Select exactly one: - [] I added/updated documentation for this change - [x] Documentation is **not needed** for this change (explain why) Internal metrics documentation only, in `client/internal/metrics/infra/README.md`: the four `connection_type` values with their derivation, and a note that pre-fix `ice` samples are not comparable with `ice_p2p`. No public API, CLI or configuration change, so no netbirdio/docs PR. ### Docs PR URL (required if "docs added" is checked) Paste the PR link from https://github.com/netbirdio/docs here: N/A ## Summary by CodeRabbit ## Summary by CodeRabbit * **New Features** * Connection metrics now distinguish direct peer-to-peer, TURN-assisted, relay, and unknown connection types. * Metrics include clearer connection and peer identification details. * **Documentation** * Updated connection timing metric values, traffic semantics, priority behavior, and historical data guidance. * **Bug Fixes** * Unset or unrecognized connection priorities are no longer incorrectly classified as peer-to-peer. * Unknown-transport metrics are skipped to prevent misleading connection data. --- client/internal/metrics/connection_type.go | 10 +++++-- client/internal/metrics/influxdb_test.go | 4 +-- client/internal/metrics/infra/README.md | 21 +++++++++++++- client/internal/peer/conn.go | 22 +++++++++++---- client/internal/peer/conn_test.go | 32 ++++++++++++++++++++++ 5 files changed, 78 insertions(+), 11 deletions(-) diff --git a/client/internal/metrics/connection_type.go b/client/internal/metrics/connection_type.go index a3406a6b8..d393e5112 100644 --- a/client/internal/metrics/connection_type.go +++ b/client/internal/metrics/connection_type.go @@ -4,11 +4,17 @@ package metrics type ConnectionType string const ( - // ConnectionTypeICE represents a direct peer-to-peer connection using ICE - ConnectionTypeICE ConnectionType = "ice" + // ConnectionTypeICEP2P represents a direct peer-to-peer connection using ICE + ConnectionTypeICEP2P ConnectionType = "ice_p2p" + + // ConnectionTypeICETurn represents an ICE connection through a TURN server + ConnectionTypeICETurn ConnectionType = "ice_turn" // ConnectionTypeRelay represents a relayed connection ConnectionTypeRelay ConnectionType = "relay" + + // ConnectionTypeUnknown represents a connection with no active transport. It is not pushed. + ConnectionTypeUnknown ConnectionType = "unknown" ) // String returns the string representation of the connection type diff --git a/client/internal/metrics/influxdb_test.go b/client/internal/metrics/influxdb_test.go index b964e31a3..6a226fe2f 100644 --- a/client/internal/metrics/influxdb_test.go +++ b/client/internal/metrics/influxdb_test.go @@ -28,7 +28,7 @@ func TestInfluxDBMetrics_RecordAndExport(t *testing.T) { WgHandshakeSuccess: time.Now().Add(-1 * time.Second), } - m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICE, false, ts) + m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICEP2P, false, ts) var buf bytes.Buffer err := m.Export(&buf) @@ -60,7 +60,7 @@ func TestInfluxDBMetrics_ExportDeterministicFieldOrder(t *testing.T) { // Record multiple times and verify consistent field order for i := 0; i < 10; i++ { - m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICE, false, ts) + m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICEP2P, false, ts) } var buf bytes.Buffer diff --git a/client/internal/metrics/infra/README.md b/client/internal/metrics/infra/README.md index 7941a30cf..0a69404df 100644 --- a/client/internal/metrics/infra/README.md +++ b/client/internal/metrics/infra/README.md @@ -56,14 +56,33 @@ Measurement: `netbird_peer_connection` Tags: - `deployment_type`: "cloud" | "selfhosted" | "unknown" -- `connection_type`: "ice" | "relay" +- `connection_type`: "ice_p2p" | "ice_turn" | "relay" (see below) - `attempt_type`: "initial" | "reconnection" - `version`: NetBird version string - `os`: Operating system (linux, darwin, windows, android, ios, etc.) - `arch`: CPU architecture (amd64, arm64, etc.) +- `peer_id`: anonymised peer identifier (truncated SHA-256 of the WireGuard public key) +- `connection_pair_id`: deterministic identifier for the peer pair, identical on both sides **Note:** `SignalingReceived` is set when the first offer or answer arrives from the remote peer (in both initial and reconnection paths). It excludes the potentially unbounded wait for the remote peer to come online. +#### `connection_type` values + +Derived from the connection priority (`conntype.ConnPriority`) by `metricsConnType` in `client/internal/peer/conn.go`: + +| Value | Priority | Traffic is | +|-------|----------|------------| +| `ice_p2p` | `ICEP2P` | direct peer-to-peer | +| `ice_turn` | `ICETurn` | relayed, through a TURN server | +| `relay` | `Relay` | relayed, through a NetBird relay | +| `unknown` | `None` or unrecognised | no active transport — **the sample is not pushed** | + +**Direct traffic is `ice_p2p` only.** `ice_turn` is relayed despite being negotiated by ICE, matching `Conn.isRelayed`. + +`None` means no transport is active: not established yet, or reset after a relay drop or a peer-state reset. Such a sample cannot be attributed to a transport, so `recordConnectionMetrics` drops it instead of pushing it — `unknown` therefore never appears in the bucket. Connection counts are counts of connections whose transport was known at sampling time. + +**Samples recorded before 0.77 used a single `ice` value** which covered `ICEP2P`, `ICETurn` *and* `None`, so historical `ice` samples overstate direct connections by an unknown amount and must not be compared with `ice_p2p`. + ### Sync Duration Measurement: `netbird_sync` diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index fe68bb15d..f3235ec7f 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -961,12 +961,9 @@ func (conn *Conn) recordConnectionMetrics() { priority := conn.currentConnPriority conn.mu.Unlock() - var connType metrics.ConnectionType - switch priority { - case conntype.Relay: - connType = metrics.ConnectionTypeRelay - default: - connType = metrics.ConnectionTypeICE + connType := metricsConnType(priority) + if connType == metrics.ConnectionTypeUnknown { + return } // Record metrics with timestamps - duration calculation happens in metrics package @@ -1067,3 +1064,16 @@ func boolToConnStatus(connected bool) guard.ConnStatus { } return guard.ConnStatusDisconnected } + +func metricsConnType(priority conntype.ConnPriority) metrics.ConnectionType { + switch priority { + case conntype.Relay: + return metrics.ConnectionTypeRelay + case conntype.ICETurn: + return metrics.ConnectionTypeICETurn + case conntype.ICEP2P: + return metrics.ConnectionTypeICEP2P + default: + return metrics.ConnectionTypeUnknown + } +} diff --git a/client/internal/peer/conn_test.go b/client/internal/peer/conn_test.go index 49979ea83..b709d5e40 100644 --- a/client/internal/peer/conn_test.go +++ b/client/internal/peer/conn_test.go @@ -11,6 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/internal/metrics" + "github.com/netbirdio/netbird/client/internal/peer/conntype" "github.com/netbirdio/netbird/client/internal/peer/dispatcher" "github.com/netbirdio/netbird/client/internal/peer/guard" "github.com/netbirdio/netbird/client/internal/peer/ice" @@ -386,3 +388,33 @@ func TestConn_onWGDisconnected_NoEscalationWithoutRosenpass(t *testing.T) { } assert.Empty(t, disconnected, "escalation must be limited to rosenpass connections") } + +func TestMetricsConnType(t *testing.T) { + tests := []struct { + name string + priority conntype.ConnPriority + expected metrics.ConnectionType + }{ + {"relay", conntype.Relay, metrics.ConnectionTypeRelay}, + {"ice over turn is relayed, not p2p", conntype.ICETurn, metrics.ConnectionTypeICETurn}, + {"direct p2p", conntype.ICEP2P, metrics.ConnectionTypeICEP2P}, + {"unset priority is unknown, not p2p", conntype.None, metrics.ConnectionTypeUnknown}, + {"unrecognised priority is unknown", conntype.ConnPriority(99), metrics.ConnectionTypeUnknown}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, metricsConnType(tc.priority)) + }) + } +} + +func TestMetricsConnType_RelayedMatchesIsRelayed(t *testing.T) { + for _, priority := range []conntype.ConnPriority{conntype.None, conntype.Relay, conntype.ICETurn, conntype.ICEP2P} { + conn := &Conn{currentConnPriority: priority} + tag := metricsConnType(priority) + relayedTag := tag == metrics.ConnectionTypeRelay || tag == metrics.ConnectionTypeICETurn + assert.Equal(t, conn.isRelayed(), relayedTag, + "priority %s: isRelayed and the %q metric tag must agree", priority, tag) + } +} From 2ee21d2b5c64a2f463414fd1b3369b606383f3d2 Mon Sep 17 00:00:00 2001 From: Pascal Fischer <32096965+pascal-fischer@users.noreply.github.com> Date: Fri, 7 Aug 2026 18:07:53 +0200 Subject: [PATCH 05/22] [management] Affected peers for user updates (#7099) --- management/server/account.go | 88 +++++---- management/server/affected_peers_jwt_test.go | 179 ++++++++++++++++++ management/server/affected_peers_user_test.go | 170 +++++++++++++++++ management/server/affectedpeers/resolver.go | 73 ++++++- .../server/affectedpeers/resolver_test.go | 2 + management/server/user.go | 80 ++++++-- 6 files changed, 535 insertions(+), 57 deletions(-) create mode 100644 management/server/affected_peers_jwt_test.go create mode 100644 management/server/affected_peers_user_test.go diff --git a/management/server/account.go b/management/server/account.go index 619036d0c..700dfa04d 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -33,6 +33,7 @@ import ( nbconfig "github.com/netbirdio/netbird/management/internals/server/config" "github.com/netbirdio/netbird/management/server/account" "github.com/netbirdio/netbird/management/server/activity" + "github.com/netbirdio/netbird/management/server/affectedpeers" nbcache "github.com/netbirdio/netbird/management/server/cache" nbcontext "github.com/netbirdio/netbird/management/server/context" "github.com/netbirdio/netbird/management/server/geolocation" @@ -1626,6 +1627,8 @@ func (am *DefaultAccountManager) SyncUserJWTGroups(ctx context.Context, userAuth var removeOldGroups []string var hasChanges bool var user *types.User + var change affectedpeers.Change + var snap *affectedpeers.Snapshot err = am.Store.ExecuteInTransaction(ctx, func(transaction store.Store) error { user, err = transaction.GetUserByUserID(ctx, store.LockingStrengthNone, userAuth.UserId) if err != nil { @@ -1664,14 +1667,25 @@ func (am *DefaultAccountManager) SyncUserJWTGroups(ctx context.Context, userAuth return fmt.Errorf("error saving user: %w", err) } + allGroupChanges := slices.Concat(addNewGroups, removeOldGroups) + // The user's auto-groups changed, so the SSH rules authorizing them ship a new + // group -> user mapping even when no peer moves between groups. + change.UserGroupIDs = allGroupChanges + + // The user's peers are the changed entity in every scenario the sync can + // produce — group membership, IPv6 assignment, SSH mappings — so they refresh + // together with every peer they can connect to, like on a regular peer update. + userPeers, err := transaction.GetUserPeers(ctx, store.LockingStrengthNone, userAuth.AccountId, userAuth.UserId) + if err != nil { + return fmt.Errorf("error getting user peers: %w", err) + } + for _, peer := range userPeers { + change.ChangedPeerIDs = append(change.ChangedPeerIDs, peer.ID) + } + // Propagate changes to peers if group propagation is enabled if settings.GroupsPropagationEnabled { - peers, err := transaction.GetUserPeers(ctx, store.LockingStrengthNone, userAuth.AccountId, userAuth.UserId) - if err != nil { - return fmt.Errorf("error getting user peers: %w", err) - } - - for _, peer := range peers { + for _, peer := range userPeers { for _, g := range addNewGroups { if err := transaction.AddPeerToGroup(ctx, userAuth.AccountId, peer.ID, g); err != nil { return fmt.Errorf("error adding peer %s to group %s: %w", peer.ID, g, err) @@ -1684,7 +1698,8 @@ func (am *DefaultAccountManager) SyncUserJWTGroups(ctx context.Context, userAuth } } - allGroupChanges := slices.Concat(addNewGroups, removeOldGroups) + change.LinkGroups = allGroupChanges + if err = am.reconcileIPv6ForGroupChanges(ctx, transaction, userAuth.AccountId, allGroupChanges); err != nil { return fmt.Errorf("reconcile IPv6 for group changes: %w", err) } @@ -1694,6 +1709,10 @@ func (am *DefaultAccountManager) SyncUserJWTGroups(ctx context.Context, userAuth } } + if snap, err = affectedpeers.Load(ctx, transaction, userAuth.AccountId, change); err != nil { + return err + } + return nil }) if err != nil { @@ -1730,20 +1749,17 @@ func (am *DefaultAccountManager) SyncUserJWTGroups(ctx context.Context, userAuth } } - removedGroupAffectsPeers, err := areGroupChangesAffectPeers(ctx, am.Store, userAuth.AccountId, removeOldGroups) - if err != nil { - return err - } - - newGroupsAffectsPeers, err := areGroupChangesAffectPeers(ctx, am.Store, userAuth.AccountId, addNewGroups) - if err != nil { - return err - } - - if removedGroupAffectsPeers || newGroupsAffectsPeers { - log.WithContext(ctx).Tracef("user %s: JWT group membership changed, updating account peers", userAuth.UserId) - am.BufferUpdateAccountPeers(ctx, userAuth.AccountId, types.UpdateReason{Resource: types.UpdateResourceUser, Operation: types.UpdateOperationUpdate}) - } + log.WithContext(ctx).Tracef("user %s: JWT group membership changed, updating affected peers", userAuth.UserId) + bgCtx := context.WithoutCancel(ctx) + go func() { + affectedPeerIDs := snap.Expand(bgCtx, userAuth.AccountId, change) + if len(affectedPeerIDs) == 0 { + return + } + if err := am.networkMapController.BufferUpdateAffectedPeers(bgCtx, userAuth.AccountId, affectedPeerIDs, types.UpdateReason{Resource: types.UpdateResourceUser, Operation: types.UpdateOperationUpdate}); err != nil { + log.WithContext(bgCtx).Errorf("failed to update affected peers after JWT group sync for account %s: %v", userAuth.AccountId, err) + } + }() return nil } @@ -2426,30 +2442,24 @@ func (am *DefaultAccountManager) reconcileIPv6ForGroupChanges(ctx context.Contex return fmt.Errorf("get account settings: %w", err) } - if len(settings.IPv6EnabledGroups) == 0 { - return nil - } - - enabledSet := make(map[string]struct{}, len(settings.IPv6EnabledGroups)) - for _, gid := range settings.IPv6EnabledGroups { - enabledSet[gid] = struct{}{} - } - - affected := false - for _, gid := range groupIDs { - if _, ok := enabledSet[gid]; ok { - affected = true - break - } - } - - if !affected { + if !ipv6ReconcileNeeded(settings, groupIDs) { return nil } return am.updatePeerIPv6Addresses(ctx, transaction, accountID, settings) } +// ipv6ReconcileNeeded reports whether changes to the given groups trigger an IPv6 +// reconciliation. +func ipv6ReconcileNeeded(settings *types.Settings, groupIDs []string) bool { + for _, groupID := range groupIDs { + if slices.Contains(settings.IPv6EnabledGroups, groupID) { + return true + } + } + return false +} + func (am *DefaultAccountManager) ensureIPv6Subnet(ctx context.Context, transaction store.Store, accountID string, settings *types.Settings, network *types.Network) error { if settings.NetworkRangeV6.IsValid() { network.NetV6 = net.IPNet{ diff --git a/management/server/affected_peers_jwt_test.go b/management/server/affected_peers_jwt_test.go new file mode 100644 index 000000000..766745cd7 --- /dev/null +++ b/management/server/affected_peers_jwt_test.go @@ -0,0 +1,179 @@ +package server + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/management/server/affectedpeers" + nbpeer "github.com/netbirdio/netbird/management/server/peer" + "github.com/netbirdio/netbird/management/server/store" + "github.com/netbirdio/netbird/management/server/types" + "github.com/netbirdio/netbird/shared/auth" +) + +// A user's auto-group change refreshes the destinations of the SSH rules authorizing +// that group — they carry the group -> user mapping — even though no peer moved +// between groups. +func TestAffectedPeers_UserGroupChange_RefreshesSSHAuthorizedDestinations(t *testing.T) { + manager, s, accountID, peerIDs, groupIDs := setupAffectedPeersTest(t) + ctx := context.Background() + + _, err := manager.SavePolicy(ctx, accountID, userID, &types.Policy{ + Enabled: true, + Rules: []*types.PolicyRule{ + { + Enabled: true, + Sources: []string{groupIDs[0]}, + Destinations: []string{groupIDs[1]}, + Protocol: types.PolicyRuleProtocolNetbirdSSH, + Action: types.PolicyTrafficActionAccept, + AuthorizedGroups: map[string][]string{groupIDs[3]: {"root"}}, + }, + }, + }, true) + require.NoError(t, err) + + result := resolveAffected(t, s, accountID, affectedpeers.Change{UserGroupIDs: []string{groupIDs[3]}}) + assert.ElementsMatch(t, []string{peerIDs[1]}, result, + "only the SSH rule's destination peers carry the changed group -> user mapping") + + result = resolveAffected(t, s, accountID, affectedpeers.Change{UserGroupIDs: []string{groupIDs[4]}}) + assert.Empty(t, result, "a group no SSH rule authorizes affects nobody") +} + +// Creating, blocking or unblocking a user changes the account's allowed-user set, which +// reaches only the destinations of the SSH rules that ship it. +func TestAffectedPeers_AllowedUsersChange_RefreshesSSHDestinations(t *testing.T) { + manager, s, accountID, peerIDs, groupIDs := setupAffectedPeersTest(t) + ctx := context.Background() + + // Ships the allowed-user set: an SSH rule naming no groups and no user. + _, err := manager.SavePolicy(ctx, accountID, userID, &types.Policy{ + Enabled: true, + Rules: []*types.PolicyRule{{ + Enabled: true, + Sources: []string{groupIDs[0]}, + Destinations: []string{groupIDs[1]}, + Protocol: types.PolicyRuleProtocolNetbirdSSH, + Action: types.PolicyTrafficActionAccept, + }}, + }, true) + require.NoError(t, err) + + // Does not ship it: an SSH rule that authorizes a specific group. + _, err = manager.SavePolicy(ctx, accountID, userID, &types.Policy{ + Enabled: true, + Rules: []*types.PolicyRule{{ + Enabled: true, + Sources: []string{groupIDs[2]}, + Destinations: []string{groupIDs[3]}, + Protocol: types.PolicyRuleProtocolNetbirdSSH, + Action: types.PolicyTrafficActionAccept, + AuthorizedGroups: map[string][]string{groupIDs[0]: {"root"}}, + }}, + }, true) + require.NoError(t, err) + + result := resolveAffected(t, s, accountID, affectedpeers.Change{AllowedUsersChanged: true}) + assert.ElementsMatch(t, []string{peerIDs[1]}, result, + "only the destinations of the rule shipping the allowed-user set refresh") +} + +// TestAffectedPeers_SyncUserJWTGroups_OnlyAffectedPeersUpdated verifies that a JWT +// auto-group change updates only the user's peers and the peers linked to the changed +// group through policies, instead of fanning out to the whole account. +func TestAffectedPeers_SyncUserJWTGroups_OnlyAffectedPeersUpdated(t *testing.T) { + manager, updateManager, account, _, peer2, peer3 := setupNetworkMapTest(t) + ctx := context.Background() + accountID := account.Id + + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + userPeer, _, _, _, err := manager.AddPeer(ctx, accountID, "", userID, &nbpeer.Peer{ + Key: key.PublicKey().String(), + Meta: nbpeer.PeerSystemMeta{Hostname: "user-peer"}, + }, false) + require.NoError(t, err) + + policies, err := manager.Store.GetAccountPolicies(ctx, store.LockingStrengthNone, accountID) + require.NoError(t, err) + for _, p := range policies { + require.NoError(t, manager.Store.DeletePolicy(ctx, accountID, p.ID)) + } + + account, err = manager.Store.GetAccount(ctx, accountID) + require.NoError(t, err) + account.Settings.JWTGroupsEnabled = true + account.Settings.JWTGroupsClaimName = "groups" + account.Settings.GroupsPropagationEnabled = true + require.NoError(t, manager.Store.SaveAccount(ctx, account)) + + require.NoError(t, manager.CreateGroup(ctx, accountID, userID, &types.Group{ID: "jwt-grp", Name: "jwt-linked", Issued: types.GroupIssuedJWT, Peers: []string{}})) + require.NoError(t, manager.CreateGroup(ctx, accountID, userID, &types.Group{ID: "jwt-dest", Name: "jwt-dest", Peers: []string{peer2.ID}})) + + _, err = manager.SavePolicy(ctx, accountID, userID, &types.Policy{ + Enabled: true, + Rules: []*types.PolicyRule{ + { + Enabled: true, + Sources: []string{"jwt-grp"}, + Destinations: []string{"jwt-dest"}, + Bidirectional: true, + Action: types.PolicyTrafficActionAccept, + }, + }, + }, true) + require.NoError(t, err) + + updUser := updateManager.CreateChannel(ctx, userPeer.ID) + upd2 := updateManager.CreateChannel(ctx, peer2.ID) + upd3 := updateManager.CreateChannel(ctx, peer3.ID) + t.Cleanup(func() { + updateManager.CloseChannel(ctx, userPeer.ID) + updateManager.CloseChannel(ctx, peer2.ID) + updateManager.CloseChannel(ctx, peer3.ID) + }) + + userAuth := auth.UserAuth{ + AccountId: accountID, + UserId: userID, + Groups: []string{"jwt-linked"}, + } + + t.Run("adding JWT group updates only linked peers", func(t *testing.T) { + drainPeerUpdates(updUser) + drainPeerUpdates(upd2) + drainPeerUpdates(upd3) + + require.NoError(t, manager.SyncUserJWTGroups(ctx, userAuth)) + + peerShouldReceiveUpdate(t, updUser) + peerShouldReceiveUpdate(t, upd2) + peerShouldNotReceiveUpdate(t, upd3) + + user, err := manager.Store.GetUserByUserID(ctx, store.LockingStrengthNone, userID) + require.NoError(t, err) + assert.Contains(t, user.AutoGroups, "jwt-grp") + }) + + t.Run("removing JWT group updates only linked peers", func(t *testing.T) { + drainPeerUpdates(updUser) + drainPeerUpdates(upd2) + drainPeerUpdates(upd3) + + userAuth.Groups = nil + require.NoError(t, manager.SyncUserJWTGroups(ctx, userAuth)) + + peerShouldReceiveUpdate(t, updUser) + peerShouldReceiveUpdate(t, upd2) + peerShouldNotReceiveUpdate(t, upd3) + + user, err := manager.Store.GetUserByUserID(ctx, store.LockingStrengthNone, userID) + require.NoError(t, err) + assert.NotContains(t, user.AutoGroups, "jwt-grp") + }) +} diff --git a/management/server/affected_peers_user_test.go b/management/server/affected_peers_user_test.go new file mode 100644 index 000000000..c0dbbb84f --- /dev/null +++ b/management/server/affected_peers_user_test.go @@ -0,0 +1,170 @@ +package server + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/management/server/activity" + nbpeer "github.com/netbirdio/netbird/management/server/peer" + "github.com/netbirdio/netbird/management/server/store" + "github.com/netbirdio/netbird/management/server/types" +) + +// A user update refreshes only the peers its auto-group change reaches, and a user +// update that changes no group membership refreshes nobody. +func TestAffectedPeers_SaveUser_OnlyAffectedPeersUpdated(t *testing.T) { + manager, updateManager, account, _, peer2, peer3 := setupNetworkMapTest(t) + ctx := context.Background() + accountID := account.Id + + const targetUserID = "target-user" + require.NoError(t, manager.Store.SaveUser(ctx, &types.User{ + Id: targetUserID, AccountID: accountID, Role: types.UserRoleUser, + })) + + key, err := wgtypes.GeneratePrivateKey() + require.NoError(t, err) + targetPeer, _, _, _, err := manager.AddPeer(ctx, accountID, "", targetUserID, &nbpeer.Peer{ + Key: key.PublicKey().String(), + Meta: nbpeer.PeerSystemMeta{Hostname: "target-peer"}, + }, false) + require.NoError(t, err) + + policies, err := manager.Store.GetAccountPolicies(ctx, store.LockingStrengthNone, accountID) + require.NoError(t, err) + for _, p := range policies { + require.NoError(t, manager.Store.DeletePolicy(ctx, accountID, p.ID)) + } + + account, err = manager.Store.GetAccount(ctx, accountID) + require.NoError(t, err) + account.Settings.GroupsPropagationEnabled = true + require.NoError(t, manager.Store.SaveAccount(ctx, account)) + + require.NoError(t, manager.CreateGroup(ctx, accountID, userID, &types.Group{ID: "ug-linked", Name: "ug-linked"})) + require.NoError(t, manager.CreateGroup(ctx, accountID, userID, &types.Group{ID: "ug-dest", Name: "ug-dest", Peers: []string{peer2.ID}})) + + _, err = manager.SavePolicy(ctx, accountID, userID, &types.Policy{ + Enabled: true, + Rules: []*types.PolicyRule{ + { + Enabled: true, + Sources: []string{"ug-linked"}, + Destinations: []string{"ug-dest"}, + Bidirectional: true, + Action: types.PolicyTrafficActionAccept, + }, + }, + }, true) + require.NoError(t, err) + + updTarget := updateManager.CreateChannel(ctx, targetPeer.ID) + upd2 := updateManager.CreateChannel(ctx, peer2.ID) + upd3 := updateManager.CreateChannel(ctx, peer3.ID) + t.Cleanup(func() { + updateManager.CloseChannel(ctx, targetPeer.ID) + updateManager.CloseChannel(ctx, peer2.ID) + updateManager.CloseChannel(ctx, peer3.ID) + }) + + t.Run("auto group change updates only linked peers", func(t *testing.T) { + drainPeerUpdates(updTarget) + drainPeerUpdates(upd2) + drainPeerUpdates(upd3) + + _, err := manager.SaveUser(ctx, accountID, activity.SystemInitiator, &types.User{ + Id: targetUserID, AccountID: accountID, Role: types.UserRoleUser, + AutoGroups: []string{"ug-linked"}, + }) + require.NoError(t, err) + + peerShouldReceiveUpdate(t, updTarget) + peerShouldReceiveUpdate(t, upd2) + peerShouldNotReceiveUpdate(t, upd3) + }) + + t.Run("update without group changes refreshes nobody", func(t *testing.T) { + drainPeerUpdates(updTarget) + drainPeerUpdates(upd2) + drainPeerUpdates(upd3) + + _, err := manager.SaveUser(ctx, accountID, activity.SystemInitiator, &types.User{ + Id: targetUserID, AccountID: accountID, Role: types.UserRoleUser, + AutoGroups: []string{"ug-linked"}, Name: "renamed", + }) + require.NoError(t, err) + + peerShouldNotReceiveUpdate(t, updTarget) + peerShouldNotReceiveUpdate(t, upd2) + peerShouldNotReceiveUpdate(t, upd3) + + user, err := manager.Store.GetUserByUserID(ctx, store.LockingStrengthNone, targetUserID) + require.NoError(t, err) + assert.Equal(t, "renamed", user.Name) + }) + + t.Run("auto group change reassigning IPv6 refreshes the changed peers and their observers", func(t *testing.T) { + account, err := manager.Store.GetAccount(ctx, accountID) + require.NoError(t, err) + account.Settings.IPv6EnabledGroups = []string{"ug-v6"} + require.NoError(t, manager.Store.SaveAccount(ctx, account)) + require.NoError(t, manager.CreateGroup(ctx, accountID, userID, &types.Group{ID: "ug-v6", Name: "ug-v6"})) + + drainPeerUpdates(updTarget) + drainPeerUpdates(upd2) + drainPeerUpdates(upd3) + + _, err = manager.SaveUser(ctx, accountID, activity.SystemInitiator, &types.User{ + Id: targetUserID, AccountID: accountID, Role: types.UserRoleUser, + AutoGroups: []string{"ug-linked", "ug-v6"}, Name: "renamed", + }) + require.NoError(t, err) + + // The reassigned peer refreshes with everyone it can reach: peer2 via the + // policy, but not peer3, which shares no group or policy with it. + peerShouldReceiveUpdate(t, updTarget) + peerShouldReceiveUpdate(t, upd2) + peerShouldNotReceiveUpdate(t, upd3) + }) + + t.Run("unblocking a user refreshes only the SSH rule destinations", func(t *testing.T) { + // An SSH rule that authorizes no group of its own ships the account's + // allowed-user set to its destinations, so those are the peers an unblock + // reaches — not the whole account. + _, err := manager.SavePolicy(ctx, accountID, userID, &types.Policy{ + Enabled: true, + Rules: []*types.PolicyRule{{ + Enabled: true, + Sources: []string{"ug-linked"}, + Destinations: []string{"ug-dest"}, + Protocol: types.PolicyRuleProtocolNetbirdSSH, + Action: types.PolicyTrafficActionAccept, + }}, + }, true) + require.NoError(t, err) + + blocked, err := manager.Store.GetUserByUserID(ctx, store.LockingStrengthNone, targetUserID) + require.NoError(t, err) + blocked.Blocked = true + require.NoError(t, manager.Store.SaveUser(ctx, blocked)) + + drainPeerUpdates(updTarget) + drainPeerUpdates(upd2) + drainPeerUpdates(upd3) + + // Same auto-groups as the previous subtest left them, so no group change and + // no IPv6 reconciliation interferes: the unblock alone drives the refresh. + _, err = manager.SaveUser(ctx, accountID, activity.SystemInitiator, &types.User{ + Id: targetUserID, AccountID: accountID, Role: types.UserRoleUser, + AutoGroups: []string{"ug-linked", "ug-v6"}, Name: "renamed", + }) + require.NoError(t, err) + + peerShouldReceiveUpdate(t, upd2) + peerShouldNotReceiveUpdate(t, upd3) + }) +} diff --git a/management/server/affectedpeers/resolver.go b/management/server/affectedpeers/resolver.go index 16a795539..cb2063ac9 100644 --- a/management/server/affectedpeers/resolver.go +++ b/management/server/affectedpeers/resolver.go @@ -18,6 +18,7 @@ import ( "context" log "github.com/sirupsen/logrus" + "golang.org/x/exp/maps" nbdns "github.com/netbirdio/netbird/dns" rpservice "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service" @@ -83,7 +84,7 @@ func (snap *Snapshot) loadCollections(ctx context.Context, s store.Store, accoun hasGroupOrPeerChange := len(c.ChangedGroupIDs) > 0 || len(c.ChangedPeerIDs) > 0 || len(c.LinkGroups) > 0 || len(c.Resources) > 0 hasNetworkObject := len(c.Routers) > 0 || len(c.Resources) > 0 || len(c.Networks) > 0 // the resource<->router bridge can fire for any of these - needsRoutersResources := hasGroupOrPeerChange || len(c.PostureCheckIDs) > 0 || len(c.Policies) > 0 || hasNetworkObject + needsRoutersResources := hasGroupOrPeerChange || len(c.PostureCheckIDs) > 0 || len(c.Policies) > 0 || hasNetworkObject || len(c.UserGroupIDs) > 0 || c.AllowedUsersChanged if needsRoutersResources { if err := snap.loadPolicyRoutersResources(ctx, s, accountID); err != nil { @@ -219,6 +220,18 @@ type Change struct { // (correct when the peer's own attributes changed, e.g. IP/status). OutputPeerIDs []string + // UserGroupIDs are groups whose USER membership changed (a user's auto-groups), + // as opposed to their peer membership. Peers ship the group -> user mapping only + // for the groups an SSH rule authorizes, so these refresh the destinations of the + // SSH rules authorizing them — independently of any peer moving between groups. + UserGroupIDs []string + + // AllowedUsersChanged marks a change to the set of users allowed to open SSH + // sessions — a user was created, blocked or unblocked. That set is account-wide, + // and peers receive it through the SSH rules that name no group or user of their + // own, so those rules' destinations refresh. + AllowedUsersChanged bool + // LinkGroups are groups used ONLY to match policies/routes/routers and walk to the // OPPOSITE side — they are never expanded to their own members. Use this when a // peer's group membership changed: pass the peer in ChangedPeerIDs and its @@ -240,6 +253,8 @@ func (c Change) isEmpty() bool { len(c.Resources) == 0 && len(c.Networks) == 0 && len(c.PostureCheckIDs) == 0 && + len(c.UserGroupIDs) == 0 && + !c.AllowedUsersChanged && len(c.DistributionGroupIDs) == 0 && len(c.RemovedPeersByGroup) == 0 && len(c.LinkGroups) == 0 && @@ -359,6 +374,9 @@ func (r *resolver) walk() { r.collectFromProxyServices() } + r.collectFromSSHAuthorizedGroups() + r.collectFromAllowedUsers() + r.collectFromChangedRoutes(r.change.Routes) r.collectFromChangedRouters(r.change.Routers) r.collectFromChangedResources(r.change.Resources) @@ -811,6 +829,59 @@ func (r *resolver) collectFromNameServers() { } } +// collectFromSSHAuthorizedGroups folds the destinations of the enabled SSH rules that +// authorize a group whose user membership changed. Those destination peers carry the +// group -> user mapping for the groups they authorize, so they refresh even when no +// peer moved between groups. +func (r *resolver) collectFromSSHAuthorizedGroups() { + if len(r.change.UserGroupIDs) == 0 { + return + } + + changed := toSet(r.change.UserGroupIDs) + for _, policy := range r.policies() { + for _, rule := range policy.Rules { + if !rule.Enabled || rule.Protocol != types.PolicyRuleProtocolNetbirdSSH { + continue + } + if !anyInSet(maps.Keys(rule.AuthorizedGroups), changed) { + continue + } + log.WithContext(r.ctx).Tracef("collectFromSSHAuthorizedGroups: rule %s authorizes a changed user group -> folding its destinations", rule.ID) + r.foldPolicySideForRule(policy, rule, sideDestination) + } + } +} + +// collectFromAllowedUsers folds the destinations of the rules that make a peer carry +// the account's allowed-user set, for a change to who is in that set. +func (r *resolver) collectFromAllowedUsers() { + if !r.change.AllowedUsersChanged { + return + } + + for _, policy := range r.policies() { + for _, rule := range policy.Rules { + if !rule.Enabled || !ruleShipsAllowedUsers(rule) { + continue + } + log.WithContext(r.ctx).Tracef("collectFromAllowedUsers: rule %s ships the allowed-user set -> folding its destinations", rule.ID) + r.foldPolicySideForRule(policy, rule, sideDestination) + } + } +} + +// ruleShipsAllowedUsers reports whether a rule makes its destination peers carry the +// account's allowed-user set. It mirrors the network map's SSH requirements except for +// the destination peer's own SSH flag, which the snapshot does not hold — so it folds a +// superset and never misses a peer. +func ruleShipsAllowedUsers(rule *types.PolicyRule) bool { + if rule.Protocol == types.PolicyRuleProtocolNetbirdSSH { + return len(rule.AuthorizedGroups) == 0 && rule.AuthorizedUser == "" + } + return types.PolicyRuleImpliesLegacySSH(rule) +} + func (r *resolver) collectFromDNSSettings() { if len(r.linkGroups) == 0 || r.snap.dnsSettings == nil { return diff --git a/management/server/affectedpeers/resolver_test.go b/management/server/affectedpeers/resolver_test.go index fe6ada347..b0c430c3c 100644 --- a/management/server/affectedpeers/resolver_test.go +++ b/management/server/affectedpeers/resolver_test.go @@ -85,6 +85,8 @@ func TestChangeIsEmpty(t *testing.T) { assert.False(t, Change{Resources: []*resourceTypes.NetworkResource{{ID: "r"}}}.isEmpty()) assert.False(t, Change{Networks: []*networkTypes.Network{{ID: "n"}}}.isEmpty()) assert.False(t, Change{PostureCheckIDs: []string{"pc"}}.isEmpty()) + assert.False(t, Change{UserGroupIDs: []string{"g"}}.isEmpty()) + assert.False(t, Change{AllowedUsersChanged: true}.isEmpty()) } func TestPolicyReferencesPostureChecks(t *testing.T) { diff --git a/management/server/user.go b/management/server/user.go index fc8400e29..7c0a3088d 100644 --- a/management/server/user.go +++ b/management/server/user.go @@ -593,7 +593,8 @@ func (am *DefaultAccountManager) SaveOrAddUsers(ctx context.Context, accountID, return nil, err } - var updateAccountPeers bool + var snaps []*affectedpeers.Snapshot + var changes []affectedpeers.Change var peersToExpire []*nbpeer.Peer var addUserEvents []func() var usersToSave = make([]*types.User, 0, len(updates)) @@ -629,20 +630,25 @@ func (am *DefaultAccountManager) SaveOrAddUsers(ctx context.Context, accountID, } err = am.Store.ExecuteInTransaction(ctx, func(transaction store.Store) error { - _, updatedUser, userPeersToExpire, userEvents, err := am.processUserUpdate( + change, updatedUser, userPeersToExpire, userEvents, err := am.processUserUpdate( ctx, transaction, groupsMap, accountID, initiatorUserID, initiatorUser, update, addIfNotExists, settings, ) if err != nil { return fmt.Errorf("failed to process update for user %s: %w", update.Id, err) } - updateAccountPeers = true - err = transaction.SaveUser(ctx, updatedUser) if err != nil { return fmt.Errorf("failed to save updated user %s: %w", update.Id, err) } + snap, err := affectedpeers.Load(ctx, transaction, accountID, change) + if err != nil { + return err + } + + snaps = append(snaps, snap) + changes = append(changes, change) usersToSave = append(usersToSave, updatedUser) addUserEvents = append(addUserEvents, userEvents...) peersToExpire = append(peersToExpire, userPeersToExpire...) @@ -683,11 +689,11 @@ func (am *DefaultAccountManager) SaveOrAddUsers(ctx context.Context, accountID, log.WithContext(ctx).Errorf("failed update expired peers: %s", err) return nil, err } - } else if updateAccountPeers { + } else if len(usersToSave) > 0 { if err = am.Store.IncrementNetworkSerial(ctx, accountID); err != nil { return nil, fmt.Errorf("failed to increment network serial: %w", err) } - am.UpdateAccountPeers(ctx, accountID, types.UpdateReason{Resource: types.UpdateResourceUser, Operation: types.UpdateOperationUpdate}) + go am.dispatchAffected(ctx, accountID, snaps, changes) } return updatedUsersInfo, globalErr @@ -759,19 +765,21 @@ func (am *DefaultAccountManager) prepareUserUpdateEvents(ctx context.Context, ac } func (am *DefaultAccountManager) processUserUpdate(ctx context.Context, transaction store.Store, groupsMap map[string]*types.Group, - accountID, initiatorUserId string, initiatorUser, update *types.User, addIfNotExists bool, settings *types.Settings) (bool, *types.User, []*nbpeer.Peer, []func(), error) { + accountID, initiatorUserId string, initiatorUser, update *types.User, addIfNotExists bool, settings *types.Settings) (affectedpeers.Change, *types.User, []*nbpeer.Peer, []func(), error) { + + var change affectedpeers.Change if update == nil { - return false, nil, nil, nil, status.Errorf(status.InvalidArgument, "provided user update is nil") + return change, nil, nil, nil, status.Errorf(status.InvalidArgument, "provided user update is nil") } oldUser, isNewUser, err := getUserOrCreateIfNotExists(ctx, transaction, accountID, update, addIfNotExists) if err != nil { - return false, nil, nil, nil, err + return change, nil, nil, nil, err } if err := validateUserUpdate(groupsMap, initiatorUser, oldUser, update); err != nil { - return false, nil, nil, nil, err + return change, nil, nil, nil, err } // only auto groups, revoked status, and integration reference can be updated for now @@ -792,13 +800,13 @@ func (am *DefaultAccountManager) processUserUpdate(ctx context.Context, transact var transferredOwnerRole bool result, err := handleOwnerRoleTransfer(ctx, transaction, initiatorUser, update) if err != nil { - return false, nil, nil, nil, err + return change, nil, nil, nil, err } transferredOwnerRole = result userPeers, err := transaction.GetUserPeers(ctx, store.LockingStrengthNone, updatedUser.AccountID, update.Id) if err != nil { - return false, nil, nil, nil, err + return change, nil, nil, nil, err } var peersToExpire []*nbpeer.Peer @@ -807,6 +815,32 @@ func (am *DefaultAccountManager) processUserUpdate(ctx context.Context, transact peersToExpire = userPeers } + // A user reaches a peer's network map only through the SSH rules: as part of a + // group -> user mapping, and as part of the account's allowed-user set. Creating, + // blocking or unblocking a user adds it to or removes it from both, so every group + // it maps into changes — including the All group that holds every active user. + // Otherwise only the auto-groups it joined or left do. + if isNewUser || oldUser.IsBlocked() != updatedUser.IsBlocked() { + change.AllowedUsersChanged = true + change.UserGroupIDs = slices.Concat(oldUser.AutoGroups, updatedUser.AutoGroups, allGroupIDs(groupsMap)) + } else { + change.UserGroupIDs = slices.Concat( + util.Difference(oldUser.AutoGroups, updatedUser.AutoGroups), + util.Difference(updatedUser.AutoGroups, oldUser.AutoGroups), + ) + } + + // The user's peers are the changed entity in every scenario the update can + // produce — group membership, IPv6 assignment, SSH mappings — so they refresh + // together with every peer they can connect to, like on a regular peer update. + // An update that changes neither the auto-groups nor the active-user set has no + // peer-visible effect and refreshes nobody. + if len(change.UserGroupIDs) > 0 || change.AllowedUsersChanged { + for _, peer := range userPeers { + change.ChangedPeerIDs = append(change.ChangedPeerIDs, peer.ID) + } + } + var removedGroups, addedGroups []string if update.AutoGroups != nil && settings.GroupsPropagationEnabled { removedGroups = util.Difference(oldUser.AutoGroups, update.AutoGroups) @@ -814,26 +848,38 @@ func (am *DefaultAccountManager) processUserUpdate(ctx context.Context, transact for _, peer := range userPeers { for _, groupID := range removedGroups { if err := transaction.RemovePeerFromGroup(ctx, peer.ID, groupID); err != nil { - return false, nil, nil, nil, fmt.Errorf("failed to remove peer %s from group %s: %w", peer.ID, groupID, err) + return change, nil, nil, nil, fmt.Errorf("failed to remove peer %s from group %s: %w", peer.ID, groupID, err) } } for _, groupID := range addedGroups { if err := transaction.AddPeerToGroup(ctx, accountID, peer.ID, groupID); err != nil { - return false, nil, nil, nil, fmt.Errorf("failed to add peer %s to group %s: %w", peer.ID, groupID, err) + return change, nil, nil, nil, fmt.Errorf("failed to add peer %s to group %s: %w", peer.ID, groupID, err) } } } allGroupChanges := slices.Concat(removedGroups, addedGroups) + change.LinkGroups = allGroupChanges + if err := am.reconcileIPv6ForGroupChanges(ctx, transaction, accountID, allGroupChanges); err != nil { - return false, nil, nil, nil, fmt.Errorf("reconcile IPv6 for group changes: %w", err) + return change, nil, nil, nil, fmt.Errorf("reconcile IPv6 for group changes: %w", err) } } - updateAccountPeers := len(userPeers) > 0 userEventsToAdd := am.prepareUserUpdateEvents(ctx, updatedUser.AccountID, initiatorUserId, oldUser, updatedUser, transferredOwnerRole, isNewUser, removedGroups, addedGroups, transaction) - return updateAccountPeers, updatedUser, peersToExpire, userEventsToAdd, nil + return change, updatedUser, peersToExpire, userEventsToAdd, nil +} + +// allGroupIDs returns the ID of the account's All group, which every active user maps +// into, as a slice so callers can concatenate it. +func allGroupIDs(groupsMap map[string]*types.Group) []string { + for _, group := range groupsMap { + if group.IsGroupAll() { + return []string{group.ID} + } + } + return nil } // getUserOrCreateIfNotExists retrieves the existing user or creates a new one if it doesn't exist. From 179e8f6e139bc857f0d8e99a82b54bd9ee07decf Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Sat, 8 Aug 2026 15:04:06 +0900 Subject: [PATCH 06/22] [infrastructure] add grafana dashboard for licensed management (#7095) --- .../dashboards/management-enterprise.json | 8857 +++++++++++++++++ 1 file changed, 8857 insertions(+) create mode 100644 infrastructure_files/observability/grafana/dashboards/management-enterprise.json diff --git a/infrastructure_files/observability/grafana/dashboards/management-enterprise.json b/infrastructure_files/observability/grafana/dashboards/management-enterprise.json new file mode 100644 index 000000000..481050a02 --- /dev/null +++ b/infrastructure_files/observability/grafana/dashboards/management-enterprise.json @@ -0,0 +1,8857 @@ +{ + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "builtIn": true, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "query": { + "datasource": { + "name": "-- Grafana --" + }, + "group": "grafana", + "kind": "DataQuery", + "spec": {}, + "version": "v0" + } + } + }, + { + "kind": "AnnotationQuery", + "spec": { + "enable": true, + "hide": false, + "iconColor": "red", + "name": "Deployments", + "query": { + "datasource": { + "name": "grafana" + }, + "group": "datasource", + "kind": "DataQuery", + "spec": { + "expr": "", + "interval": "", + "refId": "Anno", + "tags": [], + "type": "tags" + }, + "version": "v0" + } + } + } + ], + "cursorSync": "Crosshair", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, sum by(le) (rate(management_grpc_updatechannel_queue_length_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Connected grpc streams", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p99 of the update-channel queue length: how many network map updates are waiting to be delivered to a peer. The channel buffer is 100 messages - approaching it means updates start getting dropped and peers fall behind.", + "id": 1, + "links": [], + "title": "UpdateChannel Queue length max", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-10": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_total_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\", type=\"read\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HTTP read duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of read API requests (GET/OPTIONS), from the total request duration histogram tagged type=read.", + "id": 10, + "links": [], + "title": "Read HTTP API Requests p95 Duration (GET/OPTIONS)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-11": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_updatechannel_close_one_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "CloseOne", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(rate(management_updatechannel_send_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "Send", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(rate(management_updatechannel_create_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "Create", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_updatechannel_get_all_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GetAll", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_updatechannel_haschannel_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HasChannel", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_updatechannel_close_multiple_channels_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "CloseMultiple", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "F" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate per minute of update-channel operations: close one, send update, create, get all connected peers, has-channel and close multiple. Together they show how much channel churn an instance is handling.", + "id": 11, + "links": [], + "title": "Update Channel operations", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 69, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-12": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_updatechannel_create_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\", closed=\"true\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_updatechannel_create_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "__expr__" + }, + "group": "__expr__", + "kind": "DataQuery", + "spec": { + "expression": "100-$A/$B*100", + "type": "math" + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Tracks how often a new peer update channel replaces a channel that was still open (closed=true), i.e. peers reconnecting before management noticed the previous stream was gone. The plotted expression is 100 - recreated/total, so the line drops as recreations rise.", + "id": 12, + "links": [], + "title": "Percentage of Recreated channels", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "displayName", + "value": "Recreation" + } + ] + } + ] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-13": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_close_one_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "CloseOne", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_send_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "Send", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_create_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "Create", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_get_all_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GetAll", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_haschannel_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "CloseMultiple", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_close_multiple_channels_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HasChannel", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "F" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of each update-channel operation, in microseconds. These are in-memory operations behind a lock, so growth here means lock contention rather than I/O.", + "id": 13, + "links": [], + "title": "Update Channel methods p95 Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-14": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum(management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Connected grpc streams", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Total number of peers currently holding an open gRPC Sync stream across all management instances (management.grpc.connected.streams). A sudden drop means peers were disconnected and will come back as a Login/Sync reconnect storm.", + "id": 14, + "links": [], + "title": "Connected peers", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "red", + "value": 0 + }, + { + "color": "#EAB839", + "value": 8000 + }, + { + "color": "green", + "value": 11000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": true, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-15": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "sum by(instance) (management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"})", + "instant": false, + "legendFormat": "Total", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Open peer Sync streams over time, in total per instance (A) and as raw series (B). Steps down mark instance restarts or LB rebalancing; the ramp back up is the reconnect wave.", + "id": 15, + "links": [], + "title": "Connected peers historical", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-16": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_sync_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Sync duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of the Sync gRPC request - the time management needs to authenticate the peer, build its first network map and open the update channel. Primary latency SLI for peer connects.", + "id": 16, + "links": [], + "title": "gRPC Sync Request p95 Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-17": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_login_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Login duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of the Login gRPC request (peer authentication plus initial configuration and relay credentials). Rises when the IdP, the store or account locking is slow.", + "id": 17, + "links": [], + "title": "gRPC Login Request p95 Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-18": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_account_get_peer_network_map_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken to build the network map returned to a single peer, in milliseconds.", + "id": 18, + "links": [], + "title": "GetPeerNetworkMap Latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-19": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(le) (increase(management_account_network_map_object_count_objects_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the number of objects (peers, routes, firewall rules, DNS entries and so on) in the network maps sent to clients. Large maps make both calculation and serialization more expensive.", + "id": 19, + "links": [], + "title": "NetworkMap Objects", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 0, + "reverse": false, + "unit": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "login", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "sync", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (rate(management_grpc_key_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "get key", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of the three peer-facing gRPC calls: Login (authenticate, return initial config and relay credentials), Sync (open the update channel and receive network map updates) and GetServerKey (fetch the server WireGuard public key). Elevated Login/Sync rates usually mean peers are reconnecting in a loop.", + "id": 2, + "links": [], + "title": "gRPC Requests", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 51, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": true, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": {}, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-20": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_updatechannel_tosyncresponse_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken to convert a calculated network map into the gRPC SyncResponse sent to the peer, in microseconds.", + "id": 20, + "links": [], + "title": "To Sync Response", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "µs" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-23": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(le) (increase(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of per-account p95 Sync durations - each sample is one account's p95, flushed every 60s. Shows whether slow Syncs are spread across the fleet or concentrated in a few accounts.", + "id": 23, + "links": [], + "title": "GRPC Sync Latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.6, + "fill": "dark-red", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-24": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_grpc_login_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of Login gRPC request durations across all peers. The right tail is what a peer experiences on startup; samples beyond 7s are what the high-latency counters in the Debugging row track.", + "id": 24, + "links": [], + "title": "GRPC Login Latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-25": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by(environment) (rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Total", + "range": true + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Login requests, per instance (A) and in total (B). Login is the most expensive peer call (IdP plus store writes); a sustained high rate means clients are restarting or failing to hold their Sync stream.", + "id": 25, + "links": [], + "title": "gRPC Login Request rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-26": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by(environment) (rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Total", + "range": true + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Sync requests, per instance (A) and in total (B). Every Sync opens or re-opens a peer update channel, so a spike here is a reconnect storm and predicts CPU spent on network map calculation.", + "id": 26, + "links": [], + "title": "gRPC Sync Request rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-27": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_grpc_updatechannel_queue_length_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of update-channel queue lengths across all peers. Mass moving to the right shows a growing backlog of undelivered network map updates.", + "id": 27, + "links": [], + "title": "Update Channel heat map", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Greens", + "steps": 73 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-29": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(endpoint) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval])*60) > 10", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{endpoint}} > 10", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Total", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Successful (2xx) GET requests per minute broken down by endpoint, limited to endpoints above 10 rpm. Shows which read endpoints dashboard and API clients hit hardest.", + "id": 29, + "links": [], + "title": "HTTP Read request counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-30": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(endpoint) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60) > 1", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{endpoint}} > 1", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Total", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Successful (2xx) non-GET/OPTIONS requests per minute by endpoint, limited to endpoints above 1 rpm. Writes take account locks and fan out network map updates, so read this together with the Network Map row.", + "id": 30, + "links": [], + "title": "HTTP Write request counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-31": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_store_persistence_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of how long it takes to save or delete an account in the store (management.store.persistence.duration.ms). Widening buckets point at database write pressure.", + "id": 31, + "links": [], + "title": "Store Persistence latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.6, + "fill": "dark-red", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Reds", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-32": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_store_transaction_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of store transaction execution time. Nearly every API and gRPC call runs through a transaction, so this is the first place to look when latency rises everywhere at once.", + "id": 32, + "links": [], + "title": "Store Transaction latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.6, + "fill": "dark-red", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Reds", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-33": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_account_peer_meta_update_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{endpoint}} > 10", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate per minute of peers reporting changed metadata (OS, version, hostname, addresses). Each change persists the peer and can trigger an account peers update.", + "id": 33, + "links": [], + "title": "Peer meta updates counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-34": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_grpc_sync_request_blocked_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Sync request rate", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Sync requests rejected because the peer is blocked (login expired, peer disabled or account restricted). A rising line means clients retrying without valid access.", + "id": 34, + "links": [], + "title": "gRPC Sync Request Blocked rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-35": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_grpc_login_request_blocked_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Login request rate", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Login requests rejected because the peer is blocked. Persistent volume here is usually expired peer logins retrying in a loop.", + "id": 35, + "links": [], + "title": "gRPC Login Request Blocked rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-36": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_account_update_account_peers_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken by an account peers update: preparing the data and pushing a fresh network map to every peer in the account, in milliseconds. Scales with account size.", + "id": 36, + "links": [], + "title": "UpdateAccountPeers Latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-37": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_updatechannel_calc_networkmap_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken to calculate one peer's network map, in milliseconds. This is the dominant CPU cost when many peers reconnect at once.", + "id": 37, + "links": [], + "title": "Network Map Calculation", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-39": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_updatechannel_calc_posturechecks_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken to evaluate a peer's posture checks while building its network map, in microseconds.", + "id": 39, + "links": [], + "title": "Posture Check Calculation", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "µs" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-45": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "sort_desc(\n topk(\n 10,\n sum by (account_id) (\n increase(management_grpc_login_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n )\n )\n)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [ + { + "group": "organize", + "kind": "Transformation", + "spec": { + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Time": "" + } + } + } + } + ] + } + }, + "description": "Top 10 accounts by number of Login requests that exceeded the 7s high-latency threshold.", + "id": 45, + "links": [], + "title": "High Login Latency Ranking (login count per account)", + "vizConfig": { + "group": "table", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "footer": { + "reducers": [] + }, + "inspect": false, + "tooltip": { + "placement": "auto" + }, + "wrapHeaderText": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Time" + } + ] + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-46": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "sort_desc(\n topk(\n 10,\n sum by (account_id) (\n increase(management_grpc_sync_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n )\n )\n)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [ + { + "group": "organize", + "kind": "Transformation", + "spec": { + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Time": "" + } + } + } + } + ] + } + }, + "description": "Top 10 accounts by number of Sync requests that exceeded the 7s high-latency threshold. Use it to find which tenant is behind a latency spike.", + "id": 46, + "links": [], + "title": "High Sync Latency Ranking (sync count per account)", + "vizConfig": { + "group": "table", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "footer": { + "reducers": [] + }, + "inspect": false, + "tooltip": { + "placement": "auto" + }, + "wrapHeaderText": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Time" + } + ] + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-47": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "count(\n sum by (account_id) (\n increase(management_grpc_sync_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n ) > 1\n)", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of distinct accounts with more than one slow (>7s) Sync in the interval - tells you whether a latency spike is fleet-wide or limited to a few tenants.", + "id": 47, + "links": [], + "title": "Accounts with high Sync latency", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-48": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "count(\n sum by (account_id) (\n increase(management_grpc_login_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n ) > 1\n)", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of distinct accounts with more than one slow (>7s) Login in the interval.", + "id": 48, + "links": [], + "title": "Accounts with high Login latency", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-50": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${logs_datasource}" + }, + "group": "loki", + "kind": "DataQuery", + "spec": { + "direction": "backward", + "editorMode": "code", + "expr": "sort_desc(\ntopk(\n 20,\n sum by (peerID) (\n count_over_time(\n {job=\"$log_job\"} |= \"Sync took\" | regexp `peerID:\\s(?P[^,\\]]+)` [$__range]\n )\n )\n)\n)", + "queryType": "instant" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [ + { + "group": "organize", + "kind": "Transformation", + "spec": { + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value #A": "number of times ", + "peerID": "peerPubKey" + } + } + } + } + ] + } + }, + "description": "Top 20 peers by number of Sync log lines in the selected range, parsed out of the management logs to find individual peers stuck in a reconnect loop. Pick the log stream with the `log stream` variable; the query matches the `Sync took` debug line, so management must run at debug level for this panel to have data.", + "id": 50, + "links": [], + "title": "Highest Syncs by peer key", + "vizConfig": { + "group": "table", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "footer": { + "reducers": [] + }, + "inspect": false, + "tooltip": { + "placement": "auto" + }, + "wrapHeaderText": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "labels" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "peerPubKey" + }, + "properties": [ + { + "id": "custom.width", + "value": 455 + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "number of times " + } + ] + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-52": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "wsproxy_active_connections{job=~\"$job\", instance=~\"$instance\"}", + "legendFormat": "{{__name__}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Currently open WebSocket proxy connections. The wsproxy fronts gRPC for browser-based clients, so this is the number of connected web clients.", + "id": 52, + "links": [], + "title": "Proxy Active Connections", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-53": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "delta(wsproxy_bytes_transferred_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])", + "legendFormat": "{{direction}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Bytes transferred through the WebSocket proxy per interval, split by direction.", + "id": 53, + "links": [], + "title": "Proxy Traffic by Direction", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-54": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "increase(wsproxy_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{error_type}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "WebSocket proxy errors grouped by error type - dial failures, read/write errors and protocol issues on browser client connections.", + "id": 54, + "links": [], + "title": "Errors by Type", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-65": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "exemplar": false, + "expr": "management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "legendFormat": "{{host}}", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Open peer Sync streams per management instance. Use it to confirm the load balancer spreads streams evenly - a skewed instance carries proportionally more update-channel and network map work.", + "id": 65, + "links": [], + "title": "Connected Peers per Server", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-66": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(host) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval]) * 60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Successful GET requests per minute grouped by management instance - read load distribution across the fleet.", + "id": 66, + "links": [], + "title": "HTTP Read request counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-67": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(host) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Successful write requests per minute grouped by management instance - write load distribution across the fleet.", + "id": 67, + "links": [], + "title": "HTTP Write request counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_total_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\", type=\"write\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HTTP write duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of write API requests (PUT/POST/DELETE), tagged type=write. Writes acquire account locks and trigger peer updates, so they are normally slower than reads.", + "id": 7, + "links": [], + "title": "Write HTTP API Requests p95 Duration (PUT/POST/DELETE)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-74": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_pat_usage_distribution_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of personal access token usage counts, sampled once per minute per token. Mass in the high buckets means a single PAT is hammering the API.", + "id": 74, + "links": [], + "title": "PAT usage counter", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-76": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "gnatsd_connz_subscriptions{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "{{host}}", + "range": false, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Current subscriptions held by each client connected to NATS (connz). Signal instances subscribe per connected peer, so this tracks how peers are spread across signal nodes.", + "id": 76, + "links": [], + "title": "Subscriptions per node", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-77": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "gnatsd_connz_subscriptions{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Subscriptions per connected NATS client over time. Steps mark signal instances restarting and re-subscribing their peers.", + "id": 77, + "links": [], + "title": "Subscriptions per node", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-78": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "gnatsd_connz_num_connections{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of client connections on each NATS node - effectively how many signal instances are attached to that node.", + "id": 78, + "links": [], + "title": "Signal nodes connected per NATs", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-79": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(environment) (rate(gnatsd_connz_out_msgs{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of messages NATS delivers out to its clients: the peer signalling throughput flowing through the cluster.", + "id": 79, + "links": [], + "title": "Messages per Second", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HTTP request duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of incoming REST API requests by endpoint and method (management.http.request.duration.ms). Covers dashboard and API clients only - peer traffic is on the gRPC panels.", + "id": 8, + "links": [], + "title": "HTTP API Requests p95 Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-80": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(environment) (rate(gnatsd_connz_out_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of bytes NATS delivers out to its clients - signalling bandwidth, useful next to the message rate to spot unusually large payloads.", + "id": 80, + "links": [], + "title": "Data per Second", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-81": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(environment) (increase(gnatsd_varz_slow_consumers{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Increase of the NATS slow-consumer counter. Anything above zero means a signal node could not read fast enough and NATS dropped its connection, so peers lose signalling until it reconnects.", + "id": 81, + "links": [], + "title": "Slow consumers detected", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-82": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "gnatsd_varz_subscriptions{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Total subscriptions registered on each NATS server (varz). Grows roughly with the number of connected peers.", + "id": 82, + "links": [], + "title": "Total number of NATS subscriptions", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-83": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{instance}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Resident memory of the scraped processes. process_resident_memory_bytes is exposed by every Go/Prometheus process, so narrow the job and instance variables to the NATS nodes to read this panel as NATS memory.", + "id": 83, + "links": [], + "title": "NATS memory consumption", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-84": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "idelta(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[2m])", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Per-account Sync p95 samples as a heatmap (2m deltas). Use it to spot a handful of large or slow accounts pulling the overall Sync latency up.", + "id": 84, + "links": [], + "title": "GRPC Sync Latency By Account", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.6, + "fill": "dark-red", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-85": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "idelta(management_grpc_login_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[2m])", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Per-account Login p95 samples as a heatmap (2m deltas). Highlights accounts whose logins are much slower than the fleet average.", + "id": 85, + "links": [], + "title": "GRPC Login Latency By Account", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-86": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Sync duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 of the per-account Sync p95 distribution - the latency seen by the worst-served accounts rather than by the average request. Diverges from the global p95 when only a few accounts are hurting.", + "id": 86, + "links": [], + "title": "gRPC Sync Request p95 Duration (by Account)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-87": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_login_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Login duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 of the per-account Login p95 distribution - login latency for the worst-served accounts rather than the average request.", + "id": 87, + "links": [], + "title": "gRPC Login Request p95 Duration (by Account)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-88": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{environment}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Account peers updates triggered per minute (2m increase halved). Each trigger recalculates and pushes network maps to every peer in the account.", + "id": 88, + "links": [], + "title": "NetworkMap Triggers Total", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 19, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-89": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(resource) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Account peers updates over 2m intervals, grouped by the resource that triggered them - the time series behind the pie chart above.", + "id": 89, + "links": [], + "title": "NetworkMap Triggers By Source", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.9, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_idp_authenticate_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "authenticate (mgtm->auth0)", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_idp_update_user_meta_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "update user meta", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_idp_get_account_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "get account", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Requests per minute that management sends to the configured identity provider: authenticate (service token refresh), user metadata updates and account lookups. IdP throttling shows up here before it shows up as Login latency.", + "id": 9, + "links": [], + "title": "IdP Requests", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 69, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-90": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "sum by(resource) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[1h]))", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Account peers updates over the last hour, grouped by the resource that triggered them (policy, group, peer, route, user, network, ...). Shows which kind of configuration change generates the most fan-out work.", + "id": 90, + "links": [], + "title": "NetworkMap Triggers by source in last hour", + "vizConfig": { + "group": "piechart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#73BF69", + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + } + }, + "overrides": [] + }, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-91": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(resource) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[1h]))", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Network maps computed in the last hour, grouped by the resource whose change triggered the computation.", + "id": 91, + "links": [], + "title": "NetworkMaps by source in last hour", + "vizConfig": { + "group": "piechart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#73BF69", + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + } + }, + "overrides": [] + }, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-92": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{environment}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Network maps computed per minute (2m increase halved). One account peers update fans out into one map per peer, so this - not the trigger count - is the real CPU driver.", + "id": 92, + "links": [], + "title": "NetworkMaps Total", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 19, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-93": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(resource) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{resource}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Network maps computed per minute, grouped by the resource that triggered the computation.", + "id": 93, + "links": [], + "title": "NetworkMaps Total By Source", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.9, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-14" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-65" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-15" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-2" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-23" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-24" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-16" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-17" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-84" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-85" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-86" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-87" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-26" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 48 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-25" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 48 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-34" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 56 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-35" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 56 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-31" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 64 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-32" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 64 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-9" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 72 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-8" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 72 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-29" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 80 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-30" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 80 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-66" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 88 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-67" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 88 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-10" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 96 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-7" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 96 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-33" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 104 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-74" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 104 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-11" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 112 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-13" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 112 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-12" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 120 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-1" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 120 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-27" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 128 + } + } + ] + } + }, + "title": "General" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-76" + }, + "height": 8, + "width": 24, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-77" + }, + "height": 8, + "width": 24, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-78" + }, + "height": 8, + "width": 24, + "x": 0, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-79" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-80" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-81" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-82" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-83" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 40 + } + } + ] + } + }, + "title": "NATS" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-46" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-45" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-47" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-48" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-50" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 16 + } + } + ] + } + }, + "title": "Debugging" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-37" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-20" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-39" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-36" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-18" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-19" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-90" + }, + "height": 19, + "width": 6, + "x": 0, + "y": 48 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-88" + }, + "height": 10, + "width": 17, + "x": 6, + "y": 48 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-89" + }, + "height": 9, + "width": 17, + "x": 6, + "y": 58 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-91" + }, + "height": 19, + "width": 6, + "x": 0, + "y": 67 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-92" + }, + "height": 10, + "width": 17, + "x": 6, + "y": 67 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-93" + }, + "height": 9, + "width": 17, + "x": 6, + "y": 77 + } + } + ] + } + }, + "title": "Network Map" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-52" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-53" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-54" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + } + ] + } + }, + "title": "Browser Client" + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [], + "timeSettings": { + "autoRefresh": "auto", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "fiscalYearStartMonth": 0, + "from": "now-1h", + "hideTimepicker": false, + "timezone": "browser", + "to": "now" + }, + "title": "Management generic", + "variables": [ + { + "kind": "DatasourceVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "description": "Prometheus data source scraping the management metrics.", + "hide": "dontHide", + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "pluginId": "prometheus", + "refresh": "onDashboardLoad", + "regex": "", + "skipUrlSync": false + } + }, + { + "kind": "DatasourceVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "description": "Loki data source holding the management logs.", + "hide": "dontHide", + "includeAll": false, + "label": "Logs data source", + "multi": false, + "name": "logs_datasource", + "options": [], + "pluginId": "loki", + "refresh": "onDashboardLoad", + "regex": "", + "skipUrlSync": false + } + }, + { + "kind": "QueryVariable", + "spec": { + "allValue": ".*", + "allowCustomValue": true, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "definition": "label_values(job)", + "description": "Prometheus scrape job(s) exposing the management metrics.", + "hide": "dontHide", + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [], + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allValue": ".*", + "allowCustomValue": true, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "definition": "label_values(up{job=~\"$job\"},instance)", + "description": "Instance(s) of the selected job(s). Depends on the job variable.", + "hide": "dontHide", + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(up{job=~\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "definition": "label_values(job)", + "description": "Loki stream carrying the management logs (adjust the label if your setup does not use `job`).", + "hide": "dontHide", + "includeAll": false, + "label": "log stream", + "multi": false, + "name": "log_job", + "options": [], + "query": { + "datasource": { + "name": "${logs_datasource}" + }, + "group": "loki", + "kind": "DataQuery", + "spec": { + "label": "job", + "refId": "LokiVariableQueryEditor-VariableQuery", + "type": 1 + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + }, + { + "kind": "CustomVariable", + "spec": { + "allValue": "5m", + "allowCustomValue": true, + "current": { + "text": "60s", + "value": "60s" + }, + "hide": "dontHide", + "includeAll": false, + "label": "interval", + "multi": false, + "name": "interval", + "options": [], + "query": "5m,60s", + "skipUrlSync": false, + "valuesFormat": "csv" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(management_store_query_count_total,method)", + "hide": "hideVariable", + "includeAll": true, + "multi": true, + "name": "method", + "options": [], + "query": { + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(management_store_query_count_total,method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "disabled" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "definition": "label_values(aws_rds_cpuutilization_average,dimension_DBClusterIdentifier)", + "hide": "hideVariable", + "includeAll": true, + "multi": true, + "name": "dimension_DBClusterIdentifier", + "options": [], + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(aws_rds_cpuutilization_average,dimension_DBClusterIdentifier)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "disabled" + } + } + ] +} From f65f7b347ee4e7de6d98c488d3d894cd018b02b6 Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Sat, 8 Aug 2026 20:48:34 +0900 Subject: [PATCH 07/22] [management] Deny reverse proxy access to pending and blocked users (#7105) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A user in the Pending Approval state could complete SSO and reach any SSO-protected reverse proxy service distributed to a group they belong to, including the All Users group. The reverse proxy authorization path checked the session token signature, that the user exists, that the user's account matches the service's account, and group membership — never the user's account status. The REST API (`permissions/manager.go`) and peer registration both gate on that state, but the proxy gRPC service does not go through the permissions manager, so neither gate applied. A pending user is persisted as blocked and pending approval, so blocked users reached those services the same way. `ValidateSession` now denies on account status, reporting `pending_approval` or `user_blocked` so the proxy access log and the denied page carry the cause rather than a generic refusal. `GenerateSessionToken` refuses to mint a token for such a user at all, so the browser never receives a session cookie and the OIDC callback can tell the user why instead of showing "Service configuration error". `ValidateUserGroupAccess` and `ValidateTunnelPeer` close the same gap; for the tunnel path this covers a user blocked after their peer was registered, since peer group membership alone kept mesh-origin access open. A single helper produces both the denied reason for the RPC responses and the sentinel error for the error-returning callers, so the four entry points cannot drift apart. A user the store cannot resolve is denied rather than passed through. One thing deliberately left out: session cookies are validated locally by the proxy against the service public key with no management round-trip, so a cookie issued before a user is blocked stays valid until it expires (24h by default). That is a revocation-propagation problem rather than this authorization gap, and every option for it (per-request validation with a cache, short-lived tokens with refresh, push-based revocation) changes the proxy hot path or the proxy/management protocol. Worth its own ticket. --- management/internals/shared/grpc/proxy.go | 302 +++++++++++++----- .../shared/grpc/proxy_group_access_test.go | 227 +++++++++++-- .../shared/grpc/validate_session_test.go | 185 +++++++++++ management/server/http/handlers/proxy/auth.go | 17 +- .../proxy/auth_callback_integration_test.go | 103 ++++++ 5 files changed, 735 insertions(+), 99 deletions(-) diff --git a/management/internals/shared/grpc/proxy.go b/management/internals/shared/grpc/proxy.go index 8f24de116..4cae872e2 100644 --- a/management/internals/shared/grpc/proxy.go +++ b/management/internals/shared/grpc/proxy.go @@ -29,6 +29,7 @@ import ( "github.com/netbirdio/netbird/shared/management/domain" + "github.com/netbirdio/netbird/management/internals/modules/agentnetwork" "github.com/netbirdio/netbird/management/internals/modules/peers" "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs" "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy" @@ -36,7 +37,6 @@ import ( "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/sessionkey" "github.com/netbirdio/netbird/management/server/idp" "github.com/netbirdio/netbird/management/server/peer" - "github.com/netbirdio/netbird/management/internals/modules/agentnetwork" "github.com/netbirdio/netbird/management/server/types" "github.com/netbirdio/netbird/management/server/users" proxyauth "github.com/netbirdio/netbird/proxy/auth" @@ -1579,9 +1579,62 @@ func (s *ProxyServiceServer) ValidateState(state string) (verifier, redirectURL return verifier, redirectURL, nil } +// Denied reasons reported to the proxy when access is refused because of the +// account status of the user behind the request. +const ( + deniedReasonPendingApproval = "pending_approval" + deniedReasonUserBlocked = "user_blocked" + deniedReasonUserNotFound = "user_not_found" +) + +var ( + // ErrUserPendingApproval reports a user whose account still awaits approval + // by an administrator and may therefore not hold a proxy session. + ErrUserPendingApproval = errors.New("user pending approval") + + // ErrUserBlocked reports a blocked user, who may not hold a proxy session. + ErrUserBlocked = errors.New("user blocked") + + errUserUnresolved = errors.New("user could not be resolved") +) + +// checkUserStatus reports whether the user's account status permits reverse +// proxy access, returning the denied reason for the proxy access log together +// with the sentinel error callers match on. A user awaiting approval is stored +// as both pending and blocked, so the pending state is reported first: it is +// the one an administrator can act on. +func checkUserStatus(user *types.User) (string, error) { + switch { + case user == nil: + return deniedReasonUserNotFound, errUserUnresolved + case user.PendingApproval: + return deniedReasonPendingApproval, ErrUserPendingApproval + case user.IsBlocked(): + return deniedReasonUserBlocked, ErrUserBlocked + default: + return "", nil + } +} + +// userStatusDeniedReason returns the denied reason for callers that report a +// decision rather than an error, and an empty string when the user may proceed. +func userStatusDeniedReason(user *types.User) string { + reason, _ := checkUserStatus(user) + return reason +} + +// sameAccount reports whether a user belongs to a service's account. An empty +// identifier on either side never matches: two unset accounts must not compare +// equal into a grant. +func sameAccount(userAccountID, serviceAccountID string) bool { + return userAccountID != "" && serviceAccountID != "" && userAccountID == serviceAccountID +} + // GenerateSessionToken creates a signed session JWT for the given domain and // user. The user's group memberships are embedded in the token so policy-aware // middlewares on the proxy can authorise without an extra management round-trip. +// A user the store cannot resolve, or whose account is pending approval or +// blocked, gets no token at all, so the browser never receives a session cookie. func (s *ProxyServiceServer) GenerateSessionToken(ctx context.Context, domain, userID string, method proxyauth.Method) (string, error) { service, err := s.getServiceByDomain(ctx, domain) if err != nil { @@ -1592,25 +1645,37 @@ func (s *ProxyServiceServer) GenerateSessionToken(ctx context.Context, domain, u return "", fmt.Errorf("no session key configured for domain: %s", domain) } - var ( - email string - groupIDs []string - groupNames []string - ) - if s.usersManager != nil { - user, userGroups, uerr := s.usersManager.GetUserWithGroups(ctx, userID) - if uerr != nil { - log.WithContext(ctx).Debugf("session token mint: lookup user %s: %v", userID, uerr) - } else if user != nil { - email = user.Email - groupIDs, groupNames = pairGroupIDsAndNames(userGroups) - } + if s.usersManager == nil { + return "", errors.New("users manager not configured") } + user, userGroups, err := s.usersManager.GetUserWithGroups(ctx, userID) + if err != nil { + return "", fmt.Errorf("get user %s: %w", userID, err) + } + + if user == nil { + return "", fmt.Errorf("get user %s: %w", userID, errUserUnresolved) + } + + // Bind the OIDC identity to the service's account before signing anything + // with that service's session key. The proxy validates an installed cookie + // locally against the service public key, so a token minted for a user of + // another account would be honoured without a management round-trip. + if !sameAccount(user.AccountID, service.AccountID) { + return "", fmt.Errorf("user %s does not belong to the service account", userID) + } + + if _, err := checkUserStatus(user); err != nil { + return "", fmt.Errorf("session token for user %s: %w", userID, err) + } + + groupIDs, groupNames := pairGroupIDsAndNames(userGroups) + return sessionkey.SignToken( service.SessionPrivateKey, userID, - email, + user.Email, domain, method, groupIDs, @@ -1628,6 +1693,10 @@ func (s *ProxyServiceServer) ValidateUserGroupAccess(ctx context.Context, domain return fmt.Errorf("user not found: %s", userID) } + if _, err := checkUserStatus(user); err != nil { + return fmt.Errorf("user %s denied access to domain %s: %w", userID, domain, err) + } + service, err := s.getAccountServiceByDomain(ctx, user.AccountID, domain) if err != nil { return err @@ -1682,10 +1751,7 @@ func (s *ProxyServiceServer) ValidateSession(ctx context.Context, req *proto.Val sessionToken := req.GetSessionToken() if domain == "" || sessionToken == "" { - return &proto.ValidateSessionResponse{ - Valid: false, - DeniedReason: "missing domain or session_token", - }, nil + return deniedSessionResponse("missing domain or session_token"), nil } service, err := s.getServiceByDomain(ctx, domain) @@ -1695,83 +1761,49 @@ func (s *ProxyServiceServer) ValidateSession(ctx context.Context, req *proto.Val "error": err.Error(), }).Debug("ValidateSession: service not found") //nolint:nilerr - return &proto.ValidateSessionResponse{ - Valid: false, - DeniedReason: "service_not_found", - }, nil + return deniedSessionResponse("service_not_found"), nil } if err := enforceAccountScope(ctx, service.AccountID); err != nil { return nil, err } - pubKeyBytes, err := base64.StdEncoding.DecodeString(service.SessionPublicKey) - if err != nil { - log.WithFields(log.Fields{ - "domain": domain, - "error": err.Error(), - }).Error("ValidateSession: decode public key") - //nolint:nilerr - return &proto.ValidateSessionResponse{ - Valid: false, - DeniedReason: "invalid_service_config", - }, nil - } - - userID, _, _, _, _, err := proxyauth.ValidateSessionJWT(sessionToken, domain, pubKeyBytes) - if err != nil { - log.WithFields(log.Fields{ - "domain": domain, - "error": err.Error(), - }).Debug("ValidateSession: invalid session token") - //nolint:nilerr - return &proto.ValidateSessionResponse{ - Valid: false, - DeniedReason: "invalid_token", - }, nil + userID, reason := sessionTokenSubject(domain, service, sessionToken) + if reason != "" { + return deniedSessionResponse(reason), nil } user, userGroups, err := s.usersManager.GetUserWithGroups(ctx, userID) - if err != nil { + if err != nil || user == nil { log.WithFields(log.Fields{ "domain": domain, "user_id": userID, - "error": err.Error(), + "error": err, }).Debug("ValidateSession: user not found") //nolint:nilerr - return &proto.ValidateSessionResponse{ - Valid: false, - DeniedReason: "user_not_found", - }, nil + return deniedSessionResponse(deniedReasonUserNotFound), nil } - if user.AccountID != service.AccountID { + // A user from another account gets a bare response: none of their identity + // belongs in an answer to a proxy serving a different account. + if !sameAccount(user.AccountID, service.AccountID) { log.WithFields(log.Fields{ "domain": domain, "user_id": userID, "user_account": user.AccountID, "service_account": service.AccountID, }).Debug("ValidateSession: user account mismatch") - //nolint:nilerr - return &proto.ValidateSessionResponse{ - Valid: false, - DeniedReason: "account_mismatch", - }, nil + return deniedSessionResponse("account_mismatch"), nil } - if err := s.checkGroupAccess(service, user); err != nil { - log.WithFields(log.Fields{ - "domain": domain, - "user_id": userID, - "error": err.Error(), - }).Debug("ValidateSession: access denied") - groupIDs, groupNames := pairGroupIDsAndNames(userGroups) - //nolint:nilerr + groupIDs, groupNames := pairGroupIDsAndNames(userGroups) + + if reason := s.accountUserDeniedReason(domain, service, user); reason != "" { return &proto.ValidateSessionResponse{ Valid: false, UserId: user.Id, UserEmail: user.Email, - DeniedReason: "not_in_group", + DeniedReason: reason, PeerGroupIds: groupIDs, PeerGroupNames: groupNames, }, nil @@ -1783,7 +1815,6 @@ func (s *ProxyServiceServer) ValidateSession(ctx context.Context, req *proto.Val "email": user.Email, }).Debug("ValidateSession: access granted") - groupIDs, groupNames := pairGroupIDsAndNames(userGroups) return &proto.ValidateSessionResponse{ Valid: true, UserId: user.Id, @@ -1793,6 +1824,66 @@ func (s *ProxyServiceServer) ValidateSession(ctx context.Context, req *proto.Val }, nil } +// deniedSessionResponse builds a denial that carries no identity, for the +// checks that run before a user of this service's account is resolved. +func deniedSessionResponse(reason string) *proto.ValidateSessionResponse { + return &proto.ValidateSessionResponse{ + Valid: false, + DeniedReason: reason, + } +} + +// sessionTokenSubject verifies the session token against the service's session +// key and returns the user it was minted for, or the reason it cannot be +// trusted. +func sessionTokenSubject(domain string, service *rpservice.Service, sessionToken string) (userID, deniedReason string) { + pubKeyBytes, err := base64.StdEncoding.DecodeString(service.SessionPublicKey) + if err != nil { + log.WithFields(log.Fields{ + "domain": domain, + "error": err.Error(), + }).Error("ValidateSession: decode public key") + return "", "invalid_service_config" + } + + userID, _, _, _, _, err = proxyauth.ValidateSessionJWT(sessionToken, domain, pubKeyBytes) + if err != nil { + log.WithFields(log.Fields{ + "domain": domain, + "error": err.Error(), + }).Debug("ValidateSession: invalid session token") + return "", "invalid_token" + } + + return userID, "" +} + +// accountUserDeniedReason gates a user of the service's own account, returning +// an empty string when access is granted. Account status comes before group +// membership: a user awaiting approval or blocked has no access regardless of +// the groups they were auto-assigned. +func (s *ProxyServiceServer) accountUserDeniedReason(domain string, service *rpservice.Service, user *types.User) string { + if reason := userStatusDeniedReason(user); reason != "" { + log.WithFields(log.Fields{ + "domain": domain, + "user_id": user.Id, + "reason": reason, + }).Debug("ValidateSession: user status denies access") + return reason + } + + if err := s.checkGroupAccess(service, user); err != nil { + log.WithFields(log.Fields{ + "domain": domain, + "user_id": user.Id, + "error": err.Error(), + }).Debug("ValidateSession: access denied") + return "not_in_group" + } + + return "" +} + func (s *ProxyServiceServer) getServiceByDomain(ctx context.Context, domain string) (*rpservice.Service, error) { service, err := s.serviceManager.GetServiceByDomain(ctx, domain) if err == nil { @@ -1907,7 +1998,20 @@ func (s *ProxyServiceServer) ValidateTunnelPeer(ctx context.Context, req *proto. } groupIDs, groupNames := pairGroupIDsAndNames(peerGroups) - principalID, displayIdentity := s.getTunnelPeerInfo(ctx, domain, service, peer) + owner := s.resolvePeerOwner(ctx, peer, service.AccountID) + principalID, displayIdentity := s.getTunnelPeerInfo(ctx, domain, service, peer, owner) + + if reason := peerOwnerDeniedReason(peer, owner); reason != "" { + log.WithFields(log.Fields{"domain": domain, "peer_id": peer.ID, "user_id": peer.UserID, "reason": reason}).Debug("ValidateTunnelPeer: owner status denies access") + return &proto.ValidateTunnelPeerResponse{ + Valid: false, + UserId: principalID, + UserEmail: displayIdentity, + DeniedReason: reason, + PeerGroupIds: groupIDs, + PeerGroupNames: groupNames, + }, nil + } if err := checkPeerGroupAccess(service, groupIDs); err != nil { log.WithFields(log.Fields{"domain": domain, "peer_id": peer.ID, "error": err.Error()}).Debug("ValidateTunnelPeer: access denied") @@ -1944,9 +2048,55 @@ func (s *ProxyServiceServer) ValidateTunnelPeer(ctx context.Context, req *proto. }, nil } +// resolvePeerOwner returns the user a peer is linked to, once per request so +// the status gate and the identity resolution below share a single lookup. +// Unlinked peers (machine agents) have no owner. A lookup that fails returns +// nil rather than an error: both callers treat an unresolved owner the same +// way, and neither may trust one it could not read. +func (s *ProxyServiceServer) resolvePeerOwner(ctx context.Context, peer *peer.Peer, accountID string) *types.User { + if peer.UserID == "" { + return nil + } + + user, err := s.usersManager.GetUser(ctx, peer.UserID) + if err != nil { + log.WithContext(ctx).Debugf("ValidateTunnelPeer: look up owner %s of peer %s: %v", peer.UserID, peer.ID, err) + return nil + } + + // The lookup is by user ID alone, so a peer row pointing outside the + // service's account would otherwise resolve a foreign user. Leave the owner + // unresolved instead: the gate denies it, and neither the response nor the + // minted token carries an identity from another account. + if !sameAccount(user.AccountID, accountID) { + log.WithContext(ctx).Debugf("ValidateTunnelPeer: owner %s of peer %s belongs to another account", peer.UserID, peer.ID) + return nil + } + + return user +} + +// peerOwnerDeniedReason gates the mesh fast-path on the account status of the +// peer's owning user, so a user blocked after registering a peer loses +// mesh-origin access too. Unlinked peers (machine agents) have no owner to gate +// on and stay first-class callers. An owner the store cannot resolve denies: +// an unavailable lookup must not grant access. +func peerOwnerDeniedReason(peer *peer.Peer, owner *types.User) string { + if peer.UserID == "" { + return "" + } + + if owner == nil { + return deniedReasonUserNotFound + } + + return userStatusDeniedReason(owner) +} + // getTunnelPeerInfo returns the principal ID and display name for a peer, e.g. a -// user or peer ID, and peer name or user email. -func (s *ProxyServiceServer) getTunnelPeerInfo(ctx context.Context, domain string, service *rpservice.Service, peer *peer.Peer) (string, string) { +// user or peer ID, and peer name or user email. owner is the already-resolved +// user the peer is linked to, or nil. +func (s *ProxyServiceServer) getTunnelPeerInfo(ctx context.Context, domain string, service *rpservice.Service, peer *peer.Peer, owner *types.User) (string, string) { // Resolve the principal: when the peer is linked to a user, the human is the // principal so multiple peers owned by the same user share a single // identity. Unlinked peers (machine agents) are their own principal keyed on @@ -1963,10 +2113,10 @@ func (s *ProxyServiceServer) getTunnelPeerInfo(ctx context.Context, domain strin principalID := peer.UserID displayIdentity := peer.Name // Stored column first (cheap, but often empty for OIDC-provisioned users). - if user, uerr := s.usersManager.GetUser(ctx, peer.UserID); uerr == nil && user != nil { - principalID = user.Id - if user.Email != "" { - displayIdentity = user.Email + if owner != nil { + principalID = owner.Id + if owner.Email != "" { + displayIdentity = owner.Email } } // IdP enrichment wins when available — the stored email column is a diff --git a/management/internals/shared/grpc/proxy_group_access_test.go b/management/internals/shared/grpc/proxy_group_access_test.go index 532cb7cc3..cb30a07b9 100644 --- a/management/internals/shared/grpc/proxy_group_access_test.go +++ b/management/internals/shared/grpc/proxy_group_access_test.go @@ -119,11 +119,13 @@ func (m *mockReverseProxyManager) GetClusters(_ context.Context, _, _ string) ([ } type mockUsersManager struct { - users map[string]*types.User - err error + users map[string]*types.User + err error + getUserCalls int } func (m *mockUsersManager) GetUser(ctx context.Context, userID string) (*types.User, error) { + m.getUserCalls++ if m.err != nil { return nil, m.err } @@ -350,6 +352,64 @@ func TestValidateUserGroupAccess(t *testing.T) { }, expectErr: false, }, + { + name: "user pending approval denied despite group membership", + domain: "app.example.com", + userID: "user1", + proxiesByAccount: map[string][]*service.Service{ + "account1": {{ + Domain: "app.example.com", + AccountID: "account1", + Auth: service.AuthConfig{ + BearerAuth: &service.BearerAuthConfig{ + Enabled: true, + DistributionGroups: []string{"group1"}, + }, + }, + }}, + }, + users: map[string]*types.User{ + // The approval flow stores a pending user as blocked as well. + "user1": {Id: "user1", AccountID: "account1", AutoGroups: []string{"group1"}, Blocked: true, PendingApproval: true}, + }, + expectErr: true, + expectErrMsg: "user pending approval", + }, + { + name: "blocked user denied despite group membership", + domain: "app.example.com", + userID: "user1", + proxiesByAccount: map[string][]*service.Service{ + "account1": {{ + Domain: "app.example.com", + AccountID: "account1", + Auth: service.AuthConfig{ + BearerAuth: &service.BearerAuthConfig{ + Enabled: true, + DistributionGroups: []string{"group1"}, + }, + }, + }}, + }, + users: map[string]*types.User{ + "user1": {Id: "user1", AccountID: "account1", AutoGroups: []string{"group1"}, Blocked: true}, + }, + expectErr: true, + expectErrMsg: "user blocked", + }, + { + name: "blocked user denied on a service with no auth configured", + domain: "app.example.com", + userID: "user1", + proxiesByAccount: map[string][]*service.Service{ + "account1": {{Domain: "app.example.com", AccountID: "account1", Auth: service.AuthConfig{}}}, + }, + users: map[string]*types.User{ + "user1": {Id: "user1", AccountID: "account1", Blocked: true}, + }, + expectErr: true, + expectErrMsg: "user blocked", + }, { name: "proxy manager error", domain: "app.example.com", @@ -421,17 +481,18 @@ func TestValidateTunnelPeerUserEmailEnrichment(t *testing.T) { storedUserNoEmail := map[string]*types.User{userID: {Id: userID, AccountID: accountID, Email: ""}} tests := []struct { - name string - peerUserID string - storedUsers map[string]*types.User - storedErr error - noIdP bool - idpEmail string - idpHasData bool - idpErr error - expectEmail string - expectUserID string - expectIdPHit bool + name string + peerUserID string + storedUsers map[string]*types.User + storedErr error + noIdP bool + idpEmail string + idpHasData bool + idpErr error + expectEmail string + expectUserID string + expectIdPHit bool + expectDeniedReason string }{ { name: "idp email wins over stored email", @@ -490,14 +551,17 @@ func TestValidateTunnelPeerUserEmailEnrichment(t *testing.T) { expectIdPHit: true, }, { - name: "idp email when stored user missing keeps peer.UserID as principal", - peerUserID: userID, - storedUsers: map[string]*types.User{}, - idpEmail: "idp@example.com", - idpHasData: true, - expectEmail: "idp@example.com", - expectUserID: userID, - expectIdPHit: true, + // The identity still resolves from the IdP, but an owner the store + // cannot resolve denies the fast-path rather than granting it. + name: "idp email when stored user missing keeps peer.UserID as principal", + peerUserID: userID, + storedUsers: map[string]*types.User{}, + idpEmail: "idp@example.com", + idpHasData: true, + expectEmail: "idp@example.com", + expectUserID: userID, + expectIdPHit: true, + expectDeniedReason: deniedReasonUserNotFound, }, { name: "unlinked peer uses peer name and never consults idp", @@ -545,9 +609,13 @@ func TestValidateTunnelPeerUserEmailEnrichment(t *testing.T) { require.NoError(t, err) require.NotNil(t, resp) - assert.True(t, resp.GetValid(), "expected access granted") + assert.Equal(t, tt.expectDeniedReason == "", resp.GetValid(), "unexpected access decision") + assert.Equal(t, tt.expectDeniedReason, resp.GetDeniedReason(), "unexpected denied reason") assert.Equal(t, tt.expectEmail, resp.GetUserEmail()) assert.Equal(t, tt.expectUserID, resp.GetUserId()) + if tt.expectDeniedReason != "" { + assert.Empty(t, resp.GetSessionToken(), "a denied peer must not receive a session token") + } if idpMock != nil { if tt.expectIdPHit { @@ -562,6 +630,121 @@ func TestValidateTunnelPeerUserEmailEnrichment(t *testing.T) { } } +// TestDeniedReasonValues pins the wire values of the account status denied +// reasons. The proxy logs them and operators filter access logs on them, so a +// rename is a breaking change rather than an internal detail. +// TestSameAccount pins the fail-closed behaviour of the account binding: an +// unset account on either side must never compare equal into a grant. +func TestSameAccount(t *testing.T) { + assert.True(t, sameAccount("account1", "account1"), "matching accounts should bind") + assert.False(t, sameAccount("account1", "account2"), "different accounts must not bind") + assert.False(t, sameAccount("", ""), "two unset accounts must not bind") + assert.False(t, sameAccount("account1", ""), "an unset service account must not bind") + assert.False(t, sameAccount("", "account1"), "an unset user account must not bind") +} + +func TestDeniedReasonValues(t *testing.T) { + assert.Equal(t, "pending_approval", deniedReasonPendingApproval, "pending approval denied reason wire value") + assert.Equal(t, "user_blocked", deniedReasonUserBlocked, "blocked user denied reason wire value") + assert.Equal(t, "user_not_found", deniedReasonUserNotFound, "unresolved user denied reason wire value") +} + +// TestValidateTunnelPeerOwnerStatus verifies that the mesh fast-path gates on +// the account status of the peer's owning user. A peer whose owner was blocked +// after the peer registered must lose access, while an unlinked machine peer +// keeps it. +func TestValidateTunnelPeerOwnerStatus(t *testing.T) { + const ( + domain = "app.example.com" + accountID = "account1" + peerID = "peer1" + peerName = "peer-display-name" + userID = "user1" + ) + + tests := []struct { + name string + peerUserID string + owner *types.User + expectDeniedReason string + expectEmail string + }{ + { + name: "active owner allowed", + peerUserID: userID, + owner: &types.User{Id: userID, AccountID: accountID, Email: "user@example.com"}, + }, + { + name: "owner pending approval denied", + peerUserID: userID, + owner: &types.User{Id: userID, AccountID: accountID, Email: "user@example.com", Blocked: true, PendingApproval: true}, + expectDeniedReason: deniedReasonPendingApproval, + }, + { + name: "owner blocked after registering the peer denied", + peerUserID: userID, + owner: &types.User{Id: userID, AccountID: accountID, Email: "user@example.com", Blocked: true}, + expectDeniedReason: deniedReasonUserBlocked, + }, + { + name: "unlinked machine peer stays allowed", + peerUserID: "", + owner: &types.User{Id: userID, AccountID: accountID, Blocked: true}, + }, + { + // The user lookup is not account-scoped, so a peer row pointing at + // another account's user must not resolve into an owner: the peer is + // denied and the foreign email never reaches the response. + name: "owner in another account denied and not disclosed", + peerUserID: userID, + owner: &types.User{Id: userID, AccountID: "otherAccount", Email: "foreign@example.com"}, + expectDeniedReason: deniedReasonUserNotFound, + expectEmail: peerName, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + svc := &service.Service{Domain: domain, AccountID: accountID} + usersManager := &mockUsersManager{users: map[string]*types.User{userID: tt.owner}} + server := &ProxyServiceServer{ + serviceManager: &mockReverseProxyManager{ + proxiesByAccount: map[string][]*service.Service{accountID: {svc}}, + }, + peersManager: &mockTunnelPeersManager{ + peer: &peer.Peer{ID: peerID, Name: peerName, UserID: tt.peerUserID}, + }, + usersManager: usersManager, + } + + resp, err := server.ValidateTunnelPeer(context.Background(), &proto.ValidateTunnelPeerRequest{ + Domain: domain, + TunnelIp: "100.64.0.1", + }) + + require.NoError(t, err) + require.NotNil(t, resp) + assert.Equal(t, tt.expectDeniedReason, resp.GetDeniedReason(), "unexpected denied reason") + assert.Equal(t, tt.expectDeniedReason == "", resp.GetValid(), "unexpected access decision") + if tt.expectDeniedReason != "" { + assert.Empty(t, resp.GetSessionToken(), "a denied peer must not receive a session token") + } + + if tt.expectEmail != "" { + assert.Equal(t, tt.expectEmail, resp.GetUserEmail(), "unexpected identity on the response") + } + + // The status gate and the identity resolution share one lookup; + // an unlinked peer has no owner to look up at all. + wantLookups := 1 + if tt.peerUserID == "" { + wantLookups = 0 + } + assert.Equal(t, wantLookups, usersManager.getUserCalls, "owner must be resolved exactly once per request") + }) + } +} + func TestGetAccountProxyByDomain(t *testing.T) { tests := []struct { name string diff --git a/management/internals/shared/grpc/validate_session_test.go b/management/internals/shared/grpc/validate_session_test.go index d649102a1..03f200414 100644 --- a/management/internals/shared/grpc/validate_session_test.go +++ b/management/internals/shared/grpc/validate_session_test.go @@ -46,6 +46,7 @@ func setupValidateSessionTest(t *testing.T) *validateSessionTestSetup { proxyService.SetServiceManager(serviceManager) createTestProxies(t, ctx, testStore) + createStatusTestUsers(t, ctx, testStore) return &validateSessionTestSetup{ proxyService: proxyService, @@ -91,6 +92,82 @@ func createTestProxies(t *testing.T, ctx context.Context, testStore store.Store) }, } require.NoError(t, testStore.CreateService(ctx, restrictedProxy)) + + // Distributed to the account's "All" group, the configuration that hands a + // service to every user in the account. + allUsersProxy := &service.Service{ + ID: "allUsersProxyId", + AccountID: "testAccountId", + Name: "All Users Proxy", + Domain: "all-users-proxy.example.com", + Enabled: true, + SessionPrivateKey: privKey, + SessionPublicKey: pubKey, + Auth: service.AuthConfig{ + BearerAuth: &service.BearerAuthConfig{ + Enabled: true, + DistributionGroups: []string{allUsersGroupID}, + }, + }, + } + require.NoError(t, testStore.CreateService(ctx, allUsersProxy)) +} + +const ( + allUsersGroupID = "allUsersGroupId" + pendingUserID = "pendingUserId" + blockedUserID = "blockedUserId" + pendingAllUsersID = "pendingAllUsersUserId" +) + +// createStatusTestUsers adds the users whose account status must keep them out +// of a proxy session. A user awaiting approval is persisted as both blocked and +// pending approval, the way the approval flow stores one. +func createStatusTestUsers(t *testing.T, ctx context.Context, testStore store.Store) { + t.Helper() + + require.NoError(t, testStore.CreateGroup(ctx, &types.Group{ + ID: allUsersGroupID, + AccountID: "testAccountId", + Name: "All", + Issued: types.GroupIssuedAPI, + })) + + users := []*types.User{ + { + Id: pendingUserID, + AccountID: "testAccountId", + Role: types.UserRoleUser, + AutoGroups: []string{"allowedGroupId"}, + Blocked: true, + PendingApproval: true, + Issued: "api", + CreatedAt: time.Now(), + }, + { + Id: pendingAllUsersID, + AccountID: "testAccountId", + Role: types.UserRoleUser, + AutoGroups: []string{allUsersGroupID}, + Blocked: true, + PendingApproval: true, + Issued: "api", + CreatedAt: time.Now(), + }, + { + Id: blockedUserID, + AccountID: "testAccountId", + Role: types.UserRoleUser, + AutoGroups: []string{"allowedGroupId"}, + Blocked: true, + PendingApproval: false, + Issued: "api", + CreatedAt: time.Now(), + }, + } + for _, user := range users { + require.NoError(t, testStore.SaveUser(ctx, user)) + } } func generateSessionKeyPair(t *testing.T) (string, string) { @@ -149,6 +226,114 @@ func TestValidateSession_UserNotInAllowedGroup(t *testing.T) { assert.Empty(t, resp.GetPeerGroupIds(), "PeerGroupIds must mirror the resolved user's actual (empty) memberships on denial") } +// TestValidateSession_PendingApprovalUserDenied covers a user who is a member of +// the service's distribution group but is still waiting for an administrator to +// approve the account. Group membership alone must not open the service. +func TestValidateSession_PendingApprovalUserDenied(t *testing.T) { + setup := setupValidateSessionTest(t) + defer setup.cleanup() + + proxy, err := setup.store.GetServiceByID(context.Background(), store.LockingStrengthNone, "testAccountId", "restrictedProxyId") + require.NoError(t, err) + + token := createSessionToken(t, proxy.SessionPrivateKey, pendingUserID, "restricted-proxy.example.com") + + resp, err := setup.proxyService.ValidateSession(context.Background(), &proto.ValidateSessionRequest{ + Domain: "restricted-proxy.example.com", + SessionToken: token, + }) + + require.NoError(t, err) + assert.False(t, resp.Valid, "User pending approval should be denied") + assert.Equal(t, deniedReasonPendingApproval, resp.DeniedReason, "Denied reason should name the pending approval state") + assert.Equal(t, pendingUserID, resp.UserId, "Denial should identify the user it applies to") + assert.Equal(t, []string{"allowedGroupId"}, resp.GetPeerGroupIds(), "PeerGroupIds must mirror the resolved user's group memberships on denial") + assert.Equal(t, []string{"Allowed Group"}, resp.GetPeerGroupNames(), "PeerGroupNames must pair with PeerGroupIds on denial") +} + +// TestValidateSession_PendingApprovalUserInAllUsersGroupDenied covers the same +// user against a service distributed to the account's "All" group, where every +// user of the account is a member by default. +func TestValidateSession_PendingApprovalUserInAllUsersGroupDenied(t *testing.T) { + setup := setupValidateSessionTest(t) + defer setup.cleanup() + + proxy, err := setup.store.GetServiceByID(context.Background(), store.LockingStrengthNone, "testAccountId", "allUsersProxyId") + require.NoError(t, err) + + token := createSessionToken(t, proxy.SessionPrivateKey, pendingAllUsersID, "all-users-proxy.example.com") + + resp, err := setup.proxyService.ValidateSession(context.Background(), &proto.ValidateSessionRequest{ + Domain: "all-users-proxy.example.com", + SessionToken: token, + }) + + require.NoError(t, err) + assert.False(t, resp.Valid, "User pending approval should be denied even in the All Users group") + assert.Equal(t, deniedReasonPendingApproval, resp.DeniedReason, "Denied reason should name the pending approval state") + assert.Equal(t, pendingAllUsersID, resp.UserId, "Denial should identify the user it applies to") + assert.Equal(t, []string{allUsersGroupID}, resp.GetPeerGroupIds(), "PeerGroupIds must mirror the resolved user's group memberships on denial") +} + +// TestValidateSession_BlockedUserDenied covers a user blocked after having been +// approved, so PendingApproval is false and only the blocked flag is set. +func TestValidateSession_BlockedUserDenied(t *testing.T) { + setup := setupValidateSessionTest(t) + defer setup.cleanup() + + proxy, err := setup.store.GetServiceByID(context.Background(), store.LockingStrengthNone, "testAccountId", "restrictedProxyId") + require.NoError(t, err) + + token := createSessionToken(t, proxy.SessionPrivateKey, blockedUserID, "restricted-proxy.example.com") + + resp, err := setup.proxyService.ValidateSession(context.Background(), &proto.ValidateSessionRequest{ + Domain: "restricted-proxy.example.com", + SessionToken: token, + }) + + require.NoError(t, err) + assert.False(t, resp.Valid, "Blocked user should be denied") + assert.Equal(t, deniedReasonUserBlocked, resp.DeniedReason, "Denied reason should name the blocked state") + assert.Equal(t, blockedUserID, resp.UserId, "Denial should identify the user it applies to") +} + +// TestValidateSession_UserAllowedAfterApproval walks the same session token +// through the approval transition: denied while pending, allowed once an +// administrator clears both flags. +func TestValidateSession_UserAllowedAfterApproval(t *testing.T) { + setup := setupValidateSessionTest(t) + defer setup.cleanup() + + ctx := context.Background() + + proxy, err := setup.store.GetServiceByID(ctx, store.LockingStrengthNone, "testAccountId", "restrictedProxyId") + require.NoError(t, err) + + token := createSessionToken(t, proxy.SessionPrivateKey, pendingUserID, "restricted-proxy.example.com") + req := &proto.ValidateSessionRequest{ + Domain: "restricted-proxy.example.com", + SessionToken: token, + } + + resp, err := setup.proxyService.ValidateSession(ctx, req) + require.NoError(t, err) + require.False(t, resp.Valid, "User pending approval should be denied before approval") + assert.Equal(t, deniedReasonPendingApproval, resp.DeniedReason, "Denied reason should name the pending approval state") + + user, err := setup.store.GetUserByUserID(ctx, store.LockingStrengthNone, pendingUserID) + require.NoError(t, err) + user.PendingApproval = false + user.Blocked = false + require.NoError(t, setup.store.SaveUser(ctx, user)) + + resp, err = setup.proxyService.ValidateSession(ctx, req) + require.NoError(t, err) + assert.True(t, resp.Valid, "Approved user should be allowed access") + assert.Empty(t, resp.DeniedReason) + assert.Equal(t, pendingUserID, resp.UserId, "Approved user should be identified in the response") + assert.Equal(t, []string{"allowedGroupId"}, resp.GetPeerGroupIds(), "PeerGroupIds must mirror the approved user's group memberships") +} + func TestValidateSession_UserInDifferentAccount(t *testing.T) { setup := setupValidateSessionTest(t) defer setup.cleanup() diff --git a/management/server/http/handlers/proxy/auth.go b/management/server/http/handlers/proxy/auth.go index 0120fad0e..62725e8d4 100644 --- a/management/server/http/handlers/proxy/auth.go +++ b/management/server/http/handlers/proxy/auth.go @@ -2,6 +2,7 @@ package proxy import ( "context" + "errors" "net" "net/http" "net/netip" @@ -108,7 +109,7 @@ func (h *AuthCallbackHandler) handleCallback(w http.ResponseWriter, r *http.Requ redirectURL.Scheme = "https" query := redirectURL.Query() query.Set("error", "access_denied") - query.Set("error_description", "Service configuration error") + query.Set("error_description", sessionTokenErrorDescription(err)) redirectURL.RawQuery = query.Encode() http.Redirect(w, r, redirectURL.String(), http.StatusFound) return @@ -124,6 +125,20 @@ func (h *AuthCallbackHandler) handleCallback(w http.ResponseWriter, r *http.Requ http.Redirect(w, r, redirectURL.String(), http.StatusFound) } +// sessionTokenErrorDescription maps a session token failure to the text the +// proxy renders on its access denied page. Account status denials get a message +// the user can act on, while everything else stays generic so a lookup or +// signing failure does not describe management internals to the browser. +func sessionTokenErrorDescription(err error) string { + if errors.Is(err, nbgrpc.ErrUserPendingApproval) { + return "Your account is pending approval by an administrator" + } + if errors.Is(err, nbgrpc.ErrUserBlocked) { + return "Your account is blocked" + } + return "Service configuration error" +} + func extractUserIDFromToken(ctx context.Context, provider *oidc.Provider, config nbgrpc.ProxyOIDCConfig, token *oauth2.Token) string { rawIDToken, ok := token.Extra("id_token").(string) if !ok { diff --git a/management/server/http/handlers/proxy/auth_callback_integration_test.go b/management/server/http/handlers/proxy/auth_callback_integration_test.go index a24857066..51e54ebce 100644 --- a/management/server/http/handlers/proxy/auth_callback_integration_test.go +++ b/management/server/http/handlers/proxy/auth_callback_integration_test.go @@ -360,6 +360,51 @@ func createTestAccountsAndUsers(t *testing.T, ctx context.Context, testStore sto Issued: "api", } require.NoError(t, testStore.SaveUser(ctx, allowedUser)) + + // A second tenant, whose users must never be issued a token signed with + // the first tenant's service session key. + otherAccount := &types.Account{ + Id: "otherAccountId", + Domain: "other.com", + DomainCategory: "private", + IsDomainPrimaryAccount: true, + CreatedAt: time.Now(), + } + require.NoError(t, testStore.SaveAccount(ctx, otherAccount)) + + otherAccountUser := &types.User{ + Id: "otherAccountUserId", + AccountID: "otherAccountId", + Role: types.UserRoleUser, + CreatedAt: time.Now(), + Issued: "api", + } + require.NoError(t, testStore.SaveUser(ctx, otherAccountUser)) + + // A user awaiting approval is stored as blocked and pending approval, and + // carries the same group membership as the approved one. + pendingUser := &types.User{ + Id: "pendingUserId", + AccountID: "testAccountId", + Role: types.UserRoleUser, + AutoGroups: []string{"allowedGroupId"}, + Blocked: true, + PendingApproval: true, + CreatedAt: time.Now(), + Issued: "api", + } + require.NoError(t, testStore.SaveUser(ctx, pendingUser)) + + blockedUser := &types.User{ + Id: "blockedUserId", + AccountID: "testAccountId", + Role: types.UserRoleUser, + AutoGroups: []string{"allowedGroupId"}, + Blocked: true, + CreatedAt: time.Now(), + Issued: "api", + } + require.NoError(t, testStore.SaveUser(ctx, blockedUser)) } // testServiceManager is a minimal implementation for testing. @@ -490,6 +535,64 @@ func TestAuthCallback_UserAllowedToLogin(t *testing.T) { require.Empty(t, parsedLocation.Query().Get("error"), "Should not have error parameter") } +// TestAuthCallback_UserDeniedByAccountStatus asserts that a user whose account +// is pending approval or blocked never receives a session token from the OIDC +// callback, and that the redirect carries a description the proxy can render. +func TestAuthCallback_UserDeniedByAccountStatus(t *testing.T) { + tests := []struct { + name string + subject string + expectErrorDesc string + }{ + { + name: "pending approval", + subject: "pendingUserId", + expectErrorDesc: "Your account is pending approval by an administrator", + }, + { + name: "blocked", + subject: "blockedUserId", + expectErrorDesc: "Your account is blocked", + }, + { + name: "unknown to management", + subject: "userMissingFromStoreId", + expectErrorDesc: "Service configuration error", + }, + { + // The account topology stays out of the browser-visible message. + name: "belongs to another account", + subject: "otherAccountUserId", + expectErrorDesc: "Service configuration error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + setup := setupAuthCallbackTest(t) + defer setup.cleanup() + + setup.oidcServer.tokenSubject = tt.subject + + state := createTestState(t, setup.proxyService, "https://test-proxy.example.com/dashboard") + + req := httptest.NewRequest(http.MethodGet, "/reverse-proxy/callback?code=test-auth-code&state="+url.QueryEscape(state), nil) + rec := httptest.NewRecorder() + + setup.router.ServeHTTP(rec, req) + + require.Equal(t, http.StatusFound, rec.Code) + + parsedLocation, err := url.Parse(rec.Header().Get("Location")) + require.NoError(t, err) + + require.Empty(t, parsedLocation.Query().Get("session_token"), "Denied user must not receive a session token") + require.Equal(t, "access_denied", parsedLocation.Query().Get("error")) + require.Equal(t, tt.expectErrorDesc, parsedLocation.Query().Get("error_description")) + }) + } +} + func TestAuthCallback_ProxyNotFound(t *testing.T) { setup := setupAuthCallbackTest(t) defer setup.cleanup() From 5584f8ef0a8dbd7d1db49655914d7c2c39b431cf Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:27:20 +0900 Subject: [PATCH 08/22] [client] Add strict anonymization level and MAC anonymization to debug bundles (#7102) --- client/android/client.go | 16 +- client/anonymize/anonymize.go | 410 +++- client/anonymize/anonymize_test.go | 298 +++ client/cmd/debug.go | 32 +- client/cmd/root.go | 18 +- client/cmd/status.go | 8 +- client/internal/debug/debug.go | 91 +- client/internal/debug/wgshow.go | 10 +- client/internal/engine.go | 2 + client/ios/NetBirdSDK/client.go | 15 +- client/proto/daemon.pb.go | 17 +- client/proto/daemon.proto | 5 + client/server/debug.go | 2 + client/status/status.go | 8 +- .../src/contexts/DebugBundleContext.tsx | 17 +- .../settings/SettingsTroubleshooting.tsx | 88 +- client/ui/i18n/locales/en/common.json | 22 +- client/ui/services/debug.go | 22 +- shared/management/proto/management.pb.go | 2143 +++++++++-------- shared/management/proto/management.proto | 4 + 20 files changed, 2068 insertions(+), 1160 deletions(-) diff --git a/client/android/client.go b/client/android/client.go index 154bd8484..a21348f27 100644 --- a/client/android/client.go +++ b/client/android/client.go @@ -15,6 +15,7 @@ import ( log "github.com/sirupsen/logrus" + nbAnonymize "github.com/netbirdio/netbird/client/anonymize" "github.com/netbirdio/netbird/client/iface/device" "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/debug" @@ -32,6 +33,13 @@ import ( types "github.com/netbirdio/netbird/upload-server/types" ) +// AnonymizeLevelDefault and AnonymizeLevelStrict are the accepted +// anonymizeLevel values for DebugBundle. +const ( + AnonymizeLevelDefault = nbAnonymize.LevelDefaultString + AnonymizeLevelStrict = nbAnonymize.LevelStrictString +) + // ConnectionListener export internal Listener for mobile type ConnectionListener interface { peer.Listener @@ -278,8 +286,10 @@ func (c *Client) GetTunSettings() (*TunSettings, error) { } // DebugBundle generates a debug bundle, uploads it, and returns the upload key. -// It works both with and without a running engine. -func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (string, error) { +// It works both with and without a running engine. anonymizeLevel is "default" +// or "strict"; strict also anonymizes internal IP ranges, peer names, and +// WireGuard public keys, and implies anonymize. +func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool, anonymizeLevel string) (string, error) { cfg, cacheDir, cc := c.stateSnapshot() // If the engine hasn't been started, load config from disk @@ -298,6 +308,7 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (strin InternalConfig: cfg, StatusRecorder: c.recorder, TempDir: cacheDir, + StatePath: platformFiles.StateFilePath(), } if cc != nil { @@ -321,6 +332,7 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (strin deps, debug.BundleConfig{ Anonymize: anonymize, + AnonymizeLevel: nbAnonymize.ParseLevel(anonymizeLevel), IncludeSystemInfo: true, }, ) diff --git a/client/anonymize/anonymize.go b/client/anonymize/anonymize.go index c140cef89..acadb717b 100644 --- a/client/anonymize/anonymize.go +++ b/client/anonymize/anonymize.go @@ -2,6 +2,7 @@ package anonymize import ( "crypto/rand" + "encoding/base64" "fmt" "math/big" "net" @@ -15,13 +16,88 @@ import ( const anonTLD = ".domain" +// Level selects how much the anonymizer redacts. Levels are ordered: a higher +// level redacts strictly more. On the wire (protos, flags) levels travel as +// their string form. +type Level int + +const ( + // LevelDefault anonymizes public IP addresses, IPv6 ULA, domains, and MAC + // addresses. Internal IPv4 ranges (RFC 1918, CGNAT, link-local) are + // preserved so support can reason about the real topology. + LevelDefault Level = iota + // LevelStrict additionally anonymizes internal IP ranges, peer names, and + // WireGuard public keys. + LevelStrict +) + +// LevelDefaultString and LevelStrictString are the wire forms of the levels, +// for boundaries that pass levels as strings (flags, protos, mobile bindings). +const ( + LevelDefaultString = "default" + LevelStrictString = "strict" +) + +// ParseLevel maps s to a Level. Empty means LevelDefault; anything +// unrecognized maps to LevelStrict so an unknown request never yields less +// anonymization than intended. +func ParseLevel(s string) Level { + switch strings.ToLower(s) { + case "", LevelDefaultString: + return LevelDefault + default: + return LevelStrict + } +} + +// String returns the wire form of the level: "default" or "strict". +func (l Level) String() string { + if l >= LevelStrict { + return LevelStrictString + } + return LevelDefaultString +} + +// protectedDomains are NetBird-operated suffixes that stay recognizable in an +// anonymized bundle. At LevelStrict the labels in front of them (the peer +// name) are still replaced, except under netbird.io, which only hosts +// NetBird infrastructure (api, signal, flow), never peer names. +var protectedDomains = []string{"netbird.io", "netbird.selfhosted", "netbird.cloud", "netbird.stage"} + +const infraDomain = "netbird.io" + +var ( + macColonRegex = regexp.MustCompile(`\b[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}\b`) + macDashRegex = regexp.MustCompile(`\b[0-9a-fA-F]{2}(?:-[0-9a-fA-F]{2}){5}\b`) + wgKeyRegex = regexp.MustCompile(`\b[A-Za-z0-9+/]{43}=`) +) + type Anonymizer struct { ipAnonymizer map[netip.Addr]netip.Addr domainAnonymizer map[string]string - currentAnonIPv4 netip.Addr - currentAnonIPv6 netip.Addr - startAnonIPv4 netip.Addr - startAnonIPv6 netip.Addr + // domainOrder caches the keys of domainAnonymizer sorted longest-first + // for AnonymizeString; it is rebuilt when the map gains entries. + domainOrder []string + labelAnonymizer map[string]string + labelAnonymized map[string]struct{} + labelCounter uint32 + macAnonymizer map[string]string + macCounter uint32 + wgKeyAnonymizer map[string]string + wgKeyAnonymized map[string]struct{} + currentAnonIPv4 netip.Addr + currentAnonIPv6 netip.Addr + startAnonIPv4 netip.Addr + startAnonIPv6 netip.Addr + + // LevelStrict also anonymizes internal ranges (RFC 1918, CGNAT, + // link-local), replacing them from the dedicated internal pools below so + // a reader can still tell an internal address from a public one. + level Level + currentAnonInternalIPv4 netip.Addr + currentAnonInternalIPv6 netip.Addr + startAnonInternalIPv4 netip.Addr + startAnonInternalIPv6 netip.Addr domainKeyRegex *regexp.Regexp } @@ -32,25 +108,50 @@ func DefaultAddresses() (netip.Addr, netip.Addr) { return netip.AddrFrom4([4]byte{198, 51, 100, 0}), netip.MustParseAddr("2001:db8:ffff::") } +// InternalAddresses returns the pool starts used in strict mode for internal +// ranges. Both are reserved ranges that cannot collide with real addressing: +// 198.18.0.0 (RFC 2544 benchmarking), 2001:db8:1:: (RFC 3849 documentation). +func InternalAddresses() (netip.Addr, netip.Addr) { + return netip.AddrFrom4([4]byte{198, 18, 0, 0}), netip.MustParseAddr("2001:db8:1::") +} + func NewAnonymizer(startIPv4, startIPv6 netip.Addr) *Anonymizer { + internalIPv4, internalIPv6 := InternalAddresses() return &Anonymizer{ ipAnonymizer: map[netip.Addr]netip.Addr{}, domainAnonymizer: map[string]string{}, + labelAnonymizer: map[string]string{}, + labelAnonymized: map[string]struct{}{}, + macAnonymizer: map[string]string{}, + wgKeyAnonymizer: map[string]string{}, + wgKeyAnonymized: map[string]struct{}{}, currentAnonIPv4: startIPv4, currentAnonIPv6: startIPv6, startAnonIPv4: startIPv4, startAnonIPv6: startIPv6, + level: LevelDefault, + currentAnonInternalIPv4: internalIPv4, + currentAnonInternalIPv6: internalIPv6, + startAnonInternalIPv4: internalIPv4, + startAnonInternalIPv6: internalIPv6, + domainKeyRegex: regexp.MustCompile(`\bdomain=([^\s,:"]+)`), } } +// SetLevel selects the anonymization level. The zero value of a new +// Anonymizer is LevelDefault. +func (a *Anonymizer) SetLevel(level Level) { + a.level = level +} + func (a *Anonymizer) AnonymizeIP(ip netip.Addr) netip.Addr { + // Normalize 4-in-6 addresses so ::ffff:192.168.1.1 classifies and maps + // like 192.168.1.1. + ip = ip.Unmap() + if ip.IsLoopback() || - ip.IsLinkLocalUnicast() || - ip.IsLinkLocalMulticast() || - ip.IsInterfaceLocalMulticast() || - (ip.Is4() && ip.IsPrivate()) || ip.IsUnspecified() || ip.IsMulticast() || isWellKnown(ip) || @@ -59,18 +160,100 @@ func (a *Anonymizer) AnonymizeIP(ip netip.Addr) netip.Addr { return ip } + if isInternal(ip) && a.level < LevelStrict { + return ip + } + if _, ok := a.ipAnonymizer[ip]; !ok { - if ip.Is4() { - a.ipAnonymizer[ip] = a.currentAnonIPv4 - a.currentAnonIPv4 = a.currentAnonIPv4.Next() - } else { - a.ipAnonymizer[ip] = a.currentAnonIPv6 - a.currentAnonIPv6 = a.currentAnonIPv6.Next() - } + a.ipAnonymizer[ip] = a.nextAnonIP(ip) } return a.ipAnonymizer[ip] } +func (a *Anonymizer) nextAnonIP(ip netip.Addr) netip.Addr { + // At the strict level, internal addresses (including IPv6 ULA, matched + // by IsPrivate) come from the internal pools so they remain recognizable + // as internal without disclosing the real values. + if a.level >= LevelStrict && (isInternal(ip) || ip.IsPrivate()) { + if ip.Is4() { + anon := a.currentAnonInternalIPv4 + a.currentAnonInternalIPv4 = a.currentAnonInternalIPv4.Next() + return anon + } + anon := a.currentAnonInternalIPv6 + a.currentAnonInternalIPv6 = a.currentAnonInternalIPv6.Next() + return anon + } + + if ip.Is4() { + anon := a.currentAnonIPv4 + a.currentAnonIPv4 = a.currentAnonIPv4.Next() + return anon + } + anon := a.currentAnonIPv6 + a.currentAnonIPv6 = a.currentAnonIPv6.Next() + return anon +} + +// AnonymizeMAC replaces a MAC address with a consistent placeholder from the +// locally administered range starting at 02:00:00:00:00:01, at every +// anonymization level. Broadcast, multicast, all-zero, and already assigned +// placeholder addresses are preserved. The colon and dash spellings of the +// same address share one placeholder; the output keeps the input's separator. +func (a *Anonymizer) AnonymizeMAC(mac string) string { + hw, err := net.ParseMAC(mac) + if err != nil || len(hw) != 6 { + return mac + } + + if isWellKnownMAC(hw) || a.isAnonymizedMAC(hw) { + return mac + } + + key := hw.String() + anon, ok := a.macAnonymizer[key] + if !ok { + a.macCounter++ + anon = fmt.Sprintf("02:00:00:%02x:%02x:%02x", byte(a.macCounter>>16), byte(a.macCounter>>8), byte(a.macCounter)) + a.macAnonymizer[key] = anon + } + + if strings.Contains(mac, "-") { + anon = strings.ReplaceAll(anon, ":", "-") + } + return anon +} + +// isAnonymizedMAC reports whether hw is a placeholder this anonymizer already +// handed out, so a second pass over anonymized output leaves it unchanged. +func (a *Anonymizer) isAnonymizedMAC(hw net.HardwareAddr) bool { + if hw[0] != 0x02 || hw[1] != 0 || hw[2] != 0 { + return false + } + value := uint32(hw[3])<<16 | uint32(hw[4])<<8 | uint32(hw[5]) + return value <= a.macCounter +} + +// AnonymizeWGKey replaces a WireGuard public key with a consistent random +// placeholder of the same shape. Keys are only anonymized at LevelStrict; +// placeholders already handed out pass through unchanged. +func (a *Anonymizer) AnonymizeWGKey(key string) string { + if a.level < LevelStrict || !looksLikeWGKey(key) { + return key + } + if _, ok := a.wgKeyAnonymized[key]; ok { + return key + } + + anon, ok := a.wgKeyAnonymizer[key] + if !ok { + anon = generateAnonymousKey() + a.wgKeyAnonymizer[key] = anon + a.wgKeyAnonymized[anon] = struct{}{} + } + return anon +} + func (a *Anonymizer) AnonymizeUDPAddr(addr net.UDPAddr) net.UDPAddr { // Convert IP to netip.Addr ip, ok := netip.AddrFromSlice(addr.IP) @@ -89,12 +272,12 @@ func (a *Anonymizer) AnonymizeUDPAddr(addr net.UDPAddr) net.UDPAddr { // isInAnonymizedRange checks if an IP is within the range of already assigned anonymized IPs func (a *Anonymizer) isInAnonymizedRange(ip netip.Addr) bool { - if ip.Is4() && ip.Compare(a.startAnonIPv4) >= 0 && ip.Compare(a.currentAnonIPv4) <= 0 { - return true - } else if !ip.Is4() && ip.Compare(a.startAnonIPv6) >= 0 && ip.Compare(a.currentAnonIPv6) <= 0 { - return true + if ip.Is4() { + return inPoolRange(ip, a.startAnonIPv4, a.currentAnonIPv4) || + inPoolRange(ip, a.startAnonInternalIPv4, a.currentAnonInternalIPv4) } - return false + return inPoolRange(ip, a.startAnonIPv6, a.currentAnonIPv6) || + inPoolRange(ip, a.startAnonInternalIPv6, a.currentAnonInternalIPv6) } func (a *Anonymizer) AnonymizeIPString(ip string) string { @@ -118,14 +301,17 @@ func (a *Anonymizer) AnonymizeDomain(domain string) string { baseDomain = domain[:len(domain)-1] } - if strings.HasSuffix(baseDomain, "netbird.io") || - strings.HasSuffix(baseDomain, "netbird.selfhosted") || - strings.HasSuffix(baseDomain, "netbird.cloud") || - strings.HasSuffix(baseDomain, "netbird.stage") || - strings.HasSuffix(baseDomain, anonTLD) { + if strings.HasSuffix(baseDomain, anonTLD) { return domain } + if suffix := protectedSuffix(baseDomain); suffix != "" { + if a.level < LevelStrict || baseDomain == suffix || suffix == infraDomain { + return domain + } + return withTrailingDot(a.anonymizePeerName(baseDomain, suffix), hasDot) + } + parts := strings.Split(baseDomain, ".") if len(parts) < 2 { return domain @@ -141,12 +327,53 @@ func (a *Anonymizer) AnonymizeDomain(domain string) string { } result := strings.Replace(baseDomain, baseForLookup, anonymized, 1) - if hasDot { - result += "." + if a.level >= LevelStrict && len(parts) > 2 { + prefix := strings.TrimSuffix(baseDomain, "."+baseForLookup) + result = a.anonymizeLabels(prefix, "host") + "." + anonymized + // The full mapping feeds AnonymizeString so seeded FQDNs are caught + // in log lines as a whole, labels included. + a.domainAnonymizer[baseDomain] = result + } + return withTrailingDot(result, hasDot) +} + +// anonymizePeerName replaces the labels in front of a protected suffix with +// numbered peer placeholders, keeping the suffix, and records the full +// mapping for string replacement in logs. The numbering keeps a peer +// recognizable across the whole bundle without disclosing its name. +func (a *Anonymizer) anonymizePeerName(baseDomain, suffix string) string { + prefix := strings.TrimSuffix(baseDomain, "."+suffix) + result := a.anonymizeLabels(prefix, "peer") + "." + suffix + if result != baseDomain { + a.domainAnonymizer[baseDomain] = result } return result } +// anonymizeLabels replaces each dot-separated label with a consistent +// numbered placeholder ("-"). Wildcard labels and +// placeholders already handed out pass through unchanged. +func (a *Anonymizer) anonymizeLabels(prefix, placeholder string) string { + labels := strings.Split(prefix, ".") + for i, label := range labels { + if label == "*" { + continue + } + if _, ok := a.labelAnonymized[label]; ok { + continue + } + anon, ok := a.labelAnonymizer[label] + if !ok { + a.labelCounter++ + anon = fmt.Sprintf("%s-%d", placeholder, a.labelCounter) + a.labelAnonymizer[label] = anon + a.labelAnonymized[anon] = struct{}{} + } + labels[i] = anon + } + return strings.Join(labels, ".") +} + func (a *Anonymizer) AnonymizeURI(uri string) string { u, err := url.Parse(uri) if err != nil { @@ -181,16 +408,70 @@ func (a *Anonymizer) AnonymizeString(str string) string { str = ipv4Regex.ReplaceAllStringFunc(str, a.AnonymizeIPString) str = ipv6Regex.ReplaceAllStringFunc(str, a.AnonymizeIPString) - for domain, anonDomain := range a.domainAnonymizer { - str = strings.ReplaceAll(str, domain, anonDomain) + for _, domain := range a.sortedDomains() { + str = strings.ReplaceAll(str, domain, a.domainAnonymizer[domain]) } str = a.AnonymizeSchemeURI(str) str = a.AnonymizeDNSLogLine(str) + // MAC handling runs after the IP passes so preserved IPv6 addresses are + // already out of the way; the separator guard skips matches embedded in a + // longer colon- or dash-separated sequence (such as an IPv6 tail). + str = a.anonymizeMACsInString(str, macColonRegex, ':') + str = a.anonymizeMACsInString(str, macDashRegex, '-') + + if a.level >= LevelStrict { + str = wgKeyRegex.ReplaceAllStringFunc(str, a.AnonymizeWGKey) + } + return str } +// sortedDomains returns the domain mappings longest-first, so a full-FQDN +// mapping (strict level) is applied before the base-domain mapping it +// contains. The order is rebuilt only when domainAnonymizer has grown. +func (a *Anonymizer) sortedDomains() []string { + if len(a.domainOrder) == len(a.domainAnonymizer) { + return a.domainOrder + } + + a.domainOrder = a.domainOrder[:0] + for domain := range a.domainAnonymizer { + a.domainOrder = append(a.domainOrder, domain) + } + slices.SortFunc(a.domainOrder, func(x, y string) int { + if d := len(y) - len(x); d != 0 { + return d + } + return strings.Compare(x, y) + }) + return a.domainOrder +} + +// anonymizeMACsInString replaces MAC addresses matched by re, skipping +// matches that directly adjoin another sep so a six-group run inside a longer +// separated sequence is left alone. +func (a *Anonymizer) anonymizeMACsInString(str string, re *regexp.Regexp, sep byte) string { + matches := re.FindAllStringIndex(str, -1) + if len(matches) == 0 { + return str + } + + var b strings.Builder + last := 0 + for _, m := range matches { + if (m[0] > 0 && str[m[0]-1] == sep) || (m[1] < len(str) && str[m[1]] == sep) { + continue + } + b.WriteString(str[last:m[0]]) + b.WriteString(a.AnonymizeMAC(str[m[0]:m[1]])) + last = m[1] + } + b.WriteString(str[last:]) + return b.String() +} + // AnonymizeSchemeURI finds and anonymizes URIs with ws, wss, rel, rels, stun, stuns, turn, and turns schemes. func (a *Anonymizer) AnonymizeSchemeURI(text string) string { re := regexp.MustCompile(`(?i)\b(wss?://|rels?://|stuns?:|turns?:|https?://)\S+\b`) @@ -239,10 +520,79 @@ func isWellKnown(addr netip.Addr) bool { "128.0.0.0", "8000::", // 2nd split subnet for default routes } - if slices.Contains(wellKnown, addr.String()) { + return slices.Contains(wellKnown, addr.String()) +} + +// isInternal reports whether ip identifies a host only within the local +// network: IPv4 private (RFC 1918), CGNAT (RFC 6598), and link-local (v4 and +// v6). These are preserved at the default level so support can reason about +// the real topology, and replaced from the internal pools at the strict +// level. IPv6 ULA is deliberately not internal: its random global ID uniquely +// fingerprints the network, so it is anonymized at every level. +func isInternal(ip netip.Addr) bool { + return (ip.Is4() && ip.IsPrivate()) || + ip.IsLinkLocalUnicast() || + isCGNAT(ip) +} + +func inPoolRange(ip, start, current netip.Addr) bool { + return ip.Compare(start) >= 0 && ip.Compare(current) <= 0 +} + +// isWellKnownMAC reports whether hw carries no stable host identity: all-zero +// or a group address (broadcast and multicast). +func isWellKnownMAC(hw net.HardwareAddr) bool { + if hw[0]&1 == 1 { return true } + for _, b := range hw { + if b != 0 { + return false + } + } + return true +} +// looksLikeWGKey reports whether s has the shape of a WireGuard key: +// 44 base64 characters decoding to 32 bytes. +func looksLikeWGKey(s string) bool { + if len(s) != 44 || s[43] != '=' { + return false + } + decoded, err := base64.StdEncoding.DecodeString(s) + return err == nil && len(decoded) == 32 +} + +func generateAnonymousKey() string { + buf := make([]byte, 32) + if _, err := rand.Read(buf); err != nil { + return strings.Repeat("A", 43) + "=" + } + return base64.StdEncoding.EncodeToString(buf) +} + +// protectedSuffix returns the protected NetBird suffix baseDomain ends with, +// or empty. The match is label-anchored so an unrelated domain that merely +// ends in the same characters is not preserved. +func protectedSuffix(baseDomain string) string { + for _, d := range protectedDomains { + if baseDomain == d || strings.HasSuffix(baseDomain, "."+d) { + return d + } + } + return "" +} + +func withTrailingDot(domain string, hasDot bool) string { + if hasDot { + return domain + "." + } + return domain +} + +// isCGNAT reports whether addr is in 100.64.0.0/10 (RFC 6598), the range +// NetBird assigns overlay peer addresses from. +func isCGNAT(addr netip.Addr) bool { cgnatRangeStart := netip.AddrFrom4([4]byte{100, 64, 0, 0}) cgnatRange := netip.PrefixFrom(cgnatRangeStart, 10) diff --git a/client/anonymize/anonymize_test.go b/client/anonymize/anonymize_test.go index 852315fa1..7c3c7bcf8 100644 --- a/client/anonymize/anonymize_test.go +++ b/client/anonymize/anonymize_test.go @@ -1,8 +1,11 @@ package anonymize_test import ( + "bytes" + "encoding/base64" "net/netip" "regexp" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -44,6 +47,301 @@ func TestAnonymizeIP(t *testing.T) { } } +func TestParseLevel(t *testing.T) { + tests := []struct { + input string + expect anonymize.Level + }{ + {"", anonymize.LevelDefault}, + {"default", anonymize.LevelDefault}, + {"DEFAULT", anonymize.LevelDefault}, + {"strict", anonymize.LevelStrict}, + {"STRICT", anonymize.LevelStrict}, + // Unknown values must never yield less anonymization than requested. + {"garbage", anonymize.LevelStrict}, + } + + for _, tc := range tests { + t.Run("input="+tc.input, func(t *testing.T) { + assert.Equal(t, tc.expect, anonymize.ParseLevel(tc.input), "parsed level should match") + }) + } +} + +func TestAnonymizeIP_DefaultLevelInternalRanges(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + tests := []struct { + name string + ip string + expect string + }{ + {"RFC1918 10/8", "10.1.2.3", "10.1.2.3"}, + {"RFC1918 172.16/12", "172.16.5.5", "172.16.5.5"}, + {"RFC1918 192.168/16", "192.168.1.1", "192.168.1.1"}, + {"CGNAT", "100.64.0.5", "100.64.0.5"}, + {"IPv4 link-local", "169.254.1.1", "169.254.1.1"}, + {"IPv6 link-local", "fe80::1", "fe80::1"}, + // ULA is anonymized even at the default level: its random global ID + // uniquely fingerprints the network, unlike shared RFC 1918 space. + {"IPv6 ULA", "fd12:3456:789a::1", "2001:db8:ffff::"}, + // 4-in-6 addresses classify like their unmapped IPv4 form. + {"4-in-6 RFC1918", "::ffff:192.168.1.1", "192.168.1.1"}, + {"4-in-6 CGNAT", "::ffff:100.64.0.5", "100.64.0.5"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := anonymizer.AnonymizeIP(netip.MustParseAddr(tc.ip)) + assert.Equal(t, tc.expect, result.String(), "default level should preserve internal ranges except ULA") + }) + } +} + +func TestAnonymizeIP_StrictLevel(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + // Order matters: internal pool addresses are assigned sequentially. + tests := []struct { + name string + ip string + expect string + }{ + {"RFC1918 192.168/16", "192.168.1.1", "198.18.0.0"}, + {"Second RFC1918", "192.168.1.2", "198.18.0.1"}, + {"Repeated RFC1918", "192.168.1.1", "198.18.0.0"}, + {"RFC1918 10/8", "10.1.2.3", "198.18.0.2"}, + {"RFC1918 172.16/12", "172.16.5.5", "198.18.0.3"}, + {"CGNAT", "100.64.0.5", "198.18.0.4"}, + {"IPv4 link-local", "169.254.1.1", "198.18.0.5"}, + {"Public IPv4 uses public pool", "1.2.3.4", "198.51.100.0"}, + {"IPv6 link-local", "fe80::1", "2001:db8:1::"}, + {"IPv6 ULA", "fd12:3456:789a::1", "2001:db8:1::1"}, + {"Public IPv6 uses public pool", "2607:f8b0:4005:805::200e", "2001:db8:ffff::"}, + {"Loopback IPv4", "127.0.0.1", "127.0.0.1"}, + {"Loopback IPv6", "::1", "::1"}, + {"Unspecified", "0.0.0.0", "0.0.0.0"}, + {"Multicast", "224.0.0.251", "224.0.0.251"}, + {"Well known resolver", "8.8.8.8", "8.8.8.8"}, + {"Well known split marker", "128.0.0.0", "128.0.0.0"}, + {"In internal pool range", "198.18.0.3", "198.18.0.3"}, + {"In public pool range", "198.51.100.0", "198.51.100.0"}, + {"4-in-6 repeated RFC1918", "::ffff:192.168.1.1", "198.18.0.0"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := anonymizer.AnonymizeIP(netip.MustParseAddr(tc.ip)) + assert.Equal(t, tc.expect, result.String(), "strict level should replace internal ranges from the internal pools") + }) + } +} + +func TestAnonymizeString_StrictInternalIPs(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + input := "route 10.20.30.0/24 via 192.168.1.1 dev eth0 src 100.64.0.7" + firstPass := anonymizer.AnonymizeString(input) + secondPass := anonymizer.AnonymizeString(firstPass) + + assert.NotContains(t, firstPass, "10.20.30.0", "private network address should be anonymized") + assert.NotContains(t, firstPass, "192.168.1.1", "private gateway should be anonymized") + assert.NotContains(t, firstPass, "100.64.0.7", "CGNAT address should be anonymized") + assert.Contains(t, firstPass, "/24", "prefix length should be preserved") + assert.Equal(t, firstPass, secondPass, "second pass should not further anonymize the string") +} + +func TestAnonymizeMAC(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + first := anonymizer.AnonymizeMAC("aa:bb:cc:dd:ee:0f") + assert.Equal(t, "02:00:00:00:00:01", first, "first MAC should get the first placeholder") + assert.Equal(t, first, anonymizer.AnonymizeMAC("aa:bb:cc:dd:ee:0f"), "repeated MAC should map to the same placeholder") + assert.Equal(t, first, anonymizer.AnonymizeMAC("AA:BB:CC:DD:EE:0F"), "case should not affect the mapping") + assert.Equal(t, "02-00-00-00-00-01", anonymizer.AnonymizeMAC("AA-BB-CC-DD-EE-0F"), "dash form should keep its separator but share the mapping") + + second := anonymizer.AnonymizeMAC("10:22:33:44:55:66") + assert.Equal(t, "02:00:00:00:00:02", second, "second distinct MAC should get the next placeholder") + + tests := []struct { + name string + mac string + }{ + {"Broadcast", "ff:ff:ff:ff:ff:ff"}, + {"IPv4 multicast", "01:00:5e:00:00:fb"}, + {"IPv6 multicast", "33:33:00:00:00:01"}, + {"All zero", "00:00:00:00:00:00"}, + {"Assigned placeholder", "02:00:00:00:00:01"}, + {"Invalid", "not-a-mac"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.mac, anonymizer.AnonymizeMAC(tc.mac), "should be preserved") + }) + } +} + +func TestAnonymizeString_MACAddresses(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + tests := []struct { + name string + input string + expect string + }{ + { + name: "nftables ether rule", + input: "ether saddr aa:bb:cc:dd:ee:ff drop", + expect: "ether saddr 02:00:00:00:00:01 drop", + }, + { + name: "Windows dash form", + input: "Physical Address : AA-BB-CC-DD-EE-FF", + expect: "Physical Address : 02-00-00-00-00-01", + }, + { + name: "IPv6 address tail is not treated as MAC", + input: "addr fe80:0:11:22:33:44:55:66 scope link", + expect: "addr fe80:0:11:22:33:44:55:66 scope link", + }, + { + name: "broadcast MAC preserved", + input: "dst ff:ff:ff:ff:ff:ff type ARP", + expect: "dst ff:ff:ff:ff:ff:ff type ARP", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := anonymizer.AnonymizeString(tc.input) + assert.Equal(t, tc.expect, result, "MAC addresses should be anonymized at every level") + assert.Equal(t, result, anonymizer.AnonymizeString(result), "second pass should not change the result") + }) + } +} + +func TestAnonymizeWGKey(t *testing.T) { + key := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0x42}, 32)) + + t.Run("default level preserves keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + assert.Equal(t, key, anonymizer.AnonymizeWGKey(key), "default level should not touch WireGuard keys") + }) + + t.Run("strict level replaces keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + anon := anonymizer.AnonymizeWGKey(key) + assert.NotEqual(t, key, anon, "strict level should replace the key") + assert.Regexp(t, `^[A-Za-z0-9+/]{43}=$`, anon, "placeholder should keep the WireGuard key shape") + assert.Equal(t, anon, anonymizer.AnonymizeWGKey(key), "repeated key should map to the same placeholder") + assert.Equal(t, anon, anonymizer.AnonymizeWGKey(anon), "an assigned placeholder should pass through unchanged") + + assert.Equal(t, "not-a-key", anonymizer.AnonymizeWGKey("not-a-key"), "non-key values should be preserved") + }) +} + +func TestAnonymizeString_WGKeys(t *testing.T) { + key := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0x42}, 32)) + input := "peer " + key + " handshake completed" + + t.Run("default level preserves keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + assert.Equal(t, input, anonymizer.AnonymizeString(input), "default level should not touch WireGuard keys in strings") + }) + + t.Run("strict level replaces keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + firstPass := anonymizer.AnonymizeString(input) + assert.NotContains(t, firstPass, key, "the key should not survive strict anonymization") + assert.Equal(t, anonymizer.AnonymizeWGKey(key), extractKey(t, firstPass), "string replacement should be consistent with AnonymizeWGKey") + assert.Equal(t, firstPass, anonymizer.AnonymizeString(firstPass), "second pass should not change the result") + }) +} + +func extractKey(t *testing.T, logLine string) string { + t.Helper() + fields := strings.Fields(logLine) + require.Len(t, fields, 4, "log line should keep its structure") + return fields[1] +} + +func TestAnonymizeDomain_StrictLevel(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + t.Run("netbird peer name", func(t *testing.T) { + result := anonymizer.AnonymizeDomain("my-laptop.netbird.cloud") + assert.Regexp(t, `^peer-\d+\.netbird\.cloud$`, result, "peer name should be anonymized, suffix kept") + assert.NotContains(t, result, "my-laptop", "the peer name should not survive") + assert.Equal(t, result, anonymizer.AnonymizeDomain("my-laptop.netbird.cloud"), "repeated domain should map consistently") + assert.Equal(t, result, anonymizer.AnonymizeDomain(result), "an anonymized domain should pass through unchanged") + }) + + t.Run("bare netbird domain", func(t *testing.T) { + assert.Equal(t, "netbird.cloud", anonymizer.AnonymizeDomain("netbird.cloud"), "the bare protected suffix should be preserved") + }) + + t.Run("netbird infrastructure preserved", func(t *testing.T) { + assert.Equal(t, "api.netbird.io", anonymizer.AnonymizeDomain("api.netbird.io"), + "netbird.io hosts infrastructure, not peer names, and should stay readable") + }) + + t.Run("leading labels of other domains", func(t *testing.T) { + result := anonymizer.AnonymizeDomain("host1.corp.example.com") + assert.Regexp(t, `^host-\d+\.host-\d+\.anon-[a-zA-Z0-9]+\.domain$`, result, "every label should be anonymized") + for _, label := range []string{"host1", "corp", "example"} { + assert.NotContains(t, result, label, "no original label should survive") + } + assert.Equal(t, result, anonymizer.AnonymizeDomain("host1.corp.example.com"), "repeated domain should map consistently") + }) + + t.Run("same label maps consistently across domains", func(t *testing.T) { + first := anonymizer.AnonymizeDomain("shared.one.com") + second := anonymizer.AnonymizeDomain("shared.two.com") + assert.Equal(t, strings.Split(first, ".")[0], strings.Split(second, ".")[0], "the shared host label should get one placeholder") + }) + + t.Run("wildcard label preserved", func(t *testing.T) { + result := anonymizer.AnonymizeDomain("*.example.com") + assert.Regexp(t, `^\*\.anon-[a-zA-Z0-9]+\.domain$`, result, "the wildcard label should stay a wildcard") + }) +} + +func TestAnonymizeDomain_DefaultLevelKeepsPeerNames(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + assert.Equal(t, "my-laptop.netbird.cloud", anonymizer.AnonymizeDomain("my-laptop.netbird.cloud"), + "default level should preserve netbird FQDNs including the peer name") + assert.Regexp(t, `^sub\.anon-[a-zA-Z0-9]+\.domain$`, anonymizer.AnonymizeDomain("sub.example.com"), + "default level should keep subdomain labels") +} + +func TestAnonymizeString_StrictPeerNames(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + // Seed like the bundle generator does from the status: base first, then + // the full FQDN, so replacement must prefer the longer mapping. + anonBase := anonymizer.AnonymizeDomain("example.com") + anonPeer := anonymizer.AnonymizeDomain("peer1.netbird.cloud") + anonHost := anonymizer.AnonymizeDomain("host1.example.com") + + logLine := "connected to peer1.netbird.cloud via host1.example.com endpoint" + firstPass := anonymizer.AnonymizeString(logLine) + assert.NotContains(t, firstPass, "peer1", "the peer name should not survive in logs") + assert.NotContains(t, firstPass, "host1", "the host label should not survive in logs") + assert.Contains(t, firstPass, anonPeer, "the seeded peer mapping should be applied") + assert.Contains(t, firstPass, anonHost, "the seeded host mapping should be applied, not just the base mapping") + assert.NotContains(t, firstPass, "host1."+anonBase, "the base mapping must not preempt the longer FQDN mapping") + assert.Equal(t, firstPass, anonymizer.AnonymizeString(firstPass), "second pass should not change the result") +} + func TestAnonymizeDNSLogLine(t *testing.T) { anonymizer := anonymize.NewAnonymizer(netip.Addr{}, netip.Addr{}) tests := []struct { diff --git a/client/cmd/debug.go b/client/cmd/debug.go index 7ddc3afc4..893b1e248 100644 --- a/client/cmd/debug.go +++ b/client/cmd/debug.go @@ -27,8 +27,8 @@ import ( const errCloseConnection = "Failed to close connection: %v" var ( - logFileCount uint32 - systemInfoFlag bool + logFileCount uint32 + systemInfoFlag bool uploadBundleFlag bool uploadBundleURLFlag string uploadBundleInsecureFlag bool @@ -156,6 +156,11 @@ func debugConfigDump(cmd *cobra.Command, _ []string) error { // request. Returns an error if the RPC fails or if the daemon reports // an upload failure reason. func debugBundle(cmd *cobra.Command, _ []string) error { + anonymizeEnabled, anonymizeLevel, err := effectiveAnonymize() + if err != nil { + return err + } + conn, err := getClient(cmd) if err != nil { return err @@ -168,10 +173,11 @@ func debugBundle(cmd *cobra.Command, _ []string) error { client := proto.NewDaemonServiceClient(conn) request := &proto.DebugBundleRequest{ - Anonymize: anonymizeFlag, - SystemInfo: systemInfoFlag, - LogFileCount: logFileCount, - CliVersion: version.NetbirdVersion(), + Anonymize: anonymizeEnabled, + AnonymizeLevel: anonymizeLevel.String(), + SystemInfo: systemInfoFlag, + LogFileCount: logFileCount, + CliVersion: version.NetbirdVersion(), } if uploadBundleFlag { request.UploadURL = uploadBundleURLFlag @@ -229,6 +235,11 @@ func runForDuration(cmd *cobra.Command, args []string) error { return fmt.Errorf("invalid duration format: %v", err) } + anonymizeEnabled, anonymizeLevel, err := effectiveAnonymize() + if err != nil { + return err + } + conn, err := getClient(cmd) if err != nil { return err @@ -368,10 +379,11 @@ func runForDuration(cmd *cobra.Command, args []string) error { cmd.Println("Creating debug bundle...") request := &proto.DebugBundleRequest{ - Anonymize: anonymizeFlag, - SystemInfo: systemInfoFlag, - LogFileCount: logFileCount, - CliVersion: version.NetbirdVersion(), + Anonymize: anonymizeEnabled, + AnonymizeLevel: anonymizeLevel.String(), + SystemInfo: systemInfoFlag, + LogFileCount: logFileCount, + CliVersion: version.NetbirdVersion(), } if uploadBundleFlag { request.UploadURL = uploadBundleURLFlag diff --git a/client/cmd/root.go b/client/cmd/root.go index ebaae7e3e..ccad78942 100644 --- a/client/cmd/root.go +++ b/client/cmd/root.go @@ -21,6 +21,7 @@ import ( "github.com/spf13/pflag" "google.golang.org/grpc" + "github.com/netbirdio/netbird/client/anonymize" daddr "github.com/netbirdio/netbird/client/internal/daemonaddr" "github.com/netbirdio/netbird/client/internal/profilemanager" ) @@ -69,6 +70,7 @@ var ( autoConnectDisabled bool extraIFaceBlackList []string anonymizeFlag bool + anonymizeLevelFlag string dnsRouteInterval time.Duration // lazyConnEnabled is the parse target for the deprecated --enable-lazy-connection // flag. The flag is inert; the value is no longer read (use NB_LAZY_CONN instead). @@ -156,7 +158,8 @@ func init() { rootCmd.MarkFlagsMutuallyExclusive("setup-key", "setup-key-file") rootCmd.PersistentFlags().StringVar(&preSharedKey, preSharedKeyFlag, "", "Sets WireGuard PreSharedKey property. If set, then only peers that have the same key can communicate.") rootCmd.PersistentFlags().StringVarP(&hostName, "hostname", "n", "", "Sets a custom hostname for the device") - rootCmd.PersistentFlags().BoolVarP(&anonymizeFlag, "anonymize", "A", false, "anonymize IP addresses and non-netbird.io domains in logs and status output") + rootCmd.PersistentFlags().BoolVarP(&anonymizeFlag, "anonymize", "A", false, "anonymize public IP addresses, MAC addresses, and non-netbird.io domains in logs and status output; private, CGNAT, and link-local IP ranges are kept (see --anonymize-level strict)") + rootCmd.PersistentFlags().StringVar(&anonymizeLevelFlag, "anonymize-level", "", "anonymization level: \"default\" or \"strict\"; strict also anonymizes private, CGNAT, and link-local IP ranges, peer names, and WireGuard public keys. Setting this flag implies --anonymize") rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", profilemanager.DefaultConfigPath, "Overrides the default profile file location") rootCmd.AddCommand(upCmd) @@ -293,6 +296,19 @@ var CLIBackOffSettings = &backoff.ExponentialBackOff{ Clock: backoff.SystemClock, } +// effectiveAnonymize resolves the --anonymize and --anonymize-level flags: +// setting a level implies anonymization, and an invalid level is rejected. +func effectiveAnonymize() (bool, anonymize.Level, error) { + if anonymizeLevelFlag == "" { + return anonymizeFlag, anonymize.LevelDefault, nil + } + level := anonymize.ParseLevel(anonymizeLevelFlag) + if !strings.EqualFold(anonymizeLevelFlag, level.String()) { + return false, anonymize.LevelDefault, fmt.Errorf("invalid anonymize level %q: use %q or %q", anonymizeLevelFlag, anonymize.LevelDefault.String(), anonymize.LevelStrict.String()) + } + return true, level, nil +} + func getSetupKey() (string, error) { if setupKeyPath != "" && setupKey == "" { return getSetupKeyFromFile(setupKeyPath) diff --git a/client/cmd/status.go b/client/cmd/status.go index c4057ed82..f2e5bcc66 100644 --- a/client/cmd/status.go +++ b/client/cmd/status.go @@ -121,8 +121,14 @@ func statusFunc(cmd *cobra.Command, args []string) error { sessionExpiresAt = ts.AsTime().UTC() } + anonymizeEnabled, anonymizeLevel, err := effectiveAnonymize() + if err != nil { + return err + } + var outputInformationHolder = nbstatus.ConvertToStatusOutputOverview(resp.GetFullStatus(), nbstatus.ConvertOptions{ - Anonymize: anonymizeFlag, + Anonymize: anonymizeEnabled, + AnonymizeLevel: anonymizeLevel, DaemonVersion: resp.GetDaemonVersion(), DaemonStatus: nbstatus.ParseDaemonStatus(status), StatusFilter: statusFilter, diff --git a/client/internal/debug/debug.go b/client/internal/debug/debug.go index 0f81844f6..dbe22139a 100644 --- a/client/internal/debug/debug.go +++ b/client/internal/debug/debug.go @@ -34,9 +34,8 @@ import ( "github.com/netbirdio/netbird/shared/netiputil" ) -const readmeContent = `Netbird debug bundle -This debug bundle contains the following files. -If the --anonymize flag is set, the files are anonymized to protect sensitive information. +const readmeContent = `This debug bundle contains the following files. +If anonymization is enabled (--anonymize / --anonymize-level), the files are anonymized to protect sensitive information. status.txt: Anonymized status information of the NetBird client. client.log: Most recent, anonymized client log file of the NetBird client. @@ -70,21 +69,34 @@ capture.pcap: Packet capture in pcap format. Only present when capture was runni Anonymization Process -The files in this bundle have been anonymized to protect sensitive information. Here's how the anonymization was applied: +The files in this bundle have been anonymized to protect sensitive information. The level applied to this bundle is recorded at the top of this file. Here's how the anonymization was applied: IP Addresses -IPv4 addresses are replaced with addresses starting from 198.51.100.0 -IPv6 addresses are replaced with addresses starting from 100:: +Default level: +- Public IPv4 addresses are replaced with addresses starting from 198.51.100.0 +- Public IPv6 addresses are replaced with addresses starting from 2001:db8:ffff:: +- IPv6 unique local addresses (fc00::/7) are anonymized as well: their random global ID uniquely identifies the network. +- IP addresses from internal IPv4 ranges and well-known addresses are not anonymized (e.g. 8.8.8.8, 100.64.0.0/10, addresses starting with 192.168., 172.16., 10., 169.254., fe80::). + +Strict level (--anonymize-level strict), in addition to the default level: +- Private (RFC 1918), CGNAT (100.64.0.0/10), and link-local (169.254.0.0/16, fe80::/10) addresses are anonymized too. +- Internal IPv4 addresses are replaced with addresses starting from 198.18.0.0 and internal IPv6 addresses with addresses starting from 2001:db8:1::, so internal addresses remain distinguishable from public ones. +- Addresses are mapped in order of first appearance: subnet structure, allocation scheme, and gateway conventions are not preserved. Prefix lengths of networks are preserved. +- Peer names in front of NetBird domains are replaced with numbered placeholders (e.g. peer-1.netbird.cloud), and subdomain labels of other domains with host-N placeholders. +- WireGuard public keys are replaced with consistent placeholder keys. -IP addresses from non public ranges and well known addresses are not anonymized (e.g. 8.8.8.8, 100.64.0.0/10, addresses starting with 192.168., 172.16., 10., etc.). Reoccuring IP addresses are replaced with the same anonymized address. Note: The anonymized IP addresses in the status file do not match those in the log and routes files. However, the anonymized IP addresses are consistent within the status file and across the routes and log files. +MAC Addresses +MAC addresses are replaced at every anonymization level with consistent placeholders counting up from 02:00:00:00:00:01. Broadcast, multicast, and all-zero addresses are kept. At the default level a preserved IPv6 link-local address may still embed a MAC address (EUI-64); the strict level anonymizes those addresses. + Domains All domain names (except for the netbird domains) are replaced with randomly generated strings ending in ".domain". Anonymized domains are consistent across all files in the bundle. Reoccuring domain names are replaced with the same anonymized domain. +At the strict level, the peer name labels in front of netbird domains are anonymized as well. Sync Response The network_map.json file contains the following anonymized information: @@ -281,6 +293,7 @@ type BundleGenerator struct { cliVersion string anonymize bool + anonymizeLevel anonymize.Level includeSystemInfo bool logFileCount uint32 @@ -288,7 +301,10 @@ type BundleGenerator struct { } type BundleConfig struct { - Anonymize bool + Anonymize bool + // AnonymizeLevel selects how much the anonymizer redacts. + // anonymize.LevelStrict implies Anonymize. + AnonymizeLevel anonymize.Level IncludeSystemInfo bool LogFileCount uint32 } @@ -327,8 +343,11 @@ func NewBundleGenerator(deps GeneratorDependencies, cfg BundleConfig) *BundleGen uiLogOpener = openLogFile } + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(cfg.AnonymizeLevel) + return &BundleGenerator{ - anonymizer: anonymize.NewAnonymizer(anonymize.DefaultAddresses()), + anonymizer: anonymizer, internalConfig: deps.InternalConfig, statusRecorder: deps.StatusRecorder, @@ -345,7 +364,8 @@ func NewBundleGenerator(deps GeneratorDependencies, cfg BundleConfig) *BundleGen daemonVersion: deps.DaemonVersion, cliVersion: deps.CliVersion, - anonymize: cfg.Anonymize, + anonymize: cfg.Anonymize || cfg.AnonymizeLevel >= anonymize.LevelStrict, + anonymizeLevel: cfg.AnonymizeLevel, includeSystemInfo: cfg.IncludeSystemInfo, logFileCount: logFileCount, } @@ -485,7 +505,13 @@ func (g *BundleGenerator) addSystemInfo() { } func (g *BundleGenerator) addReadme() error { - readmeReader := strings.NewReader(readmeContent) + level := "none (anonymization disabled)" + if g.anonymize { + level = g.anonymizeLevel.String() + } + header := fmt.Sprintf("Netbird debug bundle\nAnonymization level applied to this bundle: %s\n", level) + + readmeReader := strings.NewReader(header + readmeContent) if err := g.addFileToZip(readmeReader, "README.txt"); err != nil { return fmt.Errorf("add README file to zip: %w", err) } @@ -507,9 +533,10 @@ func (g *BundleGenerator) addStatus() error { fullStatus := g.statusRecorder.GetFullStatus() protoFullStatus := nbstatus.ToProtoFullStatus(fullStatus) overview := nbstatus.ConvertToStatusOutputOverview(protoFullStatus, nbstatus.ConvertOptions{ - Anonymize: g.anonymize, - ProfileName: profName, - DaemonVersion: g.daemonVersion, + Anonymize: g.anonymize, + AnonymizeLevel: g.anonymizeLevel, + ProfileName: profName, + DaemonVersion: g.daemonVersion, }) overview.CliVersion = g.cliVersion statusOutput := overview.FullDetailSummary() @@ -662,7 +689,7 @@ func (g *BundleGenerator) addCommonConfigFields(configContent *strings.Builder) configContent.WriteString("NetBird Client Configuration:\n\n") if key, err := wgtypes.ParseKey(g.internalConfig.PrivateKey); err == nil { - configContent.WriteString(fmt.Sprintf("PublicKey: %s\n", key.PublicKey().String())) + configContent.WriteString(fmt.Sprintf("PublicKey: %s\n", g.anonymizer.AnonymizeWGKey(key.PublicKey().String()))) } configContent.WriteString(fmt.Sprintf("WgIface: %s\n", g.internalConfig.WgIface)) configContent.WriteString(fmt.Sprintf("WgPort: %d\n", g.internalConfig.WgPort)) @@ -952,6 +979,11 @@ func (g *BundleGenerator) addUpdateLogs() error { } baseName := filepath.Base(logFile) + data, err = g.anonymizeBytes(data) + if err != nil { + log.Warnf("skipping update log file %s: %v", baseName, err) + continue + } if err := g.addFileToZip(bytes.NewReader(data), filepath.Join("update-logs", baseName)); err != nil { return fmt.Errorf("add update log file %s to zip: %w", baseName, err) } @@ -979,6 +1011,13 @@ func (g *BundleGenerator) addCorruptedStateFiles() error { } fileName := filepath.Base(match) + // Corrupted state files usually fail structured JSON anonymization, + // so run them through the string anonymizer instead. + data, err = g.anonymizeBytes(data) + if err != nil { + log.Warnf("skipping corrupted state file %s: %v", fileName, err) + continue + } if err := g.addFileToZip(bytes.NewReader(data), "corrupted_states/"+fileName); err != nil { log.Warnf("Failed to add corrupted state file %s to zip: %v", fileName, err) continue @@ -990,6 +1029,27 @@ func (g *BundleGenerator) addCorruptedStateFiles() error { return nil } +// anonymizeBytes runs raw file content through the string anonymizer line by +// line when anonymization is enabled. It errors instead of returning partial +// content, so a caller never adds an unanonymized fallback to the bundle. +func (g *BundleGenerator) anonymizeBytes(data []byte) ([]byte, error) { + if !g.anonymize { + return data, nil + } + + var buf bytes.Buffer + scanner := bufio.NewScanner(bytes.NewReader(data)) + scanner.Buffer(make([]byte, 1024*1024), 1024*1024) + for scanner.Scan() { + buf.WriteString(g.anonymizer.AnonymizeString(scanner.Text())) + buf.WriteByte('\n') + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("anonymize content: %w", err) + } + return buf.Bytes(), nil +} + func (g *BundleGenerator) addMetrics() error { if g.clientMetrics == nil { log.Debugf("skipping metrics in debug bundle: no metrics collector") @@ -1462,6 +1522,7 @@ func anonymizeRemotePeer(peer *mgmProto.RemotePeerConfig, anonymizer *anonymize. } peer.Fqdn = anonymizer.AnonymizeDomain(peer.Fqdn) + peer.WgPubKey = anonymizer.AnonymizeWGKey(peer.WgPubKey) anonymizeSSHConfig(peer.SshConfig) } diff --git a/client/internal/debug/wgshow.go b/client/internal/debug/wgshow.go index 1e8a8a6cc..ee24902e6 100644 --- a/client/internal/debug/wgshow.go +++ b/client/internal/debug/wgshow.go @@ -35,14 +35,14 @@ func (g *BundleGenerator) toWGShowFormat(s *configurer.Stats) string { var sb strings.Builder sb.WriteString(fmt.Sprintf("interface: %s\n", s.DeviceName)) - sb.WriteString(fmt.Sprintf(" public key: %s\n", s.PublicKey)) + sb.WriteString(fmt.Sprintf(" public key: %s\n", g.anonymizer.AnonymizeWGKey(s.PublicKey))) sb.WriteString(fmt.Sprintf(" listen port: %d\n", s.ListenPort)) if s.FWMark != 0 { sb.WriteString(fmt.Sprintf(" fwmark: %#x\n", s.FWMark)) } for _, peer := range s.Peers { - sb.WriteString(fmt.Sprintf("\npeer: %s\n", peer.PublicKey)) + sb.WriteString(fmt.Sprintf("\npeer: %s\n", g.anonymizer.AnonymizeWGKey(peer.PublicKey))) if peer.Endpoint.IP != nil { if g.anonymize { anonEndpoint := g.anonymizer.AnonymizeUDPAddr(peer.Endpoint) @@ -54,7 +54,11 @@ func (g *BundleGenerator) toWGShowFormat(s *configurer.Stats) string { if len(peer.AllowedIPs) > 0 { var ipStrings []string for _, ipnet := range peer.AllowedIPs { - ipStrings = append(ipStrings, ipnet.String()) + ipStr := ipnet.String() + if g.anonymize { + ipStr = g.anonymizer.AnonymizeIPString(ipStr) + } + ipStrings = append(ipStrings, ipStr) } sb.WriteString(fmt.Sprintf(" allowed ips: %s\n", strings.Join(ipStrings, ", "))) } diff --git a/client/internal/engine.go b/client/internal/engine.go index f4f47992f..d92c360f2 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -23,6 +23,7 @@ import ( "golang.zx2c4.com/wireguard/tun/netstack" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "github.com/netbirdio/netbird/client/anonymize" nberrors "github.com/netbirdio/netbird/client/errors" "github.com/netbirdio/netbird/client/firewall" "github.com/netbirdio/netbird/client/firewall/firewalld" @@ -1385,6 +1386,7 @@ func (e *Engine) handleBundle(params *mgmProto.BundleParameters) (*mgmProto.JobR bundleJobParams := debug.BundleConfig{ Anonymize: params.Anonymize, + AnonymizeLevel: anonymize.ParseLevel(params.AnonymizeLevel), IncludeSystemInfo: true, LogFileCount: uint32(params.LogFileCount), } diff --git a/client/ios/NetBirdSDK/client.go b/client/ios/NetBirdSDK/client.go index 2d5460d03..5df6f92f1 100644 --- a/client/ios/NetBirdSDK/client.go +++ b/client/ios/NetBirdSDK/client.go @@ -14,6 +14,7 @@ import ( log "github.com/sirupsen/logrus" + nbAnonymize "github.com/netbirdio/netbird/client/anonymize" "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/auth" "github.com/netbirdio/netbird/client/internal/debug" @@ -28,6 +29,13 @@ import ( types "github.com/netbirdio/netbird/upload-server/types" ) +// AnonymizeLevelDefault and AnonymizeLevelStrict are the accepted +// anonymizeLevel values for DebugBundle. +const ( + AnonymizeLevelDefault = nbAnonymize.LevelDefaultString + AnonymizeLevelStrict = nbAnonymize.LevelStrictString +) + // ConnectionListener export internal Listener for mobile type ConnectionListener interface { peer.Listener @@ -200,8 +208,10 @@ func (c *Client) Stop() { // DebugBundle generates a debug bundle, uploads it and returns the upload key. // It works with or without a running engine: when the engine is up it reuses // the live config, sync response and client metrics; otherwise it loads the -// config from disk (or the preloaded tvOS config). -func (c *Client) DebugBundle(anonymize bool) (string, error) { +// config from disk (or the preloaded tvOS config). anonymizeLevel is "default" +// or "strict"; strict also anonymizes internal IP ranges, peer names, and +// WireGuard public keys, and implies anonymize. +func (c *Client) DebugBundle(anonymize bool, anonymizeLevel string) (string, error) { cfg, cc := c.stateSnapshot() // If the engine hasn't been started, load config so we can reach management. @@ -251,6 +261,7 @@ func (c *Client) DebugBundle(anonymize bool) (string, error) { deps, debug.BundleConfig{ Anonymize: anonymize, + AnonymizeLevel: nbAnonymize.ParseLevel(anonymizeLevel), IncludeSystemInfo: true, }, ) diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index d4deeb8ec..83243be49 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -2781,6 +2781,11 @@ type DebugBundleRequest struct { // untrusted TLS certificate. Restricted to privileged callers; for // self-hosted upload servers. UploadInsecure bool `protobuf:"varint,7,opt,name=uploadInsecure,proto3" json:"uploadInsecure,omitempty"` + // anonymizeLevel selects how much the anonymizer redacts: "default" + // (or empty) keeps internal IP ranges, "strict" also anonymizes them. + // Unknown values are treated as "strict". Only meaningful with anonymize; + // "strict" implies it. + AnonymizeLevel string `protobuf:"bytes,8,opt,name=anonymizeLevel,proto3" json:"anonymizeLevel,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2857,6 +2862,13 @@ func (x *DebugBundleRequest) GetUploadInsecure() bool { return false } +func (x *DebugBundleRequest) GetAnonymizeLevel() string { + if x != nil { + return x.AnonymizeLevel + } + return "" +} + type DebugBundleResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` @@ -7253,7 +7265,7 @@ const file_daemon_proto_rawDesc = "" + "\x12translatedHostname\x18\x04 \x01(\tR\x12translatedHostname\x128\n" + "\x0etranslatedPort\x18\x05 \x01(\v2\x10.daemon.PortInfoR\x0etranslatedPort\"G\n" + "\x17ForwardingRulesResponse\x12,\n" + - "\x05rules\x18\x01 \x03(\v2\x16.daemon.ForwardingRuleR\x05rules\"\xdc\x01\n" + + "\x05rules\x18\x01 \x03(\v2\x16.daemon.ForwardingRuleR\x05rules\"\x84\x02\n" + "\x12DebugBundleRequest\x12\x1c\n" + "\tanonymize\x18\x01 \x01(\bR\tanonymize\x12\x1e\n" + "\n" + @@ -7264,7 +7276,8 @@ const file_daemon_proto_rawDesc = "" + "\n" + "cliVersion\x18\x06 \x01(\tR\n" + "cliVersion\x12&\n" + - "\x0euploadInsecure\x18\a \x01(\bR\x0euploadInsecure\"}\n" + + "\x0euploadInsecure\x18\a \x01(\bR\x0euploadInsecure\x12&\n" + + "\x0eanonymizeLevel\x18\b \x01(\tR\x0eanonymizeLevel\"}\n" + "\x13DebugBundleResponse\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12 \n" + "\vuploadedKey\x18\x02 \x01(\tR\vuploadedKey\x120\n" + diff --git a/client/proto/daemon.proto b/client/proto/daemon.proto index 3c31156ec..18ce0e79c 100644 --- a/client/proto/daemon.proto +++ b/client/proto/daemon.proto @@ -540,6 +540,11 @@ message DebugBundleRequest { // untrusted TLS certificate. Restricted to privileged callers; for // self-hosted upload servers. bool uploadInsecure = 7; + // anonymizeLevel selects how much the anonymizer redacts: "default" + // (or empty) keeps internal IP ranges, "strict" also anonymizes them. + // Unknown values are treated as "strict". Only meaningful with anonymize; + // "strict" implies it. + string anonymizeLevel = 8; } message DebugBundleResponse { diff --git a/client/server/debug.go b/client/server/debug.go index 60a401b0e..8f4a506b4 100644 --- a/client/server/debug.go +++ b/client/server/debug.go @@ -16,6 +16,7 @@ import ( "google.golang.org/grpc/codes" gstatus "google.golang.org/grpc/status" + "github.com/netbirdio/netbird/client/anonymize" "github.com/netbirdio/netbird/client/internal/debug" "github.com/netbirdio/netbird/client/internal/ipcauth" "github.com/netbirdio/netbird/client/proto" @@ -122,6 +123,7 @@ func (s *Server) generateDebugBundle(req *proto.DebugBundleRequest, uiOpener deb }, debug.BundleConfig{ Anonymize: req.GetAnonymize(), + AnonymizeLevel: anonymize.ParseLevel(req.GetAnonymizeLevel()), IncludeSystemInfo: req.GetSystemInfo(), LogFileCount: req.GetLogFileCount(), }, diff --git a/client/status/status.go b/client/status/status.go index e8276d0fa..1c204cdb1 100644 --- a/client/status/status.go +++ b/client/status/status.go @@ -46,7 +46,10 @@ func ParseDaemonStatus(s string) DaemonStatus { // ConvertOptions holds parameters for ConvertToStatusOutputOverview. type ConvertOptions struct { - Anonymize bool + Anonymize bool + // AnonymizeLevel selects how much the anonymizer redacts. Only + // meaningful when Anonymize is set. + AnonymizeLevel anonymize.Level DaemonVersion string DaemonStatus DaemonStatus StatusFilter string @@ -217,6 +220,7 @@ func ConvertToStatusOutputOverview(pbFullStatus *proto.FullStatus, opts ConvertO if opts.Anonymize { anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(opts.AnonymizeLevel) anonymizeOverview(anonymizer, &overview) } @@ -976,6 +980,7 @@ func timeAgo(t time.Time) string { func anonymizePeerDetail(a *anonymize.Anonymizer, peer *PeerStateDetailOutput) { peer.FQDN = a.AnonymizeDomain(peer.FQDN) + peer.PubKey = a.AnonymizeWGKey(peer.PubKey) if localIP, port, err := net.SplitHostPort(peer.IceCandidateEndpoint.Local); err == nil { peer.IceCandidateEndpoint.Local = fmt.Sprintf("%s:%s", a.AnonymizeIPString(localIP), port) } @@ -1007,6 +1012,7 @@ func anonymizeOverview(a *anonymize.Anonymizer, overview *OutputOverview) { overview.SignalState.URL = a.AnonymizeURI(overview.SignalState.URL) overview.SignalState.Error = a.AnonymizeString(overview.SignalState.Error) + overview.PubKey = a.AnonymizeWGKey(overview.PubKey) overview.IP = a.AnonymizeIPString(overview.IP) overview.IPv6 = a.AnonymizeIPString(overview.IPv6) for i, detail := range overview.Relays.Details { diff --git a/client/ui/frontend/src/contexts/DebugBundleContext.tsx b/client/ui/frontend/src/contexts/DebugBundleContext.tsx index a0a131fbf..5f2ed9041 100644 --- a/client/ui/frontend/src/contexts/DebugBundleContext.tsx +++ b/client/ui/frontend/src/contexts/DebugBundleContext.tsx @@ -71,10 +71,12 @@ type BundleOptions = { hasWindow: boolean; totalSec: number; uploadUrl: string; - anonymize: boolean; + anonymizeLevel: AnonymizeLevel; systemInfo: boolean; }; +export type AnonymizeLevel = "none" | "default" | "strict"; + const startCaptureBestEffort = async (totalSec: number, pcap: CaptureState) => { try { // Mirror the CLI's safety margin: window + 30s, server caps at 10m. @@ -187,7 +189,10 @@ const runBundleFlow = async ( if (opts.uploadUrl) setStage({ kind: "uploading" }); const result = await DebugSvc.Bundle({ - anonymize: opts.anonymize, + anonymize: opts.anonymizeLevel !== "none", + // The daemon only knows "default" and "strict"; "none" is expressed + // through the anonymize flag being off. + anonymizeLevel: opts.anonymizeLevel === "strict" ? "strict" : "default", systemInfo: opts.systemInfo, uploadUrl: opts.uploadUrl, logFileCount, @@ -198,7 +203,7 @@ const runBundleFlow = async ( }; const useDebugBundle = () => { - const [anonymize, setAnonymize] = useState(false); + const [anonymizeLevel, setAnonymizeLevel] = useState("none"); const [systemInfo, setSystemInfo] = useState(true); const [upload, setUpload] = useState(true); const [trace, setTrace] = useState(true); @@ -240,7 +245,7 @@ const useDebugBundle = () => { hasWindow: capture && totalSec > 0, totalSec, uploadUrl: upload ? NETBIRD_UPLOAD_URL : "", - anonymize, + anonymizeLevel, systemInfo, }; @@ -272,8 +277,8 @@ const useDebugBundle = () => { }; return { - anonymize, - setAnonymize, + anonymizeLevel, + setAnonymizeLevel, systemInfo, setSystemInfo, upload, diff --git a/client/ui/frontend/src/modules/settings/SettingsTroubleshooting.tsx b/client/ui/frontend/src/modules/settings/SettingsTroubleshooting.tsx index 991937719..8b1774ed6 100644 --- a/client/ui/frontend/src/modules/settings/SettingsTroubleshooting.tsx +++ b/client/ui/frontend/src/modules/settings/SettingsTroubleshooting.tsx @@ -1,6 +1,6 @@ import { useId, type ReactNode } from "react"; import { Trans, useTranslation } from "react-i18next"; -import { CircleCheckBig, FolderOpen, Loader2 } from "lucide-react"; +import { ChevronDown, CircleCheckBig, FolderOpen, Info, Loader2 } from "lucide-react"; import { Browser } from "@wailsio/runtime"; import { Debug as DebugSvc } from "@bindings/services"; import type { DebugBundleResult } from "@bindings/services/models.js"; @@ -8,13 +8,22 @@ import { Button } from "@/components/buttons/Button"; import { DialogActions } from "@/components/dialog/DialogActions"; import { DialogDescription } from "@/components/dialog/DialogDescription"; import { DialogHeading } from "@/components/dialog/DialogHeading"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuRadioGroup, + DropdownMenuRadioItem, + DropdownMenuTrigger, +} from "@/components/DropdownMenu"; import FancyToggleSwitch from "@/components/switches/FancyToggleSwitch"; import HelpText from "@/components/typography/HelpText.tsx"; import { Input } from "@/components/inputs/Input"; import { Label } from "@/components/typography/Label"; import { SquareIcon } from "@/components/SquareIcon"; +import { Tooltip } from "@/components/Tooltip"; +import { cn } from "@/lib/cn"; import { formatRemaining } from "@/lib/formatters"; -import type { DebugStage } from "@/contexts/DebugBundleContext"; +import type { AnonymizeLevel, DebugStage } from "@/contexts/DebugBundleContext"; import { useDebugBundleContext } from "@/contexts/DebugBundleContext"; import { SectionGroup, SettingsBottomBar } from "@/modules/settings/SettingsSection.tsx"; @@ -24,8 +33,8 @@ export function SettingsTroubleshooting() { const { t } = useTranslation(); const durationId = useId(); const { - anonymize, - setAnonymize, + anonymizeLevel, + setAnonymizeLevel, systemInfo, setSystemInfo, upload, @@ -55,12 +64,71 @@ export function SettingsTroubleshooting() { return ( - +
+
+
+ } + > + + + + + + {t("settings.troubleshooting.anonymize.help")} + +
+
+ + + + + + setAnonymizeLevel(v as AnonymizeLevel)} + > + + {t("settings.troubleshooting.anonymize.none")} + + + {t("settings.troubleshooting.anonymize.default")} + + + {t("settings.troubleshooting.anonymize.strict")} + + + + +
+ Date: Mon, 10 Aug 2026 09:28:15 +0000 Subject: [PATCH 09/22] [client] Declare the xdg-utils dependency for the netbird-ui packages (#7126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Describe your changes The UI shells out to xdg-open to launch the external browser for the SSO verification page, which the embedded webview cannot open inline, and to reveal the debug bundle in the file manager. client/ui/build/linux/nfpm/nfpm.yaml lists xdg-utils for every package format, but the released packages are built from the goreleaser configs, where it was missing: the GTK and WebKitGTK dependencies carried over and xdg-utils did not. Add it to all four nfpm dependency lists. ## Issue ticket number and link ## Stack ### Checklist - [x] Is it a bug fix - [ ] Is a typo/documentation fix - [ ] Is a feature enhancement - [ ] It is a refactor - [ ] Created tests that fail without the change (if possible) - [ ] I ran and tested this change locally — I did not rely on CI to find out whether it works - [ ] This PR has a single purpose (not a fix + refactor + feature in one) - [ ] This change is a trivial fix, **OR** it links an issue the NetBird team agreed on beforehand. Changes to the public API, gRPC protocols, functionality behavior, CLI / service flags, or new features always need that agreement first. See [CONTRIBUTING.md](https://github.com/netbirdio/netbird/blob/main/CONTRIBUTING.md#ticket-first-pr-second). > By submitting this pull request, you confirm that you have read and agree to the terms of the [Contributor License Agreement](https://github.com/netbirdio/netbird/blob/main/CONTRIBUTOR_LICENSE_AGREEMENT.md). ## Documentation Select exactly one: - [ ] I added/updated documentation for this change - [x] Documentation is **not needed** for this change (explain why) ### Docs PR URL (required if "docs added" is checked) Paste the PR link from https://github.com/netbirdio/docs here: https://github.com/netbirdio/docs/pull/__ ## Summary by CodeRabbit * **Bug Fixes** * Added required desktop integration support to Debian and RPM packages. * Ensured GTK3 packages include the same runtime support for opening links and files through the system. --- .goreleaser_ui.yaml | 2 ++ .goreleaser_ui_gtk3.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.goreleaser_ui.yaml b/.goreleaser_ui.yaml index ca5148823..c61b8c474 100644 --- a/.goreleaser_ui.yaml +++ b/.goreleaser_ui.yaml @@ -96,6 +96,7 @@ nfpms: - netbird (>= 0.75.0) - libgtk-4-1 (>= 4.14) - libwebkitgtk-6.0-4 + - xdg-utils - maintainer: Netbird description: Netbird client UI. @@ -119,6 +120,7 @@ nfpms: - netbird >= 0.75.0 - (gtk4 >= 4.14 or libgtk-4-1 >= 4.14) - (webkitgtk6.0 or libwebkitgtk-6_0-4) + - xdg-utils rpm: signature: diff --git a/.goreleaser_ui_gtk3.yaml b/.goreleaser_ui_gtk3.yaml index d0a7de6e2..732fd578d 100644 --- a/.goreleaser_ui_gtk3.yaml +++ b/.goreleaser_ui_gtk3.yaml @@ -71,6 +71,7 @@ nfpms: - netbird (>= 0.75.0) - libgtk-3-0 - libwebkit2gtk-4.1-0 + - xdg-utils - maintainer: Netbird description: Netbird client UI. @@ -95,6 +96,7 @@ nfpms: - netbird >= 0.75.0 - (gtk3 or libgtk-3-0) - (webkit2gtk4.1 or libwebkit2gtk-4_1-0) + - xdg-utils rpm: signature: From 664a3d026cb3b56bef75980b14ab9e6d3ab279e0 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 10 Aug 2026 10:58:50 +0000 Subject: [PATCH 10/22] [client] Fix credentials for the gtk3 package uploads (#7125) --- .goreleaser_ui_gtk3.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.goreleaser_ui_gtk3.yaml b/.goreleaser_ui_gtk3.yaml index 732fd578d..5770ef348 100644 --- a/.goreleaser_ui_gtk3.yaml +++ b/.goreleaser_ui_gtk3.yaml @@ -114,6 +114,13 @@ uploads: # The gtk3 packages reuse the netbird-ui package name, so they live in # dedicated repo paths (deb distribution `gtk3`, yum path `yum-gtk3`) that # legacy distros point their repo config at. + # + # GoReleaser derives the credential env var from the upload name, so these + # would look for UPLOAD_DEBIAN-GTK3_SECRET / UPLOAD_YUM-GTK3_SECRET. The + # release workflow only exports UPLOAD_DEBIAN_SECRET / UPLOAD_YUM_SECRET, and + # a missing secret is a silent skip rather than a failure -- the packages + # reached the GitHub release but never the package repositories. Point + # `password` at the exported vars so both uploads authenticate. - name: debian-gtk3 skip: "{{ .Env.SKIP_PUBLISH }}" ids: @@ -121,6 +128,7 @@ uploads: mode: archive target: https://pkgs.wiretrustee.com/debian/pool/{{ .ArtifactName }};deb.distribution=gtk3;deb.component=main;deb.architecture={{ if .Arm }}armhf{{ else }}{{ .Arch }}{{ end }};deb.package= username: dev@wiretrustee.com + password: "{{ .Env.UPLOAD_DEBIAN_SECRET }}" method: PUT - name: yum-gtk3 @@ -130,4 +138,5 @@ uploads: mode: archive target: https://pkgs.wiretrustee.com/yum-gtk3/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }} username: dev@wiretrustee.com + password: "{{ .Env.UPLOAD_YUM_SECRET }}" method: PUT From f9abe2727fe3563e581a29ac9b59de9206004766 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:41:09 +0900 Subject: [PATCH 11/22] [client] Do not misroute WireGuard packets to the STUN handler (#7059) --- client/iface/bind/ice_bind.go | 70 +++++++-- client/iface/bind/stun_filter_test.go | 215 ++++++++++++++++++++++++++ 2 files changed, 269 insertions(+), 16 deletions(-) create mode 100644 client/iface/bind/stun_filter_test.go diff --git a/client/iface/bind/ice_bind.go b/client/iface/bind/ice_bind.go index 156450c61..2d35b9c6f 100644 --- a/client/iface/bind/ice_bind.go +++ b/client/iface/bind/ice_bind.go @@ -22,6 +22,16 @@ import ( nbnet "github.com/netbirdio/netbird/client/net" ) +const ( + // wgMsgTypeHandshakeInitiation is the lowest WireGuard message type. + wgMsgTypeHandshakeInitiation uint32 = 1 + // wgMsgTypeTransport is the highest WireGuard message type. + wgMsgTypeTransport uint32 = 4 + // wgMinMsgSize is the smallest WireGuard message: transport data with an empty + // payload, which is what a keepalive is. + wgMinMsgSize = 32 +) + type receiverCreator struct { iceBind *ICEBind } @@ -216,8 +226,15 @@ func (s *ICEBind) createReceiverFn(pc wgConn.BatchReader, conn *net.UDPConn, rxO for i := 0; i < numMsgs; i++ { msg := &(*msgs)[i] - // todo: handle err - if ok, _ := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr); ok { + if ok, err := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr); ok { + if err != nil { + log.Debugf("failed to handle STUN packet from %s: %v", msg.Addr, err) + } + // WireGuard reuses sizes and eps across reads and only skips a slot + // whose size is below the minimum message size. Leaving a consumed + // slot untouched makes it process this buffer again under the + // previous packet's length and endpoint. + sizes[i] = 0 continue } sizes[i] = msg.N @@ -271,11 +288,16 @@ func (s *ICEBind) createOrUpdateMux() { func (s *ICEBind) filterOutStunMessages(buffers [][]byte, n int, addr net.Addr) (bool, error) { for i := range buffers { - if !stun.IsMessage(buffers[i]) { + if n > len(buffers[i]) { + continue + } + pkt := buffers[i][:n] + + if isWireGuardMsg(pkt) || !stun.IsMessage(pkt) { continue } - msg, err := s.parseSTUNMessage(buffers[i][:n]) + msg, err := s.parseSTUNMessage(pkt) if err != nil { buffers[i] = []byte{} return true, err @@ -347,18 +369,34 @@ func putMessages(msgs *[]ipv6.Message, msgsPool *sync.Pool) { msgsPool.Put(msgs) } -func isTransportPkg(buffers [][]byte, n int) bool { - // The first buffer should contain at least 4 bytes for type - if len(buffers[0]) < 4 { - return true +// isWireGuardMsg reports whether the packet carries a WireGuard message header: a +// little-endian uint32 message type in the range 1..4, which leaves the three bytes +// after the type byte zero, in a packet long enough to hold any WireGuard message. +// +// A well formed STUN message cannot take that shape. Its length field sits in the two +// bytes the type must leave zero, and for a message of at least wgMinMsgSize bytes that +// field holds at least 12, so the two framings do not overlap. The test has to be this +// tight because stun.IsMessage only looks at the magic cookie, which in a WireGuard +// message overlaps the receiver index: a session whose index happens to equal the cookie +// would otherwise have all of its inbound data misrouted to the STUN handler until the +// next rekey. +func isWireGuardMsg(pkt []byte) bool { + if len(pkt) < wgMinMsgSize { + return false } - // WireGuard packet type is a little-endian uint32 at start - packetType := binary.LittleEndian.Uint32(buffers[0][:4]) - - // Check if packetType matches known WireGuard message types - if packetType == 4 && n > 32 { - return true - } - return false + msgType := binary.LittleEndian.Uint32(pkt[:4]) + return msgType >= wgMsgTypeHandshakeInitiation && msgType <= wgMsgTypeTransport +} + +// isTransportPkg reports whether the packet is WireGuard transport data carrying a +// payload, which is what counts as peer activity. A keepalive holds no payload and is +// exactly wgMinMsgSize bytes. +func isTransportPkg(buffers [][]byte, n int) bool { + if n < 4 || n > len(buffers[0]) { + return false + } + + msgType := binary.LittleEndian.Uint32(buffers[0][:4]) + return msgType == wgMsgTypeTransport && n > wgMinMsgSize } diff --git a/client/iface/bind/stun_filter_test.go b/client/iface/bind/stun_filter_test.go new file mode 100644 index 000000000..0e118e0fd --- /dev/null +++ b/client/iface/bind/stun_filter_test.go @@ -0,0 +1,215 @@ +//go:build !js + +package bind + +import ( + "encoding/binary" + "net" + "testing" + "time" + + "github.com/pion/stun/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/ipv4" + wgConn "golang.zx2c4.com/wireguard/conn" +) + +// magicCookieBytes is the STUN magic cookie as it appears on the wire. In a +// WireGuard message the same offset holds the receiver (or sender) index, which is +// a random uint32, so a session can draw exactly this value. +var magicCookieBytes = []byte{0x21, 0x12, 0xA4, 0x42} + +const testBufSize = 1500 + +// wgMsg builds a WireGuard message of the given type and size, with the index field +// at bytes 4:8 set to index. +func wgMsg(msgType uint32, size int, index []byte) []byte { + pkt := make([]byte, size) + binary.LittleEndian.PutUint32(pkt[:4], msgType) + copy(pkt[4:8], index) + return pkt +} + +// intoBuffer copies pkt into a full-size receive buffer, the way the kernel read +// does, so tests see the same buffer/length split as the hot path. +func intoBuffer(pkt []byte) [][]byte { + buf := make([]byte, testBufSize) + copy(buf, pkt) + return [][]byte{buf} +} + +func TestFilterOutStunMessages_PassesWireGuardWithCookieShapedIndex(t *testing.T) { + tests := []struct { + name string + msgType uint32 + size int + }{ + {"transport data", wgMsgTypeTransport, 128}, + {"keepalive", wgMsgTypeTransport, wgMinMsgSize}, + {"handshake initiation", wgMsgTypeHandshakeInitiation, 148}, + {"handshake response", 2, 92}, + {"cookie reply", 3, 64}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pkt := wgMsg(tc.msgType, tc.size, magicCookieBytes) + require.True(t, stun.IsMessage(pkt), "precondition: pion sees this as STUN") + + buffers := intoBuffer(pkt) + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, tc.size, &net.UDPAddr{}) + assert.NoError(t, err) + assert.False(t, filtered, "WireGuard message must be handed to WireGuard, not the STUN handler") + assert.Len(t, buffers[0], testBufSize, "buffer must be left intact for WireGuard") + }) + } +} + +func TestFilterOutStunMessages_FiltersRealSTUNMessage(t *testing.T) { + msg, err := stun.Build(stun.BindingRequest, stun.TransactionID, stun.Fingerprint) + require.NoError(t, err) + + buffers := intoBuffer(msg.Raw) + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, len(msg.Raw), &net.UDPAddr{}) + assert.NoError(t, err) + assert.True(t, filtered, "STUN message must be consumed by the STUN handler") + assert.Empty(t, buffers[0], "consumed buffer must be emptied so WireGuard does not see it") +} + +// TestIsWireGuardMsg_DisjointFromSTUN locks the invariant the filter relies on: a +// well formed STUN message long enough to be a WireGuard message always has a +// non-zero length field, so it cannot be mistaken for a WireGuard header. +func TestIsWireGuardMsg_DisjointFromSTUN(t *testing.T) { + types := []stun.MessageType{ + stun.BindingRequest, + stun.BindingSuccess, + stun.BindingError, + {Method: stun.MethodBinding, Class: stun.ClassIndication}, + } + + for _, msgType := range types { + // Long enough that the length guard is not what makes this pass. + msg, err := stun.Build(msgType, stun.TransactionID, + stun.NewUsername("remoteUfrag:localUfrag"), stun.Fingerprint) + require.NoError(t, err) + require.GreaterOrEqual(t, len(msg.Raw), wgMinMsgSize, "precondition: %s", msgType) + assert.False(t, isWireGuardMsg(msg.Raw), + "%s must not look like a WireGuard message", msgType) + } +} + +func TestIsWireGuardMsg(t *testing.T) { + tests := []struct { + name string + pkt []byte + want bool + }{ + {"transport data", wgMsg(wgMsgTypeTransport, 128, nil), true}, + {"handshake initiation", wgMsg(wgMsgTypeHandshakeInitiation, 148, nil), true}, + {"unknown type 5", wgMsg(5, 128, nil), false}, + {"type 0", wgMsg(0, 128, nil), false}, + {"non-zero reserved byte", []byte{0x04, 0x00, 0x01, 0x00}, false}, + {"too short", []byte{0x04, 0x00, 0x00}, false}, + {"empty", nil, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isWireGuardMsg(tc.pkt), "wrong classification for %s", tc.name) + }) + } +} + +// TestFilterOutStunMessages_IgnoresBytesBeyondPacket guards against classifying on +// buffer contents left over from an earlier, longer packet. +func TestFilterOutStunMessages_IgnoresBytesBeyondPacket(t *testing.T) { + buf := make([]byte, testBufSize) + copy(buf[4:8], magicCookieBytes) + buffers := [][]byte{buf} + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, 2, &net.UDPAddr{}) + assert.NoError(t, err) + assert.False(t, filtered, "a 2 byte packet must not be classified from stale buffer bytes") +} + +// TestReceiveFn_ClearsSizeOfConsumedPacket covers the accounting WireGuard relies +// on: sizes is reused across reads, so a slot whose packet was consumed as STUN must +// be reported as empty. Otherwise WireGuard reprocesses the same buffer under the +// previous packet's length, which for a WireGuard-shaped packet means it is handled +// twice. +func TestReceiveFn_ClearsSizeOfConsumedPacket(t *testing.T) { + conn := listenUDP(t, "udp4", "127.0.0.1:0") + defer conn.Close() + + recvFn := receiverCreator{setupICEBind(t)}.CreateReceiverFn( + ipv4.NewPacketConn(conn), conn, false, createMsgPool(), + ) + + msg, err := stun.Build(stun.BindingRequest, stun.TransactionID, stun.Fingerprint) + require.NoError(t, err) + + sender := listenUDP(t, "udp4", "127.0.0.1:0") + defer sender.Close() + _, err = sender.WriteTo(msg.Raw, conn.LocalAddr()) + require.NoError(t, err) + + require.NoError(t, conn.SetReadDeadline(time.Now().Add(3*time.Second))) + + bufs := [][]byte{make([]byte, 1500)} + // A leftover size from an earlier read, which is what makes the missing reset + // observable. + sizes := []int{148} + eps := make([]wgConn.Endpoint, 1) + + n, err := recvFn(bufs, sizes, eps) + require.NoError(t, err) + require.Equal(t, 1, n) + assert.Zero(t, sizes[0], "consumed STUN packet must not leave a size behind for WireGuard") +} + +func TestIsTransportPkg(t *testing.T) { + tests := []struct { + name string + pkt []byte + n int + want bool + }{ + {"transport data with payload", wgMsg(wgMsgTypeTransport, 128, nil), 128, true}, + {"keepalive", wgMsg(wgMsgTypeTransport, wgMinMsgSize, nil), wgMinMsgSize, false}, + {"handshake initiation", wgMsg(wgMsgTypeHandshakeInitiation, 148, nil), 148, false}, + {"stale type bytes beyond packet", wgMsg(wgMsgTypeTransport, 128, nil), 2, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isTransportPkg(intoBuffer(tc.pkt), tc.n), + "wrong activity classification for %s", tc.name) + }) + } +} + +// TestFilterOutStunMessages_ConsumesSTUNWithWireGuardShapedType covers the one STUN +// encoding whose leading bytes collide with a WireGuard message type: method 0x080 as a +// request encodes to 0x0200, so the type byte reads as a handshake response and the byte +// after it is zero. Only the length check keeps such a message out of WireGuard's hands. +// pion implements no method in that range, so this is a synthetic worst case rather than +// traffic ICE produces. +func TestFilterOutStunMessages_ConsumesSTUNWithWireGuardShapedType(t *testing.T) { + msg, err := stun.Build(stun.NewType(stun.Method(0x080), stun.ClassRequest), stun.TransactionID) + require.NoError(t, err) + require.Equal(t, []byte{0x02, 0x00, 0x00, 0x00}, msg.Raw[:4], + "precondition: the leading bytes read as a WireGuard message type") + + buffers := intoBuffer(msg.Raw) + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, len(msg.Raw), &net.UDPAddr{}) + assert.NoError(t, err) + assert.True(t, filtered, "STUN message must be consumed despite its WireGuard-shaped type") +} From 1ca26d8faa64cfb58aced2b8b150cd85a86d69dd Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 10 Aug 2026 14:41:50 +0000 Subject: [PATCH 12/22] [client] Bump the netbirdio/wails fork to drop the native WebView2 (#7128) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Describe your changes The native_webview2loader build tag embedded Microsoft's WebView2Loader.dll via //go:embed. Those DLLs are gitignored in the fork and go mod vendor resolves embed patterns regardless of build constraints, so vendoring the module failed on the missing files when packaging for openSUSE. The fork now removes that branch along with the go-winloader dependency, which drops out of the module graph here. The default GoWebView2Loader path is unaffected. ## Issue ticket number and link ## Stack ### Checklist - [x] Is it a bug fix - [ ] Is a typo/documentation fix - [ ] Is a feature enhancement - [ ] It is a refactor - [ ] Created tests that fail without the change (if possible) - [ ] I ran and tested this change locally — I did not rely on CI to find out whether it works - [ ] This PR has a single purpose (not a fix + refactor + feature in one) - [ ] This change is a trivial fix, **OR** it links an issue the NetBird team agreed on beforehand. Changes to the public API, gRPC protocols, functionality behavior, CLI / service flags, or new features always need that agreement first. See [CONTRIBUTING.md](https://github.com/netbirdio/netbird/blob/main/CONTRIBUTING.md#ticket-first-pr-second). > By submitting this pull request, you confirm that you have read and agree to the terms of the [Contributor License Agreement](https://github.com/netbirdio/netbird/blob/main/CONTRIBUTOR_LICENSE_AGREEMENT.md). ## Documentation Select exactly one: - [ ] I added/updated documentation for this change - [x] Documentation is **not needed** for this change (explain why) ### Docs PR URL (required if "docs added" is checked) Paste the PR link from https://github.com/netbirdio/docs here: https://github.com/netbirdio/docs/pull/__ ## Summary by CodeRabbit * **Chores** * Updated the application framework revision. * Removed an unused dependency. --- go.mod | 3 +-- go.sum | 7 ++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index f8a1a84b0..36f0a87b0 100644 --- a/go.mod +++ b/go.mod @@ -239,7 +239,6 @@ require ( github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect - github.com/jchv/go-winloader v0.0.0-20250406163304-c1995be93bd1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -340,4 +339,4 @@ replace github.com/dexidp/dex/api/v2 => github.com/netbirdio/dex/api/v2 v2.0.0-2 replace github.com/mailru/easyjson => github.com/netbirdio/easyjson v0.9.0 -replace github.com/wailsapp/wails/v3 => github.com/netbirdio/wails/v3 v3.0.0-beta.3.0.20260807055527-fc03f984d701 +replace github.com/wailsapp/wails/v3 => github.com/netbirdio/wails/v3 v3.0.0-beta.3.0.20260810103952-24e716aea4db diff --git a/go.sum b/go.sum index f6b32e32a..c219b0231 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,6 @@ github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jchv/go-winloader v0.0.0-20250406163304-c1995be93bd1 h1:njuLRcjAuMKr7kI3D85AXWkw6/+v9PwtV6M6o11sWHQ= -github.com/jchv/go-winloader v0.0.0-20250406163304-c1995be93bd1/go.mod h1:alcuEEnZsY1WQsagKhZDsoPCRoOijYqhZvPwLG0kzVs= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -490,8 +488,8 @@ github.com/netbirdio/service v0.0.0-20240911161631-f62744f42502 h1:3tHlFmhTdX9ax github.com/netbirdio/service v0.0.0-20240911161631-f62744f42502/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM= github.com/netbirdio/signal-dispatcher/dispatcher v0.0.0-20250805121659-6b4ac470ca45 h1:ujgviVYmx243Ksy7NdSwrdGPSRNE3pb8kEDSpH0QuAQ= github.com/netbirdio/signal-dispatcher/dispatcher v0.0.0-20250805121659-6b4ac470ca45/go.mod h1:5/sjFmLb8O96B5737VCqhHyGRzNFIaN/Bu7ZodXc3qQ= -github.com/netbirdio/wails/v3 v3.0.0-beta.3.0.20260807055527-fc03f984d701 h1:QL9nupfRom0L9jcY7N9l/Bc6QK2PtC6pHzC+ftpTqpw= -github.com/netbirdio/wails/v3 v3.0.0-beta.3.0.20260807055527-fc03f984d701/go.mod h1:BzATbK71VFikMMMCo434wAi0QcaI03P+xeaWgDvQvjw= +github.com/netbirdio/wails/v3 v3.0.0-beta.3.0.20260810103952-24e716aea4db h1:gBOE2r4AW1soSmpYJC5/n9/1L8UQ8+HLjed8CY/TzZY= +github.com/netbirdio/wails/v3 v3.0.0-beta.3.0.20260810103952-24e716aea4db/go.mod h1:bsdahLwBQxXjlmdPPeQyrTcDJfcqAr/ymFj0RXhwtWI= github.com/netbirdio/wireguard-go v0.0.0-20260628102922-2834bebf6c1a h1:3CWK+yTvRKOcC0Q8VCTGy4l60TEb27CQVS7LkMxwjmw= github.com/netbirdio/wireguard-go v0.0.0-20260628102922-2834bebf6c1a/go.mod h1:rpwXGsirqLqN2L0JDJQlwOboGHmptD5ZD6T2VmcqhTw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -793,7 +791,6 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From e8671a811dbc615a953d23e31f478a14f6be3cbc Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:41:31 +0900 Subject: [PATCH 13/22] [client, relay] Migrate relay QUIC tracer to qlog and bump quic-go to 0.59.1 (#7124) --- go.mod | 2 +- go.sum | 4 +- shared/relay/client/dialer/quic/quic.go | 23 --- shared/relay/client/dialer/quic/quic_test.go | 145 +++++++++++++++++++ shared/relay/client/dialer/quic/tracer.go | 70 +++++++++ 5 files changed, 218 insertions(+), 26 deletions(-) create mode 100644 shared/relay/client/dialer/quic/quic_test.go create mode 100644 shared/relay/client/dialer/quic/tracer.go diff --git a/go.mod b/go.mod index 36f0a87b0..f98073417 100644 --- a/go.mod +++ b/go.mod @@ -99,7 +99,7 @@ require ( github.com/pires/go-proxyproto v0.11.0 github.com/pkg/sftp v1.13.9 github.com/prometheus/client_golang v1.23.2 - github.com/quic-go/quic-go v0.55.0 + github.com/quic-go/quic-go v0.59.1 github.com/redis/go-redis/v9 v9.7.3 github.com/rs/xid v1.3.0 github.com/shirou/gopsutil/v4 v4.25.8 diff --git a/go.sum b/go.sum index c219b0231..31e8b5454 100644 --- a/go.sum +++ b/go.sum @@ -580,8 +580,8 @@ github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEo github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= -github.com/quic-go/quic-go v0.55.0 h1:zccPQIqYCXDt5NmcEabyYvOnomjs8Tlwl7tISjJh9Mk= -github.com/quic-go/quic-go v0.55.0/go.mod h1:DR51ilwU1uE164KuWXhinFcKWGlEjzys2l8zUl5Ss1U= +github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic= +github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= diff --git a/shared/relay/client/dialer/quic/quic.go b/shared/relay/client/dialer/quic/quic.go index 2e8de8af3..cbf6ea30f 100644 --- a/shared/relay/client/dialer/quic/quic.go +++ b/shared/relay/client/dialer/quic/quic.go @@ -9,7 +9,6 @@ import ( "time" "github.com/quic-go/quic-go" - "github.com/quic-go/quic-go/logging" log "github.com/sirupsen/logrus" nbnet "github.com/netbirdio/netbird/client/net" @@ -80,28 +79,6 @@ func (d Dialer) Dial(ctx context.Context, address, serverName string) (net.Conn, return conn, nil } -// connectionTracer returns a QUIC tracer that logs the DPLPMTUD result and the -// reason a relay connection closed, so the path MTU settled on and teardown -// cause are visible in logs. Lines carry the relay address as a structured -// field, matching the rest of the relay client logging. -func connectionTracer(addr string) func(context.Context, logging.Perspective, quic.ConnectionID) *logging.ConnectionTracer { - relayLog := log.WithField("relay", addr) - return func(context.Context, logging.Perspective, quic.ConnectionID) *logging.ConnectionTracer { - return &logging.ConnectionTracer{ - UpdatedMTU: func(mtu logging.ByteCount, done bool) { - if done { - relayLog.Infof("QUIC path MTU settled at %d", mtu) - return - } - relayLog.Debugf("QUIC path MTU probing at %d", mtu) - }, - ClosedConnection: func(err error) { - relayLog.Debugf("QUIC connection closed: %v", err) - }, - } - } -} - func prepareURL(address string) (string, error) { var host string var defaultPort string diff --git a/shared/relay/client/dialer/quic/quic_test.go b/shared/relay/client/dialer/quic/quic_test.go new file mode 100644 index 000000000..af210eba2 --- /dev/null +++ b/shared/relay/client/dialer/quic/quic_test.go @@ -0,0 +1,145 @@ +package quic + +import ( + "testing" + + "github.com/quic-go/quic-go/qlog" + "github.com/quic-go/quic-go/qlogwriter" + log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" +) + +func TestCloseReason(t *testing.T) { + transportErr := qlog.TransportErrorCode(0x2) // CONNECTION_REFUSED + appErr := qlog.ApplicationErrorCode(42) + + tests := []struct { + name string + event qlog.ConnectionClosed + want string + }{ + { + // A close carrying nothing but an initiator still reads sensibly. + name: "initiator only", + event: qlog.ConnectionClosed{Initiator: qlog.InitiatorLocal}, + want: "closed by local", + }, + { + name: "transport error with trigger", + event: qlog.ConnectionClosed{ + Initiator: qlog.InitiatorRemote, + ConnectionError: &transportErr, + Trigger: qlog.ConnectionCloseTriggerIdleTimeout, + }, + want: "closed by remote, transport error: CONNECTION_REFUSED, trigger: idle_timeout", + }, + { + name: "application error with reason", + event: qlog.ConnectionClosed{ + Initiator: qlog.InitiatorLocal, + ApplicationError: &appErr, + Reason: "bye", + }, + want: "closed by local, application error: 42, reason: bye", + }, + { + // Transport and application errors are mutually exclusive in + // practice; if both are set the transport code wins. + name: "transport error takes precedence over application error", + event: qlog.ConnectionClosed{ + Initiator: qlog.InitiatorLocal, + ConnectionError: &transportErr, + ApplicationError: &appErr, + }, + want: "closed by local, transport error: CONNECTION_REFUSED", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := closeReason(tt.event); got != tt.want { + t.Errorf("closeReason() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestLogSinkRecordEvent(t *testing.T) { + tests := []struct { + name string + event qlogwriter.Event + wantLevel log.Level + wantMsg string + }{ + { + name: "settled MTU is logged at info", + event: qlog.MTUUpdated{Value: 1400, Done: true}, + wantLevel: log.InfoLevel, + wantMsg: "QUIC path MTU settled at 1400", + }, + { + // Probing fires repeatedly during discovery, so it stays at debug. + name: "MTU probe is logged at debug", + event: qlog.MTUUpdated{Value: 1300, Done: false}, + wantLevel: log.DebugLevel, + wantMsg: "QUIC path MTU probing at 1300", + }, + { + name: "connection closed is logged at debug", + event: qlog.ConnectionClosed{Initiator: qlog.InitiatorRemote}, + wantLevel: log.DebugLevel, + wantMsg: "QUIC connection closed: closed by remote", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger, hook := test.NewNullLogger() + logger.SetLevel(log.DebugLevel) + recorder := logSink{log: logger.WithField("relay", "relay.example.com:443")} + + recorder.RecordEvent(tt.event) + + entries := hook.AllEntries() + if len(entries) != 1 { + t.Fatalf("got %d log entries, want 1", len(entries)) + } + if entries[0].Level != tt.wantLevel { + t.Errorf("level = %v, want %v", entries[0].Level, tt.wantLevel) + } + if entries[0].Message != tt.wantMsg { + t.Errorf("message = %q, want %q", entries[0].Message, tt.wantMsg) + } + if relay := entries[0].Data["relay"]; relay != "relay.example.com:443" { + t.Errorf("relay field = %v, want relay.example.com:443", relay) + } + }) + } +} + +// Events the relay client does not care about must not produce log lines. +func TestLogSinkIgnoresUnhandledEvents(t *testing.T) { + logger, hook := test.NewNullLogger() + logger.SetLevel(log.DebugLevel) + recorder := logSink{log: logger.WithField("relay", "relay.example.com:443")} + + recorder.RecordEvent(qlog.PacketLost{}) + + if entries := hook.AllEntries(); len(entries) != 0 { + t.Errorf("got %d log entries, want 0", len(entries)) + } +} + +func TestLogSinkSupportsSchemas(t *testing.T) { + trace := logSink{log: log.WithField("relay", "relay.example.com:443")} + + if !trace.SupportsSchemas(qlog.EventSchema) { + t.Errorf("SupportsSchemas(%q) = false, want true", qlog.EventSchema) + } + if trace.SupportsSchemas("urn:ietf:params:qlog:events:http3-12") { + t.Error("SupportsSchemas() = true for an unrelated schema, want false") + } + if trace.AddProducer() == nil { + t.Error("AddProducer() = nil, want a recorder") + } +} diff --git a/shared/relay/client/dialer/quic/tracer.go b/shared/relay/client/dialer/quic/tracer.go new file mode 100644 index 000000000..63be09f0c --- /dev/null +++ b/shared/relay/client/dialer/quic/tracer.go @@ -0,0 +1,70 @@ +package quic + +import ( + "context" + "fmt" + "strings" + + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/qlog" + "github.com/quic-go/quic-go/qlogwriter" + log "github.com/sirupsen/logrus" +) + +// logSink implements both qlogwriter.Trace and qlogwriter.Recorder, forwarding +// the few qlog events the relay client cares about to logrus instead of +// writing a qlog file. It holds no mutable state and logrus entries are safe +// to share, so one value can serve every producer on the connection. +type logSink struct { + log *log.Entry +} + +func (s logSink) AddProducer() qlogwriter.Recorder { return s } + +func (s logSink) SupportsSchemas(schema string) bool { return schema == qlog.EventSchema } + +func (s logSink) RecordEvent(event qlogwriter.Event) { + switch e := event.(type) { + case qlog.MTUUpdated: + if e.Done { + s.log.Infof("QUIC path MTU settled at %d", e.Value) + return + } + s.log.Debugf("QUIC path MTU probing at %d", e.Value) + case qlog.ConnectionClosed: + s.log.Debugf("QUIC connection closed: %s", closeReason(e)) + } +} + +func (s logSink) Close() error { return nil } + +// connectionTracer returns a QUIC tracer that logs the DPLPMTUD result and the +// reason a relay connection closed, so the path MTU settled on and teardown +// cause are visible in logs. Lines carry the relay address as a structured +// field, matching the rest of the relay client logging. +func connectionTracer(addr string) func(context.Context, bool, quic.ConnectionID) qlogwriter.Trace { + relayLog := log.WithField("relay", addr) + return func(context.Context, bool, quic.ConnectionID) qlogwriter.Trace { + return logSink{log: relayLog} + } +} + +// closeReason renders a ConnectionClosed event as a single line. The event +// carries the error as separate initiator, code, trigger and reason fields, +// any of which may be unset. +func closeReason(e qlog.ConnectionClosed) string { + parts := []string{fmt.Sprintf("closed by %s", e.Initiator)} + switch { + case e.ConnectionError != nil: + parts = append(parts, fmt.Sprintf("transport error: %s", *e.ConnectionError)) + case e.ApplicationError != nil: + parts = append(parts, fmt.Sprintf("application error: %d", *e.ApplicationError)) + } + if e.Trigger != "" { + parts = append(parts, fmt.Sprintf("trigger: %s", e.Trigger)) + } + if e.Reason != "" { + parts = append(parts, fmt.Sprintf("reason: %s", e.Reason)) + } + return strings.Join(parts, ", ") +} From ebfdf7d7b8c6b475c66cd7790d2f2f2a5e9275e4 Mon Sep 17 00:00:00 2001 From: Brad Ison Date: Mon, 10 Aug 2026 19:06:55 +0200 Subject: [PATCH 14/22] [management] Rework Agent Network endpoint identity and settings bootstrap (#7085) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Store the per-account gateway endpoint as {domain, proxy_address} with a global unique index on the full hostname; dedicated = (domain == proxy_address). Bootstrap becomes an explicit POST carrying exactly one of proxy_address (server allocates an adjective-noun label beneath it) or endpoint (claimed verbatim, address-first); provider create loses its bootstrap side effect. PUT is a full replace with every field required — the immutable identity fields must be echoed unchanged and a mismatch is rejected with 422. A guarded DELETE releases the endpoint: refused with 412 while providers exist or a proxy is actively serving the endpoint hostname (matched case-insensitively); re-creating bootstraps fresh. A self-addressed pin excludes its address from the account's cluster allow list, and the live mapping update path now addresses the serving proxy from the synthesized service. Existing rows are migrated on all three store engines. --- e2e/agentnetwork/chat_test.go | 8 +- e2e/agentnetwork/custom_pricing_test.go | 13 +- e2e/agentnetwork/guardrail_block_test.go | 13 +- .../guardrail_groupswitch_test.go | 1 - .../guardrail_multipolicy_test.go | 15 +- .../guardrail_pergroup_providers_test.go | 18 +- e2e/agentnetwork/guardrail_test.go | 7 +- e2e/agentnetwork/main_test.go | 10 + e2e/agentnetwork/management_test.go | 63 +-- e2e/agentnetwork/settings_bootstrap_test.go | 145 +++++-- e2e/agentnetwork/skiptls_test.go | 2 - e2e/agentnetwork/vllm_test.go | 11 +- e2e/harness/agentnetwork.go | 22 +- .../handlers/budget_handler_test.go | 4 +- .../handlers/providers_handler.go | 7 +- .../agentnetwork/handlers/settings_handler.go | 68 +++- .../handlers/settings_handler_test.go | 366 ++++++++++++++--- .../agentnetwork/labelgen/adjectives.go | 37 ++ .../modules/agentnetwork/labelgen/labelgen.go | 17 + .../agentnetwork/labelgen/labelgen_test.go | 79 ++++ .../internals/modules/agentnetwork/manager.go | 377 ++++++++++++------ .../agentnetwork/provider_bootstrap_test.go | 134 ------- .../modules/agentnetwork/reconcile.go | 77 ++-- .../modules/agentnetwork/reconcile_test.go | 97 ++++- .../agentnetwork/settings_bootstrap_test.go | 225 +++++++++++ .../modules/agentnetwork/synthesizer.go | 64 ++- .../synthesizer_realstore_test.go | 2 +- .../modules/agentnetwork/synthesizer_test.go | 7 +- .../modules/agentnetwork/types/settings.go | 136 +++++-- .../reverseproxy/domain/manager/manager.go | 37 +- .../domain/manager/manager_test.go | 150 ++++++- management/server/activity/codes.go | 4 + .../agentnetwork_budgetrule_realstack_test.go | 81 ++-- .../server/agentnetwork_realstack_test.go | 10 +- .../migration/migration_agentnetwork.go | 112 ++++++ management/server/migration/migration_test.go | 122 ++++++ management/server/store/sql_store.go | 24 ++ .../server/store/sql_store_agentnetwork.go | 73 +++- .../sql_store_agentnetwork_budgetrule_test.go | 6 +- management/server/store/store.go | 9 +- management/server/store/store_mock.go | 70 +++- .../agent_network_chain_realstack_test.go | 8 +- shared/management/client/rest/agentnetwork.go | 68 +++- .../client/rest/agentnetwork_test.go | 96 ++++- shared/management/http/api/openapi.yml | 129 +++++- shared/management/http/api/types.gen.go | 56 ++- 46 files changed, 2392 insertions(+), 688 deletions(-) create mode 100644 management/internals/modules/agentnetwork/labelgen/adjectives.go delete mode 100644 management/internals/modules/agentnetwork/provider_bootstrap_test.go create mode 100644 management/internals/modules/agentnetwork/settings_bootstrap_test.go create mode 100644 management/server/migration/migration_agentnetwork.go diff --git a/e2e/agentnetwork/chat_test.go b/e2e/agentnetwork/chat_test.go index ed94623fe..792f67c89 100644 --- a/e2e/agentnetwork/chat_test.go +++ b/e2e/agentnetwork/chat_test.go @@ -438,14 +438,10 @@ func TestProvidersMatrix(t *testing.T) { // Create every provider, all enabled, each with a unique model string so the // proxy's connect-time snapshot carries them all and model→provider routing // is unambiguous (provider toggles after connect don't reconcile to the - // proxy, so we enable everything up front). The first create bootstraps the - // cluster. + // proxy, so we enable everything up front). ids := make([]string, 0, len(matrix)) - for i, pc := range matrix { + for _, pc := range matrix { req := providerRequest(pc) - if i == 0 { - req.BootstrapCluster = ptr(harness.AgentNetworkCluster) - } prov, perr := srv.CreateProvider(ctx, req) require.NoError(t, perr, "create provider %s", pc.name) ids = append(ids, prov.Id) diff --git a/e2e/agentnetwork/custom_pricing_test.go b/e2e/agentnetwork/custom_pricing_test.go index 33788a16a..e3750258f 100644 --- a/e2e/agentnetwork/custom_pricing_test.go +++ b/e2e/agentnetwork/custom_pricing_test.go @@ -82,13 +82,12 @@ func provisionPricedProvider(t *testing.T, ctx context.Context, name string, mod // need NOT be in the catalog — the operator names it and prices it here. dummyKey := "sk-price-e2e" prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: name, - ProviderId: "openai_api", - UpstreamUrl: vllm.URL, - ApiKey: &dummyKey, - Enabled: ptr(true), - BootstrapCluster: ptr(harness.AgentNetworkCluster), - Models: &models, + Name: name, + ProviderId: "openai_api", + UpstreamUrl: vllm.URL, + ApiKey: &dummyKey, + Enabled: ptr(true), + Models: &models, }) require.NoError(t, err, "create provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) diff --git a/e2e/agentnetwork/guardrail_block_test.go b/e2e/agentnetwork/guardrail_block_test.go index c4b22ae25..302b5107e 100644 --- a/e2e/agentnetwork/guardrail_block_test.go +++ b/e2e/agentnetwork/guardrail_block_test.go @@ -113,15 +113,14 @@ func runPathRoutedGuardrailCase(t *testing.T, tc pathRoutedGuardrailCase) { // Catch-all provider (no models) so the router forwards any model; a static // bearer key means the router injects a static auth header instead of minting - // a GCP token. Bootstraps the cluster if it isn't already. + // a GCP token. staticKey := "static-e2e-token" prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: tc.name, - ProviderId: tc.catalogID, - UpstreamUrl: vllm.URL, - ApiKey: &staticKey, - Enabled: ptr(true), - BootstrapCluster: ptr(harness.AgentNetworkCluster), + Name: tc.name, + ProviderId: tc.catalogID, + UpstreamUrl: vllm.URL, + ApiKey: &staticKey, + Enabled: ptr(true), }) require.NoError(t, err, "create %s provider", tc.name) t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) diff --git a/e2e/agentnetwork/guardrail_groupswitch_test.go b/e2e/agentnetwork/guardrail_groupswitch_test.go index 5f2dce52e..7108bb587 100644 --- a/e2e/agentnetwork/guardrail_groupswitch_test.go +++ b/e2e/agentnetwork/guardrail_groupswitch_test.go @@ -73,7 +73,6 @@ func TestGuardrailGroupSwitchTakesEffectAfterTTL(t *testing.T) { {Id: modelA, InputPer1k: 0.001, OutputPer1k: 0.001}, {Id: modelB, InputPer1k: 0.001, OutputPer1k: 0.001}, }, - BootstrapCluster: ptr(harness.AgentNetworkCluster), }) require.NoError(t, err, "create provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) diff --git a/e2e/agentnetwork/guardrail_multipolicy_test.go b/e2e/agentnetwork/guardrail_multipolicy_test.go index 1d664fb58..2bb56b3bc 100644 --- a/e2e/agentnetwork/guardrail_multipolicy_test.go +++ b/e2e/agentnetwork/guardrail_multipolicy_test.go @@ -61,15 +61,14 @@ func TestGuardrailMultiPolicyModelAllowlist(t *testing.T) { } // pRestricted declares the two guardrailed models so routing is deterministic - // (model -> provider). Created first, so it carries the bootstrap cluster. + // (model -> provider). pRestricted, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: "restricted", - ProviderId: "openai_api", - UpstreamUrl: vllm.URL, - ApiKey: &staticKey, - Enabled: ptr(true), - Models: models(modelSelected, modelOther), - BootstrapCluster: ptr(harness.AgentNetworkCluster), + Name: "restricted", + ProviderId: "openai_api", + UpstreamUrl: vllm.URL, + ApiKey: &staticKey, + Enabled: ptr(true), + Models: models(modelSelected, modelOther), }) require.NoError(t, err, "create restricted provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), pRestricted.Id) }) diff --git a/e2e/agentnetwork/guardrail_pergroup_providers_test.go b/e2e/agentnetwork/guardrail_pergroup_providers_test.go index eddae65c3..0e98330c4 100644 --- a/e2e/agentnetwork/guardrail_pergroup_providers_test.go +++ b/e2e/agentnetwork/guardrail_pergroup_providers_test.go @@ -115,7 +115,7 @@ func TestGuardrailPerGroupAllowlist_AllProviders(t *testing.T) { staticKey := "static-e2e-token" enabled := true - for i, c := range cases { + for _, c := range cases { req := api.AgentNetworkProviderRequest{ Name: "e2e-pergroup-" + c.name, ProviderId: c.catalogID, @@ -124,9 +124,6 @@ func TestGuardrailPerGroupAllowlist_AllProviders(t *testing.T) { Enabled: ptr(true), Models: c.models, } - if i == 0 { - req.BootstrapCluster = ptr(harness.AgentNetworkCluster) - } prov, perr := srv.CreateProvider(ctx, req) require.NoError(t, perr, "create provider %s", c.name) c.providerID = prov.Id @@ -283,13 +280,12 @@ func TestGuardrailMultiGroupUser(t *testing.T) { // P1 — union scenario: two restricting policies, one per group. p1, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: "e2e-mg-union", - ProviderId: "openai_api", - UpstreamUrl: vllm.URL, - ApiKey: &staticKey, - Enabled: ptr(true), - Models: priced(unionA, unionB, unionC), - BootstrapCluster: ptr(harness.AgentNetworkCluster), + Name: "e2e-mg-union", + ProviderId: "openai_api", + UpstreamUrl: vllm.URL, + ApiKey: &staticKey, + Enabled: ptr(true), + Models: priced(unionA, unionB, unionC), }) require.NoError(t, err, "create union provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), p1.Id) }) diff --git a/e2e/agentnetwork/guardrail_test.go b/e2e/agentnetwork/guardrail_test.go index 6a2487a88..ddc95d62d 100644 --- a/e2e/agentnetwork/guardrail_test.go +++ b/e2e/agentnetwork/guardrail_test.go @@ -115,14 +115,11 @@ func TestModelAllowlistEnforced(t *testing.T) { }) require.NoError(t, err, "mint setup key") - // Providers with their configured (allowed) models; the first bootstraps the cluster. + // Providers with their configured (allowed) models ids := make([]string, 0, len(providers)) allowed := make([]string, 0, len(providers)) - for i, pc := range providers { + for _, pc := range providers { req := providerRequest(pc) - if i == 0 { - req.BootstrapCluster = ptr(harness.AgentNetworkCluster) - } prov, perr := srv.CreateProvider(ctx, req) require.NoError(t, perr, "create provider %s", pc.name) id := prov.Id diff --git a/e2e/agentnetwork/main_test.go b/e2e/agentnetwork/main_test.go index 17c5e00be..cc366b3fb 100644 --- a/e2e/agentnetwork/main_test.go +++ b/e2e/agentnetwork/main_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/netbirdio/netbird/e2e/harness" + "github.com/netbirdio/netbird/shared/management/http/api" ) // srv is the shared combined server for the package, ready (PAT-authenticated) @@ -42,5 +43,14 @@ func run(m *testing.M) int { return 1 } + // Bootstrap the account's agent-network endpoint once for the package: + // providers no longer have settings side effects, and every data-plane + // test expects the shared account pinned to the combined proxy cluster. + cluster := harness.AgentNetworkCluster + if _, err := srv.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ProxyAddress: &cluster}); err != nil { + fmt.Fprintf(os.Stderr, "e2e: bootstrap agent-network endpoint: %v\n", err) + return 1 + } + return m.Run() } diff --git a/e2e/agentnetwork/management_test.go b/e2e/agentnetwork/management_test.go index 6962f9796..9e3176d68 100644 --- a/e2e/agentnetwork/management_test.go +++ b/e2e/agentnetwork/management_test.go @@ -21,11 +21,10 @@ func ptr[T any](v T) *T { return &v } func newProvider(t *testing.T, ctx context.Context, name string) api.AgentNetworkProvider { t.Helper() prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: name, - ProviderId: "openai_api", - UpstreamUrl: "https://api.openai.com", - ApiKey: ptr("sk-dummy-e2e-key"), - BootstrapCluster: ptr("eu.proxy.netbird.test"), + Name: name, + ProviderId: "openai_api", + UpstreamUrl: "https://api.openai.com", + ApiKey: ptr("sk-dummy-e2e-key"), }) require.NoError(t, err, "create provider %q", name) t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) @@ -57,17 +56,11 @@ func TestProviderLifecycle(t *testing.T) { }} } - for i, pc := range cases { - i, pc := i, pc + for _, pc := range cases { + pc := pc t.Run(pc.name, func(t *testing.T) { req := providerRequest(pc) req.Name = "lc-" + pc.name - // Bootstrap the cluster on the first create in case the matrix has - // not run (e.g. no provider keys → settings not yet bootstrapped). - if i == 0 { - req.BootstrapCluster = ptr(harness.AgentNetworkCluster) - } - prov, err := srv.CreateProvider(ctx, req) require.NoError(t, err, "create %s provider", pc.name) t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) @@ -137,45 +130,65 @@ func TestProviderValidation(t *testing.T) { requireClientError(t, err) } -// TestSettingsRoundTrip flips the collection toggles and confirms cluster / -// subdomain stay immutable, then restores the original state. +// TestSettingsRoundTrip flips the collection toggles and confirms the +// endpoint and proxy address stay immutable, then restores the original +// state. A second bootstrap attempt must be rejected as a conflict. func TestSettingsRoundTrip(t *testing.T) { ctx := context.Background() - // Settings are bootstrapped on first provider create. - newProvider(t, ctx, "Settings Bootstrap") - + // The package's TestMain bootstrapped the shared account's endpoint. before, err := srv.GetSettings(ctx) require.NoError(t, err, "get settings") - require.NotEmpty(t, before.Cluster, "settings must carry an assigned cluster") + require.NotEmpty(t, before.Endpoint, "settings must carry the bootstrapped endpoint") + require.NotEmpty(t, before.ProxyAddress, "settings must carry the bootstrapped proxy address") + + require.NotNil(t, before.AccessLogRetentionDays, "bootstrapped settings must carry a retention") + beforeRetention := *before.AccessLogRetentionDays flipped, err := srv.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ + Endpoint: before.Endpoint, + ProxyAddress: before.ProxyAddress, EnableLogCollection: !before.EnableLogCollection, EnablePromptCollection: !before.EnablePromptCollection, RedactPii: !before.RedactPii, + AccessLogRetentionDays: beforeRetention, }) require.NoError(t, err, "update settings") assert.Equal(t, !before.EnableLogCollection, flipped.EnableLogCollection, "log collection toggle must flip") assert.Equal(t, !before.EnablePromptCollection, flipped.EnablePromptCollection, "prompt collection toggle must flip") - assert.Equal(t, before.Cluster, flipped.Cluster, "cluster must be immutable across updates") - assert.Equal(t, before.Subdomain, flipped.Subdomain, "subdomain must be immutable across updates") + require.NotNil(t, flipped.AccessLogRetentionDays) + assert.Equal(t, beforeRetention, *flipped.AccessLogRetentionDays, + "retention sent unchanged must round-trip, not reset to the zero value") + assert.Equal(t, before.Endpoint, flipped.Endpoint, "endpoint must be immutable across updates") + assert.Equal(t, before.ProxyAddress, flipped.ProxyAddress, "proxy address must be immutable across updates") - // A cluster different from the pinned one must be rejected; echoing the - // pinned one back is valid. + // The account is already bootstrapped: a second bootstrap is a conflict, + // whatever shape it asks for. + _, err = srv.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + Endpoint: ptr("attacker.cluster.invalid"), + }) + requireClientError(t, err) + + // The identity fields ride along on the PUT as a required echo: a request + // carrying a different endpoint is rejected without applying anything. _, err = srv.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - Cluster: ptr("attacker.cluster.invalid"), + Endpoint: "other.cluster.invalid", + ProxyAddress: before.ProxyAddress, EnableLogCollection: before.EnableLogCollection, EnablePromptCollection: before.EnablePromptCollection, RedactPii: before.RedactPii, + AccessLogRetentionDays: beforeRetention, }) requireClientError(t, err) // Restore the original toggles. _, err = srv.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - Cluster: ptr(before.Cluster), + Endpoint: before.Endpoint, + ProxyAddress: before.ProxyAddress, EnableLogCollection: before.EnableLogCollection, EnablePromptCollection: before.EnablePromptCollection, RedactPii: before.RedactPii, + AccessLogRetentionDays: beforeRetention, }) require.NoError(t, err, "restore settings") } diff --git a/e2e/agentnetwork/settings_bootstrap_test.go b/e2e/agentnetwork/settings_bootstrap_test.go index ea56f7064..806ce82f4 100644 --- a/e2e/agentnetwork/settings_bootstrap_test.go +++ b/e2e/agentnetwork/settings_bootstrap_test.go @@ -4,6 +4,7 @@ package agentnetwork import ( "context" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -14,7 +15,8 @@ import ( ) // harnessStartFresh boots a dedicated combined server with its own fresh -// account and registers its teardown on t. +// account and registers its teardown on t. Unlike the shared srv, the fresh +// account has NOT had its agent-network endpoint bootstrapped. func harnessStartFresh(ctx context.Context, t *testing.T) (*harness.Combined, error) { t.Helper() fresh, err := harness.StartCombined(ctx) @@ -28,16 +30,16 @@ func harnessStartFresh(ctx context.Context, t *testing.T) (*harness.Combined, er return fresh, nil } -// TestSettingsBootstrapViaPut covers the settings-first bootstrap path on an +// TestSettingsBootstrapViaPost covers the explicit bootstrap contract on an // account that has never been bootstrapped: the GET reads as the defaults -// with an empty cluster/subdomain/endpoint, a PUT without a cluster has -// nothing to pin and fails, and a PUT carrying a cluster creates the row and -// pins it immutably. The shared srv cannot provide that starting state (any -// provider-creating test bootstraps it, and test order is deliberately not -// relied on), so this boots a dedicated combined server — the image is -// already built and cached by TestMain's StartCombined, so the extra cost is -// one container start. -func TestSettingsBootstrapViaPut(t *testing.T) { +// with an empty endpoint/proxy_address, a PUT has no row to update and fails, +// and a POST creates the row and assigns the immutable endpoint — labeled +// beneath a proxy address here, with the toggle overrides from the same +// request applied. The shared srv cannot provide that starting state +// (TestMain bootstraps it), so this boots a dedicated combined server — the +// image is already built and cached by TestMain's StartCombined, so the extra +// cost is one container start. +func TestSettingsBootstrapViaPost(t *testing.T) { ctx := context.Background() fresh, err := harnessStartFresh(ctx, t) @@ -47,32 +49,35 @@ func TestSettingsBootstrapViaPut(t *testing.T) { // as an error and not as a null body. before, err := fresh.GetSettings(ctx) require.NoError(t, err, "get settings on a fresh account must succeed") - assert.Empty(t, before.Cluster, "cluster must be empty before bootstrap") - assert.Empty(t, before.Subdomain, "subdomain must be empty before bootstrap") - assert.Empty(t, before.Endpoint, "endpoint must be empty before bootstrap, not a bare dot") + assert.Empty(t, before.Endpoint, "endpoint must be empty before bootstrap") + assert.Empty(t, before.ProxyAddress, "proxy address must be empty before bootstrap") + assert.False(t, before.Dedicated, "an unbootstrapped account has no serving shape") assert.True(t, before.EnableLogCollection, "defaults must show log collection on, matching bootstrap") assert.False(t, before.EnablePromptCollection, "defaults must show prompt collection off") - // A PUT without a cluster has nothing to pin the account to. + // A PUT has no row to update yet — bootstrap is the explicit POST. _, err = fresh.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - EnableLogCollection: true, + EnableLogCollection: true, + AccessLogRetentionDays: 30, }) requireClientError(t, err) - // A PUT carrying a cluster bootstraps the account and applies the - // mutable fields from the same request. Every toggle is set away from - // its bootstrap default so each assertion can actually fail. + // A POST with a proxy address bootstraps a labeled endpoint and applies + // the toggles from the same request. Every toggle is set away from its + // bootstrap default so each assertion can actually fail. const cluster = "e2e.bootstrap.netbird.selfhosted" - bootstrapped, err := fresh.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - Cluster: ptr(cluster), - EnableLogCollection: false, - EnablePromptCollection: true, - RedactPii: true, + bootstrapped, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + ProxyAddress: ptr(cluster), + EnableLogCollection: ptr(false), + EnablePromptCollection: ptr(true), + RedactPii: ptr(true), }) - require.NoError(t, err, "bootstrap settings via PUT must succeed") - assert.Equal(t, cluster, bootstrapped.Cluster, "cluster must be pinned from the request") - require.NotEmpty(t, bootstrapped.Subdomain, "subdomain must be assigned at bootstrap") - assert.Equal(t, bootstrapped.Subdomain+"."+cluster, bootstrapped.Endpoint, "endpoint must combine subdomain and cluster") + require.NoError(t, err, "bootstrap settings via POST must succeed") + assert.Equal(t, cluster, bootstrapped.ProxyAddress, "proxy address must be pinned from the request") + require.NotEmpty(t, bootstrapped.Endpoint, "endpoint must be assigned at bootstrap") + assert.True(t, strings.HasSuffix(bootstrapped.Endpoint, "."+cluster), + "labeled endpoint must hang one label beneath the proxy address: %s", bootstrapped.Endpoint) + assert.False(t, bootstrapped.Dedicated, "a labeled pin is not dedicated") assert.False(t, bootstrapped.EnableLogCollection, "log collection from the bootstrap request must override the default") assert.True(t, bootstrapped.EnablePromptCollection, "prompt collection from the bootstrap request must apply") assert.True(t, bootstrapped.RedactPii, "redact toggle from the bootstrap request must apply") @@ -85,30 +90,90 @@ func TestSettingsBootstrapViaPut(t *testing.T) { assert.Equal(t, bootstrapped.EnablePromptCollection, after.EnablePromptCollection, "prompt collection must persist") assert.Equal(t, bootstrapped.RedactPii, after.RedactPii, "redact toggle must persist") - // Once bootstrapped, later updates may omit the cluster entirely. + // Once bootstrapped, PUT updates the toggles. The identity fields ride + // along as a required echo of the assigned values; a matching echo is + // accepted and never written. persisted, err := fresh.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ + Endpoint: bootstrapped.Endpoint, + ProxyAddress: bootstrapped.ProxyAddress, EnableLogCollection: true, EnablePromptCollection: false, RedactPii: true, + AccessLogRetentionDays: 21, }) - require.NoError(t, err, "post-bootstrap update without cluster must succeed") - assert.Equal(t, cluster, persisted.Cluster, "omitted cluster must keep the pinned value") + require.NoError(t, err, "post-bootstrap update must succeed") + require.NotNil(t, persisted.AccessLogRetentionDays) + assert.Equal(t, 21, *persisted.AccessLogRetentionDays, "retention from the update must apply") + assert.Equal(t, bootstrapped.Endpoint, persisted.Endpoint, "endpoint must survive updates untouched") + assert.Equal(t, cluster, persisted.ProxyAddress, "proxy address must survive updates untouched") assert.True(t, persisted.EnableLogCollection, "post-bootstrap toggle must apply") assert.False(t, persisted.EnablePromptCollection, "post-bootstrap toggle must apply") - // The cluster is immutable: a different value is rejected rather than - // silently ignored, and the rejected update must not disturb anything. + // The endpoint is immutable: a PUT carrying a different endpoint is + // rejected, and a second bootstrap is rejected as a conflict. Neither + // rejected write may disturb anything. _, err = fresh.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - Cluster: ptr("other.cluster.invalid"), - EnableLogCollection: false, + Endpoint: "other.cluster.invalid", + ProxyAddress: persisted.ProxyAddress, + EnableLogCollection: persisted.EnableLogCollection, + EnablePromptCollection: persisted.EnablePromptCollection, + RedactPii: persisted.RedactPii, + AccessLogRetentionDays: 21, + }) + requireClientError(t, err) + + _, err = fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + Endpoint: ptr("other.cluster.invalid"), }) requireClientError(t, err) final, err := fresh.GetSettings(ctx) - require.NoError(t, err, "get settings after the rejected cluster change must succeed") - assert.Equal(t, persisted.Cluster, final.Cluster, "rejected update must not change the cluster") - assert.Equal(t, persisted.Endpoint, final.Endpoint, "rejected update must not change the endpoint") - assert.Equal(t, persisted.EnableLogCollection, final.EnableLogCollection, "rejected update must not apply its toggles") - assert.Equal(t, persisted.EnablePromptCollection, final.EnablePromptCollection, "rejected update must not apply its toggles") - assert.Equal(t, persisted.RedactPii, final.RedactPii, "rejected update must not apply its toggles") + require.NoError(t, err, "get settings after the rejected bootstrap must succeed") + assert.Equal(t, persisted.Endpoint, final.Endpoint, "rejected bootstrap must not change the endpoint") + assert.Equal(t, persisted.ProxyAddress, final.ProxyAddress, "rejected bootstrap must not change the proxy address") + assert.Equal(t, persisted.EnableLogCollection, final.EnableLogCollection, "rejected bootstrap must not apply its toggles") + assert.Equal(t, persisted.EnablePromptCollection, final.EnablePromptCollection, "rejected bootstrap must not apply its toggles") + assert.Equal(t, persisted.RedactPii, final.RedactPii, "rejected bootstrap must not apply its toggles") +} + +// TestSettingsBootstrapSelfAddressed covers the dedicated shape end to end: +// a POST carrying an endpoint claims the hostname verbatim, the proxy address +// equals it, and the pin reads as dedicated — the address-first flow a +// self-hosted operator uses before deploying the proxy that will declare it. +// The tail covers the recovery path the guarded DELETE exists for: with no +// providers and no proxy at the address, the claim can be released and a +// fresh bootstrap succeeds — the fix for a typo'd immutable endpoint. +func TestSettingsBootstrapSelfAddressed(t *testing.T) { + ctx := context.Background() + + fresh, err := harnessStartFresh(ctx, t) + require.NoError(t, err, "start dedicated combined server") + + created, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + Endpoint: ptr("gw.e2e.netbird.selfhosted"), + }) + require.NoError(t, err, "self-addressed bootstrap must succeed") + assert.Equal(t, "gw.e2e.netbird.selfhosted", created.Endpoint, "endpoint must be claimed verbatim") + assert.Equal(t, created.Endpoint, created.ProxyAddress, "self-addressed: proxy address is the endpoint") + assert.True(t, created.Dedicated, "a self-addressed pin is dedicated") + + // No providers exist and no proxy declares the address, so both delete + // guards are clear: the delete releases the claim and the account reads + // as unbootstrapped defaults again. + require.NoError(t, fresh.DeleteSettings(ctx), "guarded delete with both guards clear must succeed") + + after, err := fresh.GetSettings(ctx) + require.NoError(t, err, "get settings after delete must succeed") + assert.Empty(t, after.Endpoint, "a deleted account must read as unbootstrapped") + + // A second delete has nothing to remove. + requireClientError(t, fresh.DeleteSettings(ctx)) + + // Re-creating is a fresh bootstrap — the released hostname is free to be + // claimed again, or a different one chosen. + recreated, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + Endpoint: ptr("gw2.e2e.netbird.selfhosted"), + }) + require.NoError(t, err, "bootstrap after delete must succeed") + assert.Equal(t, "gw2.e2e.netbird.selfhosted", recreated.Endpoint, "the fresh bootstrap claims the new hostname") } diff --git a/e2e/agentnetwork/skiptls_test.go b/e2e/agentnetwork/skiptls_test.go index 1f57605e3..85af8c604 100644 --- a/e2e/agentnetwork/skiptls_test.go +++ b/e2e/agentnetwork/skiptls_test.go @@ -66,9 +66,7 @@ func TestProviderSkipTLSVerification(t *testing.T) { } } - // First create bootstraps the account cluster. insecureReq := newReq("skip-tls", insecureModel, true) - insecureReq.BootstrapCluster = ptr(harness.AgentNetworkCluster) insecureProv, err := srv.CreateProvider(ctx, insecureReq) require.NoError(t, err, "create skip-tls provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), insecureProv.Id) }) diff --git a/e2e/agentnetwork/vllm_test.go b/e2e/agentnetwork/vllm_test.go index cd598f1ed..6f5117b2f 100644 --- a/e2e/agentnetwork/vllm_test.go +++ b/e2e/agentnetwork/vllm_test.go @@ -57,12 +57,11 @@ func TestVLLMProvider(t *testing.T) { // is enumerated so the router dispatches this model string to this provider. dummyKey := "sk-vllm-e2e" prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: "vllm", - ProviderId: "vllm", - UpstreamUrl: vllm.URL, - ApiKey: &dummyKey, - Enabled: ptr(true), - BootstrapCluster: ptr(harness.AgentNetworkCluster), + Name: "vllm", + ProviderId: "vllm", + UpstreamUrl: vllm.URL, + ApiKey: &dummyKey, + Enabled: ptr(true), Models: &[]api.AgentNetworkProviderModel{ {Id: harness.VLLMModel, InputPer1k: 0.001, OutputPer1k: 0.002}, }, diff --git a/e2e/harness/agentnetwork.go b/e2e/harness/agentnetwork.go index 078e697af..e85475dff 100644 --- a/e2e/harness/agentnetwork.go +++ b/e2e/harness/agentnetwork.go @@ -126,17 +126,33 @@ func (c *Combined) DeleteGuardrail(ctx context.Context, id string) error { return anDelete(ctx, c, "/api/agent-network/guardrails/"+id) } -// GetSettings returns the account's agent-network settings row. It exists only -// after the first provider create bootstraps it. +// CreateSettings bootstraps the account's agent-network settings row, +// assigning the immutable endpoint. Exactly one of req.ProxyAddress (labeled +// endpoint beneath that cluster) and req.Endpoint (self-addressed dedicated +// endpoint) must be set; a second bootstrap returns a conflict. +func (c *Combined) CreateSettings(ctx context.Context, req api.AgentNetworkSettingsCreateRequest) (api.AgentNetworkSettings, error) { + return anRequest[api.AgentNetworkSettings](ctx, c, http.MethodPost, "/api/agent-network/settings", req) +} + +// GetSettings returns the account's agent-network settings row. Before the +// CreateSettings bootstrap it reads as the defaults with an empty endpoint. func (c *Combined) GetSettings(ctx context.Context) (api.AgentNetworkSettings, error) { return anRequest[api.AgentNetworkSettings](ctx, c, http.MethodGet, "/api/agent-network/settings", nil) } -// UpdateSettings applies the mutable collection toggles. +// UpdateSettings applies the mutable collection toggles. The request must +// echo the assigned endpoint and proxy address unchanged — the server rejects +// a PUT that tries to change them. func (c *Combined) UpdateSettings(ctx context.Context, req api.AgentNetworkSettingsRequest) (api.AgentNetworkSettings, error) { return anRequest[api.AgentNetworkSettings](ctx, c, http.MethodPut, "/api/agent-network/settings", req) } +// DeleteSettings removes the account's settings row, releasing the endpoint. +// Refused while providers exist or a proxy is actively serving the endpoint. +func (c *Combined) DeleteSettings(ctx context.Context) error { + return anDelete(ctx, c, "/api/agent-network/settings") +} + // ListConsumption returns the account's consumption rows (possibly empty). func (c *Combined) ListConsumption(ctx context.Context) ([]api.AgentNetworkConsumption, error) { return anRequest[[]api.AgentNetworkConsumption](ctx, c, http.MethodGet, "/api/agent-network/consumption", nil) diff --git a/management/internals/modules/agentnetwork/handlers/budget_handler_test.go b/management/internals/modules/agentnetwork/handlers/budget_handler_test.go index 4038761c5..3a7709461 100644 --- a/management/internals/modules/agentnetwork/handlers/budget_handler_test.go +++ b/management/internals/modules/agentnetwork/handlers/budget_handler_test.go @@ -102,8 +102,8 @@ func TestSettingsHandler_GetExposesCollectionToggles(t *testing.T) { require.NoError(t, f.store.SaveAgentNetworkSettings(context.Background(), &agentNetworkTypes.Settings{ AccountID: testAccountID, - Cluster: "eu.proxy.netbird.io", - Subdomain: "violet", + Domain: "violet.eu.proxy.netbird.io", + ProxyAddress: "eu.proxy.netbird.io", EnableLogCollection: true, EnablePromptCollection: true, RedactPii: false, diff --git a/management/internals/modules/agentnetwork/handlers/providers_handler.go b/management/internals/modules/agentnetwork/handlers/providers_handler.go index c05363101..0d8a44ca3 100644 --- a/management/internals/modules/agentnetwork/handlers/providers_handler.go +++ b/management/internals/modules/agentnetwork/handlers/providers_handler.go @@ -155,12 +155,7 @@ func (h *handler) createProvider(w http.ResponseWriter, r *http.Request) { provider := types.NewProvider(userAuth.AccountId) provider.FromAPIRequest(&req) - bootstrapCluster := "" - if req.BootstrapCluster != nil { - bootstrapCluster = *req.BootstrapCluster - } - - created, err := h.manager.CreateProvider(r.Context(), userAuth.UserId, provider, bootstrapCluster) + created, err := h.manager.CreateProvider(r.Context(), userAuth.UserId, provider) if err != nil { util.WriteError(r.Context(), err, w) return diff --git a/management/internals/modules/agentnetwork/handlers/settings_handler.go b/management/internals/modules/agentnetwork/handlers/settings_handler.go index 171750838..0a365f9ce 100644 --- a/management/internals/modules/agentnetwork/handlers/settings_handler.go +++ b/management/internals/modules/agentnetwork/handlers/settings_handler.go @@ -12,13 +12,55 @@ import ( "github.com/netbirdio/netbird/shared/management/http/util" ) -// addSettingsEndpoints registers the Agent Network settings routes. The -// settings row is bootstrapped server-side on first provider create or on the -// first PUT carrying a cluster; GET reads it and PUT applies a partial update -// of the mutable collection toggles (cluster/subdomain stay immutable). +// addSettingsEndpoints registers the Agent Network settings routes. POST +// bootstraps the settings row, assigning the account's immutable endpoint; +// GET reads it (defaults with an empty endpoint before bootstrap); PUT +// carries every field, replacing the mutable collection toggles and rejecting +// any change to the identity fields; DELETE removes the row — guarded so it +// stays a bootstrap-repair operation — releasing the endpoint for a fresh +// bootstrap. func (h *handler) addSettingsEndpoints(router *mux.Router) { router.HandleFunc("/agent-network/settings", h.getSettings).Methods("GET", "OPTIONS") + router.HandleFunc("/agent-network/settings", h.createSettings).Methods("POST", "OPTIONS") router.HandleFunc("/agent-network/settings", h.updateSettings).Methods("PUT", "OPTIONS") + router.HandleFunc("/agent-network/settings", h.deleteSettings).Methods("DELETE", "OPTIONS") +} + +// createSettings bootstraps the account's settings row. Exactly one of +// proxy_address (labeled endpoint; the server allocates the label) and +// endpoint (self-addressed, claimed verbatim) must be provided; optional +// collection toggles ride along with defaults for omitted fields. +func (h *handler) createSettings(w http.ResponseWriter, r *http.Request) { + userAuth, err := nbcontext.GetUserAuthFromContext(r.Context()) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + + var req api.AgentNetworkSettingsCreateRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + util.WriteErrorResponse("couldn't parse JSON request", http.StatusBadRequest, w) + return + } + + settings := types.DefaultSettings(userAuth.AccountId) + settings.FromAPICreateRequest(&req) + + proxyAddress := "" + if req.ProxyAddress != nil { + proxyAddress = *req.ProxyAddress + } + endpoint := "" + if req.Endpoint != nil { + endpoint = *req.Endpoint + } + + created, err := h.manager.CreateSettings(r.Context(), userAuth.UserId, settings, proxyAddress, endpoint) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + util.WriteJSONObject(r.Context(), w, created.ToAPIResponse()) } // updateSettings replaces the mutable settings fields on the account's row. @@ -48,6 +90,24 @@ func (h *handler) updateSettings(w http.ResponseWriter, r *http.Request) { util.WriteJSONObject(r.Context(), w, updated.ToAPIResponse()) } +// deleteSettings removes the account's settings row, releasing the endpoint. +// The manager refuses (412) while providers exist or a proxy is actively +// serving the endpoint; a later POST bootstraps fresh, allocating a new +// endpoint. +func (h *handler) deleteSettings(w http.ResponseWriter, r *http.Request) { + userAuth, err := nbcontext.GetUserAuthFromContext(r.Context()) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + + if err := h.manager.DeleteSettings(r.Context(), userAuth.AccountId, userAuth.UserId); err != nil { + util.WriteError(r.Context(), err, w) + return + } + util.WriteJSONObject(r.Context(), w, util.EmptyObject{}) +} + // getSettings returns the account's agent-network settings. Accounts that // haven't been bootstrapped yet read as the defaults with an empty cluster, // subdomain and endpoint; the manager synthesises that view. diff --git a/management/internals/modules/agentnetwork/handlers/settings_handler_test.go b/management/internals/modules/agentnetwork/handlers/settings_handler_test.go index 636ec5b26..400208e1c 100644 --- a/management/internals/modules/agentnetwork/handlers/settings_handler_test.go +++ b/management/internals/modules/agentnetwork/handlers/settings_handler_test.go @@ -1,20 +1,25 @@ package handlers import ( + "context" "encoding/json" + "fmt" "net/http" + "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + rpproxy "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy" "github.com/netbirdio/netbird/shared/management/http/api" ) // TestSettingsHandler_GetUnbootstrappedReturnsDefaults pins the settings-read // convention shared with the account and DNS settings endpoints: settings // always read as a JSON object. Before bootstrap that object carries the -// defaults with an empty cluster/subdomain/endpoint (the "not bootstrapped" +// defaults with an empty endpoint/proxy_address (the "not bootstrapped" // signal) and no timestamps — never a 404 and never the legacy null body. func TestSettingsHandler_GetUnbootstrappedReturnsDefaults(t *testing.T) { f := newAgentNetworkHandlerFixture(t) @@ -27,9 +32,9 @@ func TestSettingsHandler_GetUnbootstrappedReturnsDefaults(t *testing.T) { var got api.AgentNetworkSettings require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) - assert.Empty(t, got.Cluster, "cluster must be empty until bootstrapped") - assert.Empty(t, got.Subdomain, "subdomain must be empty until bootstrapped") - assert.Empty(t, got.Endpoint, "endpoint must be empty until bootstrapped, not a bare dot") + assert.Empty(t, got.Endpoint, "endpoint must be empty until bootstrapped") + assert.Empty(t, got.ProxyAddress, "proxy address must be empty until bootstrapped") + assert.False(t, got.Dedicated, "an unbootstrapped account has no serving shape") assert.True(t, got.EnableLogCollection, "defaults must show log collection on, matching bootstrap") assert.False(t, got.EnablePromptCollection, "defaults must show prompt collection off") assert.False(t, got.RedactPii, "defaults must show redaction off") @@ -39,62 +44,149 @@ func TestSettingsHandler_GetUnbootstrappedReturnsDefaults(t *testing.T) { assert.Nil(t, got.UpdatedAt, "no timestamps before a row exists") } -// TestSettingsHandler_PutBootstrapsWithCluster covers the settings-first -// bootstrap path: a PUT carrying a cluster on an unbootstrapped account -// creates the row (cluster pinned, subdomain assigned) and applies the -// mutable fields from the same request. -func TestSettingsHandler_PutBootstrapsWithCluster(t *testing.T) { +// TestSettingsHandler_PostBootstrapsLabeled covers the labeled bootstrap +// shape: a POST carrying a proxy_address allocates a label beneath it, so the +// endpoint hangs one label under the shared cluster's address and the pin is +// not dedicated. Toggles riding along apply; omitted ones keep defaults. +func TestSettingsHandler_PostBootstrapsLabeled(t *testing.T) { f := newAgentNetworkHandlerFixture(t) - rec := f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "eu.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": true, "redact_pii": false, "access_log_retention_days": 30}`) - require.Equal(t, http.StatusOK, rec.Code, "bootstrap PUT must succeed: %s", rec.Body.String()) + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "enable_prompt_collection": true, "access_log_retention_days": 14}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) var got api.AgentNetworkSettings require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) - assert.Equal(t, "eu.proxy.netbird.io", got.Cluster, "cluster must be pinned from the request") - assert.NotEmpty(t, got.Subdomain, "subdomain must be assigned at bootstrap") - assert.Equal(t, got.Subdomain+".eu.proxy.netbird.io", got.Endpoint, "endpoint must combine subdomain and cluster") - assert.True(t, got.EnableLogCollection, "toggle from the bootstrap request must apply") + assert.Equal(t, "eu.proxy.netbird.io", got.ProxyAddress, "proxy address must be pinned from the request") + require.NotEmpty(t, got.Endpoint, "endpoint must be allocated at bootstrap") + assert.True(t, strings.HasSuffix(got.Endpoint, ".eu.proxy.netbird.io"), + "labeled endpoint must hang off the proxy address: %s", got.Endpoint) + label := strings.TrimSuffix(got.Endpoint, ".eu.proxy.netbird.io") + assert.NotContains(t, label, ".", "the allocated label must be a single DNS label: %s", label) + assert.False(t, got.Dedicated, "a labeled pin is not dedicated") + assert.True(t, got.EnableLogCollection, "omitted toggle must keep its default") assert.True(t, got.EnablePromptCollection, "toggle from the bootstrap request must apply") require.NotNil(t, got.AccessLogRetentionDays) - assert.Equal(t, 30, *got.AccessLogRetentionDays, "retention from the bootstrap request must apply") + assert.Equal(t, 14, *got.AccessLogRetentionDays, "retention from the bootstrap request must apply") + assert.NotNil(t, got.CreatedAt, "a persisted row carries timestamps") // The row is now readable via GET. rec = f.do(t, http.MethodGet, "/agent-network/settings", "") require.Equal(t, http.StatusOK, rec.Code, "GET after bootstrap must succeed") + var read api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &read)) + assert.Equal(t, got.Endpoint, read.Endpoint, "GET must return the bootstrapped endpoint") } -// TestSettingsHandler_PutWithoutClusterOnUnbootstrapped pins that a PUT -// without a cluster cannot conjure a settings row out of nothing — there is -// no cluster to pin — and surfaces as 404 like the GET. -func TestSettingsHandler_PutWithoutClusterOnUnbootstrapped(t *testing.T) { +// TestSettingsHandler_PostBootstrapsSelfAddressed covers the dedicated shape: +// a POST carrying an endpoint claims the hostname verbatim, the proxy address +// equals it, and the pin reads as dedicated. The claim is legitimate before +// any proxy declares the address (address-first). +func TestSettingsHandler_PostBootstrapsSelfAddressed(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"endpoint": "Brave-Otter.Gateway.Example.com"}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + + var got api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, "brave-otter.gateway.example.com", got.Endpoint, + "endpoint must be claimed verbatim, lowercased") + assert.Equal(t, got.Endpoint, got.ProxyAddress, "self-addressed: the proxy address is the endpoint") + assert.True(t, got.Dedicated, "a self-addressed pin is dedicated") + assert.True(t, got.EnableLogCollection, "omitted toggles must keep their defaults") +} + +// TestSettingsHandler_PostRequiresExactlyOneIdentityField pins the request +// contract: proxy_address and endpoint are mutually exclusive and one is +// required — both or neither is a validation error, not a guess. +func TestSettingsHandler_PostRequiresExactlyOneIdentityField(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", `{}`) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, + "empty POST must be rejected: got %d body=%s", rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "endpoint": "brave-otter.gateway.example.com"}`) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, + "POST with both identity fields must be rejected: got %d body=%s", rec.Code, rec.Body.String()) +} + +// TestSettingsHandler_PostRejectsMalformedHostnames pins per-write input +// validation: shapes canonicalization cannot repair — trailing dots, embedded +// whitespace, empty labels — are rejected with a validation error instead of +// landing in an immutable column. +func TestSettingsHandler_PostRejectsMalformedHostnames(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + for name, body := range map[string]string{ + "trailing dot": `{"endpoint": "gateway.example.com."}`, + "leading dot": `{"endpoint": ".gateway.example.com"}`, + "inner whitespace": `{"endpoint": "gate way.example.com"}`, + "empty label": `{"proxy_address": "eu..proxy.netbird.io"}`, + } { + rec := f.do(t, http.MethodPost, "/agent-network/settings", body) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, + "%s must be rejected: got %d body=%s", name, rec.Code, rec.Body.String()) + } +} + +// TestSettingsHandler_PostConflictsOnSecondBootstrap pins that bootstrap is a +// one-time create: a second POST returns 409 and leaves the row untouched. +func TestSettingsHandler_PostConflictsOnSecondBootstrap(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", `{"proxy_address": "eu.proxy.netbird.io"}`) + require.Equal(t, http.StatusOK, rec.Code, "first bootstrap must succeed: %s", rec.Body.String()) + var first api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &first)) + + rec = f.do(t, http.MethodPost, "/agent-network/settings", `{"proxy_address": "us.proxy.netbird.io"}`) + assert.Equal(t, http.StatusConflict, rec.Code, + "second bootstrap must 409: got %d body=%s", rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodGet, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code) + var got api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, first.Endpoint, got.Endpoint, "the original endpoint must survive the rejected bootstrap") + assert.Equal(t, first.ProxyAddress, got.ProxyAddress, "the original proxy address must survive") +} + +// TestSettingsHandler_PutBeforeBootstrapIs404 pins that a PUT cannot conjure a +// settings row out of nothing — bootstrap is the explicit POST — and the +// error points the caller there. +func TestSettingsHandler_PutBeforeBootstrapIs404(t *testing.T) { f := newAgentNetworkHandlerFixture(t) rec := f.do(t, http.MethodPut, "/agent-network/settings", `{"enable_log_collection": false, "enable_prompt_collection": false, "redact_pii": false}`) assert.Equal(t, http.StatusNotFound, rec.Code, - "cluster-less PUT on an unbootstrapped account must 404: got %d body=%s", rec.Code, rec.Body.String()) - assert.Contains(t, rec.Body.String(), "cluster", - "the error must point the caller at the bootstrap paths: %s", rec.Body.String()) + "PUT on an unbootstrapped account must 404: got %d body=%s", rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), "/api/agent-network/settings", + "the error must point the caller at the bootstrap POST: %s", rec.Body.String()) } // TestSettingsHandler_PutReplacesMutableFields pins the update contract shared -// with the other PUT endpoints: the request replaces every mutable field, so a -// toggle absent from the JSON lands as its zero value rather than being -// preserved. Cluster and subdomain survive untouched. +// with the other PUT endpoints: the request carries every field, replacing the +// mutable ones. The identity fields ride along as a required echo of the +// assigned values — compared, never written — so the endpoint and proxy +// address survive every accepted update. func TestSettingsHandler_PutReplacesMutableFields(t *testing.T) { f := newAgentNetworkHandlerFixture(t) - rec := f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "eu.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": true, "redact_pii": true, "access_log_retention_days": 14}`) - require.Equal(t, http.StatusOK, rec.Code, "bootstrap PUT must succeed: %s", rec.Body.String()) + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "enable_prompt_collection": true, "redact_pii": true, "access_log_retention_days": 14}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) var before api.AgentNetworkSettings require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &before)) - rec = f.do(t, http.MethodPut, "/agent-network/settings", - `{"enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false}`) + rec = f.do(t, http.MethodPut, "/agent-network/settings", fmt.Sprintf( + `{"endpoint": %q, "proxy_address": %q, "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false, "access_log_retention_days": 7}`, + before.Endpoint, before.ProxyAddress)) require.Equal(t, http.StatusOK, rec.Code, "update PUT must succeed: %s", rec.Body.String()) var got api.AgentNetworkSettings @@ -103,35 +195,201 @@ func TestSettingsHandler_PutReplacesMutableFields(t *testing.T) { assert.False(t, got.EnablePromptCollection, "sent toggle must apply") assert.False(t, got.RedactPii, "sent toggle must apply") require.NotNil(t, got.AccessLogRetentionDays) - assert.Equal(t, 0, *got.AccessLogRetentionDays, - "retention absent from the request must land as the zero value — PUT replaces all mutable fields") - assert.Equal(t, before.Cluster, got.Cluster, "cluster must survive updates untouched") - assert.Equal(t, before.Subdomain, got.Subdomain, "subdomain must survive updates untouched") + assert.Equal(t, 7, *got.AccessLogRetentionDays, "sent retention must apply") + assert.Equal(t, before.Endpoint, got.Endpoint, "endpoint must survive updates untouched") + assert.Equal(t, before.ProxyAddress, got.ProxyAddress, "proxy address must survive updates untouched") } -// TestSettingsHandler_PutRejectsClusterChange pins cluster immutability: once -// assigned, a differing cluster is rejected as a validation error instead of -// being silently ignored, so callers never observe a value other than the one -// they sent. Echoing the assigned cluster back stays valid, which lets -// declarative clients send their full desired state idempotently. -func TestSettingsHandler_PutRejectsClusterChange(t *testing.T) { +// TestSettingsHandler_PutRejectsChangedIdentity pins the immutability contract: +// the PUT carries the identity fields like every other field, but they are an +// echo — a request carrying a different endpoint or proxy address is rejected +// as a validation error and the row is left untouched. The comparison is +// lenient about casing (the stored values are normalized lowercase), so a +// client replaying a GET response with different casing is not rejected. +func TestSettingsHandler_PutRejectsChangedIdentity(t *testing.T) { f := newAgentNetworkHandlerFixture(t) - rec := f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "eu.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false}`) - require.Equal(t, http.StatusOK, rec.Code, "bootstrap PUT must succeed: %s", rec.Body.String()) + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "enable_prompt_collection": true}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + var before api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &before)) - rec = f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "us.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false}`) - assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, - "cluster change must be rejected as a validation error: got %d body=%s", rec.Code, rec.Body.String()) + for name, body := range map[string]string{ + "changed endpoint": fmt.Sprintf( + `{"endpoint": "other.gateway.example.com", "proxy_address": %q, "enable_log_collection": false, "enable_prompt_collection": false, "redact_pii": false, "access_log_retention_days": 30}`, + before.ProxyAddress), + "changed proxy_address": fmt.Sprintf( + `{"endpoint": %q, "proxy_address": "us.proxy.netbird.io", "enable_log_collection": false, "enable_prompt_collection": false, "redact_pii": false, "access_log_retention_days": 30}`, + before.Endpoint), + "omitted identity": `{"enable_log_collection": false, "enable_prompt_collection": false, "redact_pii": false, "access_log_retention_days": 30}`, + } { + rec = f.do(t, http.MethodPut, "/agent-network/settings", body) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, + "%s must be rejected: got %d body=%s", name, rec.Code, rec.Body.String()) + } - rec = f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "eu.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": true}`) - require.Equal(t, http.StatusOK, rec.Code, "echoing the assigned cluster must stay valid: %s", rec.Body.String()) + // The rejected updates must not have applied anything — toggles included. + rec = f.do(t, http.MethodGet, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code) + var got api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, before.Endpoint, got.Endpoint, "rejected PUT must not change the endpoint") + assert.Equal(t, before.ProxyAddress, got.ProxyAddress, "rejected PUT must not change the proxy address") + assert.True(t, got.EnablePromptCollection, "rejected PUT must not apply its toggles") + + // An uppercased echo of the assigned values still names the same host and + // must be accepted. + rec = f.do(t, http.MethodPut, "/agent-network/settings", fmt.Sprintf( + `{"endpoint": %q, "proxy_address": %q, "enable_log_collection": true, "enable_prompt_collection": true, "redact_pii": false, "access_log_retention_days": 30}`, + strings.ToUpper(before.Endpoint), strings.ToUpper(before.ProxyAddress))) + assert.Equal(t, http.StatusOK, rec.Code, + "an uppercased identity echo must be accepted: got %d body=%s", rec.Code, rec.Body.String()) +} + +// TestSettingsHandler_PutOmittedRetentionLandsAsZero documents a residual the +// required-ness of access_log_retention_days does not remove. Marking the field +// required changes the generated client type from *int to int, so a generated +// client cannot omit it — but nothing validates OpenAPI required-ness at +// runtime, so a hand-rolled body without the field still decodes as 0, which +// the API documents as "keep indefinitely". +// +// That is the same latitude the three booleans already have, so it is left +// consistent rather than special-cased. This test exists to make the gap +// explicit: if request validation is ever added, this expectation is what +// changes. +func TestSettingsHandler_PutOmittedRetentionLandsAsZero(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "access_log_retention_days": 14}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + var before api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &before)) + + rec = f.do(t, http.MethodPut, "/agent-network/settings", fmt.Sprintf( + `{"endpoint": %q, "proxy_address": %q, "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false}`, + before.Endpoint, before.ProxyAddress)) + require.Equal(t, http.StatusOK, rec.Code, "update PUT must succeed: %s", rec.Body.String()) var got api.AgentNetworkSettings require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) - assert.Equal(t, "eu.proxy.netbird.io", got.Cluster, "cluster must be unchanged") - assert.True(t, got.RedactPii, "toggle sent alongside the echoed cluster must apply") + require.NotNil(t, got.AccessLogRetentionDays) + assert.Equal(t, 0, *got.AccessLogRetentionDays, + "a non-conforming body that omits retention still replaces it with the zero value") +} + +// TestSettingsHandler_DeleteBeforeBootstrapIs404 pins that DELETE on an +// account with no settings row is a 404, mirroring the PUT. +func TestSettingsHandler_DeleteBeforeBootstrapIs404(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodDelete, "/agent-network/settings", "") + assert.Equal(t, http.StatusNotFound, rec.Code, + "DELETE on an unbootstrapped account must 404: got %d body=%s", rec.Code, rec.Body.String()) +} + +// TestSettingsHandler_DeleteBlockedByProviders pins the first delete guard: +// while any provider exists for the account, the delete is refused with 412 +// and the row survives. Providers route through the endpoint — the guard +// keeps DELETE a bootstrap-repair operation rather than a way to abandon a +// configured gateway. +func TestSettingsHandler_DeleteBlockedByProviders(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", `{"proxy_address": "eu.proxy.netbird.io"}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + var before api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &before)) + + f.seedProvider(t, "prov-guard") + + rec = f.do(t, http.MethodDelete, "/agent-network/settings", "") + assert.Equal(t, http.StatusPreconditionFailed, rec.Code, + "delete with a provider present must be refused: got %d body=%s", rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodGet, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code) + var got api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, before.Endpoint, got.Endpoint, "the refused delete must leave the row intact") +} + +// TestSettingsHandler_DeleteBlockedByActiveProxy pins the second delete +// guard: while a proxy is actively serving the endpoint — an active proxy +// row declaring the endpoint hostname as its cluster address, the dedicated +// shape — the delete is refused with 412. A proxy that has disconnected no +// longer blocks: the guard is about a live serving path, not history. +// +// The proxy declares its address with mixed casing on purpose: Connect +// stores the declared address verbatim while the settings row is normalized +// lowercase, and hostnames are case-insensitive, so the guard must match +// across the casing difference rather than be sidestepped by it. +func TestSettingsHandler_DeleteBlockedByActiveProxy(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + const endpoint = "gw.dedicated.example.com" + rec := f.do(t, http.MethodPost, "/agent-network/settings", fmt.Sprintf(`{"endpoint": %q}`, endpoint)) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + + now := time.Now() + accountID := testAccountID + proxyRow := &rpproxy.Proxy{ + ID: "proxy-guard", + SessionID: "sess-1", + ClusterAddress: "GW.Dedicated.Example.Com", + AccountID: &accountID, + LastSeen: now, + ConnectedAt: &now, + Status: rpproxy.StatusConnected, + } + require.NoError(t, f.store.SaveProxy(context.Background(), proxyRow)) + + rec = f.do(t, http.MethodDelete, "/agent-network/settings", "") + assert.Equal(t, http.StatusPreconditionFailed, rec.Code, + "delete with an active proxy at the endpoint must be refused: got %d body=%s", rec.Code, rec.Body.String()) + + // Once the proxy disconnects it no longer serves the endpoint, so the + // delete goes through. + require.NoError(t, f.store.DisconnectProxy(context.Background(), proxyRow.ID, proxyRow.SessionID)) + rec = f.do(t, http.MethodDelete, "/agent-network/settings", "") + assert.Equal(t, http.StatusOK, rec.Code, + "delete after the proxy disconnected must succeed: got %d body=%s", rec.Code, rec.Body.String()) +} + +// TestSettingsHandler_DeleteReleasesEndpointForFreshBootstrap pins the +// full-reset semantic that gives replace-on-change clients (e.g. Terraform's +// RequiresReplace) a real path: with both guards clear the delete succeeds, +// the account reads as the defaults again, and a fresh bootstrap draws a +// fresh label. The released hostname is not reserved — a fresh draw may even +// legitimately re-pick it — so the assertions check the new row's shape, not +// that the label differs. +func TestSettingsHandler_DeleteReleasesEndpointForFreshBootstrap(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "enable_prompt_collection": true}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + + rec = f.do(t, http.MethodDelete, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code, + "delete with both guards clear must succeed: got %d body=%s", rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodGet, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code) + var after api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &after)) + assert.Empty(t, after.Endpoint, "a deleted account must read as unbootstrapped defaults") + assert.False(t, after.EnablePromptCollection, "the deleted row's toggles must not linger") + + rec = f.do(t, http.MethodPost, "/agent-network/settings", `{"proxy_address": "eu.proxy.netbird.io"}`) + require.Equal(t, http.StatusOK, rec.Code, "re-bootstrap after delete must succeed: %s", rec.Body.String()) + var second api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &second)) + require.NotEmpty(t, second.Endpoint, "the fresh bootstrap must allocate an endpoint") + assert.True(t, strings.HasSuffix(second.Endpoint, ".eu.proxy.netbird.io"), + "the fresh endpoint must hang beneath the requested proxy address: %s", second.Endpoint) + assert.False(t, second.EnablePromptCollection, + "the fresh row must carry bootstrap defaults, not the deleted row's toggles") + assert.NotNil(t, second.CreatedAt, "the fresh row is persisted and carries timestamps") } diff --git a/management/internals/modules/agentnetwork/labelgen/adjectives.go b/management/internals/modules/agentnetwork/labelgen/adjectives.go new file mode 100644 index 000000000..5058c2f1c --- /dev/null +++ b/management/internals/modules/agentnetwork/labelgen/adjectives.go @@ -0,0 +1,37 @@ +package labelgen + +// adjectives is the descriptor half of a generated label. It pairs with the +// noun pool in words.go to form `-` labels, and is kept +// separate because words.go is almost entirely nouns — drawing both halves +// from it produced unreadable pairs like "millet-hammock". Entries are +// lowercase ASCII, 4-12 chars, free of hyphens and digits, screened for +// offensive/brand/region-specific terms, and disjoint from the noun pool +// (enforced by TestAdjectives_AreDisjointFromNouns). +var adjectives = []string{ + "able", "active", "adept", "agile", "airy", "alert", "amiable", "ample", + "ancient", "ardent", "artful", "astute", "balmy", "blithe", "bold", "bonny", + "brave", "breezy", "brisk", "bubbly", "buoyant", "bushy", "candid", "canny", + "cheery", "chilly", "chipper", "chunky", "civil", "classic", "clever", "comely", + "compact", "cordial", "cosmic", "courtly", "crafty", "creamy", "crisp", "cuddly", + "curious", "dainty", "dapper", "daring", "dashing", "deft", "dewy", "diligent", + "downy", "dreamy", "dulcet", "durable", "dusky", "eager", "earnest", "earthy", + "easy", "elated", "elegant", "epic", "fabled", "faithful", "fancy", "fearless", + "feisty", "fervent", "fleet", "fluffy", "fond", "frisky", "frosty", "gallant", + "genial", "genteel", "gentle", "giddy", "gilded", "glad", "glassy", "gleaming", + "glossy", "graceful", "grand", "grainy", "hale", "hardy", "hearty", "hefty", + "honest", "hopeful", "humble", "hushed", "immense", "jaunty", "jolly", "jovial", + "joyful", "jubilant", "keen", "kindly", "kindred", "lanky", "leafy", "limber", + "lively", "lofty", "loyal", "lucent", "lucid", "luminous", "lush", "maroon", + "mellow", "merry", "mighty", "mindful", "mirthful", "misty", "modest", "muted", + "nifty", "nimble", "noble", "patient", "peaceful", "pearly", "peppy", "perky", + "petite", "placid", "playful", "pleasant", "plucky", "plush", "polite", "posh", + "prancing", "pristine", "prompt", "proud", "prudent", "quaint", "quick", "quirky", + "radiant", "ready", "regal", "restful", "robust", "rosy", "ruddy", "rugged", + "sandy", "satin", "saucy", "savvy", "sedate", "serene", "shady", "shiny", + "silken", "silky", "sincere", "sleek", "slender", "smart", "smooth", "snappy", + "snug", "soaring", "sparkly", "spiffy", "spirited", "sprightly", "spry", "stalwart", + "stately", "steady", "sterling", "stoic", "stormy", "stout", "sturdy", "sunlit", + "supple", "svelte", "tawny", "tender", "tidy", "timeless", "trusty", "upbeat", + "urbane", "valiant", "vast", "vernal", "vibrant", "vintage", "whimsy", "willing", + "windy", "winsome", "wintry", "witty", "worthy", "zesty", "zippy", +} diff --git a/management/internals/modules/agentnetwork/labelgen/labelgen.go b/management/internals/modules/agentnetwork/labelgen/labelgen.go index b45ff4ea8..549767096 100644 --- a/management/internals/modules/agentnetwork/labelgen/labelgen.go +++ b/management/internals/modules/agentnetwork/labelgen/labelgen.go @@ -64,3 +64,20 @@ func PickUnique(rng *rand.Rand, taken map[string]struct{}, fallbackSuffix string w := pool[rng.Intn(len(pool))] return fmt.Sprintf("%s-%s", w, fallbackSuffix) } + +// PickTuple returns an adjective-noun label such as "brave-otter". It is still +// a single DNS label. +// +// Unlike PickUnique it takes no `taken` set and has no fallback suffix. The +// noun pool holds 857 entries, which is ample per cluster but a hard ceiling +// once labels must be unique across one shared zone; pairing an adjective with +// a noun spans len(adjectives) * 857 instead. Uniqueness is enforced by a +// database constraint and retried by the caller, rather than guessed from a +// pre-read set that a concurrent allocation can invalidate. +func PickTuple(rng *rand.Rand) string { + nouns := uniqueWords() + if len(nouns) == 0 || len(adjectives) == 0 { + return "" + } + return adjectives[rng.Intn(len(adjectives))] + "-" + nouns[rng.Intn(len(nouns))] +} diff --git a/management/internals/modules/agentnetwork/labelgen/labelgen_test.go b/management/internals/modules/agentnetwork/labelgen/labelgen_test.go index f03a3501d..7e12fc133 100644 --- a/management/internals/modules/agentnetwork/labelgen/labelgen_test.go +++ b/management/internals/modules/agentnetwork/labelgen/labelgen_test.go @@ -99,3 +99,82 @@ func TestUniqueWords_DropsDuplicates(t *testing.T) { } assert.GreaterOrEqual(t, len(pool), 500, "Pool must contain at least 500 unique words") } + +// TestPickTuple_ShapeAndPoolMembership locks the wire-visible shape: an +// adjective and a noun, each from its own pool, joined by a single hyphen so +// the result stays one DNS label. +func TestPickTuple_ShapeAndPoolMembership(t *testing.T) { + nouns := uniqueWords() + inNouns := make(map[string]struct{}, len(nouns)) + for _, w := range nouns { + inNouns[w] = struct{}{} + } + inAdjectives := make(map[string]struct{}, len(adjectives)) + for _, a := range adjectives { + inAdjectives[a] = struct{}{} + } + + rng := rand.New(rand.NewSource(7)) + for i := 0; i < 200; i++ { + got := PickTuple(rng) + + parts := strings.Split(got, "-") + require.Len(t, parts, 2, "PickTuple must produce exactly two hyphen-joined words; got %q", got) + + _, adjOK := inAdjectives[parts[0]] + assert.True(t, adjOK, "First half must be an adjective; %q not in adjectives (from %q)", parts[0], got) + _, nounOK := inNouns[parts[1]] + assert.True(t, nounOK, "Second half must be a noun; %q not in words (from %q)", parts[1], got) + + assert.LessOrEqual(t, len(got), 63, "Label must fit a DNS label; got %q (%d chars)", got, len(got)) + } +} + +// TestAdjectives_AreDisjointFromNouns keeps the namespace a clean product and +// prevents nonsense like "azure-azure": a handful of the noun pool's entries +// are adjectival, and any overlap would let the same word land on both sides. +func TestAdjectives_AreDisjointFromNouns(t *testing.T) { + nouns := make(map[string]struct{}, len(uniqueWords())) + for _, w := range uniqueWords() { + nouns[w] = struct{}{} + } + for _, a := range adjectives { + _, clash := nouns[a] + assert.False(t, clash, "Adjective %q also appears in the noun pool; remove it from one list", a) + } +} + +// TestAdjectives_AreDNSSafeAndDeduplicated mirrors the curation contract stated +// in words.go: lowercase ASCII, 4-12 chars, no digits or hyphens, no repeats. +func TestAdjectives_AreDNSSafeAndDeduplicated(t *testing.T) { + seen := make(map[string]struct{}, len(adjectives)) + for _, a := range adjectives { + _, dup := seen[a] + assert.False(t, dup, "Duplicate adjective %q", a) + seen[a] = struct{}{} + + assert.Regexp(t, `^[a-z]{4,12}$`, a, "Adjective %q must be 4-12 lowercase ASCII letters", a) + } + assert.Greater(t, len(adjectives), 150, "Adjective pool too small to give a useful namespace") +} + +// TestPickTuple_DeterministicWithSeededRng documents that generation is a pure +// function of the rng, which is what makes allocation retries reproducible in tests. +func TestPickTuple_DeterministicWithSeededRng(t *testing.T) { + a := PickTuple(rand.New(rand.NewSource(42))) + b := PickTuple(rand.New(rand.NewSource(42))) + assert.Equal(t, a, b, "Same seed must yield the same tuple") +} + +// TestPickTuple_SpansALargeNamespace guards the reason we moved to tuples: a +// single-word pool caps the GLOBAL namespace at 857. Drawing many tuples must +// yield overwhelmingly distinct values. +func TestPickTuple_SpansALargeNamespace(t *testing.T) { + rng := rand.New(rand.NewSource(11)) + seen := make(map[string]struct{}, 2000) + for i := 0; i < 2000; i++ { + seen[PickTuple(rng)] = struct{}{} + } + assert.Greater(t, len(seen), 1900, + "2000 draws should be nearly all distinct across a ~200k namespace; got %d unique", len(seen)) +} diff --git a/management/internals/modules/agentnetwork/manager.go b/management/internals/modules/agentnetwork/manager.go index ba2c06826..379672989 100644 --- a/management/internals/modules/agentnetwork/manager.go +++ b/management/internals/modules/agentnetwork/manager.go @@ -22,7 +22,6 @@ import ( "github.com/netbirdio/netbird/management/server/permissions/modules" "github.com/netbirdio/netbird/management/server/permissions/operations" "github.com/netbirdio/netbird/management/server/store" - "github.com/netbirdio/netbird/shared/management/proto" "github.com/netbirdio/netbird/shared/management/status" ) @@ -48,7 +47,7 @@ func ensureSessionKeys(p *types.Provider) error { type Manager interface { GetAllProviders(ctx context.Context, accountID, userID string) ([]*types.Provider, error) GetProvider(ctx context.Context, accountID, userID, providerID string) (*types.Provider, error) - CreateProvider(ctx context.Context, userID string, provider *types.Provider, bootstrapCluster string) (*types.Provider, error) + CreateProvider(ctx context.Context, userID string, provider *types.Provider) (*types.Provider, error) UpdateProvider(ctx context.Context, userID string, provider *types.Provider) (*types.Provider, error) DeleteProvider(ctx context.Context, accountID, userID, providerID string) error @@ -71,7 +70,9 @@ type Manager interface { DeleteBudgetRule(ctx context.Context, accountID, userID, ruleID string) error GetSettings(ctx context.Context, accountID, userID string) (*types.Settings, error) + CreateSettings(ctx context.Context, userID string, settings *types.Settings, proxyAddress, endpoint string) (*types.Settings, error) UpdateSettings(ctx context.Context, userID string, settings *types.Settings) (*types.Settings, error) + DeleteSettings(ctx context.Context, accountID, userID string) error ListConsumption(ctx context.Context, accountID, userID string) ([]*types.Consumption, error) ListAccessLogs(ctx context.Context, accountID, userID string, filter types.AgentNetworkAccessLogFilter) ([]*types.AgentNetworkAccessLog, int64, error) @@ -123,11 +124,10 @@ type managerImpl struct { proxyController proxy.Controller // reconcileCache holds the last set of synthesised proxy mappings - // per account so reconcile can emit precise Create/Update/Delete - // updates instead of a full re-push on every mutation. Keyed by - // accountID, then by synthesised service ID. + // per account, each paired with the proxy that served it, so a change + // of serving proxy can be diffed without re-deriving it. reconcileMu sync.Mutex - reconcileCache map[string]map[string]*proto.ProxyMapping + reconcileCache map[string]map[string]syntheticMapping // labelRngMu guards labelRng. PickUnique consumes math/rand.Source // state; concurrent provider creates would otherwise race. @@ -151,7 +151,7 @@ func NewManager( accountManager: accountManager, permissionsManager: permissionsManager, proxyController: proxyController, - reconcileCache: make(map[string]map[string]*proto.ProxyMapping), + reconcileCache: make(map[string]map[string]syntheticMapping), labelRng: rand.New(rand.NewSource(time.Now().UnixNano())), } } @@ -170,19 +170,14 @@ func (m *managerImpl) GetProvider(ctx context.Context, accountID, userID, provid return m.store.GetAgentNetworkProviderByID(ctx, store.LockingStrengthNone, accountID, providerID) } -// CreateProvider persists a new provider for the account. bootstrapCluster -// is used only when the per-account agent-network Settings row hasn't -// been created yet; otherwise it is ignored (the cluster is pinned on -// Settings and every provider in the account routes through it). -func (m *managerImpl) CreateProvider(ctx context.Context, userID string, provider *types.Provider, bootstrapCluster string) (*types.Provider, error) { +// CreateProvider persists a new provider for the account. Providers have no +// settings side effects: the account's endpoint is bootstrapped separately and +// explicitly via CreateSettings, and every provider in the account routes +// through it. +func (m *managerImpl) CreateProvider(ctx context.Context, userID string, provider *types.Provider) (*types.Provider, error) { if err := m.requirePermission(ctx, provider.AccountID, userID, modules.AgentNetworkProviders, operations.Create); err != nil { return nil, err } - if strings.TrimSpace(bootstrapCluster) != "" { - if err := m.requireSettingsBootstrapPermission(ctx, provider.AccountID, userID); err != nil { - return nil, err - } - } // An empty api_key would silently produce a synthesised service // that 401s on every upstream request. Surface the misconfiguration @@ -206,16 +201,6 @@ func (m *managerImpl) CreateProvider(ctx context.Context, userID string, provide return nil, fmt.Errorf("save agent network provider: %w", err) } - if strings.TrimSpace(bootstrapCluster) != "" { - if _, err := m.bootstrapSettingsIfNeeded(ctx, m.store, provider.AccountID, bootstrapCluster); err != nil { - // The provider create has already succeeded; logging the - // bootstrap miss matches the plan's PoC behaviour. The synth - // path treats a missing settings row as a no-op, and the next - // provider create retries the bootstrap. - log.WithContext(ctx).Debugf("agent-network bootstrap settings for account %s on cluster %s: %v", provider.AccountID, bootstrapCluster, err) - } - } - m.accountManager.StoreEvent(ctx, userID, provider.ID, provider.AccountID, activity.AgentNetworkProviderCreated, provider.EventMeta()) m.reconcile(ctx, provider.AccountID) @@ -560,52 +545,44 @@ func (m *managerImpl) DeleteBudgetRule(ctx context.Context, accountID, userID, r } // UpdateSettings replaces the mutable account-level settings — the collection -// toggles and retention — on the account's row. When the account has no -// settings row yet, a non-empty settings.Cluster bootstraps one (same path as -// first provider create); without it the update fails with NotFound. On an -// existing row the cluster and subdomain are immutable: a differing -// settings.Cluster is rejected rather than silently ignored so callers never -// observe a value other than what they sent. Because the collection toggles -// change the synthesised service config (prompt-capture gating, access-log -// emission), a reconcile is triggered so the proxy and peer network maps -// converge on the new state. +// toggles and retention — on the account's row. The identity fields (Domain, +// ProxyAddress) are assigned at bootstrap (CreateSettings) and immutable: the +// request carries them, matching the PUT convention of every other endpoint, +// but they are only compared against the stored row — a request carrying +// different values is rejected, and the stored values are never overwritten. +// When the account has no settings row yet the update fails with NotFound. +// Because the collection toggles change the synthesised service config +// (prompt-capture gating, access-log emission), a reconcile is triggered so +// the proxy and peer network maps converge on the new state. func (m *managerImpl) UpdateSettings(ctx context.Context, userID string, settings *types.Settings) (*types.Settings, error) { if err := m.requirePermission(ctx, settings.AccountID, userID, modules.AgentNetworkSettings, operations.Update); err != nil { return nil, err } - requestedCluster := strings.TrimSpace(settings.Cluster) - // The row lock from LockingStrengthUpdate only holds for the duration of - // the surrounding transaction, so the read, the cluster-immutability - // check, and the save must share one — otherwise concurrent PUTs could - // interleave between them. + // the surrounding transaction, so the read and the save must share one — + // otherwise concurrent PUTs could interleave between them. var updated *types.Settings err := m.store.ExecuteInTransaction(ctx, func(tx store.Store) error { existing, err := tx.GetAgentNetworkSettings(ctx, store.LockingStrengthUpdate, settings.AccountID) switch { case err == nil: - if requestedCluster != "" && requestedCluster != existing.Cluster { - return status.Errorf(status.InvalidArgument, "cluster is immutable once assigned (current: %s)", existing.Cluster) - } case isNotFound(err): - if requestedCluster == "" { - return status.Errorf(status.NotFound, "agent network settings have not been bootstrapped yet; pass cluster to bootstrap them, or create a provider with bootstrap_cluster set") - } - // Bootstrapping pins the cluster and subdomain — a settings - // create on top of the update the caller already passed, matching - // the gate on the provider-create bootstrap path. - if err := m.requirePermission(ctx, settings.AccountID, userID, modules.AgentNetworkSettings, operations.Create); err != nil { - return err - } - existing, err = m.bootstrapSettingsIfNeeded(ctx, tx, settings.AccountID, requestedCluster) - if err != nil { - return err - } + return status.Errorf(status.NotFound, "agent network settings have not been bootstrapped yet; POST /api/agent-network/settings to bootstrap them") default: return fmt.Errorf("get agent network settings: %w", err) } + // The identity echo is compared leniently (trimmed, case-insensitive): + // the stored values are normalized lowercase, and a client replaying a + // GET response must never be rejected over casing it didn't choose. + if !hostnamesEquivalent(settings.Domain, existing.Domain) { + return status.Errorf(status.InvalidArgument, "endpoint is immutable: it must match the assigned endpoint %q; delete the settings to release it and bootstrap again", existing.Domain) + } + if !hostnamesEquivalent(settings.ProxyAddress, existing.ProxyAddress) { + return status.Errorf(status.InvalidArgument, "proxy_address is immutable: it must match the assigned proxy address %q; delete the settings to release it and bootstrap again", existing.ProxyAddress) + } + existing.EnableLogCollection = settings.EnableLogCollection existing.EnablePromptCollection = settings.EnablePromptCollection existing.RedactPii = settings.RedactPii @@ -632,6 +609,83 @@ func (m *managerImpl) UpdateSettings(ctx context.Context, userID string, setting return updated, nil } +// hostnamesEquivalent reports whether a caller-supplied hostname names the +// same host as a stored (normalized, lowercase) one: equal after trimming and +// case folding. No structural validation — an arbitrary mismatch and a +// malformed value are both simply "not the assigned value". +func hostnamesEquivalent(supplied, stored string) bool { + return strings.EqualFold(strings.TrimSpace(supplied), stored) +} + +// DeleteSettings removes the account's settings row, releasing the endpoint. +// Two guards make this a bootstrap-repair operation rather than a way to tear +// down a serving gateway, both re-checked under the row lock: +// +// - No Agent Network providers may exist for the account. Providers route +// through the endpoint; delete them first. +// - No proxy may be actively serving the endpoint — that is, no active proxy +// declares the endpoint hostname as its cluster address. This is the +// dedicated (self-addressed) shape's guard: the proxy at the address IS +// this account's gateway. A labeled endpoint hangs beneath a shared +// cluster's address, and with the account's providers already gone the +// shared proxy serves nothing of the account's, so the parent cluster +// being up does not block the delete. +// +// Bootstrapping again after a delete allocates fresh — the released hostname +// is not reserved. That full-reset semantic is what gives clients that model +// immutability as replace-on-change (e.g. Terraform's RequiresReplace) a real +// path: tear down providers, delete, re-create. +func (m *managerImpl) DeleteSettings(ctx context.Context, accountID, userID string) error { + if err := m.requirePermission(ctx, accountID, userID, modules.AgentNetworkSettings, operations.Delete); err != nil { + return err + } + + var deleted *types.Settings + err := m.store.ExecuteInTransaction(ctx, func(tx store.Store) error { + existing, err := tx.GetAgentNetworkSettings(ctx, store.LockingStrengthUpdate, accountID) + switch { + case err == nil: + case isNotFound(err): + return status.Errorf(status.NotFound, "agent network settings have not been bootstrapped yet; there is nothing to delete") + default: + return fmt.Errorf("get agent network settings: %w", err) + } + + providers, err := tx.GetAccountAgentNetworkProviders(ctx, store.LockingStrengthNone, accountID) + if err != nil { + return fmt.Errorf("get agent network providers: %w", err) + } + if len(providers) > 0 { + return status.Errorf(status.PreconditionFailed, "agent network settings cannot be deleted while %d provider(s) exist; delete the providers first", len(providers)) + } + + serving, err := tx.HasActiveProxyAtClusterAddress(ctx, existing.Domain) + if err != nil { + return fmt.Errorf("check for a proxy serving the endpoint: %w", err) + } + if serving { + return status.Errorf(status.PreconditionFailed, "agent network settings cannot be deleted while a proxy is actively serving the endpoint %q", existing.Domain) + } + + if err := tx.DeleteAgentNetworkSettings(ctx, accountID); err != nil { + return fmt.Errorf("delete agent network settings: %w", err) + } + deleted = existing + return nil + }) + if err != nil { + return err + } + + m.accountManager.StoreEvent(ctx, userID, accountID, accountID, activity.AgentNetworkSettingsDeleted, map[string]any{ + "endpoint": deleted.Domain, + "proxy_address": deleted.ProxyAddress, + }) + m.reconcile(ctx, accountID) + + return nil +} + // isNotFound reports whether err is a status.NotFound error. func isNotFound(err error) bool { var sErr *status.Error @@ -678,74 +732,162 @@ func (m *managerImpl) GetSettings(ctx context.Context, accountID, userID string) } } -// requireSettingsBootstrapPermission gates the one-time settings bootstrap a -// first provider create performs. Pinning the account's cluster and subdomain -// is a settings write, so it needs the settings permission on top of the -// provider one. No-op once the settings row exists. -func (m *managerImpl) requireSettingsBootstrapPermission(ctx context.Context, accountID, userID string) error { - _, err := m.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, accountID) - if err == nil { - return nil - } - if !isNotFound(err) { - return fmt.Errorf("get agent network settings: %w", err) - } - return m.requirePermission(ctx, accountID, userID, modules.AgentNetworkSettings, operations.Create) -} +// maxDomainAllocationAttempts bounds the label search when bootstrapping a +// labeled endpoint. Package-level (rather than function-local) so tests can +// assert on the exhaustion path without duplicating the literal. +const maxDomainAllocationAttempts = 10 -// bootstrapSettingsIfNeeded creates the per-account agent-network -// settings row when missing. The cluster comes from the create-time -// hint the dashboard sends (auto-picked from the active cluster list); -// the subdomain is picked from the curated wordlist avoiding -// collisions on the same cluster. Idempotent: if a row already exists -// it is returned untouched and the hint is ignored. st is the store to -// operate on — pass the transaction store when calling from within one. -func (m *managerImpl) bootstrapSettingsIfNeeded(ctx context.Context, st store.Store, accountID, providerCluster string) (*types.Settings, error) { - if accountID == "" { - return nil, fmt.Errorf("bootstrap settings: account id is required") +// CreateSettings bootstraps the per-account settings row, assigning the +// account's immutable endpoint. Exactly one of proxyAddress and endpoint must +// be non-empty: proxyAddress allocates a labeled endpoint one label beneath +// the given cluster address; endpoint claims the given hostname verbatim as a +// self-addressed (dedicated) endpoint — a legitimate claim before any proxy +// declares the address (address-first). settings carries the account ID and +// the initial collection toggles; its identity fields are assigned here. +func (m *managerImpl) CreateSettings(ctx context.Context, userID string, settings *types.Settings, proxyAddress, endpoint string) (*types.Settings, error) { + if settings == nil || settings.AccountID == "" { + return nil, status.Errorf(status.InvalidArgument, "account id is required") } - if strings.TrimSpace(providerCluster) == "" { - return nil, fmt.Errorf("bootstrap settings: provider cluster is required") + if err := m.requirePermission(ctx, settings.AccountID, userID, modules.AgentNetworkSettings, operations.Create); err != nil { + return nil, err } - existing, err := st.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, accountID) - if err == nil { - return existing, nil + hasProxyAddress := strings.TrimSpace(proxyAddress) != "" + hasEndpoint := strings.TrimSpace(endpoint) != "" + if hasProxyAddress == hasEndpoint { + return nil, status.Errorf(status.InvalidArgument, "exactly one of proxy_address and endpoint is required") } - if !isNotFound(err) { + + // Fail fast on an existing row for a clean 409; the insert below stays + // the authority against concurrent bootstraps (the primary key wins). + if _, err := m.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, settings.AccountID); err == nil { + return nil, status.Errorf(status.AlreadyExists, "agent network settings already bootstrapped for account %s", settings.AccountID) + } else if !isNotFound(err) { return nil, fmt.Errorf("get agent network settings: %w", err) } - siblings, err := st.GetAgentNetworkSettingsByCluster(ctx, store.LockingStrengthNone, providerCluster) - if err != nil { - return nil, fmt.Errorf("list agent network settings on cluster: %w", err) - } - taken := make(map[string]struct{}, len(siblings)) - for _, s := range siblings { - taken[s.Subdomain] = struct{}{} - } - - suffix := accountID - if len(suffix) > 4 { - suffix = suffix[:4] - } - - m.labelRngMu.Lock() - subdomain := labelgen.PickUnique(m.labelRng, taken, suffix) - m.labelRngMu.Unlock() - now := time.Now().UTC() - settings := types.DefaultSettings(accountID) - settings.Cluster = providerCluster - settings.Subdomain = subdomain settings.CreatedAt = now settings.UpdatedAt = now - if err := st.SaveAgentNetworkSettings(ctx, settings); err != nil { - return nil, fmt.Errorf("save agent network settings: %w", err) + + var err error + if hasEndpoint { + err = m.bootstrapSelfAddressed(ctx, settings, endpoint) + } else { + err = m.bootstrapLabeled(ctx, settings, proxyAddress) } + if err != nil { + return nil, err + } + + m.accountManager.StoreEvent(ctx, userID, settings.AccountID, settings.AccountID, activity.AgentNetworkSettingsUpdated, map[string]any{ + "bootstrapped": true, + "endpoint": settings.Domain, + "dedicated": settings.Dedicated(), + }) + m.reconcile(ctx, settings.AccountID) + return settings, nil } +// bootstrapSelfAddressed claims the given hostname as the account's endpoint, +// served only by a proxy declaring exactly that address (Domain == +// ProxyAddress). The domain unique index is the arbiter of availability. +func (m *managerImpl) bootstrapSelfAddressed(ctx context.Context, settings *types.Settings, endpoint string) error { + hostname, err := types.NormalizeHostname(endpoint) + if err != nil { + return status.Errorf(status.InvalidArgument, "invalid endpoint: %s", err) + } + + settings.Domain = hostname + settings.ProxyAddress = hostname + if err := m.store.CreateAgentNetworkSettings(ctx, settings); err != nil { + if isUniqueConstraintError(err) { + // The violation is either the account primary key (a concurrent + // bootstrap for the same account won) or the domain index + // (another account holds the hostname). Distinguish by re-read. + if _, getErr := m.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, settings.AccountID); getErr == nil { + return status.Errorf(status.AlreadyExists, "agent network settings already bootstrapped for account %s", settings.AccountID) + } + return status.Errorf(status.AlreadyExists, "endpoint %s is already taken", hostname) + } + return fmt.Errorf("create agent network settings: %w", err) + } + return nil +} + +// bootstrapLabeled allocates a labeled endpoint one label beneath the given +// cluster address: Domain =