diff --git a/.goreleaser_ui.yaml b/.goreleaser_ui.yaml index ca5148823..c61b8c474 100644 --- a/.goreleaser_ui.yaml +++ b/.goreleaser_ui.yaml @@ -96,6 +96,7 @@ nfpms: - netbird (>= 0.75.0) - libgtk-4-1 (>= 4.14) - libwebkitgtk-6.0-4 + - xdg-utils - maintainer: Netbird description: Netbird client UI. @@ -119,6 +120,7 @@ nfpms: - netbird >= 0.75.0 - (gtk4 >= 4.14 or libgtk-4-1 >= 4.14) - (webkitgtk6.0 or libwebkitgtk-6_0-4) + - xdg-utils rpm: signature: diff --git a/.goreleaser_ui_gtk3.yaml b/.goreleaser_ui_gtk3.yaml index d0a7de6e2..a6cfd199e 100644 --- a/.goreleaser_ui_gtk3.yaml +++ b/.goreleaser_ui_gtk3.yaml @@ -43,19 +43,17 @@ archives: - netbird-ui-gtk3 nfpms: - # Same package_name as the GTK4 packages -- the two are mutually-exclusive - # alternatives served from separate repo paths (see uploads below); a given - # distro points at exactly one of them. The file names must still differ: - # the Debian pool is shared storage keyed by file name, so a default-named - # gtk3 .deb would overwrite the stable one. + # Mutually-exclusive alternative to the GTK4 netbird-ui package -- both + # ship the same /usr/bin/netbird-ui from the shared stable/yum repos, so + # this one carries its own name and conflicts with the GTK4 package. - maintainer: Netbird description: Netbird client UI. homepage: https://netbird.io/ license: BSD-3-Clause vendor: NetBird id: netbird_ui_deb_gtk3 - package_name: netbird-ui - file_name_template: "{{ .PackageName }}-gtk3_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + package_name: netbird-ui-gtk3 + file_name_template: "{{ .PackageName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" builds: - netbird-ui-gtk3 formats: @@ -67,10 +65,15 @@ nfpms: dst: /usr/share/applications/org.wails.netbird.desktop - src: client/ui/build/appicon.png dst: /usr/share/pixmaps/netbird.png + conflicts: + - netbird-ui + replaces: + - netbird-ui dependencies: - netbird (>= 0.75.0) - libgtk-3-0 - libwebkit2gtk-4.1-0 + - xdg-utils - maintainer: Netbird description: Netbird client UI. @@ -78,8 +81,8 @@ nfpms: license: BSD-3-Clause vendor: NetBird id: netbird_ui_rpm_gtk3 - package_name: netbird-ui - file_name_template: "{{ .PackageName }}-gtk3_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + package_name: netbird-ui-gtk3 + file_name_template: "{{ .PackageName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" builds: - netbird-ui-gtk3 formats: @@ -91,10 +94,15 @@ nfpms: dst: /usr/share/applications/org.wails.netbird.desktop - src: client/ui/build/appicon.png dst: /usr/share/pixmaps/netbird.png + # No `replaces` here: nfpm maps it to rpm Obsoletes, which would make + # dnf swap installed GTK4 netbird-ui packages for this one on upgrade. + conflicts: + - netbird-ui dependencies: - netbird >= 0.75.0 - (gtk3 or libgtk-3-0) - (webkit2gtk4.1 or libwebkit2gtk-4_1-0) + - xdg-utils rpm: signature: @@ -109,23 +117,20 @@ changelog: disable: true uploads: - # The gtk3 packages reuse the netbird-ui package name, so they live in - # dedicated repo paths (deb distribution `gtk3`, yum path `yum-gtk3`) that - # legacy distros point their repo config at. - - name: debian-gtk3 + - name: debian skip: "{{ .Env.SKIP_PUBLISH }}" ids: - netbird_ui_deb_gtk3 mode: archive - target: https://pkgs.wiretrustee.com/debian/pool/{{ .ArtifactName }};deb.distribution=gtk3;deb.component=main;deb.architecture={{ if .Arm }}armhf{{ else }}{{ .Arch }}{{ end }};deb.package= + target: https://pkgs.wiretrustee.com/debian/pool/{{ .ArtifactName }};deb.distribution=stable;deb.component=main;deb.architecture={{ if .Arm }}armhf{{ else }}{{ .Arch }}{{ end }};deb.package= username: dev@wiretrustee.com method: PUT - - name: yum-gtk3 + - name: yum skip: "{{ .Env.SKIP_PUBLISH }}" ids: - netbird_ui_rpm_gtk3 mode: archive - target: https://pkgs.wiretrustee.com/yum-gtk3/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }} + target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }} username: dev@wiretrustee.com method: PUT diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db5097a48..233e7a442 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,6 +112,7 @@ aligns with our security standards and design expectations. - [Test suite](#test-suite) - [Checklist before submitting a PR](#checklist-before-submitting-a-pr) - [When we close a PR](#when-we-close-a-pr) + - [Translations](#translations) - [Other project repositories](#other-project-repositories) - [Contributor License Agreement](#contributor-license-agreement) @@ -612,6 +613,17 @@ A closed PR is not a rejected idea. Take it back to the [discussion](https://github.com/netbirdio/netbird/discussions), settle the approach, and reopen the work from there. +## Translations + +Desktop UI translations are not contributed through pull requests. Translate on +[Crowdin](https://crowdin.com/project/netbird) instead: no ticket needed, just +join the project and pick your language. Crowdin syncs with this repository and +opens the service PRs itself, so hand-edited locale files would conflict with +the next sync. Style, terminology, and review guidance live in +[client/ui/i18n/TRANSLATING.md](client/ui/i18n/TRANSLATING.md). To request a +language the project does not offer yet, ask on the Crowdin project page or in +a [discussion](https://github.com/netbirdio/netbird/discussions). + ## Other project repositories NetBird project is composed of 3 main repositories: diff --git a/client/android/client.go b/client/android/client.go index 5bbccda3d..bd34cc92a 100644 --- a/client/android/client.go +++ b/client/android/client.go @@ -15,6 +15,7 @@ import ( log "github.com/sirupsen/logrus" + nbAnonymize "github.com/netbirdio/netbird/client/anonymize" "github.com/netbirdio/netbird/client/iface/device" "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/debug" @@ -34,6 +35,13 @@ import ( types "github.com/netbirdio/netbird/upload-server/types" ) +// AnonymizeLevelDefault and AnonymizeLevelStrict are the accepted +// anonymizeLevel values for DebugBundle. +const ( + AnonymizeLevelDefault = nbAnonymize.LevelDefaultString + AnonymizeLevelStrict = nbAnonymize.LevelStrictString +) + // TunAdapter export internal TunAdapter for mobile type TunAdapter interface { device.TunAdapter @@ -303,8 +311,10 @@ func (c *Client) NotifyNetworkChange() { } // DebugBundle generates a debug bundle, uploads it, and returns the upload key. -// It works both with and without a running engine. -func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (string, error) { +// It works both with and without a running engine. anonymizeLevel is "default" +// or "strict"; strict also anonymizes internal IP ranges, peer names, and +// WireGuard public keys, and implies anonymize. +func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool, anonymizeLevel string) (string, error) { cfg, cacheDir, cc := c.stateSnapshot() // If the engine hasn't been started, load config from disk @@ -323,6 +333,7 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (strin InternalConfig: cfg, StatusRecorder: c.recorder, TempDir: cacheDir, + StatePath: platformFiles.StateFilePath(), } if cc != nil { @@ -346,6 +357,7 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (strin deps, debug.BundleConfig{ Anonymize: anonymize, + AnonymizeLevel: nbAnonymize.ParseLevel(anonymizeLevel), IncludeSystemInfo: true, }, ) diff --git a/client/anonymize/anonymize.go b/client/anonymize/anonymize.go index c140cef89..acadb717b 100644 --- a/client/anonymize/anonymize.go +++ b/client/anonymize/anonymize.go @@ -2,6 +2,7 @@ package anonymize import ( "crypto/rand" + "encoding/base64" "fmt" "math/big" "net" @@ -15,13 +16,88 @@ import ( const anonTLD = ".domain" +// Level selects how much the anonymizer redacts. Levels are ordered: a higher +// level redacts strictly more. On the wire (protos, flags) levels travel as +// their string form. +type Level int + +const ( + // LevelDefault anonymizes public IP addresses, IPv6 ULA, domains, and MAC + // addresses. Internal IPv4 ranges (RFC 1918, CGNAT, link-local) are + // preserved so support can reason about the real topology. + LevelDefault Level = iota + // LevelStrict additionally anonymizes internal IP ranges, peer names, and + // WireGuard public keys. + LevelStrict +) + +// LevelDefaultString and LevelStrictString are the wire forms of the levels, +// for boundaries that pass levels as strings (flags, protos, mobile bindings). +const ( + LevelDefaultString = "default" + LevelStrictString = "strict" +) + +// ParseLevel maps s to a Level. Empty means LevelDefault; anything +// unrecognized maps to LevelStrict so an unknown request never yields less +// anonymization than intended. +func ParseLevel(s string) Level { + switch strings.ToLower(s) { + case "", LevelDefaultString: + return LevelDefault + default: + return LevelStrict + } +} + +// String returns the wire form of the level: "default" or "strict". +func (l Level) String() string { + if l >= LevelStrict { + return LevelStrictString + } + return LevelDefaultString +} + +// protectedDomains are NetBird-operated suffixes that stay recognizable in an +// anonymized bundle. At LevelStrict the labels in front of them (the peer +// name) are still replaced, except under netbird.io, which only hosts +// NetBird infrastructure (api, signal, flow), never peer names. +var protectedDomains = []string{"netbird.io", "netbird.selfhosted", "netbird.cloud", "netbird.stage"} + +const infraDomain = "netbird.io" + +var ( + macColonRegex = regexp.MustCompile(`\b[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}\b`) + macDashRegex = regexp.MustCompile(`\b[0-9a-fA-F]{2}(?:-[0-9a-fA-F]{2}){5}\b`) + wgKeyRegex = regexp.MustCompile(`\b[A-Za-z0-9+/]{43}=`) +) + type Anonymizer struct { ipAnonymizer map[netip.Addr]netip.Addr domainAnonymizer map[string]string - currentAnonIPv4 netip.Addr - currentAnonIPv6 netip.Addr - startAnonIPv4 netip.Addr - startAnonIPv6 netip.Addr + // domainOrder caches the keys of domainAnonymizer sorted longest-first + // for AnonymizeString; it is rebuilt when the map gains entries. + domainOrder []string + labelAnonymizer map[string]string + labelAnonymized map[string]struct{} + labelCounter uint32 + macAnonymizer map[string]string + macCounter uint32 + wgKeyAnonymizer map[string]string + wgKeyAnonymized map[string]struct{} + currentAnonIPv4 netip.Addr + currentAnonIPv6 netip.Addr + startAnonIPv4 netip.Addr + startAnonIPv6 netip.Addr + + // LevelStrict also anonymizes internal ranges (RFC 1918, CGNAT, + // link-local), replacing them from the dedicated internal pools below so + // a reader can still tell an internal address from a public one. + level Level + currentAnonInternalIPv4 netip.Addr + currentAnonInternalIPv6 netip.Addr + startAnonInternalIPv4 netip.Addr + startAnonInternalIPv6 netip.Addr domainKeyRegex *regexp.Regexp } @@ -32,25 +108,50 @@ func DefaultAddresses() (netip.Addr, netip.Addr) { return netip.AddrFrom4([4]byte{198, 51, 100, 0}), netip.MustParseAddr("2001:db8:ffff::") } +// InternalAddresses returns the pool starts used in strict mode for internal +// ranges. Both are reserved ranges that cannot collide with real addressing: +// 198.18.0.0 (RFC 2544 benchmarking), 2001:db8:1:: (RFC 3849 documentation). +func InternalAddresses() (netip.Addr, netip.Addr) { + return netip.AddrFrom4([4]byte{198, 18, 0, 0}), netip.MustParseAddr("2001:db8:1::") +} + func NewAnonymizer(startIPv4, startIPv6 netip.Addr) *Anonymizer { + internalIPv4, internalIPv6 := InternalAddresses() return &Anonymizer{ ipAnonymizer: map[netip.Addr]netip.Addr{}, domainAnonymizer: map[string]string{}, + labelAnonymizer: map[string]string{}, + labelAnonymized: map[string]struct{}{}, + macAnonymizer: map[string]string{}, + wgKeyAnonymizer: map[string]string{}, + wgKeyAnonymized: map[string]struct{}{}, currentAnonIPv4: startIPv4, currentAnonIPv6: startIPv6, startAnonIPv4: startIPv4, startAnonIPv6: startIPv6, + level: LevelDefault, + currentAnonInternalIPv4: internalIPv4, + currentAnonInternalIPv6: internalIPv6, + startAnonInternalIPv4: internalIPv4, + startAnonInternalIPv6: internalIPv6, + domainKeyRegex: regexp.MustCompile(`\bdomain=([^\s,:"]+)`), } } +// SetLevel selects the anonymization level. The zero value of a new +// Anonymizer is LevelDefault. +func (a *Anonymizer) SetLevel(level Level) { + a.level = level +} + func (a *Anonymizer) AnonymizeIP(ip netip.Addr) netip.Addr { + // Normalize 4-in-6 addresses so ::ffff:192.168.1.1 classifies and maps + // like 192.168.1.1. + ip = ip.Unmap() + if ip.IsLoopback() || - ip.IsLinkLocalUnicast() || - ip.IsLinkLocalMulticast() || - ip.IsInterfaceLocalMulticast() || - (ip.Is4() && ip.IsPrivate()) || ip.IsUnspecified() || ip.IsMulticast() || isWellKnown(ip) || @@ -59,18 +160,100 @@ func (a *Anonymizer) AnonymizeIP(ip netip.Addr) netip.Addr { return ip } + if isInternal(ip) && a.level < LevelStrict { + return ip + } + if _, ok := a.ipAnonymizer[ip]; !ok { - if ip.Is4() { - a.ipAnonymizer[ip] = a.currentAnonIPv4 - a.currentAnonIPv4 = a.currentAnonIPv4.Next() - } else { - a.ipAnonymizer[ip] = a.currentAnonIPv6 - a.currentAnonIPv6 = a.currentAnonIPv6.Next() - } + a.ipAnonymizer[ip] = a.nextAnonIP(ip) } return a.ipAnonymizer[ip] } +func (a *Anonymizer) nextAnonIP(ip netip.Addr) netip.Addr { + // At the strict level, internal addresses (including IPv6 ULA, matched + // by IsPrivate) come from the internal pools so they remain recognizable + // as internal without disclosing the real values. + if a.level >= LevelStrict && (isInternal(ip) || ip.IsPrivate()) { + if ip.Is4() { + anon := a.currentAnonInternalIPv4 + a.currentAnonInternalIPv4 = a.currentAnonInternalIPv4.Next() + return anon + } + anon := a.currentAnonInternalIPv6 + a.currentAnonInternalIPv6 = a.currentAnonInternalIPv6.Next() + return anon + } + + if ip.Is4() { + anon := a.currentAnonIPv4 + a.currentAnonIPv4 = a.currentAnonIPv4.Next() + return anon + } + anon := a.currentAnonIPv6 + a.currentAnonIPv6 = a.currentAnonIPv6.Next() + return anon +} + +// AnonymizeMAC replaces a MAC address with a consistent placeholder from the +// locally administered range starting at 02:00:00:00:00:01, at every +// anonymization level. Broadcast, multicast, all-zero, and already assigned +// placeholder addresses are preserved. The colon and dash spellings of the +// same address share one placeholder; the output keeps the input's separator. +func (a *Anonymizer) AnonymizeMAC(mac string) string { + hw, err := net.ParseMAC(mac) + if err != nil || len(hw) != 6 { + return mac + } + + if isWellKnownMAC(hw) || a.isAnonymizedMAC(hw) { + return mac + } + + key := hw.String() + anon, ok := a.macAnonymizer[key] + if !ok { + a.macCounter++ + anon = fmt.Sprintf("02:00:00:%02x:%02x:%02x", byte(a.macCounter>>16), byte(a.macCounter>>8), byte(a.macCounter)) + a.macAnonymizer[key] = anon + } + + if strings.Contains(mac, "-") { + anon = strings.ReplaceAll(anon, ":", "-") + } + return anon +} + +// isAnonymizedMAC reports whether hw is a placeholder this anonymizer already +// handed out, so a second pass over anonymized output leaves it unchanged. +func (a *Anonymizer) isAnonymizedMAC(hw net.HardwareAddr) bool { + if hw[0] != 0x02 || hw[1] != 0 || hw[2] != 0 { + return false + } + value := uint32(hw[3])<<16 | uint32(hw[4])<<8 | uint32(hw[5]) + return value <= a.macCounter +} + +// AnonymizeWGKey replaces a WireGuard public key with a consistent random +// placeholder of the same shape. Keys are only anonymized at LevelStrict; +// placeholders already handed out pass through unchanged. +func (a *Anonymizer) AnonymizeWGKey(key string) string { + if a.level < LevelStrict || !looksLikeWGKey(key) { + return key + } + if _, ok := a.wgKeyAnonymized[key]; ok { + return key + } + + anon, ok := a.wgKeyAnonymizer[key] + if !ok { + anon = generateAnonymousKey() + a.wgKeyAnonymizer[key] = anon + a.wgKeyAnonymized[anon] = struct{}{} + } + return anon +} + func (a *Anonymizer) AnonymizeUDPAddr(addr net.UDPAddr) net.UDPAddr { // Convert IP to netip.Addr ip, ok := netip.AddrFromSlice(addr.IP) @@ -89,12 +272,12 @@ func (a *Anonymizer) AnonymizeUDPAddr(addr net.UDPAddr) net.UDPAddr { // isInAnonymizedRange checks if an IP is within the range of already assigned anonymized IPs func (a *Anonymizer) isInAnonymizedRange(ip netip.Addr) bool { - if ip.Is4() && ip.Compare(a.startAnonIPv4) >= 0 && ip.Compare(a.currentAnonIPv4) <= 0 { - return true - } else if !ip.Is4() && ip.Compare(a.startAnonIPv6) >= 0 && ip.Compare(a.currentAnonIPv6) <= 0 { - return true + if ip.Is4() { + return inPoolRange(ip, a.startAnonIPv4, a.currentAnonIPv4) || + inPoolRange(ip, a.startAnonInternalIPv4, a.currentAnonInternalIPv4) } - return false + return inPoolRange(ip, a.startAnonIPv6, a.currentAnonIPv6) || + inPoolRange(ip, a.startAnonInternalIPv6, a.currentAnonInternalIPv6) } func (a *Anonymizer) AnonymizeIPString(ip string) string { @@ -118,14 +301,17 @@ func (a *Anonymizer) AnonymizeDomain(domain string) string { baseDomain = domain[:len(domain)-1] } - if strings.HasSuffix(baseDomain, "netbird.io") || - strings.HasSuffix(baseDomain, "netbird.selfhosted") || - strings.HasSuffix(baseDomain, "netbird.cloud") || - strings.HasSuffix(baseDomain, "netbird.stage") || - strings.HasSuffix(baseDomain, anonTLD) { + if strings.HasSuffix(baseDomain, anonTLD) { return domain } + if suffix := protectedSuffix(baseDomain); suffix != "" { + if a.level < LevelStrict || baseDomain == suffix || suffix == infraDomain { + return domain + } + return withTrailingDot(a.anonymizePeerName(baseDomain, suffix), hasDot) + } + parts := strings.Split(baseDomain, ".") if len(parts) < 2 { return domain @@ -141,12 +327,53 @@ func (a *Anonymizer) AnonymizeDomain(domain string) string { } result := strings.Replace(baseDomain, baseForLookup, anonymized, 1) - if hasDot { - result += "." + if a.level >= LevelStrict && len(parts) > 2 { + prefix := strings.TrimSuffix(baseDomain, "."+baseForLookup) + result = a.anonymizeLabels(prefix, "host") + "." + anonymized + // The full mapping feeds AnonymizeString so seeded FQDNs are caught + // in log lines as a whole, labels included. + a.domainAnonymizer[baseDomain] = result + } + return withTrailingDot(result, hasDot) +} + +// anonymizePeerName replaces the labels in front of a protected suffix with +// numbered peer placeholders, keeping the suffix, and records the full +// mapping for string replacement in logs. The numbering keeps a peer +// recognizable across the whole bundle without disclosing its name. +func (a *Anonymizer) anonymizePeerName(baseDomain, suffix string) string { + prefix := strings.TrimSuffix(baseDomain, "."+suffix) + result := a.anonymizeLabels(prefix, "peer") + "." + suffix + if result != baseDomain { + a.domainAnonymizer[baseDomain] = result } return result } +// anonymizeLabels replaces each dot-separated label with a consistent +// numbered placeholder ("-"). Wildcard labels and +// placeholders already handed out pass through unchanged. +func (a *Anonymizer) anonymizeLabels(prefix, placeholder string) string { + labels := strings.Split(prefix, ".") + for i, label := range labels { + if label == "*" { + continue + } + if _, ok := a.labelAnonymized[label]; ok { + continue + } + anon, ok := a.labelAnonymizer[label] + if !ok { + a.labelCounter++ + anon = fmt.Sprintf("%s-%d", placeholder, a.labelCounter) + a.labelAnonymizer[label] = anon + a.labelAnonymized[anon] = struct{}{} + } + labels[i] = anon + } + return strings.Join(labels, ".") +} + func (a *Anonymizer) AnonymizeURI(uri string) string { u, err := url.Parse(uri) if err != nil { @@ -181,16 +408,70 @@ func (a *Anonymizer) AnonymizeString(str string) string { str = ipv4Regex.ReplaceAllStringFunc(str, a.AnonymizeIPString) str = ipv6Regex.ReplaceAllStringFunc(str, a.AnonymizeIPString) - for domain, anonDomain := range a.domainAnonymizer { - str = strings.ReplaceAll(str, domain, anonDomain) + for _, domain := range a.sortedDomains() { + str = strings.ReplaceAll(str, domain, a.domainAnonymizer[domain]) } str = a.AnonymizeSchemeURI(str) str = a.AnonymizeDNSLogLine(str) + // MAC handling runs after the IP passes so preserved IPv6 addresses are + // already out of the way; the separator guard skips matches embedded in a + // longer colon- or dash-separated sequence (such as an IPv6 tail). + str = a.anonymizeMACsInString(str, macColonRegex, ':') + str = a.anonymizeMACsInString(str, macDashRegex, '-') + + if a.level >= LevelStrict { + str = wgKeyRegex.ReplaceAllStringFunc(str, a.AnonymizeWGKey) + } + return str } +// sortedDomains returns the domain mappings longest-first, so a full-FQDN +// mapping (strict level) is applied before the base-domain mapping it +// contains. The order is rebuilt only when domainAnonymizer has grown. +func (a *Anonymizer) sortedDomains() []string { + if len(a.domainOrder) == len(a.domainAnonymizer) { + return a.domainOrder + } + + a.domainOrder = a.domainOrder[:0] + for domain := range a.domainAnonymizer { + a.domainOrder = append(a.domainOrder, domain) + } + slices.SortFunc(a.domainOrder, func(x, y string) int { + if d := len(y) - len(x); d != 0 { + return d + } + return strings.Compare(x, y) + }) + return a.domainOrder +} + +// anonymizeMACsInString replaces MAC addresses matched by re, skipping +// matches that directly adjoin another sep so a six-group run inside a longer +// separated sequence is left alone. +func (a *Anonymizer) anonymizeMACsInString(str string, re *regexp.Regexp, sep byte) string { + matches := re.FindAllStringIndex(str, -1) + if len(matches) == 0 { + return str + } + + var b strings.Builder + last := 0 + for _, m := range matches { + if (m[0] > 0 && str[m[0]-1] == sep) || (m[1] < len(str) && str[m[1]] == sep) { + continue + } + b.WriteString(str[last:m[0]]) + b.WriteString(a.AnonymizeMAC(str[m[0]:m[1]])) + last = m[1] + } + b.WriteString(str[last:]) + return b.String() +} + // AnonymizeSchemeURI finds and anonymizes URIs with ws, wss, rel, rels, stun, stuns, turn, and turns schemes. func (a *Anonymizer) AnonymizeSchemeURI(text string) string { re := regexp.MustCompile(`(?i)\b(wss?://|rels?://|stuns?:|turns?:|https?://)\S+\b`) @@ -239,10 +520,79 @@ func isWellKnown(addr netip.Addr) bool { "128.0.0.0", "8000::", // 2nd split subnet for default routes } - if slices.Contains(wellKnown, addr.String()) { + return slices.Contains(wellKnown, addr.String()) +} + +// isInternal reports whether ip identifies a host only within the local +// network: IPv4 private (RFC 1918), CGNAT (RFC 6598), and link-local (v4 and +// v6). These are preserved at the default level so support can reason about +// the real topology, and replaced from the internal pools at the strict +// level. IPv6 ULA is deliberately not internal: its random global ID uniquely +// fingerprints the network, so it is anonymized at every level. +func isInternal(ip netip.Addr) bool { + return (ip.Is4() && ip.IsPrivate()) || + ip.IsLinkLocalUnicast() || + isCGNAT(ip) +} + +func inPoolRange(ip, start, current netip.Addr) bool { + return ip.Compare(start) >= 0 && ip.Compare(current) <= 0 +} + +// isWellKnownMAC reports whether hw carries no stable host identity: all-zero +// or a group address (broadcast and multicast). +func isWellKnownMAC(hw net.HardwareAddr) bool { + if hw[0]&1 == 1 { return true } + for _, b := range hw { + if b != 0 { + return false + } + } + return true +} +// looksLikeWGKey reports whether s has the shape of a WireGuard key: +// 44 base64 characters decoding to 32 bytes. +func looksLikeWGKey(s string) bool { + if len(s) != 44 || s[43] != '=' { + return false + } + decoded, err := base64.StdEncoding.DecodeString(s) + return err == nil && len(decoded) == 32 +} + +func generateAnonymousKey() string { + buf := make([]byte, 32) + if _, err := rand.Read(buf); err != nil { + return strings.Repeat("A", 43) + "=" + } + return base64.StdEncoding.EncodeToString(buf) +} + +// protectedSuffix returns the protected NetBird suffix baseDomain ends with, +// or empty. The match is label-anchored so an unrelated domain that merely +// ends in the same characters is not preserved. +func protectedSuffix(baseDomain string) string { + for _, d := range protectedDomains { + if baseDomain == d || strings.HasSuffix(baseDomain, "."+d) { + return d + } + } + return "" +} + +func withTrailingDot(domain string, hasDot bool) string { + if hasDot { + return domain + "." + } + return domain +} + +// isCGNAT reports whether addr is in 100.64.0.0/10 (RFC 6598), the range +// NetBird assigns overlay peer addresses from. +func isCGNAT(addr netip.Addr) bool { cgnatRangeStart := netip.AddrFrom4([4]byte{100, 64, 0, 0}) cgnatRange := netip.PrefixFrom(cgnatRangeStart, 10) diff --git a/client/anonymize/anonymize_test.go b/client/anonymize/anonymize_test.go index 852315fa1..7c3c7bcf8 100644 --- a/client/anonymize/anonymize_test.go +++ b/client/anonymize/anonymize_test.go @@ -1,8 +1,11 @@ package anonymize_test import ( + "bytes" + "encoding/base64" "net/netip" "regexp" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -44,6 +47,301 @@ func TestAnonymizeIP(t *testing.T) { } } +func TestParseLevel(t *testing.T) { + tests := []struct { + input string + expect anonymize.Level + }{ + {"", anonymize.LevelDefault}, + {"default", anonymize.LevelDefault}, + {"DEFAULT", anonymize.LevelDefault}, + {"strict", anonymize.LevelStrict}, + {"STRICT", anonymize.LevelStrict}, + // Unknown values must never yield less anonymization than requested. + {"garbage", anonymize.LevelStrict}, + } + + for _, tc := range tests { + t.Run("input="+tc.input, func(t *testing.T) { + assert.Equal(t, tc.expect, anonymize.ParseLevel(tc.input), "parsed level should match") + }) + } +} + +func TestAnonymizeIP_DefaultLevelInternalRanges(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + tests := []struct { + name string + ip string + expect string + }{ + {"RFC1918 10/8", "10.1.2.3", "10.1.2.3"}, + {"RFC1918 172.16/12", "172.16.5.5", "172.16.5.5"}, + {"RFC1918 192.168/16", "192.168.1.1", "192.168.1.1"}, + {"CGNAT", "100.64.0.5", "100.64.0.5"}, + {"IPv4 link-local", "169.254.1.1", "169.254.1.1"}, + {"IPv6 link-local", "fe80::1", "fe80::1"}, + // ULA is anonymized even at the default level: its random global ID + // uniquely fingerprints the network, unlike shared RFC 1918 space. + {"IPv6 ULA", "fd12:3456:789a::1", "2001:db8:ffff::"}, + // 4-in-6 addresses classify like their unmapped IPv4 form. + {"4-in-6 RFC1918", "::ffff:192.168.1.1", "192.168.1.1"}, + {"4-in-6 CGNAT", "::ffff:100.64.0.5", "100.64.0.5"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := anonymizer.AnonymizeIP(netip.MustParseAddr(tc.ip)) + assert.Equal(t, tc.expect, result.String(), "default level should preserve internal ranges except ULA") + }) + } +} + +func TestAnonymizeIP_StrictLevel(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + // Order matters: internal pool addresses are assigned sequentially. + tests := []struct { + name string + ip string + expect string + }{ + {"RFC1918 192.168/16", "192.168.1.1", "198.18.0.0"}, + {"Second RFC1918", "192.168.1.2", "198.18.0.1"}, + {"Repeated RFC1918", "192.168.1.1", "198.18.0.0"}, + {"RFC1918 10/8", "10.1.2.3", "198.18.0.2"}, + {"RFC1918 172.16/12", "172.16.5.5", "198.18.0.3"}, + {"CGNAT", "100.64.0.5", "198.18.0.4"}, + {"IPv4 link-local", "169.254.1.1", "198.18.0.5"}, + {"Public IPv4 uses public pool", "1.2.3.4", "198.51.100.0"}, + {"IPv6 link-local", "fe80::1", "2001:db8:1::"}, + {"IPv6 ULA", "fd12:3456:789a::1", "2001:db8:1::1"}, + {"Public IPv6 uses public pool", "2607:f8b0:4005:805::200e", "2001:db8:ffff::"}, + {"Loopback IPv4", "127.0.0.1", "127.0.0.1"}, + {"Loopback IPv6", "::1", "::1"}, + {"Unspecified", "0.0.0.0", "0.0.0.0"}, + {"Multicast", "224.0.0.251", "224.0.0.251"}, + {"Well known resolver", "8.8.8.8", "8.8.8.8"}, + {"Well known split marker", "128.0.0.0", "128.0.0.0"}, + {"In internal pool range", "198.18.0.3", "198.18.0.3"}, + {"In public pool range", "198.51.100.0", "198.51.100.0"}, + {"4-in-6 repeated RFC1918", "::ffff:192.168.1.1", "198.18.0.0"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := anonymizer.AnonymizeIP(netip.MustParseAddr(tc.ip)) + assert.Equal(t, tc.expect, result.String(), "strict level should replace internal ranges from the internal pools") + }) + } +} + +func TestAnonymizeString_StrictInternalIPs(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + input := "route 10.20.30.0/24 via 192.168.1.1 dev eth0 src 100.64.0.7" + firstPass := anonymizer.AnonymizeString(input) + secondPass := anonymizer.AnonymizeString(firstPass) + + assert.NotContains(t, firstPass, "10.20.30.0", "private network address should be anonymized") + assert.NotContains(t, firstPass, "192.168.1.1", "private gateway should be anonymized") + assert.NotContains(t, firstPass, "100.64.0.7", "CGNAT address should be anonymized") + assert.Contains(t, firstPass, "/24", "prefix length should be preserved") + assert.Equal(t, firstPass, secondPass, "second pass should not further anonymize the string") +} + +func TestAnonymizeMAC(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + first := anonymizer.AnonymizeMAC("aa:bb:cc:dd:ee:0f") + assert.Equal(t, "02:00:00:00:00:01", first, "first MAC should get the first placeholder") + assert.Equal(t, first, anonymizer.AnonymizeMAC("aa:bb:cc:dd:ee:0f"), "repeated MAC should map to the same placeholder") + assert.Equal(t, first, anonymizer.AnonymizeMAC("AA:BB:CC:DD:EE:0F"), "case should not affect the mapping") + assert.Equal(t, "02-00-00-00-00-01", anonymizer.AnonymizeMAC("AA-BB-CC-DD-EE-0F"), "dash form should keep its separator but share the mapping") + + second := anonymizer.AnonymizeMAC("10:22:33:44:55:66") + assert.Equal(t, "02:00:00:00:00:02", second, "second distinct MAC should get the next placeholder") + + tests := []struct { + name string + mac string + }{ + {"Broadcast", "ff:ff:ff:ff:ff:ff"}, + {"IPv4 multicast", "01:00:5e:00:00:fb"}, + {"IPv6 multicast", "33:33:00:00:00:01"}, + {"All zero", "00:00:00:00:00:00"}, + {"Assigned placeholder", "02:00:00:00:00:01"}, + {"Invalid", "not-a-mac"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.mac, anonymizer.AnonymizeMAC(tc.mac), "should be preserved") + }) + } +} + +func TestAnonymizeString_MACAddresses(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + tests := []struct { + name string + input string + expect string + }{ + { + name: "nftables ether rule", + input: "ether saddr aa:bb:cc:dd:ee:ff drop", + expect: "ether saddr 02:00:00:00:00:01 drop", + }, + { + name: "Windows dash form", + input: "Physical Address : AA-BB-CC-DD-EE-FF", + expect: "Physical Address : 02-00-00-00-00-01", + }, + { + name: "IPv6 address tail is not treated as MAC", + input: "addr fe80:0:11:22:33:44:55:66 scope link", + expect: "addr fe80:0:11:22:33:44:55:66 scope link", + }, + { + name: "broadcast MAC preserved", + input: "dst ff:ff:ff:ff:ff:ff type ARP", + expect: "dst ff:ff:ff:ff:ff:ff type ARP", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := anonymizer.AnonymizeString(tc.input) + assert.Equal(t, tc.expect, result, "MAC addresses should be anonymized at every level") + assert.Equal(t, result, anonymizer.AnonymizeString(result), "second pass should not change the result") + }) + } +} + +func TestAnonymizeWGKey(t *testing.T) { + key := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0x42}, 32)) + + t.Run("default level preserves keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + assert.Equal(t, key, anonymizer.AnonymizeWGKey(key), "default level should not touch WireGuard keys") + }) + + t.Run("strict level replaces keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + anon := anonymizer.AnonymizeWGKey(key) + assert.NotEqual(t, key, anon, "strict level should replace the key") + assert.Regexp(t, `^[A-Za-z0-9+/]{43}=$`, anon, "placeholder should keep the WireGuard key shape") + assert.Equal(t, anon, anonymizer.AnonymizeWGKey(key), "repeated key should map to the same placeholder") + assert.Equal(t, anon, anonymizer.AnonymizeWGKey(anon), "an assigned placeholder should pass through unchanged") + + assert.Equal(t, "not-a-key", anonymizer.AnonymizeWGKey("not-a-key"), "non-key values should be preserved") + }) +} + +func TestAnonymizeString_WGKeys(t *testing.T) { + key := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0x42}, 32)) + input := "peer " + key + " handshake completed" + + t.Run("default level preserves keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + assert.Equal(t, input, anonymizer.AnonymizeString(input), "default level should not touch WireGuard keys in strings") + }) + + t.Run("strict level replaces keys", func(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + firstPass := anonymizer.AnonymizeString(input) + assert.NotContains(t, firstPass, key, "the key should not survive strict anonymization") + assert.Equal(t, anonymizer.AnonymizeWGKey(key), extractKey(t, firstPass), "string replacement should be consistent with AnonymizeWGKey") + assert.Equal(t, firstPass, anonymizer.AnonymizeString(firstPass), "second pass should not change the result") + }) +} + +func extractKey(t *testing.T, logLine string) string { + t.Helper() + fields := strings.Fields(logLine) + require.Len(t, fields, 4, "log line should keep its structure") + return fields[1] +} + +func TestAnonymizeDomain_StrictLevel(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + t.Run("netbird peer name", func(t *testing.T) { + result := anonymizer.AnonymizeDomain("my-laptop.netbird.cloud") + assert.Regexp(t, `^peer-\d+\.netbird\.cloud$`, result, "peer name should be anonymized, suffix kept") + assert.NotContains(t, result, "my-laptop", "the peer name should not survive") + assert.Equal(t, result, anonymizer.AnonymizeDomain("my-laptop.netbird.cloud"), "repeated domain should map consistently") + assert.Equal(t, result, anonymizer.AnonymizeDomain(result), "an anonymized domain should pass through unchanged") + }) + + t.Run("bare netbird domain", func(t *testing.T) { + assert.Equal(t, "netbird.cloud", anonymizer.AnonymizeDomain("netbird.cloud"), "the bare protected suffix should be preserved") + }) + + t.Run("netbird infrastructure preserved", func(t *testing.T) { + assert.Equal(t, "api.netbird.io", anonymizer.AnonymizeDomain("api.netbird.io"), + "netbird.io hosts infrastructure, not peer names, and should stay readable") + }) + + t.Run("leading labels of other domains", func(t *testing.T) { + result := anonymizer.AnonymizeDomain("host1.corp.example.com") + assert.Regexp(t, `^host-\d+\.host-\d+\.anon-[a-zA-Z0-9]+\.domain$`, result, "every label should be anonymized") + for _, label := range []string{"host1", "corp", "example"} { + assert.NotContains(t, result, label, "no original label should survive") + } + assert.Equal(t, result, anonymizer.AnonymizeDomain("host1.corp.example.com"), "repeated domain should map consistently") + }) + + t.Run("same label maps consistently across domains", func(t *testing.T) { + first := anonymizer.AnonymizeDomain("shared.one.com") + second := anonymizer.AnonymizeDomain("shared.two.com") + assert.Equal(t, strings.Split(first, ".")[0], strings.Split(second, ".")[0], "the shared host label should get one placeholder") + }) + + t.Run("wildcard label preserved", func(t *testing.T) { + result := anonymizer.AnonymizeDomain("*.example.com") + assert.Regexp(t, `^\*\.anon-[a-zA-Z0-9]+\.domain$`, result, "the wildcard label should stay a wildcard") + }) +} + +func TestAnonymizeDomain_DefaultLevelKeepsPeerNames(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + + assert.Equal(t, "my-laptop.netbird.cloud", anonymizer.AnonymizeDomain("my-laptop.netbird.cloud"), + "default level should preserve netbird FQDNs including the peer name") + assert.Regexp(t, `^sub\.anon-[a-zA-Z0-9]+\.domain$`, anonymizer.AnonymizeDomain("sub.example.com"), + "default level should keep subdomain labels") +} + +func TestAnonymizeString_StrictPeerNames(t *testing.T) { + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(anonymize.LevelStrict) + + // Seed like the bundle generator does from the status: base first, then + // the full FQDN, so replacement must prefer the longer mapping. + anonBase := anonymizer.AnonymizeDomain("example.com") + anonPeer := anonymizer.AnonymizeDomain("peer1.netbird.cloud") + anonHost := anonymizer.AnonymizeDomain("host1.example.com") + + logLine := "connected to peer1.netbird.cloud via host1.example.com endpoint" + firstPass := anonymizer.AnonymizeString(logLine) + assert.NotContains(t, firstPass, "peer1", "the peer name should not survive in logs") + assert.NotContains(t, firstPass, "host1", "the host label should not survive in logs") + assert.Contains(t, firstPass, anonPeer, "the seeded peer mapping should be applied") + assert.Contains(t, firstPass, anonHost, "the seeded host mapping should be applied, not just the base mapping") + assert.NotContains(t, firstPass, "host1."+anonBase, "the base mapping must not preempt the longer FQDN mapping") + assert.Equal(t, firstPass, anonymizer.AnonymizeString(firstPass), "second pass should not change the result") +} + func TestAnonymizeDNSLogLine(t *testing.T) { anonymizer := anonymize.NewAnonymizer(netip.Addr{}, netip.Addr{}) tests := []struct { diff --git a/client/cmd/debug.go b/client/cmd/debug.go index 7ddc3afc4..893b1e248 100644 --- a/client/cmd/debug.go +++ b/client/cmd/debug.go @@ -27,8 +27,8 @@ import ( const errCloseConnection = "Failed to close connection: %v" var ( - logFileCount uint32 - systemInfoFlag bool + logFileCount uint32 + systemInfoFlag bool uploadBundleFlag bool uploadBundleURLFlag string uploadBundleInsecureFlag bool @@ -156,6 +156,11 @@ func debugConfigDump(cmd *cobra.Command, _ []string) error { // request. Returns an error if the RPC fails or if the daemon reports // an upload failure reason. func debugBundle(cmd *cobra.Command, _ []string) error { + anonymizeEnabled, anonymizeLevel, err := effectiveAnonymize() + if err != nil { + return err + } + conn, err := getClient(cmd) if err != nil { return err @@ -168,10 +173,11 @@ func debugBundle(cmd *cobra.Command, _ []string) error { client := proto.NewDaemonServiceClient(conn) request := &proto.DebugBundleRequest{ - Anonymize: anonymizeFlag, - SystemInfo: systemInfoFlag, - LogFileCount: logFileCount, - CliVersion: version.NetbirdVersion(), + Anonymize: anonymizeEnabled, + AnonymizeLevel: anonymizeLevel.String(), + SystemInfo: systemInfoFlag, + LogFileCount: logFileCount, + CliVersion: version.NetbirdVersion(), } if uploadBundleFlag { request.UploadURL = uploadBundleURLFlag @@ -229,6 +235,11 @@ func runForDuration(cmd *cobra.Command, args []string) error { return fmt.Errorf("invalid duration format: %v", err) } + anonymizeEnabled, anonymizeLevel, err := effectiveAnonymize() + if err != nil { + return err + } + conn, err := getClient(cmd) if err != nil { return err @@ -368,10 +379,11 @@ func runForDuration(cmd *cobra.Command, args []string) error { cmd.Println("Creating debug bundle...") request := &proto.DebugBundleRequest{ - Anonymize: anonymizeFlag, - SystemInfo: systemInfoFlag, - LogFileCount: logFileCount, - CliVersion: version.NetbirdVersion(), + Anonymize: anonymizeEnabled, + AnonymizeLevel: anonymizeLevel.String(), + SystemInfo: systemInfoFlag, + LogFileCount: logFileCount, + CliVersion: version.NetbirdVersion(), } if uploadBundleFlag { request.UploadURL = uploadBundleURLFlag diff --git a/client/cmd/root.go b/client/cmd/root.go index ebaae7e3e..ccad78942 100644 --- a/client/cmd/root.go +++ b/client/cmd/root.go @@ -21,6 +21,7 @@ import ( "github.com/spf13/pflag" "google.golang.org/grpc" + "github.com/netbirdio/netbird/client/anonymize" daddr "github.com/netbirdio/netbird/client/internal/daemonaddr" "github.com/netbirdio/netbird/client/internal/profilemanager" ) @@ -69,6 +70,7 @@ var ( autoConnectDisabled bool extraIFaceBlackList []string anonymizeFlag bool + anonymizeLevelFlag string dnsRouteInterval time.Duration // lazyConnEnabled is the parse target for the deprecated --enable-lazy-connection // flag. The flag is inert; the value is no longer read (use NB_LAZY_CONN instead). @@ -156,7 +158,8 @@ func init() { rootCmd.MarkFlagsMutuallyExclusive("setup-key", "setup-key-file") rootCmd.PersistentFlags().StringVar(&preSharedKey, preSharedKeyFlag, "", "Sets WireGuard PreSharedKey property. If set, then only peers that have the same key can communicate.") rootCmd.PersistentFlags().StringVarP(&hostName, "hostname", "n", "", "Sets a custom hostname for the device") - rootCmd.PersistentFlags().BoolVarP(&anonymizeFlag, "anonymize", "A", false, "anonymize IP addresses and non-netbird.io domains in logs and status output") + rootCmd.PersistentFlags().BoolVarP(&anonymizeFlag, "anonymize", "A", false, "anonymize public IP addresses, MAC addresses, and non-netbird.io domains in logs and status output; private, CGNAT, and link-local IP ranges are kept (see --anonymize-level strict)") + rootCmd.PersistentFlags().StringVar(&anonymizeLevelFlag, "anonymize-level", "", "anonymization level: \"default\" or \"strict\"; strict also anonymizes private, CGNAT, and link-local IP ranges, peer names, and WireGuard public keys. Setting this flag implies --anonymize") rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", profilemanager.DefaultConfigPath, "Overrides the default profile file location") rootCmd.AddCommand(upCmd) @@ -293,6 +296,19 @@ var CLIBackOffSettings = &backoff.ExponentialBackOff{ Clock: backoff.SystemClock, } +// effectiveAnonymize resolves the --anonymize and --anonymize-level flags: +// setting a level implies anonymization, and an invalid level is rejected. +func effectiveAnonymize() (bool, anonymize.Level, error) { + if anonymizeLevelFlag == "" { + return anonymizeFlag, anonymize.LevelDefault, nil + } + level := anonymize.ParseLevel(anonymizeLevelFlag) + if !strings.EqualFold(anonymizeLevelFlag, level.String()) { + return false, anonymize.LevelDefault, fmt.Errorf("invalid anonymize level %q: use %q or %q", anonymizeLevelFlag, anonymize.LevelDefault.String(), anonymize.LevelStrict.String()) + } + return true, level, nil +} + func getSetupKey() (string, error) { if setupKeyPath != "" && setupKey == "" { return getSetupKeyFromFile(setupKeyPath) diff --git a/client/cmd/status.go b/client/cmd/status.go index c4057ed82..f2e5bcc66 100644 --- a/client/cmd/status.go +++ b/client/cmd/status.go @@ -121,8 +121,14 @@ func statusFunc(cmd *cobra.Command, args []string) error { sessionExpiresAt = ts.AsTime().UTC() } + anonymizeEnabled, anonymizeLevel, err := effectiveAnonymize() + if err != nil { + return err + } + var outputInformationHolder = nbstatus.ConvertToStatusOutputOverview(resp.GetFullStatus(), nbstatus.ConvertOptions{ - Anonymize: anonymizeFlag, + Anonymize: anonymizeEnabled, + AnonymizeLevel: anonymizeLevel, DaemonVersion: resp.GetDaemonVersion(), DaemonStatus: nbstatus.ParseDaemonStatus(status), StatusFilter: statusFilter, diff --git a/client/iface/bind/ice_bind.go b/client/iface/bind/ice_bind.go index 156450c61..2d35b9c6f 100644 --- a/client/iface/bind/ice_bind.go +++ b/client/iface/bind/ice_bind.go @@ -22,6 +22,16 @@ import ( nbnet "github.com/netbirdio/netbird/client/net" ) +const ( + // wgMsgTypeHandshakeInitiation is the lowest WireGuard message type. + wgMsgTypeHandshakeInitiation uint32 = 1 + // wgMsgTypeTransport is the highest WireGuard message type. + wgMsgTypeTransport uint32 = 4 + // wgMinMsgSize is the smallest WireGuard message: transport data with an empty + // payload, which is what a keepalive is. + wgMinMsgSize = 32 +) + type receiverCreator struct { iceBind *ICEBind } @@ -216,8 +226,15 @@ func (s *ICEBind) createReceiverFn(pc wgConn.BatchReader, conn *net.UDPConn, rxO for i := 0; i < numMsgs; i++ { msg := &(*msgs)[i] - // todo: handle err - if ok, _ := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr); ok { + if ok, err := s.filterOutStunMessages(msg.Buffers, msg.N, msg.Addr); ok { + if err != nil { + log.Debugf("failed to handle STUN packet from %s: %v", msg.Addr, err) + } + // WireGuard reuses sizes and eps across reads and only skips a slot + // whose size is below the minimum message size. Leaving a consumed + // slot untouched makes it process this buffer again under the + // previous packet's length and endpoint. + sizes[i] = 0 continue } sizes[i] = msg.N @@ -271,11 +288,16 @@ func (s *ICEBind) createOrUpdateMux() { func (s *ICEBind) filterOutStunMessages(buffers [][]byte, n int, addr net.Addr) (bool, error) { for i := range buffers { - if !stun.IsMessage(buffers[i]) { + if n > len(buffers[i]) { + continue + } + pkt := buffers[i][:n] + + if isWireGuardMsg(pkt) || !stun.IsMessage(pkt) { continue } - msg, err := s.parseSTUNMessage(buffers[i][:n]) + msg, err := s.parseSTUNMessage(pkt) if err != nil { buffers[i] = []byte{} return true, err @@ -347,18 +369,34 @@ func putMessages(msgs *[]ipv6.Message, msgsPool *sync.Pool) { msgsPool.Put(msgs) } -func isTransportPkg(buffers [][]byte, n int) bool { - // The first buffer should contain at least 4 bytes for type - if len(buffers[0]) < 4 { - return true +// isWireGuardMsg reports whether the packet carries a WireGuard message header: a +// little-endian uint32 message type in the range 1..4, which leaves the three bytes +// after the type byte zero, in a packet long enough to hold any WireGuard message. +// +// A well formed STUN message cannot take that shape. Its length field sits in the two +// bytes the type must leave zero, and for a message of at least wgMinMsgSize bytes that +// field holds at least 12, so the two framings do not overlap. The test has to be this +// tight because stun.IsMessage only looks at the magic cookie, which in a WireGuard +// message overlaps the receiver index: a session whose index happens to equal the cookie +// would otherwise have all of its inbound data misrouted to the STUN handler until the +// next rekey. +func isWireGuardMsg(pkt []byte) bool { + if len(pkt) < wgMinMsgSize { + return false } - // WireGuard packet type is a little-endian uint32 at start - packetType := binary.LittleEndian.Uint32(buffers[0][:4]) - - // Check if packetType matches known WireGuard message types - if packetType == 4 && n > 32 { - return true - } - return false + msgType := binary.LittleEndian.Uint32(pkt[:4]) + return msgType >= wgMsgTypeHandshakeInitiation && msgType <= wgMsgTypeTransport +} + +// isTransportPkg reports whether the packet is WireGuard transport data carrying a +// payload, which is what counts as peer activity. A keepalive holds no payload and is +// exactly wgMinMsgSize bytes. +func isTransportPkg(buffers [][]byte, n int) bool { + if n < 4 || n > len(buffers[0]) { + return false + } + + msgType := binary.LittleEndian.Uint32(buffers[0][:4]) + return msgType == wgMsgTypeTransport && n > wgMinMsgSize } diff --git a/client/iface/bind/stun_filter_test.go b/client/iface/bind/stun_filter_test.go new file mode 100644 index 000000000..0e118e0fd --- /dev/null +++ b/client/iface/bind/stun_filter_test.go @@ -0,0 +1,215 @@ +//go:build !js + +package bind + +import ( + "encoding/binary" + "net" + "testing" + "time" + + "github.com/pion/stun/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/ipv4" + wgConn "golang.zx2c4.com/wireguard/conn" +) + +// magicCookieBytes is the STUN magic cookie as it appears on the wire. In a +// WireGuard message the same offset holds the receiver (or sender) index, which is +// a random uint32, so a session can draw exactly this value. +var magicCookieBytes = []byte{0x21, 0x12, 0xA4, 0x42} + +const testBufSize = 1500 + +// wgMsg builds a WireGuard message of the given type and size, with the index field +// at bytes 4:8 set to index. +func wgMsg(msgType uint32, size int, index []byte) []byte { + pkt := make([]byte, size) + binary.LittleEndian.PutUint32(pkt[:4], msgType) + copy(pkt[4:8], index) + return pkt +} + +// intoBuffer copies pkt into a full-size receive buffer, the way the kernel read +// does, so tests see the same buffer/length split as the hot path. +func intoBuffer(pkt []byte) [][]byte { + buf := make([]byte, testBufSize) + copy(buf, pkt) + return [][]byte{buf} +} + +func TestFilterOutStunMessages_PassesWireGuardWithCookieShapedIndex(t *testing.T) { + tests := []struct { + name string + msgType uint32 + size int + }{ + {"transport data", wgMsgTypeTransport, 128}, + {"keepalive", wgMsgTypeTransport, wgMinMsgSize}, + {"handshake initiation", wgMsgTypeHandshakeInitiation, 148}, + {"handshake response", 2, 92}, + {"cookie reply", 3, 64}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pkt := wgMsg(tc.msgType, tc.size, magicCookieBytes) + require.True(t, stun.IsMessage(pkt), "precondition: pion sees this as STUN") + + buffers := intoBuffer(pkt) + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, tc.size, &net.UDPAddr{}) + assert.NoError(t, err) + assert.False(t, filtered, "WireGuard message must be handed to WireGuard, not the STUN handler") + assert.Len(t, buffers[0], testBufSize, "buffer must be left intact for WireGuard") + }) + } +} + +func TestFilterOutStunMessages_FiltersRealSTUNMessage(t *testing.T) { + msg, err := stun.Build(stun.BindingRequest, stun.TransactionID, stun.Fingerprint) + require.NoError(t, err) + + buffers := intoBuffer(msg.Raw) + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, len(msg.Raw), &net.UDPAddr{}) + assert.NoError(t, err) + assert.True(t, filtered, "STUN message must be consumed by the STUN handler") + assert.Empty(t, buffers[0], "consumed buffer must be emptied so WireGuard does not see it") +} + +// TestIsWireGuardMsg_DisjointFromSTUN locks the invariant the filter relies on: a +// well formed STUN message long enough to be a WireGuard message always has a +// non-zero length field, so it cannot be mistaken for a WireGuard header. +func TestIsWireGuardMsg_DisjointFromSTUN(t *testing.T) { + types := []stun.MessageType{ + stun.BindingRequest, + stun.BindingSuccess, + stun.BindingError, + {Method: stun.MethodBinding, Class: stun.ClassIndication}, + } + + for _, msgType := range types { + // Long enough that the length guard is not what makes this pass. + msg, err := stun.Build(msgType, stun.TransactionID, + stun.NewUsername("remoteUfrag:localUfrag"), stun.Fingerprint) + require.NoError(t, err) + require.GreaterOrEqual(t, len(msg.Raw), wgMinMsgSize, "precondition: %s", msgType) + assert.False(t, isWireGuardMsg(msg.Raw), + "%s must not look like a WireGuard message", msgType) + } +} + +func TestIsWireGuardMsg(t *testing.T) { + tests := []struct { + name string + pkt []byte + want bool + }{ + {"transport data", wgMsg(wgMsgTypeTransport, 128, nil), true}, + {"handshake initiation", wgMsg(wgMsgTypeHandshakeInitiation, 148, nil), true}, + {"unknown type 5", wgMsg(5, 128, nil), false}, + {"type 0", wgMsg(0, 128, nil), false}, + {"non-zero reserved byte", []byte{0x04, 0x00, 0x01, 0x00}, false}, + {"too short", []byte{0x04, 0x00, 0x00}, false}, + {"empty", nil, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isWireGuardMsg(tc.pkt), "wrong classification for %s", tc.name) + }) + } +} + +// TestFilterOutStunMessages_IgnoresBytesBeyondPacket guards against classifying on +// buffer contents left over from an earlier, longer packet. +func TestFilterOutStunMessages_IgnoresBytesBeyondPacket(t *testing.T) { + buf := make([]byte, testBufSize) + copy(buf[4:8], magicCookieBytes) + buffers := [][]byte{buf} + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, 2, &net.UDPAddr{}) + assert.NoError(t, err) + assert.False(t, filtered, "a 2 byte packet must not be classified from stale buffer bytes") +} + +// TestReceiveFn_ClearsSizeOfConsumedPacket covers the accounting WireGuard relies +// on: sizes is reused across reads, so a slot whose packet was consumed as STUN must +// be reported as empty. Otherwise WireGuard reprocesses the same buffer under the +// previous packet's length, which for a WireGuard-shaped packet means it is handled +// twice. +func TestReceiveFn_ClearsSizeOfConsumedPacket(t *testing.T) { + conn := listenUDP(t, "udp4", "127.0.0.1:0") + defer conn.Close() + + recvFn := receiverCreator{setupICEBind(t)}.CreateReceiverFn( + ipv4.NewPacketConn(conn), conn, false, createMsgPool(), + ) + + msg, err := stun.Build(stun.BindingRequest, stun.TransactionID, stun.Fingerprint) + require.NoError(t, err) + + sender := listenUDP(t, "udp4", "127.0.0.1:0") + defer sender.Close() + _, err = sender.WriteTo(msg.Raw, conn.LocalAddr()) + require.NoError(t, err) + + require.NoError(t, conn.SetReadDeadline(time.Now().Add(3*time.Second))) + + bufs := [][]byte{make([]byte, 1500)} + // A leftover size from an earlier read, which is what makes the missing reset + // observable. + sizes := []int{148} + eps := make([]wgConn.Endpoint, 1) + + n, err := recvFn(bufs, sizes, eps) + require.NoError(t, err) + require.Equal(t, 1, n) + assert.Zero(t, sizes[0], "consumed STUN packet must not leave a size behind for WireGuard") +} + +func TestIsTransportPkg(t *testing.T) { + tests := []struct { + name string + pkt []byte + n int + want bool + }{ + {"transport data with payload", wgMsg(wgMsgTypeTransport, 128, nil), 128, true}, + {"keepalive", wgMsg(wgMsgTypeTransport, wgMinMsgSize, nil), wgMinMsgSize, false}, + {"handshake initiation", wgMsg(wgMsgTypeHandshakeInitiation, 148, nil), 148, false}, + {"stale type bytes beyond packet", wgMsg(wgMsgTypeTransport, 128, nil), 2, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isTransportPkg(intoBuffer(tc.pkt), tc.n), + "wrong activity classification for %s", tc.name) + }) + } +} + +// TestFilterOutStunMessages_ConsumesSTUNWithWireGuardShapedType covers the one STUN +// encoding whose leading bytes collide with a WireGuard message type: method 0x080 as a +// request encodes to 0x0200, so the type byte reads as a handshake response and the byte +// after it is zero. Only the length check keeps such a message out of WireGuard's hands. +// pion implements no method in that range, so this is a synthetic worst case rather than +// traffic ICE produces. +func TestFilterOutStunMessages_ConsumesSTUNWithWireGuardShapedType(t *testing.T) { + msg, err := stun.Build(stun.NewType(stun.Method(0x080), stun.ClassRequest), stun.TransactionID) + require.NoError(t, err) + require.Equal(t, []byte{0x02, 0x00, 0x00, 0x00}, msg.Raw[:4], + "precondition: the leading bytes read as a WireGuard message type") + + buffers := intoBuffer(msg.Raw) + bind := &ICEBind{} + + filtered, err := bind.filterOutStunMessages(buffers, len(msg.Raw), &net.UDPAddr{}) + assert.NoError(t, err) + assert.True(t, filtered, "STUN message must be consumed despite its WireGuard-shaped type") +} diff --git a/client/internal/debug/debug.go b/client/internal/debug/debug.go index 0f81844f6..dbe22139a 100644 --- a/client/internal/debug/debug.go +++ b/client/internal/debug/debug.go @@ -34,9 +34,8 @@ import ( "github.com/netbirdio/netbird/shared/netiputil" ) -const readmeContent = `Netbird debug bundle -This debug bundle contains the following files. -If the --anonymize flag is set, the files are anonymized to protect sensitive information. +const readmeContent = `This debug bundle contains the following files. +If anonymization is enabled (--anonymize / --anonymize-level), the files are anonymized to protect sensitive information. status.txt: Anonymized status information of the NetBird client. client.log: Most recent, anonymized client log file of the NetBird client. @@ -70,21 +69,34 @@ capture.pcap: Packet capture in pcap format. Only present when capture was runni Anonymization Process -The files in this bundle have been anonymized to protect sensitive information. Here's how the anonymization was applied: +The files in this bundle have been anonymized to protect sensitive information. The level applied to this bundle is recorded at the top of this file. Here's how the anonymization was applied: IP Addresses -IPv4 addresses are replaced with addresses starting from 198.51.100.0 -IPv6 addresses are replaced with addresses starting from 100:: +Default level: +- Public IPv4 addresses are replaced with addresses starting from 198.51.100.0 +- Public IPv6 addresses are replaced with addresses starting from 2001:db8:ffff:: +- IPv6 unique local addresses (fc00::/7) are anonymized as well: their random global ID uniquely identifies the network. +- IP addresses from internal IPv4 ranges and well-known addresses are not anonymized (e.g. 8.8.8.8, 100.64.0.0/10, addresses starting with 192.168., 172.16., 10., 169.254., fe80::). + +Strict level (--anonymize-level strict), in addition to the default level: +- Private (RFC 1918), CGNAT (100.64.0.0/10), and link-local (169.254.0.0/16, fe80::/10) addresses are anonymized too. +- Internal IPv4 addresses are replaced with addresses starting from 198.18.0.0 and internal IPv6 addresses with addresses starting from 2001:db8:1::, so internal addresses remain distinguishable from public ones. +- Addresses are mapped in order of first appearance: subnet structure, allocation scheme, and gateway conventions are not preserved. Prefix lengths of networks are preserved. +- Peer names in front of NetBird domains are replaced with numbered placeholders (e.g. peer-1.netbird.cloud), and subdomain labels of other domains with host-N placeholders. +- WireGuard public keys are replaced with consistent placeholder keys. -IP addresses from non public ranges and well known addresses are not anonymized (e.g. 8.8.8.8, 100.64.0.0/10, addresses starting with 192.168., 172.16., 10., etc.). Reoccuring IP addresses are replaced with the same anonymized address. Note: The anonymized IP addresses in the status file do not match those in the log and routes files. However, the anonymized IP addresses are consistent within the status file and across the routes and log files. +MAC Addresses +MAC addresses are replaced at every anonymization level with consistent placeholders counting up from 02:00:00:00:00:01. Broadcast, multicast, and all-zero addresses are kept. At the default level a preserved IPv6 link-local address may still embed a MAC address (EUI-64); the strict level anonymizes those addresses. + Domains All domain names (except for the netbird domains) are replaced with randomly generated strings ending in ".domain". Anonymized domains are consistent across all files in the bundle. Reoccuring domain names are replaced with the same anonymized domain. +At the strict level, the peer name labels in front of netbird domains are anonymized as well. Sync Response The network_map.json file contains the following anonymized information: @@ -281,6 +293,7 @@ type BundleGenerator struct { cliVersion string anonymize bool + anonymizeLevel anonymize.Level includeSystemInfo bool logFileCount uint32 @@ -288,7 +301,10 @@ type BundleGenerator struct { } type BundleConfig struct { - Anonymize bool + Anonymize bool + // AnonymizeLevel selects how much the anonymizer redacts. + // anonymize.LevelStrict implies Anonymize. + AnonymizeLevel anonymize.Level IncludeSystemInfo bool LogFileCount uint32 } @@ -327,8 +343,11 @@ func NewBundleGenerator(deps GeneratorDependencies, cfg BundleConfig) *BundleGen uiLogOpener = openLogFile } + anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(cfg.AnonymizeLevel) + return &BundleGenerator{ - anonymizer: anonymize.NewAnonymizer(anonymize.DefaultAddresses()), + anonymizer: anonymizer, internalConfig: deps.InternalConfig, statusRecorder: deps.StatusRecorder, @@ -345,7 +364,8 @@ func NewBundleGenerator(deps GeneratorDependencies, cfg BundleConfig) *BundleGen daemonVersion: deps.DaemonVersion, cliVersion: deps.CliVersion, - anonymize: cfg.Anonymize, + anonymize: cfg.Anonymize || cfg.AnonymizeLevel >= anonymize.LevelStrict, + anonymizeLevel: cfg.AnonymizeLevel, includeSystemInfo: cfg.IncludeSystemInfo, logFileCount: logFileCount, } @@ -485,7 +505,13 @@ func (g *BundleGenerator) addSystemInfo() { } func (g *BundleGenerator) addReadme() error { - readmeReader := strings.NewReader(readmeContent) + level := "none (anonymization disabled)" + if g.anonymize { + level = g.anonymizeLevel.String() + } + header := fmt.Sprintf("Netbird debug bundle\nAnonymization level applied to this bundle: %s\n", level) + + readmeReader := strings.NewReader(header + readmeContent) if err := g.addFileToZip(readmeReader, "README.txt"); err != nil { return fmt.Errorf("add README file to zip: %w", err) } @@ -507,9 +533,10 @@ func (g *BundleGenerator) addStatus() error { fullStatus := g.statusRecorder.GetFullStatus() protoFullStatus := nbstatus.ToProtoFullStatus(fullStatus) overview := nbstatus.ConvertToStatusOutputOverview(protoFullStatus, nbstatus.ConvertOptions{ - Anonymize: g.anonymize, - ProfileName: profName, - DaemonVersion: g.daemonVersion, + Anonymize: g.anonymize, + AnonymizeLevel: g.anonymizeLevel, + ProfileName: profName, + DaemonVersion: g.daemonVersion, }) overview.CliVersion = g.cliVersion statusOutput := overview.FullDetailSummary() @@ -662,7 +689,7 @@ func (g *BundleGenerator) addCommonConfigFields(configContent *strings.Builder) configContent.WriteString("NetBird Client Configuration:\n\n") if key, err := wgtypes.ParseKey(g.internalConfig.PrivateKey); err == nil { - configContent.WriteString(fmt.Sprintf("PublicKey: %s\n", key.PublicKey().String())) + configContent.WriteString(fmt.Sprintf("PublicKey: %s\n", g.anonymizer.AnonymizeWGKey(key.PublicKey().String()))) } configContent.WriteString(fmt.Sprintf("WgIface: %s\n", g.internalConfig.WgIface)) configContent.WriteString(fmt.Sprintf("WgPort: %d\n", g.internalConfig.WgPort)) @@ -952,6 +979,11 @@ func (g *BundleGenerator) addUpdateLogs() error { } baseName := filepath.Base(logFile) + data, err = g.anonymizeBytes(data) + if err != nil { + log.Warnf("skipping update log file %s: %v", baseName, err) + continue + } if err := g.addFileToZip(bytes.NewReader(data), filepath.Join("update-logs", baseName)); err != nil { return fmt.Errorf("add update log file %s to zip: %w", baseName, err) } @@ -979,6 +1011,13 @@ func (g *BundleGenerator) addCorruptedStateFiles() error { } fileName := filepath.Base(match) + // Corrupted state files usually fail structured JSON anonymization, + // so run them through the string anonymizer instead. + data, err = g.anonymizeBytes(data) + if err != nil { + log.Warnf("skipping corrupted state file %s: %v", fileName, err) + continue + } if err := g.addFileToZip(bytes.NewReader(data), "corrupted_states/"+fileName); err != nil { log.Warnf("Failed to add corrupted state file %s to zip: %v", fileName, err) continue @@ -990,6 +1029,27 @@ func (g *BundleGenerator) addCorruptedStateFiles() error { return nil } +// anonymizeBytes runs raw file content through the string anonymizer line by +// line when anonymization is enabled. It errors instead of returning partial +// content, so a caller never adds an unanonymized fallback to the bundle. +func (g *BundleGenerator) anonymizeBytes(data []byte) ([]byte, error) { + if !g.anonymize { + return data, nil + } + + var buf bytes.Buffer + scanner := bufio.NewScanner(bytes.NewReader(data)) + scanner.Buffer(make([]byte, 1024*1024), 1024*1024) + for scanner.Scan() { + buf.WriteString(g.anonymizer.AnonymizeString(scanner.Text())) + buf.WriteByte('\n') + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("anonymize content: %w", err) + } + return buf.Bytes(), nil +} + func (g *BundleGenerator) addMetrics() error { if g.clientMetrics == nil { log.Debugf("skipping metrics in debug bundle: no metrics collector") @@ -1462,6 +1522,7 @@ func anonymizeRemotePeer(peer *mgmProto.RemotePeerConfig, anonymizer *anonymize. } peer.Fqdn = anonymizer.AnonymizeDomain(peer.Fqdn) + peer.WgPubKey = anonymizer.AnonymizeWGKey(peer.WgPubKey) anonymizeSSHConfig(peer.SshConfig) } diff --git a/client/internal/debug/wgshow.go b/client/internal/debug/wgshow.go index 1e8a8a6cc..ee24902e6 100644 --- a/client/internal/debug/wgshow.go +++ b/client/internal/debug/wgshow.go @@ -35,14 +35,14 @@ func (g *BundleGenerator) toWGShowFormat(s *configurer.Stats) string { var sb strings.Builder sb.WriteString(fmt.Sprintf("interface: %s\n", s.DeviceName)) - sb.WriteString(fmt.Sprintf(" public key: %s\n", s.PublicKey)) + sb.WriteString(fmt.Sprintf(" public key: %s\n", g.anonymizer.AnonymizeWGKey(s.PublicKey))) sb.WriteString(fmt.Sprintf(" listen port: %d\n", s.ListenPort)) if s.FWMark != 0 { sb.WriteString(fmt.Sprintf(" fwmark: %#x\n", s.FWMark)) } for _, peer := range s.Peers { - sb.WriteString(fmt.Sprintf("\npeer: %s\n", peer.PublicKey)) + sb.WriteString(fmt.Sprintf("\npeer: %s\n", g.anonymizer.AnonymizeWGKey(peer.PublicKey))) if peer.Endpoint.IP != nil { if g.anonymize { anonEndpoint := g.anonymizer.AnonymizeUDPAddr(peer.Endpoint) @@ -54,7 +54,11 @@ func (g *BundleGenerator) toWGShowFormat(s *configurer.Stats) string { if len(peer.AllowedIPs) > 0 { var ipStrings []string for _, ipnet := range peer.AllowedIPs { - ipStrings = append(ipStrings, ipnet.String()) + ipStr := ipnet.String() + if g.anonymize { + ipStr = g.anonymizer.AnonymizeIPString(ipStr) + } + ipStrings = append(ipStrings, ipStr) } sb.WriteString(fmt.Sprintf(" allowed ips: %s\n", strings.Join(ipStrings, ", "))) } diff --git a/client/internal/engine.go b/client/internal/engine.go index 67390f3c7..5380651a5 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -23,6 +23,7 @@ import ( "golang.zx2c4.com/wireguard/tun/netstack" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "github.com/netbirdio/netbird/client/anonymize" nberrors "github.com/netbirdio/netbird/client/errors" "github.com/netbirdio/netbird/client/firewall" "github.com/netbirdio/netbird/client/firewall/firewalld" @@ -1394,6 +1395,7 @@ func (e *Engine) handleBundle(params *mgmProto.BundleParameters) (*mgmProto.JobR bundleJobParams := debug.BundleConfig{ Anonymize: params.Anonymize, + AnonymizeLevel: anonymize.ParseLevel(params.AnonymizeLevel), IncludeSystemInfo: true, LogFileCount: uint32(params.LogFileCount), } diff --git a/client/internal/metrics/connection_type.go b/client/internal/metrics/connection_type.go index a3406a6b8..d393e5112 100644 --- a/client/internal/metrics/connection_type.go +++ b/client/internal/metrics/connection_type.go @@ -4,11 +4,17 @@ package metrics type ConnectionType string const ( - // ConnectionTypeICE represents a direct peer-to-peer connection using ICE - ConnectionTypeICE ConnectionType = "ice" + // ConnectionTypeICEP2P represents a direct peer-to-peer connection using ICE + ConnectionTypeICEP2P ConnectionType = "ice_p2p" + + // ConnectionTypeICETurn represents an ICE connection through a TURN server + ConnectionTypeICETurn ConnectionType = "ice_turn" // ConnectionTypeRelay represents a relayed connection ConnectionTypeRelay ConnectionType = "relay" + + // ConnectionTypeUnknown represents a connection with no active transport. It is not pushed. + ConnectionTypeUnknown ConnectionType = "unknown" ) // String returns the string representation of the connection type diff --git a/client/internal/metrics/influxdb_test.go b/client/internal/metrics/influxdb_test.go index b964e31a3..6a226fe2f 100644 --- a/client/internal/metrics/influxdb_test.go +++ b/client/internal/metrics/influxdb_test.go @@ -28,7 +28,7 @@ func TestInfluxDBMetrics_RecordAndExport(t *testing.T) { WgHandshakeSuccess: time.Now().Add(-1 * time.Second), } - m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICE, false, ts) + m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICEP2P, false, ts) var buf bytes.Buffer err := m.Export(&buf) @@ -60,7 +60,7 @@ func TestInfluxDBMetrics_ExportDeterministicFieldOrder(t *testing.T) { // Record multiple times and verify consistent field order for i := 0; i < 10; i++ { - m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICE, false, ts) + m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICEP2P, false, ts) } var buf bytes.Buffer diff --git a/client/internal/metrics/infra/README.md b/client/internal/metrics/infra/README.md index 7941a30cf..0a69404df 100644 --- a/client/internal/metrics/infra/README.md +++ b/client/internal/metrics/infra/README.md @@ -56,14 +56,33 @@ Measurement: `netbird_peer_connection` Tags: - `deployment_type`: "cloud" | "selfhosted" | "unknown" -- `connection_type`: "ice" | "relay" +- `connection_type`: "ice_p2p" | "ice_turn" | "relay" (see below) - `attempt_type`: "initial" | "reconnection" - `version`: NetBird version string - `os`: Operating system (linux, darwin, windows, android, ios, etc.) - `arch`: CPU architecture (amd64, arm64, etc.) +- `peer_id`: anonymised peer identifier (truncated SHA-256 of the WireGuard public key) +- `connection_pair_id`: deterministic identifier for the peer pair, identical on both sides **Note:** `SignalingReceived` is set when the first offer or answer arrives from the remote peer (in both initial and reconnection paths). It excludes the potentially unbounded wait for the remote peer to come online. +#### `connection_type` values + +Derived from the connection priority (`conntype.ConnPriority`) by `metricsConnType` in `client/internal/peer/conn.go`: + +| Value | Priority | Traffic is | +|-------|----------|------------| +| `ice_p2p` | `ICEP2P` | direct peer-to-peer | +| `ice_turn` | `ICETurn` | relayed, through a TURN server | +| `relay` | `Relay` | relayed, through a NetBird relay | +| `unknown` | `None` or unrecognised | no active transport — **the sample is not pushed** | + +**Direct traffic is `ice_p2p` only.** `ice_turn` is relayed despite being negotiated by ICE, matching `Conn.isRelayed`. + +`None` means no transport is active: not established yet, or reset after a relay drop or a peer-state reset. Such a sample cannot be attributed to a transport, so `recordConnectionMetrics` drops it instead of pushing it — `unknown` therefore never appears in the bucket. Connection counts are counts of connections whose transport was known at sampling time. + +**Samples recorded before 0.77 used a single `ice` value** which covered `ICEP2P`, `ICETurn` *and* `None`, so historical `ice` samples overstate direct connections by an unknown amount and must not be compared with `ice_p2p`. + ### Sync Duration Measurement: `netbird_sync` diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index d4b2264b5..a3c320027 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -312,6 +312,8 @@ func (conn *Conn) Close(signalToRemote bool) { if conn.wgWatcherCancel != nil { conn.wgWatcherCancel() + conn.wgWatcher = nil + conn.wgWatcherCancel = nil } conn.workerRelay.CloseConn() if conn.workerICE != nil { @@ -964,12 +966,9 @@ func (conn *Conn) recordConnectionMetrics() { priority := conn.currentConnPriority conn.mu.Unlock() - var connType metrics.ConnectionType - switch priority { - case conntype.Relay: - connType = metrics.ConnectionTypeRelay - default: - connType = metrics.ConnectionTypeICE + connType := metricsConnType(priority) + if connType == metrics.ConnectionTypeUnknown { + return } // Record metrics with timestamps - duration calculation happens in metrics package @@ -1070,3 +1069,16 @@ func boolToConnStatus(connected bool) guard.ConnStatus { } return guard.ConnStatusDisconnected } + +func metricsConnType(priority conntype.ConnPriority) metrics.ConnectionType { + switch priority { + case conntype.Relay: + return metrics.ConnectionTypeRelay + case conntype.ICETurn: + return metrics.ConnectionTypeICETurn + case conntype.ICEP2P: + return metrics.ConnectionTypeICEP2P + default: + return metrics.ConnectionTypeUnknown + } +} diff --git a/client/internal/peer/conn_test.go b/client/internal/peer/conn_test.go index 49979ea83..b709d5e40 100644 --- a/client/internal/peer/conn_test.go +++ b/client/internal/peer/conn_test.go @@ -11,6 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/netbirdio/netbird/client/iface" + "github.com/netbirdio/netbird/client/internal/metrics" + "github.com/netbirdio/netbird/client/internal/peer/conntype" "github.com/netbirdio/netbird/client/internal/peer/dispatcher" "github.com/netbirdio/netbird/client/internal/peer/guard" "github.com/netbirdio/netbird/client/internal/peer/ice" @@ -386,3 +388,33 @@ func TestConn_onWGDisconnected_NoEscalationWithoutRosenpass(t *testing.T) { } assert.Empty(t, disconnected, "escalation must be limited to rosenpass connections") } + +func TestMetricsConnType(t *testing.T) { + tests := []struct { + name string + priority conntype.ConnPriority + expected metrics.ConnectionType + }{ + {"relay", conntype.Relay, metrics.ConnectionTypeRelay}, + {"ice over turn is relayed, not p2p", conntype.ICETurn, metrics.ConnectionTypeICETurn}, + {"direct p2p", conntype.ICEP2P, metrics.ConnectionTypeICEP2P}, + {"unset priority is unknown, not p2p", conntype.None, metrics.ConnectionTypeUnknown}, + {"unrecognised priority is unknown", conntype.ConnPriority(99), metrics.ConnectionTypeUnknown}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, metricsConnType(tc.priority)) + }) + } +} + +func TestMetricsConnType_RelayedMatchesIsRelayed(t *testing.T) { + for _, priority := range []conntype.ConnPriority{conntype.None, conntype.Relay, conntype.ICETurn, conntype.ICEP2P} { + conn := &Conn{currentConnPriority: priority} + tag := metricsConnType(priority) + relayedTag := tag == metrics.ConnectionTypeRelay || tag == metrics.ConnectionTypeICETurn + assert.Equal(t, conn.isRelayed(), relayedTag, + "priority %s: isRelayed and the %q metric tag must agree", priority, tag) + } +} diff --git a/client/ios/NetBirdSDK/client.go b/client/ios/NetBirdSDK/client.go index 350d68740..1c3ad7705 100644 --- a/client/ios/NetBirdSDK/client.go +++ b/client/ios/NetBirdSDK/client.go @@ -14,6 +14,7 @@ import ( log "github.com/sirupsen/logrus" + nbAnonymize "github.com/netbirdio/netbird/client/anonymize" "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/auth" "github.com/netbirdio/netbird/client/internal/debug" @@ -30,6 +31,13 @@ import ( types "github.com/netbirdio/netbird/upload-server/types" ) +// AnonymizeLevelDefault and AnonymizeLevelStrict are the accepted +// anonymizeLevel values for DebugBundle. +const ( + AnonymizeLevelDefault = nbAnonymize.LevelDefaultString + AnonymizeLevelStrict = nbAnonymize.LevelStrictString +) + // RouteListener export internal RouteListener for mobile type NetworkChangeListener interface { listener.NetworkChangeListener @@ -224,8 +232,10 @@ func (c *Client) Stop() { // DebugBundle generates a debug bundle, uploads it and returns the upload key. // It works with or without a running engine: when the engine is up it reuses // the live config, sync response and client metrics; otherwise it loads the -// config from disk (or the preloaded tvOS config). -func (c *Client) DebugBundle(anonymize bool) (string, error) { +// config from disk (or the preloaded tvOS config). anonymizeLevel is "default" +// or "strict"; strict also anonymizes internal IP ranges, peer names, and +// WireGuard public keys, and implies anonymize. +func (c *Client) DebugBundle(anonymize bool, anonymizeLevel string) (string, error) { cfg, cc := c.stateSnapshot() // If the engine hasn't been started, load config so we can reach management. @@ -275,6 +285,7 @@ func (c *Client) DebugBundle(anonymize bool) (string, error) { deps, debug.BundleConfig{ Anonymize: anonymize, + AnonymizeLevel: nbAnonymize.ParseLevel(anonymizeLevel), IncludeSystemInfo: true, }, ) diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index d4deeb8ec..83243be49 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -2781,6 +2781,11 @@ type DebugBundleRequest struct { // untrusted TLS certificate. Restricted to privileged callers; for // self-hosted upload servers. UploadInsecure bool `protobuf:"varint,7,opt,name=uploadInsecure,proto3" json:"uploadInsecure,omitempty"` + // anonymizeLevel selects how much the anonymizer redacts: "default" + // (or empty) keeps internal IP ranges, "strict" also anonymizes them. + // Unknown values are treated as "strict". Only meaningful with anonymize; + // "strict" implies it. + AnonymizeLevel string `protobuf:"bytes,8,opt,name=anonymizeLevel,proto3" json:"anonymizeLevel,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2857,6 +2862,13 @@ func (x *DebugBundleRequest) GetUploadInsecure() bool { return false } +func (x *DebugBundleRequest) GetAnonymizeLevel() string { + if x != nil { + return x.AnonymizeLevel + } + return "" +} + type DebugBundleResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` @@ -7253,7 +7265,7 @@ const file_daemon_proto_rawDesc = "" + "\x12translatedHostname\x18\x04 \x01(\tR\x12translatedHostname\x128\n" + "\x0etranslatedPort\x18\x05 \x01(\v2\x10.daemon.PortInfoR\x0etranslatedPort\"G\n" + "\x17ForwardingRulesResponse\x12,\n" + - "\x05rules\x18\x01 \x03(\v2\x16.daemon.ForwardingRuleR\x05rules\"\xdc\x01\n" + + "\x05rules\x18\x01 \x03(\v2\x16.daemon.ForwardingRuleR\x05rules\"\x84\x02\n" + "\x12DebugBundleRequest\x12\x1c\n" + "\tanonymize\x18\x01 \x01(\bR\tanonymize\x12\x1e\n" + "\n" + @@ -7264,7 +7276,8 @@ const file_daemon_proto_rawDesc = "" + "\n" + "cliVersion\x18\x06 \x01(\tR\n" + "cliVersion\x12&\n" + - "\x0euploadInsecure\x18\a \x01(\bR\x0euploadInsecure\"}\n" + + "\x0euploadInsecure\x18\a \x01(\bR\x0euploadInsecure\x12&\n" + + "\x0eanonymizeLevel\x18\b \x01(\tR\x0eanonymizeLevel\"}\n" + "\x13DebugBundleResponse\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12 \n" + "\vuploadedKey\x18\x02 \x01(\tR\vuploadedKey\x120\n" + diff --git a/client/proto/daemon.proto b/client/proto/daemon.proto index 3c31156ec..18ce0e79c 100644 --- a/client/proto/daemon.proto +++ b/client/proto/daemon.proto @@ -540,6 +540,11 @@ message DebugBundleRequest { // untrusted TLS certificate. Restricted to privileged callers; for // self-hosted upload servers. bool uploadInsecure = 7; + // anonymizeLevel selects how much the anonymizer redacts: "default" + // (or empty) keeps internal IP ranges, "strict" also anonymizes them. + // Unknown values are treated as "strict". Only meaningful with anonymize; + // "strict" implies it. + string anonymizeLevel = 8; } message DebugBundleResponse { diff --git a/client/server/debug.go b/client/server/debug.go index 60a401b0e..8f4a506b4 100644 --- a/client/server/debug.go +++ b/client/server/debug.go @@ -16,6 +16,7 @@ import ( "google.golang.org/grpc/codes" gstatus "google.golang.org/grpc/status" + "github.com/netbirdio/netbird/client/anonymize" "github.com/netbirdio/netbird/client/internal/debug" "github.com/netbirdio/netbird/client/internal/ipcauth" "github.com/netbirdio/netbird/client/proto" @@ -122,6 +123,7 @@ func (s *Server) generateDebugBundle(req *proto.DebugBundleRequest, uiOpener deb }, debug.BundleConfig{ Anonymize: req.GetAnonymize(), + AnonymizeLevel: anonymize.ParseLevel(req.GetAnonymizeLevel()), IncludeSystemInfo: req.GetSystemInfo(), LogFileCount: req.GetLogFileCount(), }, diff --git a/client/ssh/server/command_execution_windows.go b/client/ssh/server/command_execution_windows.go index e1ba777f6..feb8daa26 100644 --- a/client/ssh/server/command_execution_windows.go +++ b/client/ssh/server/command_execution_windows.go @@ -243,7 +243,7 @@ func (s *Server) setUserEnvironmentVariables(envMap map[string]string, userProfi // prepareCommandEnv prepares environment variables for command execution on Windows func (s *Server) prepareCommandEnv(logger *log.Entry, localUser *user.User, session ssh.Session) []string { - username, domain := s.parseUsername(localUser.Username) + username, domain := parseUsername(localUser.Username) userEnv, err := s.getUserEnvironment(logger, username, domain) if err != nil { log.Debugf("failed to get user environment for %s\\%s, using fallback: %v", domain, username, err) @@ -383,7 +383,7 @@ func (s *Server) executeCommandWithPty(logger *log.Entry, session ssh.Session, _ return false } - username, domain := s.parseUsername(localUser.Username) + username, domain := parseUsername(localUser.Username) shell := getUserShell(localUser.Uid) req := PtyExecutionRequest{ diff --git a/client/ssh/server/port_forwarding.go b/client/ssh/server/port_forwarding.go index a47fdb48a..81d3b9173 100644 --- a/client/ssh/server/port_forwarding.go +++ b/client/ssh/server/port_forwarding.go @@ -133,7 +133,12 @@ func (s *Server) checkPrivilegedPortAccess(forwardType string, port uint32, resu return nil } - if result.User != nil && isPrivilegedUsername(result.User.Username) { + // Only uid 0 may bind below the threshold, which is the kernel's own rule and + // is asked directly rather than through isPrivilegedOrUnknown: that helper + // reports an account it cannot evaluate as privileged, which is safe for a + // refusal and unsafe for a grant such as this one. Windows has returned + // above, so Uid here is a Unix uid and never a SID. + if result.User != nil && result.User.Uid == "0" { return nil } diff --git a/client/ssh/server/privileges_other.go b/client/ssh/server/privileges_other.go new file mode 100644 index 000000000..89440dea8 --- /dev/null +++ b/client/ssh/server/privileges_other.go @@ -0,0 +1,16 @@ +//go:build !windows + +package server + +// isProcessElevated is only meaningful on Windows; other platforms use the +// effective UID check in isCurrentProcessPrivileged. +func isProcessElevated() bool { + return false +} + +// isWindowsAccountPrivilegedOrUnknown is only reachable on Windows. Report +// privileged on other platforms so a caller refusing privileged accounts fails +// closed. +func isWindowsAccountPrivilegedOrUnknown(string) bool { + return true +} diff --git a/client/ssh/server/privileges_windows.go b/client/ssh/server/privileges_windows.go new file mode 100644 index 000000000..41ea00fd7 --- /dev/null +++ b/client/ssh/server/privileges_windows.go @@ -0,0 +1,228 @@ +//go:build windows + +package server + +import ( + "fmt" + "strings" + "unsafe" + + log "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" +) + +var ( + netapi32 = windows.NewLazySystemDLL("netapi32.dll") + procNetUserGetLocalGroups = netapi32.NewProc("NetUserGetLocalGroups") +) + +const ( + // lgIncludeIndirect makes NetUserGetLocalGroups also return local groups + // the user belongs to through a global group. + lgIncludeIndirect = 0x1 + maxPreferredLength = 0xFFFFFFFF +) + +// localGroupUsersInfo0 mirrors LOCALGROUP_USERS_INFO_0. +type localGroupUsersInfo0 struct { + name *uint16 +} + +// isProcessElevated reports whether the current process token is elevated +// (TokenElevation): true for elevated administrators, the built-in +// Administrator, administrators with UAC disabled, and SYSTEM; false for +// standard users and administrators running with a UAC-filtered token. +func isProcessElevated() bool { + return windows.GetCurrentProcessToken().IsElevated() +} + +// isWindowsAccountPrivilegedOrUnknown reports whether the account is privileged +// on this machine: a well-known service account, a built-in Administrator +// (RID 500), or a member of the local Administrators group, directly or through +// nested groups. +// +// An account whose privilege cannot be determined counts as privileged, which +// is why the name says "or unknown". That is fail-closed for a caller that +// refuses privileged accounts, and fail-open for a caller that grants something +// to them, so only the former may use this. +func isWindowsAccountPrivilegedOrUnknown(username string) bool { + sid, _, _, err := windows.LookupSID("", username) + if err != nil { + log.Warnf("privilege check: SID lookup for %q failed, treating as privileged: %v", username, err) + return true + } + + if isPrivilegedUserSID(sid) { + return true + } + + member, err := isLocalAdminsMember(username) + if err != nil { + log.Warnf("privilege check: cannot determine Administrators membership for %q, treating as privileged: %v", username, err) + return true + } + return member +} + +// isPrivilegedUserSID reports whether the SID itself identifies a privileged +// principal, without consulting group membership. +func isPrivilegedUserSID(sid *windows.SID) bool { + wellKnown := []windows.WELL_KNOWN_SID_TYPE{ + windows.WinLocalSystemSid, + windows.WinLocalServiceSid, + windows.WinNetworkServiceSid, + windows.WinBuiltinAdministratorsSid, + } + for _, sidType := range wellKnown { + if sid.IsWellKnown(sidType) { + return true + } + } + return isBuiltinAdministratorSID(sid) +} + +// isBuiltinAdministratorSID reports whether the SID is a machine or domain +// built-in Administrator account (S-1-5-21-...-500). RID 500 is reserved for +// that account; it can be renamed but cannot be removed from the +// Administrators group. +func isBuiltinAdministratorSID(sid *windows.SID) bool { + if sid.IdentifierAuthority() != windows.SECURITY_NT_AUTHORITY { + return false + } + count := sid.SubAuthorityCount() + if count < 2 || sid.SubAuthority(0) != 21 { + return false + } + return sid.SubAuthority(uint32(count-1)) == 500 +} + +// isLocalAdminsMember reports whether the account is a member of the local +// Administrators group. +// +// Local accounts are checked against the local SAM, which is authoritative for +// them and, unlike a token, cannot under-report: UAC filters the tokens of +// local administrators, and a filtered token carries Administrators as +// deny-only, which a membership check on the token would read as "not a +// member". Domain accounts are exempt from that filtering, so for them an S4U +// token is preferred because its group list is LSA's transitive expansion and +// therefore covers nested and universal groups plus the machine's own local +// groups. NetUserGetLocalGroups expands only one global-group hop but needs no +// logon, so it serves as the fallback when no token can be obtained. +func isLocalAdminsMember(username string) (bool, error) { + adminSid, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + if err != nil { + return false, fmt.Errorf("create Administrators SID: %w", err) + } + + account, domain := parseUsername(username) + if NewPrivilegeDropper().isLocalUser(domain) { + return localGroupsContainSID(account, adminSid) + } + + member, s4uErr := s4uTokenIsMember(account, domain, adminSid) + if s4uErr == nil { + return member, nil + } + log.Debugf("privilege check: S4U membership check for %q failed, falling back to local group enumeration: %v", username, s4uErr) + + member, err = localGroupsContainSID(buildUserCpn(account, domain), adminSid) + if err != nil { + return false, fmt.Errorf("S4U check: %w; local group enumeration: %w", s4uErr, err) + } + return member, nil +} + +// s4uTokenIsMember obtains an S4U token for the account and checks whether the +// given SID is enabled in it. +func s4uTokenIsMember(account, domain string, sid *windows.SID) (bool, error) { + token, err := generateS4UUserToken(log.NewEntry(log.StandardLogger()), account, domain) + if err != nil { + return false, err + } + defer func() { + if err := windows.CloseHandle(token); err != nil { + log.Debugf("close S4U token: %v", err) + } + }() + return windows.Token(token).IsMember(sid) +} + +// localGroupsContainSID reports whether the wanted group is among the local +// groups the account belongs to, directly or through a global group. +// +// The wanted SID is resolved to its group name once and compared against the +// enumerated names. Well-known SIDs resolve from a static table, so that lookup +// needs no domain controller, and it keeps the comparison correct for a renamed +// or localized group because both sides then carry the new name. Resolving each +// enumerated name back to a SID instead would add a lookup per group that can +// block until it times out while a domain controller is unreachable, and cannot +// change the outcome: the names enumerated here are local groups of this +// machine, whose names are unique, so a name match identifies the group. +// +// A failure to resolve the wanted SID is returned rather than reported as +// "not a member", so a privilege check built on this fails closed. +func localGroupsContainSID(username string, want *windows.SID) (bool, error) { + wantName, _, _, err := want.LookupAccount("") + if err != nil { + return false, fmt.Errorf("resolve group SID %s to a name: %w", want, err) + } + + groups, err := netUserGetLocalGroups(username) + if err != nil { + return false, err + } + + for _, group := range groups { + if strings.EqualFold(group, wantName) { + return true, nil + } + } + return false, nil +} + +// netUserGetLocalGroups returns the names of the local groups the account is a +// member of, including indirect membership through global groups. +func netUserGetLocalGroups(username string) ([]string, error) { + name16, err := windows.UTF16PtrFromString(username) + if err != nil { + return nil, fmt.Errorf("convert username: %w", err) + } + + var buf *byte + var entriesRead, totalEntries uint32 + status, _, _ := procNetUserGetLocalGroups.Call( + 0, // local server + uintptr(unsafe.Pointer(name16)), + 0, // level 0: LOCALGROUP_USERS_INFO_0 + lgIncludeIndirect, + uintptr(unsafe.Pointer(&buf)), + maxPreferredLength, + uintptr(unsafe.Pointer(&entriesRead)), + uintptr(unsafe.Pointer(&totalEntries)), + ) + if status != 0 { + return nil, fmt.Errorf("NetUserGetLocalGroups for %q: status %d", username, status) + } + if buf == nil { + return nil, nil + } + defer func() { + if err := windows.NetApiBufferFree(buf); err != nil { + log.Debugf("free NetApi buffer: %v", err) + } + }() + + // MAX_PREFERRED_LENGTH makes the API allocate as much as it needs, so a + // short read is not expected. Report it rather than silently returning a + // subset of the account's groups. + if entriesRead != totalEntries { + return nil, fmt.Errorf("NetUserGetLocalGroups for %q returned %d of %d groups", username, entriesRead, totalEntries) + } + + entries := unsafe.Slice((*localGroupUsersInfo0)(unsafe.Pointer(buf)), entriesRead) + groups := make([]string, 0, entriesRead) + for _, entry := range entries { + groups = append(groups, windows.UTF16PtrToString(entry.name)) + } + return groups, nil +} diff --git a/client/ssh/server/privileges_windows_test.go b/client/ssh/server/privileges_windows_test.go new file mode 100644 index 000000000..983fccdf7 --- /dev/null +++ b/client/ssh/server/privileges_windows_test.go @@ -0,0 +1,293 @@ +//go:build windows + +package server + +import ( + "os/user" + "testing" + "unsafe" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" +) + +// filterNormalAccount limits NetUserEnum to normal user accounts. +const filterNormalAccount = 0x2 + +// TOKEN_ELEVATION_TYPE values. +const ( + tokenElevationTypeDefault = 1 + tokenElevationTypeFull = 2 + tokenElevationTypeLimited = 3 +) + +// tokenElevationType reads TokenElevationType from a token. +func tokenElevationType(token windows.Token) (uint32, error) { + var elevationType, returnedLen uint32 + err := windows.GetTokenInformation(token, windows.TokenElevationType, + (*byte)(unsafe.Pointer(&elevationType)), uint32(unsafe.Sizeof(elevationType)), &returnedLen) + if err != nil { + return 0, err + } + return elevationType, nil +} + +// userInfo0 mirrors USER_INFO_0. +type userInfo0 struct { + name *uint16 +} + +func mustParseSID(t *testing.T, s string) *windows.SID { + t.Helper() + sid, err := windows.StringToSid(s) + require.NoError(t, err, "parse SID %s", s) + return sid +} + +// localAccountNames returns the names of the local user accounts. +func localAccountNames(t *testing.T) []string { + t.Helper() + + var buf *byte + var entriesRead, totalEntries, resume uint32 + err := windows.NetUserEnum(nil, 0, filterNormalAccount, &buf, maxPreferredLength, + &entriesRead, &totalEntries, &resume) + require.NoError(t, err, "enumerate local users") + t.Cleanup(func() { + require.NoError(t, windows.NetApiBufferFree(buf), "free NetApi buffer") + }) + + entries := unsafe.Slice((*userInfo0)(unsafe.Pointer(buf)), entriesRead) + names := make([]string, 0, entriesRead) + for _, entry := range entries { + names = append(names, windows.UTF16PtrToString(entry.name)) + } + return names +} + +// localAccountNameByRID returns the name of the local account carrying the +// given RID. Accounts such as Administrator and Guest can be renamed and are +// localized, so tests must not name them literally. +func localAccountNameByRID(t *testing.T, rid uint32) string { + t.Helper() + + for _, name := range localAccountNames(t) { + sid, _, _, err := windows.LookupSID("", name) + if err != nil { + continue + } + if sid.IdentifierAuthority() != windows.SECURITY_NT_AUTHORITY { + continue + } + count := sid.SubAuthorityCount() + if count < 2 || sid.SubAuthority(0) != 21 { + continue + } + if sid.SubAuthority(uint32(count-1)) == rid { + return name + } + } + + t.Fatalf("no local account with RID %d", rid) + return "" +} + +// wellKnownAccountName resolves a well-known SID to the qualified account name +// the local system uses for it, which is localized. +func wellKnownAccountName(t *testing.T, sidType windows.WELL_KNOWN_SID_TYPE) string { + t.Helper() + + sid, err := windows.CreateWellKnownSid(sidType) + require.NoError(t, err, "create well-known SID") + name, domain, _, err := sid.LookupAccount("") + require.NoError(t, err, "resolve %s to an account name", sid) + if domain == "" { + return name + } + return domain + `\` + name +} + +func TestIsBuiltinAdministratorSID(t *testing.T) { + tests := []struct { + name string + sid string + want bool + }{ + {"machine_administrator", "S-1-5-21-1111111111-2222222222-3333333333-500", true}, + {"domain_administrator", "S-1-5-21-3390233681-4087452608-412898826-500", true}, + {"regular_user", "S-1-5-21-1111111111-2222222222-3333333333-1001", false}, + {"guest_account", "S-1-5-21-1111111111-2222222222-3333333333-501", false}, + {"domain_admins_group", "S-1-5-21-1111111111-2222222222-3333333333-512", false}, + {"system", "S-1-5-18", false}, + {"administrators_group", "S-1-5-32-544", false}, + {"non_nt_authority", "S-1-1-0", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isBuiltinAdministratorSID(mustParseSID(t, tt.sid)) + assert.Equal(t, tt.want, result, "RID 500 detection for %s", tt.sid) + }) + } +} + +func TestIsPrivilegedUserSID(t *testing.T) { + tests := []struct { + name string + sid string + want bool + }{ + {"local_system", "S-1-5-18", true}, + {"local_service", "S-1-5-19", true}, + {"network_service", "S-1-5-20", true}, + {"administrators_group", "S-1-5-32-544", true}, + {"builtin_administrator", "S-1-5-21-1111111111-2222222222-3333333333-500", true}, + {"regular_user", "S-1-5-21-1111111111-2222222222-3333333333-1001", false}, + {"users_group", "S-1-5-32-545", false}, + {"everyone", "S-1-1-0", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isPrivilegedUserSID(mustParseSID(t, tt.sid)) + assert.Equal(t, tt.want, result, "SID privilege classification for %s", tt.sid) + }) + } +} + +func TestIsWindowsAccountPrivilegedOrUnknown(t *testing.T) { + tests := []struct { + name string + username string + want bool + }{ + {"system", wellKnownAccountName(t, windows.WinLocalSystemSid), true}, + {"local_service", wellKnownAccountName(t, windows.WinLocalServiceSid), true}, + {"network_service", wellKnownAccountName(t, windows.WinNetworkServiceSid), true}, + {"administrators_group", wellKnownAccountName(t, windows.WinBuiltinAdministratorsSid), true}, + // The built-in Administrator (RID 500) and Guest (RID 501) accounts + // exist on every Windows installation, though they may be disabled. + {"builtin_administrator", localAccountNameByRID(t, 500), true}, + {"guest", localAccountNameByRID(t, 501), false}, + // Unresolvable accounts fail closed. + {"nonexistent_user", "netbird-no-such-user", true}, + {"empty_username", "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isWindowsAccountPrivilegedOrUnknown(tt.username) + assert.Equal(t, tt.want, result, "account privilege classification for %q", tt.username) + }) + } +} + +func TestIsProcessElevated(t *testing.T) { + elevated := isProcessElevated() + + // TokenElevationType is a second, independent view of the same token: + // Full means elevated and Limited means a filtered administrator, while + // Default covers both a standard user and an administrator with no linked + // token (UAC off, the built-in Administrator, SYSTEM), so it implies nothing. + elevationType, err := tokenElevationType(windows.GetCurrentProcessToken()) + require.NoError(t, err, "read token elevation type") + + adminSid, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + require.NoError(t, err, "create Administrators SID") + + // Token(0) makes CheckTokenMembership evaluate the caller's own token. It + // counts only enabled SIDs, so a filtered administrator reports false here. + member, err := windows.Token(0).IsMember(adminSid) + require.NoError(t, err, "check own Administrators membership") + + t.Logf("elevated=%v elevationType=%d memberOfAdministrators=%v", elevated, elevationType, member) + + switch elevationType { + case tokenElevationTypeFull: + assert.True(t, elevated, "a token of elevation type Full must report elevated") + case tokenElevationTypeLimited: + assert.False(t, elevated, "a filtered administrator token must not report elevated") + } + + // Administrators enabled in the token means the token wields administrative + // rights, which is what elevation reports. + if member { + assert.True(t, elevated, "token with enabled Administrators membership must report elevated") + } +} + +// TestS4UMembershipAgreesWithLocalGroups exercises the S4U token path used +// for domain accounts. S4U logons need the TCB privilege, so the test runs +// only as SYSTEM (which is how CI executes the suite). For local accounts the +// token's Administrators membership must agree with the SAM enumeration. +func TestS4UMembershipAgreesWithLocalGroups(t *testing.T) { + system, err := windows.CreateWellKnownSid(windows.WinLocalSystemSid) + require.NoError(t, err, "create SYSTEM SID") + current, err := user.Current() + require.NoError(t, err, "get current user") + if current.Uid != system.String() { + t.Skipf("S4U logon requires SYSTEM (running as %s)", current.Username) + } + + adminSid, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + require.NoError(t, err, "create Administrators SID") + + checked := 0 + for _, name := range localAccountNames(t) { + viaToken, err := s4uTokenIsMember(name, ".", adminSid) + if err != nil { + // Disabled or logon-restricted accounts cannot get an S4U logon. + t.Logf("skipping %s: %v", name, err) + continue + } + viaSAM, err := localGroupsContainSID(name, adminSid) + require.NoError(t, err, "enumerate local groups for %s", name) + + assert.Equal(t, viaSAM, viaToken, "S4U token and SAM enumeration must agree on Administrators membership for %s", name) + checked++ + } + // Ineligible accounts are skipped, so without this the test could report + // success while comparing nothing at all. + require.Positive(t, checked, "no local account completed an S4U logon, so nothing was compared") + t.Logf("checked %d local accounts via S4U", checked) +} + +// TestLocalGroupsContainSID_Administrator checks the positive case against the +// built-in Administrator, a member of Administrators on every installation. +func TestLocalGroupsContainSID_Administrator(t *testing.T) { + adminSid, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + require.NoError(t, err, "create Administrators SID") + + administrator := localAccountNameByRID(t, 500) + member, err := localGroupsContainSID(administrator, adminSid) + require.NoError(t, err, "enumerate local groups for %s", administrator) + assert.True(t, member, "%s is a member of the Administrators group", administrator) +} + +// TestLocalGroupsContainSID_UnresolvableGroupFailsClosed covers a wanted SID +// that resolves to no group: the error must surface rather than being reported +// as "not a member", so the privilege check treats the account as privileged. +func TestLocalGroupsContainSID_UnresolvableGroupFailsClosed(t *testing.T) { + unknown := mustParseSID(t, "S-1-5-21-1111111111-2222222222-3333333333-4444") + + _, err := localGroupsContainSID(localAccountNameByRID(t, 500), unknown) + require.Error(t, err, "must report an error when the wanted group cannot be identified") +} + +func TestLocalGroupsContainSID_Guest(t *testing.T) { + guestsSid, err := windows.CreateWellKnownSid(windows.WinBuiltinGuestsSid) + require.NoError(t, err, "create Guests SID") + adminsSid, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + require.NoError(t, err, "create Administrators SID") + + guest := localAccountNameByRID(t, 501) + + inGuests, err := localGroupsContainSID(guest, guestsSid) + require.NoError(t, err, "enumerate local groups for %s", guest) + assert.True(t, inGuests, "%s is a member of the Guests group", guest) + + inAdmins, err := localGroupsContainSID(guest, adminsSid) + require.NoError(t, err, "enumerate local groups for %s", guest) + assert.False(t, inAdmins, "%s is not a member of the Administrators group", guest) +} diff --git a/client/ssh/server/server_config_test.go b/client/ssh/server/server_config_test.go index f70e29963..983bd7a43 100644 --- a/client/ssh/server/server_config_test.go +++ b/client/ssh/server/server_config_test.go @@ -239,6 +239,7 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { forwardType string port uint32 username string + uid string expectError bool errorMsg string skipOnWindows bool @@ -248,6 +249,7 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { forwardType: "remote", port: 80, username: "testuser", + uid: "1000", expectError: true, errorMsg: "cannot bind to privileged port", skipOnWindows: true, @@ -257,6 +259,7 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { forwardType: "tcpip-forward", port: 443, username: "testuser", + uid: "1000", expectError: true, errorMsg: "cannot bind to privileged port", skipOnWindows: true, @@ -266,6 +269,7 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { forwardType: "remote", port: 8080, username: "testuser", + uid: "1000", expectError: false, }, { @@ -273,6 +277,7 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { forwardType: "remote", port: 0, username: "testuser", + uid: "1000", expectError: false, }, { @@ -280,13 +285,35 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { forwardType: "remote", port: 22, username: "root", + uid: "0", expectError: false, }, + { + // Only uid 0 is privileged, whatever the account is called. + name: "uid 0 under another name may bind a privileged port", + forwardType: "remote", + port: 22, + username: "toor", + uid: "0", + expectError: false, + skipOnWindows: true, + }, + { + name: "account named root without uid 0 may not", + forwardType: "remote", + port: 22, + username: "root", + uid: "1000", + expectError: true, + errorMsg: "cannot bind to privileged port", + skipOnWindows: true, + }, { name: "local forward privileged port allowed for non-root", forwardType: "local", port: 80, username: "testuser", + uid: "1000", expectError: false, }, } @@ -299,7 +326,7 @@ func TestServer_PrivilegedPortAccess(t *testing.T) { result := PrivilegeCheckResult{ Allowed: true, - User: &user.User{Username: tt.username}, + User: &user.User{Username: tt.username, Uid: tt.uid}, } err := server.checkPrivilegedPortAccess(tt.forwardType, tt.port, result) @@ -420,6 +447,13 @@ func TestServer_PortConflictHandling(t *testing.T) { func TestServer_IsPrivilegedUser(t *testing.T) { + // Windows classification depends on account SIDs and group membership, and + // the accounts involved carry localized, renameable names. It is covered by + // TestIsWindowsAccountPrivileged, which resolves them from well-known SIDs. + if runtime.GOOS == "windows" { + t.Skip("covered by TestIsWindowsAccountPrivileged") + } + tests := []struct { username string expected bool @@ -440,44 +474,16 @@ func TestServer_IsPrivilegedUser(t *testing.T) { expected: false, description: "empty username should not be privileged", }, - } - - // Add Windows-specific tests - if runtime.GOOS == "windows" { - tests = append(tests, []struct { - username string - expected bool - description string - }{ - { - username: "Administrator", - expected: true, - description: "Administrator should be considered privileged on Windows", - }, - { - username: "administrator", - expected: true, - description: "administrator should be considered privileged on Windows (case insensitive)", - }, - }...) - } else { - // On non-Windows systems, Administrator should not be privileged - tests = append(tests, []struct { - username string - expected bool - description string - }{ - { - username: "Administrator", - expected: false, - description: "Administrator should not be privileged on non-Windows systems", - }, - }...) + { + username: "Administrator", + expected: false, + description: "Administrator should not be privileged on non-Windows systems", + }, } for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { - result := isPrivilegedUsername(tt.username) + result := isPrivilegedOrUnknown(tt.username) assert.Equal(t, tt.expected, result, tt.description) }) } diff --git a/client/ssh/server/sftp_windows.go b/client/ssh/server/sftp_windows.go index dc532b9e7..25cd17298 100644 --- a/client/ssh/server/sftp_windows.go +++ b/client/ssh/server/sftp_windows.go @@ -17,7 +17,7 @@ import ( // createSftpCommand creates a Windows SFTP command with user switching. // The caller must close the returned token handle after starting the process. func (s *Server) createSftpCommand(targetUser *user.User, sess ssh.Session) (*exec.Cmd, windows.Token, error) { - username, domain := s.parseUsername(targetUser.Username) + username, domain := parseUsername(targetUser.Username) netbirdPath, err := os.Executable() if err != nil { diff --git a/client/ssh/server/user_utils.go b/client/ssh/server/user_utils.go index bc2aa2d7d..6c8142b30 100644 --- a/client/ssh/server/user_utils.go +++ b/client/ssh/server/user_utils.go @@ -16,11 +16,6 @@ var ( ErrPrivilegedUserSwitch = errors.New("cannot switch to privileged user - current user lacks required privileges") ) -// isPlatformUnix returns true for Unix-like platforms (Linux, macOS, etc.) -func isPlatformUnix() bool { - return getCurrentOS() != "windows" -} - // Dependency injection variables for testing - allows mocking dynamic runtime checks var ( getCurrentUser = currentUserWithGetent @@ -29,6 +24,9 @@ var ( getIsProcessPrivileged = isCurrentProcessPrivileged getEuid = os.Geteuid + + getProcessElevated = isProcessElevated + getWindowsAccountPrivilegedOrUnknown = isWindowsAccountPrivilegedOrUnknown ) const ( @@ -65,6 +63,13 @@ type PrivilegeCheckResult struct { RequiresUserSwitching bool } +// privilegeCheckContext holds all context needed for privilege checking +type privilegeCheckContext struct { + currentUser *user.User + currentUserPrivileged bool + allowRoot bool +} + // CheckPrivileges performs comprehensive privilege checking for all SSH features. // This is the single source of truth for privilege decisions across the SSH server. func (s *Server) CheckPrivileges(req PrivilegeCheckRequest) PrivilegeCheckResult { @@ -75,7 +80,7 @@ func (s *Server) CheckPrivileges(req PrivilegeCheckRequest) PrivilegeCheckResult // Handle empty username case - but still check root access controls if req.RequestedUsername == "" { - if isPrivilegedUsername(context.currentUser.Username) && !context.allowRoot { + if isPrivilegedOrUnknown(context.currentUser.Username) && !context.allowRoot { return PrivilegeCheckResult{ Allowed: false, Error: &PrivilegedUserError{Username: context.currentUser.Username}, @@ -135,7 +140,7 @@ func (s *Server) checkUserRequest(ctx *privilegeCheckContext, req PrivilegeCheck needsUserSwitching := !isSameResolvedUser(resolvedUser, ctx.currentUser) - if isPrivilegedUsername(resolvedUser.Username) && !ctx.allowRoot { + if isPrivilegedOrUnknown(resolvedUser.Username) && !ctx.allowRoot { return PrivilegeCheckResult{ Allowed: false, Error: &PrivilegedUserError{Username: resolvedUser.Username}, @@ -175,6 +180,42 @@ func (s *Server) resolveRequestedUser(requestedUsername string) (*user.User, err return u, nil } +// SetAllowRootLogin configures root login access +func (s *Server) SetAllowRootLogin(allow bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.allowRootLogin = allow +} + +// userNameLookup performs user lookup with root login permission check +func (s *Server) userNameLookup(username string) (*user.User, error) { + result, err := s.userPrivilegeCheck(username) + if err != nil { + return nil, err + } + return result.User, nil +} + +// userPrivilegeCheck performs user lookup with full privilege check result +func (s *Server) userPrivilegeCheck(username string) (PrivilegeCheckResult, error) { + result := s.CheckPrivileges(PrivilegeCheckRequest{ + RequestedUsername: username, + FeatureSupportsUserSwitch: true, + FeatureName: FeatureSSHLogin, + }) + + if !result.Allowed { + return result, result.Error + } + + return result, nil +} + +// isPlatformUnix returns true for Unix-like platforms (Linux, macOS, etc.) +func isPlatformUnix() bool { + return getCurrentOS() != "windows" +} + // isSameResolvedUser compares two resolved user identities func isSameResolvedUser(user1, user2 *user.User) bool { if user1 == nil || user2 == nil { @@ -183,13 +224,6 @@ func isSameResolvedUser(user1, user2 *user.User) bool { return user1.Uid == user2.Uid } -// privilegeCheckContext holds all context needed for privilege checking -type privilegeCheckContext struct { - currentUser *user.User - currentUserPrivileged bool - allowRoot bool -} - // isSameUser checks if two usernames refer to the same user // SECURITY: This function must be conservative - it should only return true // when we're certain both usernames refer to the exact same user identity @@ -253,159 +287,30 @@ func isWindowsSameUser(requestedUsername, currentUsername string) bool { return strings.EqualFold(reqDomain, curDomain) } -// SetAllowRootLogin configures root login access -func (s *Server) SetAllowRootLogin(allow bool) { - s.mu.Lock() - defer s.mu.Unlock() - s.allowRootLogin = allow -} - -// userNameLookup performs user lookup with root login permission check -func (s *Server) userNameLookup(username string) (*user.User, error) { - result := s.CheckPrivileges(PrivilegeCheckRequest{ - RequestedUsername: username, - FeatureSupportsUserSwitch: true, - FeatureName: FeatureSSHLogin, - }) - - if !result.Allowed { - return nil, result.Error - } - - return result.User, nil -} - -// userPrivilegeCheck performs user lookup with full privilege check result -func (s *Server) userPrivilegeCheck(username string) (PrivilegeCheckResult, error) { - result := s.CheckPrivileges(PrivilegeCheckRequest{ - RequestedUsername: username, - FeatureSupportsUserSwitch: true, - FeatureName: FeatureSSHLogin, - }) - - if !result.Allowed { - return result, result.Error - } - - return result, nil -} - -// isPrivilegedUsername checks if the given username represents a privileged user across platforms. -// On Unix: root -// On Windows: Administrator, SYSTEM (case-insensitive) -// Handles domain-qualified usernames like "DOMAIN\Administrator" or "user@domain.com" -func isPrivilegedUsername(username string) bool { +// isPrivilegedOrUnknown reports whether the given username represents a +// privileged user, or on Windows an account whose privilege could not be +// determined. +// On Unix: root. +// On Windows: well-known service accounts, built-in Administrator accounts, +// and members of the local Administrators group; handles domain-qualified +// usernames like "DOMAIN\user" or "user@domain.com". An account that cannot be +// resolved or evaluated is reported as privileged. +// +// Use this to refuse privileged accounts, never to grant them anything: the +// undetermined case is safe for a refusal and unsafe for a grant. +func isPrivilegedOrUnknown(username string) bool { if getCurrentOS() != "windows" { return username == "root" } - - bareUsername := username - // Handle Windows domain format: DOMAIN\username - if idx := strings.LastIndex(username, `\`); idx != -1 { - bareUsername = username[idx+1:] - } - // Handle email-style format: username@domain.com - if idx := strings.Index(bareUsername, "@"); idx != -1 { - bareUsername = bareUsername[:idx] - } - - return isWindowsPrivilegedUser(bareUsername) -} - -// isWindowsPrivilegedUser checks if a bare username (domain already stripped) represents a Windows privileged account -func isWindowsPrivilegedUser(bareUsername string) bool { - // common privileged usernames (case insensitive) - privilegedNames := []string{ - "administrator", - "admin", - "root", - "system", - "localsystem", - "networkservice", - "localservice", - } - - usernameLower := strings.ToLower(bareUsername) - for _, privilegedName := range privilegedNames { - if usernameLower == privilegedName { - return true - } - } - - // computer accounts (ending with $) are not privileged by themselves - // They only gain privileges through group membership or specific SIDs - - if targetUser, err := lookupUser(bareUsername); err == nil { - return isWindowsPrivilegedSID(targetUser.Uid) - } - - return false -} - -// isWindowsPrivilegedSID checks if a Windows SID represents a privileged account -func isWindowsPrivilegedSID(sid string) bool { - privilegedSIDs := []string{ - "S-1-5-18", // Local System (SYSTEM) - "S-1-5-19", // Local Service (NT AUTHORITY\LOCAL SERVICE) - "S-1-5-20", // Network Service (NT AUTHORITY\NETWORK SERVICE) - "S-1-5-32-544", // Administrators group (BUILTIN\Administrators) - "S-1-5-500", // Built-in Administrator account (local machine RID 500) - } - - for _, privilegedSID := range privilegedSIDs { - if sid == privilegedSID { - return true - } - } - - // Check for domain administrator accounts (RID 500 in any domain) - // Format: S-1-5-21-domain-domain-domain-500 - // This is reliable as RID 500 is reserved for the domain Administrator account - if strings.HasPrefix(sid, "S-1-5-21-") && strings.HasSuffix(sid, "-500") { - return true - } - - // Check for other well-known privileged RIDs in domain contexts - // RID 512 = Domain Admins group, RID 516 = Domain Controllers group - if strings.HasPrefix(sid, "S-1-5-21-") { - if strings.HasSuffix(sid, "-512") || // Domain Admins group - strings.HasSuffix(sid, "-516") || // Domain Controllers group - strings.HasSuffix(sid, "-519") { // Enterprise Admins group - return true - } - } - - return false + return getWindowsAccountPrivilegedOrUnknown(username) } // isCurrentProcessPrivileged checks if the current process is running with elevated privileges. // On Unix systems, this means running as root (UID 0). -// On Windows, this means running as Administrator or SYSTEM. +// On Windows, this means the process token is elevated (administrators, SYSTEM). func isCurrentProcessPrivileged() bool { if getCurrentOS() == "windows" { - return isWindowsElevated() + return getProcessElevated() } return getEuid() == 0 } - -// isWindowsElevated checks if the current process is running with elevated privileges on Windows -func isWindowsElevated() bool { - currentUser, err := getCurrentUser() - if err != nil { - log.Errorf("failed to get current user for privilege check, assuming non-privileged: %v", err) - return false - } - - if isWindowsPrivilegedSID(currentUser.Uid) { - log.Debugf("Windows user switching supported: running as privileged SID %s", currentUser.Uid) - return true - } - - if isPrivilegedUsername(currentUser.Username) { - log.Debugf("Windows user switching supported: running as privileged username %s", currentUser.Username) - return true - } - - log.Debugf("Windows user switching not supported: not running as privileged user (current: %s)", currentUser.Uid) - return false -} diff --git a/client/ssh/server/user_utils_test.go b/client/ssh/server/user_utils_test.go index 637dc10d0..2fa9b68ee 100644 --- a/client/ssh/server/user_utils_test.go +++ b/client/ssh/server/user_utils_test.go @@ -4,6 +4,7 @@ import ( "errors" "os/user" "runtime" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -27,8 +28,8 @@ func setupTestDependencies(currentUser *user.User, currentUserErr error, os stri originalLookupUser := lookupUser originalGetCurrentOS := getCurrentOS originalGetEuid := getEuid - - // Reset caches to ensure clean test state + originalGetProcessElevated := getProcessElevated + originalGetWindowsAccountPrivilegedOrUnknown := getWindowsAccountPrivilegedOrUnknown // Set test values - inject platform dependencies getCurrentUser = func() (*user.User, error) { @@ -53,16 +54,31 @@ func setupTestDependencies(currentUser *user.User, currentUserErr error, os stri return euid } - // Mock privilege detection based on the test user - getIsProcessPrivileged = func() bool { + // Simulate the Windows token elevation check based on the fixture user: + // the built-in Administrator (RID 500) and SYSTEM run elevated. + getProcessElevated = func() bool { if currentUser == nil { return false } - // Check both username and SID for Windows systems - if os == "windows" && isWindowsPrivilegedSID(currentUser.Uid) { + return currentUser.Uid == "S-1-5-18" || strings.HasSuffix(currentUser.Uid, "-500") + } + + // Simulate the Windows account classifier for the fixture accounts. + // "root" does not exist on Windows; the real classifier fails closed on + // unresolvable accounts, so it counts as privileged here too. + getWindowsAccountPrivilegedOrUnknown = func(username string) bool { + bare := username + if idx := strings.LastIndex(bare, `\`); idx != -1 { + bare = bare[idx+1:] + } + if idx := strings.Index(bare, "@"); idx != -1 { + bare = bare[:idx] + } + switch strings.ToLower(bare) { + case "administrator", "system", "root": return true } - return isPrivilegedUsername(currentUser.Username) + return false } // Return cleanup function @@ -71,10 +87,8 @@ func setupTestDependencies(currentUser *user.User, currentUserErr error, os stri lookupUser = originalLookupUser getCurrentOS = originalGetCurrentOS getEuid = originalGetEuid - - getIsProcessPrivileged = isCurrentProcessPrivileged - - // Reset caches after test + getProcessElevated = originalGetProcessElevated + getWindowsAccountPrivilegedOrUnknown = originalGetWindowsAccountPrivilegedOrUnknown } } @@ -421,6 +435,9 @@ func TestUsedFallback_MeansNoPrivilegeDropping(t *testing.T) { } func TestPrivilegedUsernameDetection(t *testing.T) { + // Windows classification is syscall-backed (SID resolution, group + // membership) and is covered by privileges_windows_test.go; here only the + // Unix logic and the platform dispatch are exercised. tests := []struct { name string username string @@ -432,25 +449,9 @@ func TestPrivilegedUsernameDetection(t *testing.T) { {"unix_regular_user", "alice", "linux", false}, {"unix_root_capital", "Root", "linux", false}, // Case-sensitive - // Windows tests + // Windows dispatch to the (mocked) account classifier {"windows_administrator", "Administrator", "windows", true}, - {"windows_system", "SYSTEM", "windows", true}, - {"windows_admin", "admin", "windows", true}, - {"windows_admin_lowercase", "administrator", "windows", true}, // Case-insensitive - {"windows_domain_admin", "DOMAIN\\Administrator", "windows", true}, - {"windows_email_admin", "admin@domain.com", "windows", true}, {"windows_regular_user", "alice", "windows", false}, - {"windows_domain_user", "DOMAIN\\alice", "windows", false}, - {"windows_localsystem", "localsystem", "windows", true}, - {"windows_networkservice", "networkservice", "windows", true}, - {"windows_localservice", "localservice", "windows", true}, - - // Computer accounts (these depend on current user context in real implementation) - {"windows_computer_account", "WIN2K19-C2$", "windows", false}, // Computer account by itself not privileged - {"windows_domain_computer", "DOMAIN\\COMPUTER$", "windows", false}, // Domain computer account - - // Cross-platform - {"root_on_windows", "root", "windows", true}, // Root should be privileged everywhere } for _, tt := range tests { @@ -459,50 +460,8 @@ func TestPrivilegedUsernameDetection(t *testing.T) { cleanup := setupTestDependencies(nil, nil, tt.platform, 1000, nil, nil) defer cleanup() - result := isPrivilegedUsername(tt.username) - assert.Equal(t, tt.privileged, result) - }) - } -} - -func TestWindowsPrivilegedSIDDetection(t *testing.T) { - tests := []struct { - name string - sid string - privileged bool - description string - }{ - // Well-known system accounts - {"system_account", "S-1-5-18", true, "Local System (SYSTEM)"}, - {"local_service", "S-1-5-19", true, "Local Service"}, - {"network_service", "S-1-5-20", true, "Network Service"}, - {"administrators_group", "S-1-5-32-544", true, "Administrators group"}, - {"builtin_administrator", "S-1-5-500", true, "Built-in Administrator"}, - - // Domain accounts - {"domain_administrator", "S-1-5-21-1234567890-1234567890-1234567890-500", true, "Domain Administrator (RID 500)"}, - {"domain_admins_group", "S-1-5-21-1234567890-1234567890-1234567890-512", true, "Domain Admins group"}, - {"domain_controllers_group", "S-1-5-21-1234567890-1234567890-1234567890-516", true, "Domain Controllers group"}, - {"enterprise_admins_group", "S-1-5-21-1234567890-1234567890-1234567890-519", true, "Enterprise Admins group"}, - - // Regular users - {"regular_user", "S-1-5-21-1234567890-1234567890-1234567890-1001", false, "Regular domain user"}, - {"another_regular_user", "S-1-5-21-1234567890-1234567890-1234567890-1234", false, "Another regular user"}, - {"local_user", "S-1-5-21-1234567890-1234567890-1234567890-1000", false, "Local regular user"}, - - // Groups that are not privileged - {"domain_users", "S-1-5-21-1234567890-1234567890-1234567890-513", false, "Domain Users group"}, - {"power_users", "S-1-5-32-547", false, "Power Users group"}, - - // Invalid SIDs - {"malformed_sid", "S-1-5-invalid", false, "Malformed SID"}, - {"empty_sid", "", false, "Empty SID"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := isWindowsPrivilegedSID(tt.sid) - assert.Equal(t, tt.privileged, result, "Failed for %s: %s", tt.description, tt.sid) + result := isPrivilegedOrUnknown(tt.username) + assert.Equal(t, tt.privileged, result, "privilege classification for %s on %s", tt.username, tt.platform) }) } } diff --git a/client/ssh/server/userswitching_windows.go b/client/ssh/server/userswitching_windows.go index 260e1301e..9e8cd5b30 100644 --- a/client/ssh/server/userswitching_windows.go +++ b/client/ssh/server/userswitching_windows.go @@ -91,7 +91,7 @@ func validateUsernameFormat(username string) error { func (s *Server) createExecutorCommand(logger *log.Entry, session ssh.Session, localUser *user.User, hasPty bool) (*exec.Cmd, func(), error) { logger.Debugf("creating Windows executor command for user %s (Pty: %v)", localUser.Username, hasPty) - username, _ := s.parseUsername(localUser.Username) + username, _ := parseUsername(localUser.Username) if err := validateUsername(username); err != nil { return nil, nil, fmt.Errorf("invalid username %q: %w", username, err) } @@ -102,7 +102,7 @@ func (s *Server) createExecutorCommand(logger *log.Entry, session ssh.Session, l // createUserSwitchCommand creates a command with Windows user switching. // Returns the command and a cleanup function that must be called after starting the process. func (s *Server) createUserSwitchCommand(logger *log.Entry, session ssh.Session, localUser *user.User) (*exec.Cmd, func(), error) { - username, domain := s.parseUsername(localUser.Username) + username, domain := parseUsername(localUser.Username) shell := getUserShell(localUser.Uid) @@ -138,7 +138,7 @@ func (s *Server) createUserSwitchCommand(logger *log.Entry, session ssh.Session, } // parseUsername extracts username and domain from a Windows username -func (s *Server) parseUsername(fullUsername string) (username, domain string) { +func parseUsername(fullUsername string) (username, domain string) { // Handle DOMAIN\username format if idx := strings.LastIndex(fullUsername, `\`); idx != -1 { domain = fullUsername[:idx] diff --git a/client/status/status.go b/client/status/status.go index e8276d0fa..1c204cdb1 100644 --- a/client/status/status.go +++ b/client/status/status.go @@ -46,7 +46,10 @@ func ParseDaemonStatus(s string) DaemonStatus { // ConvertOptions holds parameters for ConvertToStatusOutputOverview. type ConvertOptions struct { - Anonymize bool + Anonymize bool + // AnonymizeLevel selects how much the anonymizer redacts. Only + // meaningful when Anonymize is set. + AnonymizeLevel anonymize.Level DaemonVersion string DaemonStatus DaemonStatus StatusFilter string @@ -217,6 +220,7 @@ func ConvertToStatusOutputOverview(pbFullStatus *proto.FullStatus, opts ConvertO if opts.Anonymize { anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses()) + anonymizer.SetLevel(opts.AnonymizeLevel) anonymizeOverview(anonymizer, &overview) } @@ -976,6 +980,7 @@ func timeAgo(t time.Time) string { func anonymizePeerDetail(a *anonymize.Anonymizer, peer *PeerStateDetailOutput) { peer.FQDN = a.AnonymizeDomain(peer.FQDN) + peer.PubKey = a.AnonymizeWGKey(peer.PubKey) if localIP, port, err := net.SplitHostPort(peer.IceCandidateEndpoint.Local); err == nil { peer.IceCandidateEndpoint.Local = fmt.Sprintf("%s:%s", a.AnonymizeIPString(localIP), port) } @@ -1007,6 +1012,7 @@ func anonymizeOverview(a *anonymize.Anonymizer, overview *OutputOverview) { overview.SignalState.URL = a.AnonymizeURI(overview.SignalState.URL) overview.SignalState.Error = a.AnonymizeString(overview.SignalState.Error) + overview.PubKey = a.AnonymizeWGKey(overview.PubKey) overview.IP = a.AnonymizeIPString(overview.IP) overview.IPv6 = a.AnonymizeIPString(overview.IPv6) for i, detail := range overview.Relays.Details { diff --git a/client/ui/frontend/src/contexts/DebugBundleContext.tsx b/client/ui/frontend/src/contexts/DebugBundleContext.tsx index a0a131fbf..5f2ed9041 100644 --- a/client/ui/frontend/src/contexts/DebugBundleContext.tsx +++ b/client/ui/frontend/src/contexts/DebugBundleContext.tsx @@ -71,10 +71,12 @@ type BundleOptions = { hasWindow: boolean; totalSec: number; uploadUrl: string; - anonymize: boolean; + anonymizeLevel: AnonymizeLevel; systemInfo: boolean; }; +export type AnonymizeLevel = "none" | "default" | "strict"; + const startCaptureBestEffort = async (totalSec: number, pcap: CaptureState) => { try { // Mirror the CLI's safety margin: window + 30s, server caps at 10m. @@ -187,7 +189,10 @@ const runBundleFlow = async ( if (opts.uploadUrl) setStage({ kind: "uploading" }); const result = await DebugSvc.Bundle({ - anonymize: opts.anonymize, + anonymize: opts.anonymizeLevel !== "none", + // The daemon only knows "default" and "strict"; "none" is expressed + // through the anonymize flag being off. + anonymizeLevel: opts.anonymizeLevel === "strict" ? "strict" : "default", systemInfo: opts.systemInfo, uploadUrl: opts.uploadUrl, logFileCount, @@ -198,7 +203,7 @@ const runBundleFlow = async ( }; const useDebugBundle = () => { - const [anonymize, setAnonymize] = useState(false); + const [anonymizeLevel, setAnonymizeLevel] = useState("none"); const [systemInfo, setSystemInfo] = useState(true); const [upload, setUpload] = useState(true); const [trace, setTrace] = useState(true); @@ -240,7 +245,7 @@ const useDebugBundle = () => { hasWindow: capture && totalSec > 0, totalSec, uploadUrl: upload ? NETBIRD_UPLOAD_URL : "", - anonymize, + anonymizeLevel, systemInfo, }; @@ -272,8 +277,8 @@ const useDebugBundle = () => { }; return { - anonymize, - setAnonymize, + anonymizeLevel, + setAnonymizeLevel, systemInfo, setSystemInfo, upload, diff --git a/client/ui/frontend/src/modules/settings/SettingsTroubleshooting.tsx b/client/ui/frontend/src/modules/settings/SettingsTroubleshooting.tsx index 991937719..8b1774ed6 100644 --- a/client/ui/frontend/src/modules/settings/SettingsTroubleshooting.tsx +++ b/client/ui/frontend/src/modules/settings/SettingsTroubleshooting.tsx @@ -1,6 +1,6 @@ import { useId, type ReactNode } from "react"; import { Trans, useTranslation } from "react-i18next"; -import { CircleCheckBig, FolderOpen, Loader2 } from "lucide-react"; +import { ChevronDown, CircleCheckBig, FolderOpen, Info, Loader2 } from "lucide-react"; import { Browser } from "@wailsio/runtime"; import { Debug as DebugSvc } from "@bindings/services"; import type { DebugBundleResult } from "@bindings/services/models.js"; @@ -8,13 +8,22 @@ import { Button } from "@/components/buttons/Button"; import { DialogActions } from "@/components/dialog/DialogActions"; import { DialogDescription } from "@/components/dialog/DialogDescription"; import { DialogHeading } from "@/components/dialog/DialogHeading"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuRadioGroup, + DropdownMenuRadioItem, + DropdownMenuTrigger, +} from "@/components/DropdownMenu"; import FancyToggleSwitch from "@/components/switches/FancyToggleSwitch"; import HelpText from "@/components/typography/HelpText.tsx"; import { Input } from "@/components/inputs/Input"; import { Label } from "@/components/typography/Label"; import { SquareIcon } from "@/components/SquareIcon"; +import { Tooltip } from "@/components/Tooltip"; +import { cn } from "@/lib/cn"; import { formatRemaining } from "@/lib/formatters"; -import type { DebugStage } from "@/contexts/DebugBundleContext"; +import type { AnonymizeLevel, DebugStage } from "@/contexts/DebugBundleContext"; import { useDebugBundleContext } from "@/contexts/DebugBundleContext"; import { SectionGroup, SettingsBottomBar } from "@/modules/settings/SettingsSection.tsx"; @@ -24,8 +33,8 @@ export function SettingsTroubleshooting() { const { t } = useTranslation(); const durationId = useId(); const { - anonymize, - setAnonymize, + anonymizeLevel, + setAnonymizeLevel, systemInfo, setSystemInfo, upload, @@ -55,12 +64,71 @@ export function SettingsTroubleshooting() { return ( - +
+
+
+ } + > + + + + + + {t("settings.troubleshooting.anonymize.help")} + +
+
+ + + + + + setAnonymizeLevel(v as AnonymizeLevel)} + > + + {t("settings.troubleshooting.anonymize.none")} + + + {t("settings.troubleshooting.anonymize.default")} + + + {t("settings.troubleshooting.anonymize.strict")} + + + + +
+ .** Join the project, pick your language, and translate in the editor. Each string carries a context note (the `description` from the source file) telling you what it is and where it shows up, and the project's glossary, style guide, and QA checks mirror this document. -> 💡 **The one habit that matters most:** read each key's `description` before translating it. Labels are terse and ambiguous on their own; the `description` tells you what the string is, where it shows up, what to keep verbatim, and what it actually means. +> 💡 **The one habit that matters most:** read each string's context before translating it. Labels are terse and ambiguous on their own; the context tells you what the string is, where it shows up, what to keep verbatim, and what it actually means. + +--- + +## How contributions flow + +```text +i18n/locales/en/common.json ──sync──▶ Crowdin ──service PR──▶ i18n/locales//common.json +``` + +- `i18n/locales/en/common.json` is the source of truth. New and changed strings sync to Crowdin automatically (see `crowdin.yml` in the repository root). +- Crowdin opens and updates a service pull request with the translated bundles, keeping the source's file shape and key order. Keys nobody has translated yet are left out of the export; the app falls back to English for them at runtime. Maintainers review and merge that PR. +- Don't hand-edit `i18n/locales//common.json` in your own PRs: the next sync would conflict with or overwrite your changes. Translate on Crowdin instead. +- Missing your language? Request it on the Crowdin project page or in a [GitHub discussion](https://github.com/netbirdio/netbird/discussions). When a language first ships, a maintainer adds its row to `i18n/locales/_index.json` with `code`, `displayName` (the native name), and `englishName`, which puts it in the app's language picker. + +**Prefer translating with an AI agent?** That still works: drive it with *"Read `i18n/TRANSLATING.md` and translate the UI to Russian"* as before, but deliver the result to Crowdin instead of a pull request. Download your language's file from the Crowdin editor, let the agent translate it, and upload it back (the editor's offline translation flow). Crowdin runs its QA checks on upload, and the next service PR carries the strings into the repo. --- @@ -30,25 +45,6 @@ A **business zero-trust VPN** — an encrypted **overlay mesh** between a compan --- -## The files - -``` -i18n/locales/_index.json shipped-language list -i18n/locales/en/common.json source of truth — message + description -i18n/locales//common.json a target — message only -``` - -Chrome-extension JSON, each key → `{ "message", "description" }`. You translate the **`message`**. - -| ✅ Do | ❌ Don't | -|---|---| -| Keep **every key** from `en`, in the same order | Translate, rename, reorder, drop, or add keys (they're identifiers; the set grows over time) | -| Put **only `message`** in target bundles | Copy `description` into a target bundle | -| Give every key a non-empty `message` | Leave keys missing or empty | -| Save valid UTF-8 JSON, no BOM | Add trailing commas or break the JSON | - ---- - ## Hard rules — get these exactly right These are the usual ways a translation *breaks the app*, not just reads oddly. @@ -58,7 +54,7 @@ These are the usual ways a translation *breaks the app*, not just reads oddly. | Copy `{placeholders}` verbatim — `{version}`, `{count}`, `{name}`… | Translate the word inside the braces (`{verbleibend}` breaks it) | | Reposition a placeholder so the sentence flows | Drop or duplicate a placeholder | | Preserve every `\n`, leading/trailing space, and trailing `...` | Trim "invisible" spaces or the `...` (they're load-bearing) | -| Keep `®` in WireGuard® and quotes around `{name}` | Strip punctuation the description flags | +| Keep `®` in WireGuard® and quotes around `{name}` | Strip punctuation the context flags | **Plurals:** the app has only a *one / other* split — the singular key fires only when `count == 1`; the `{count}` key covers everything else (0, 2, 5, 100…). Languages with more than two forms (ru, pl, uk) can't be fully correct here — use the form that fits the widest range (Russian genitive plural: `минут` / `часов` / `дней`). Don't invent extra keys or cram multiple forms into one string. When no single form fits every value — a unit label after a number field, say — reach for a number-agnostic form (an abbreviation, or wording that reads the same for 1 and 100) instead of forcing a plural the *one / other* split can't supply. @@ -78,13 +74,15 @@ When a brand sits beside a common noun, keep its exact spelling but join them th > **Use the word that language's IT users actually say.** Translate when a natural, common term exists; keep the English term *only* when the literal translation would be awkward or no one in that field really uses it. -Apply each term **consistently** — same English term → same translation everywhere — and keep a term once you've settled it. Whether a term stays English or takes a native word is **language-dependent**: a technical loanword (e.g. *Daemon*, *Handshake*) often stays, an everyday word (e.g. *Latency*, *Public key*) usually localizes, and some (*Exit Node*, *Peer*) go either way depending on the language. Decide per term with the rule above — a foreign origin alone is no reason to keep English. **Your main reference is the existing bundles:** match how a term was already rendered for your language rather than re-deciding it. +Apply each term **consistently** — same English term → same translation everywhere — and keep a term once you've settled it. Whether a term stays English or takes a native word is **language-dependent**: a technical loanword (e.g. *Daemon*, *Handshake*) often stays, an everyday word (e.g. *Latency*, *Public key*) usually localizes, and some (*Exit Node*, *Peer*) go either way depending on the language. Decide per term with the rule above — a foreign origin alone is no reason to keep English. **Your main reference is the existing translation:** match how a term was already rendered for your language rather than re-deciding it. Two checks before you commit a term: - **Prefer established localized wording.** If a widely used tool in this space (for example WireGuard) ships your language, its wording for a shared term such as *handshake* is what users already expect — look at the translated app, not just English docs. For generic UI verbs and formal address, follow your OS vendor's style guide (Microsoft / Apple / Google). - **Watch for false friends.** A literal translation can collide with a *different* established term in your field — confirm your word doesn't already mean something else in this domain before using it. +These tiers are mirrored in the Crowdin project glossary, so the editor highlights them inline. When you settle a new Tier C term for your language, add its translation to the glossary entry so it sticks for everyone who comes after you. + --- ## Style @@ -98,7 +96,7 @@ Two checks before you commit a term: Where it reads naturally, aim to keep each string **roughly the same length** as the English — the UI is tight and over-long strings can wrap or truncate. It's a soft preference, not a rule: if your language simply needs more words, use them. -A few habits that keep a bundle reading like one product rather than a word-for-word port: +A few habits that keep a translation reading like one product rather than a word-for-word port: - **Translate meaning, not words.** Render what a string *does*. An idiom or an awkward source phrase should become natural in your language, not a literal calque. - **Keep one voice within a family.** Sibling strings — the connection states, every settings *help* caption, every "… Failed" title — should share a grammatical form. If one member sounds wrong in that form, re-voice the whole family rather than leave one odd sibling. @@ -107,27 +105,26 @@ A few habits that keep a bundle reading like one product rather than a word-for- --- -## Procedure +## Reviewing a language -**New language** — read `en/common.json` *with* descriptions → settle your Tier C terms → write `i18n/locales//common.json` (same keys and order as `en`, `message` only, placeholders & brands preserved) → add a row to `_index.json` (`{"code","displayName"` = native name`,"englishName"}`) → run the QA list. Use the locale-code style the existing entries use (e.g. `fr`, `pt`, `zh-CN`). +**On Crowdin:** proofread in the editor — context, glossary highlights, and QA flags sit inline next to each string. -**Review (de / hu / …)** — read source and target side by side; for each key check glossary conformance (e.g. de `Exit-Node` → `Exit Node`, hu `Kilépő csomópont` → `Exit Node`), placeholder/`\n` integrity, consistency, tone, and that the meaning matches the English `description`. Fix in place, then report what you changed (especially term standardizations) so a native speaker can sanity-check. +**In the repo** — e.g. driving an AI agent with *"Read `i18n/TRANSLATING.md` and review the existing German translation"* — read source and target side by side; for each key check glossary conformance (e.g. de `Exit-Node` → `Exit Node`, hu `Kilépő csomópont` → `Exit Node`), placeholder/`\n` integrity, consistency, tone, and that the meaning matches the English `description`. Report what you found, and apply the fixes **on Crowdin** — direct edits to the locale files are overwritten by the next sync. --- ## QA before you finish -- [ ] Valid JSON · **every `en` key** present, same order · **no `description`** fields - [ ] Every `{placeholder}`, `\n`, and intentional space preserved · `...` / `… Failed` / `{name}` quotes kept -- [ ] Tier A/B left intact · Tier C applied consistently (and matching the existing bundle for your language) +- [ ] Tier A/B left intact · Tier C applied consistently (and matching the existing translation for your language) - [ ] Buttons & tray short · locale punctuation and capitalization applied -- [ ] New language added to `_index.json` +- [ ] Crowdin QA flags resolved (variables, glossary terms, punctuation) - [ ] **Tested in the running app** ↓ --- ## Test it in the app -A bundle can pass every check above and still read wrong on screen. **Run the app, switch to your language, and click through the real surfaces** — tray menu, main window, every Settings tab, the dialogs. Watch for text overflow or truncation, labels that are technically right but wrong *for what the control does*, leaked placeholders, and terms that drift between screens. +A translation can pass every check above and still read wrong on screen. **Run the app, switch to your language, and click through the real surfaces** — tray menu, main window, every Settings tab, the dialogs. Watch for text overflow or truncation, labels that are technically right but wrong *for what the control does*, leaked placeholders, and terms that drift between screens. How to run the app and switch language: see the project README. Can't run it (e.g. a headless agent)? Say so in your summary — don't silently skip this step. diff --git a/client/ui/i18n/locales/en/common.json b/client/ui/i18n/locales/en/common.json index 9769e772f..694444497 100644 --- a/client/ui/i18n/locales/en/common.json +++ b/client/ui/i18n/locales/en/common.json @@ -1013,11 +1013,27 @@ }, "settings.troubleshooting.anonymize.label": { "message": "Anonymize Sensitive Information", - "description": "Toggle label: anonymize sensitive information in the bundle." + "description": "Label for the anonymization level dropdown (None, Default, Strict)." }, "settings.troubleshooting.anonymize.help": { - "message": "Hides public IP addresses and non-NetBird domains from logs.", - "description": "Helper text for anonymizing logs (hides public IPs and non-NetBird domains)." + "message": "Hides IP addresses, domains, and other sensitive values.", + "description": "Helper text under the anonymization dropdown. The level details live in the info tooltip." + }, + "settings.troubleshooting.anonymize.info": { + "message": "Default keeps internal IPv4 addresses and peer names readable for support. Strict additionally anonymizes private (RFC 1918), CGNAT, and link-local IP addresses, peer names, and WireGuard public keys. Recurring values map to the same placeholder, so peers stay distinguishable. Use Strict when sharing the bundle outside your organization.", + "description": "Info tooltip explaining the anonymization levels. 'RFC 1918', 'CGNAT', 'link-local', and 'WireGuard' are technical terms — keep them." + }, + "settings.troubleshooting.anonymize.none": { + "message": "None", + "description": "Dropdown option: no anonymization." + }, + "settings.troubleshooting.anonymize.default": { + "message": "Default", + "description": "Dropdown option: default anonymization level." + }, + "settings.troubleshooting.anonymize.strict": { + "message": "Strict", + "description": "Dropdown option: strict anonymization level." }, "settings.troubleshooting.systemInfo.label": { "message": "Include System Information", diff --git a/client/ui/services/debug.go b/client/ui/services/debug.go index 034086747..d1f6555a8 100644 --- a/client/ui/services/debug.go +++ b/client/ui/services/debug.go @@ -15,10 +15,13 @@ import ( ) type DebugBundleParams struct { - Anonymize bool `json:"anonymize"` - SystemInfo bool `json:"systemInfo"` - UploadURL string `json:"uploadUrl"` - LogFileCount uint32 `json:"logFileCount"` + Anonymize bool `json:"anonymize"` + // AnonymizeLevel is "default" or "strict"; strict also anonymizes + // private IP ranges, peer names, and WireGuard public keys. + AnonymizeLevel string `json:"anonymizeLevel"` + SystemInfo bool `json:"systemInfo"` + UploadURL string `json:"uploadUrl"` + LogFileCount uint32 `json:"logFileCount"` } // DebugBundleResult: Path is set for local-only bundles, UploadedKey on upload @@ -48,11 +51,12 @@ func (s *Debug) Bundle(ctx context.Context, p DebugBundleParams) (DebugBundleRes return DebugBundleResult{}, err } resp, err := cli.DebugBundle(ctx, &proto.DebugBundleRequest{ - Anonymize: p.Anonymize, - SystemInfo: p.SystemInfo, - UploadURL: p.UploadURL, - LogFileCount: p.LogFileCount, - CliVersion: version.NetbirdVersion(), + Anonymize: p.Anonymize, + AnonymizeLevel: p.AnonymizeLevel, + SystemInfo: p.SystemInfo, + UploadURL: p.UploadURL, + LogFileCount: p.LogFileCount, + CliVersion: version.NetbirdVersion(), }) if err != nil { return DebugBundleResult{}, err diff --git a/crowdin.yml b/crowdin.yml new file mode 100644 index 000000000..efdbceb8d --- /dev/null +++ b/crowdin.yml @@ -0,0 +1,11 @@ +skip_untranslated_strings: true +skip_untranslated_files: true +import_eq_suggestions: true + +files: + - source: /client/ui/i18n/locales/en/common.json + translation: /client/ui/i18n/locales/%two_letters_code%/common.json + type: chrome + languages_mapping: + two_letters_code: + zh-CN: zh-CN diff --git a/e2e/agentnetwork/chat_test.go b/e2e/agentnetwork/chat_test.go index ed94623fe..792f67c89 100644 --- a/e2e/agentnetwork/chat_test.go +++ b/e2e/agentnetwork/chat_test.go @@ -438,14 +438,10 @@ func TestProvidersMatrix(t *testing.T) { // Create every provider, all enabled, each with a unique model string so the // proxy's connect-time snapshot carries them all and model→provider routing // is unambiguous (provider toggles after connect don't reconcile to the - // proxy, so we enable everything up front). The first create bootstraps the - // cluster. + // proxy, so we enable everything up front). ids := make([]string, 0, len(matrix)) - for i, pc := range matrix { + for _, pc := range matrix { req := providerRequest(pc) - if i == 0 { - req.BootstrapCluster = ptr(harness.AgentNetworkCluster) - } prov, perr := srv.CreateProvider(ctx, req) require.NoError(t, perr, "create provider %s", pc.name) ids = append(ids, prov.Id) diff --git a/e2e/agentnetwork/custom_pricing_test.go b/e2e/agentnetwork/custom_pricing_test.go index 33788a16a..e3750258f 100644 --- a/e2e/agentnetwork/custom_pricing_test.go +++ b/e2e/agentnetwork/custom_pricing_test.go @@ -82,13 +82,12 @@ func provisionPricedProvider(t *testing.T, ctx context.Context, name string, mod // need NOT be in the catalog — the operator names it and prices it here. dummyKey := "sk-price-e2e" prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: name, - ProviderId: "openai_api", - UpstreamUrl: vllm.URL, - ApiKey: &dummyKey, - Enabled: ptr(true), - BootstrapCluster: ptr(harness.AgentNetworkCluster), - Models: &models, + Name: name, + ProviderId: "openai_api", + UpstreamUrl: vllm.URL, + ApiKey: &dummyKey, + Enabled: ptr(true), + Models: &models, }) require.NoError(t, err, "create provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) diff --git a/e2e/agentnetwork/guardrail_block_test.go b/e2e/agentnetwork/guardrail_block_test.go index c4b22ae25..302b5107e 100644 --- a/e2e/agentnetwork/guardrail_block_test.go +++ b/e2e/agentnetwork/guardrail_block_test.go @@ -113,15 +113,14 @@ func runPathRoutedGuardrailCase(t *testing.T, tc pathRoutedGuardrailCase) { // Catch-all provider (no models) so the router forwards any model; a static // bearer key means the router injects a static auth header instead of minting - // a GCP token. Bootstraps the cluster if it isn't already. + // a GCP token. staticKey := "static-e2e-token" prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: tc.name, - ProviderId: tc.catalogID, - UpstreamUrl: vllm.URL, - ApiKey: &staticKey, - Enabled: ptr(true), - BootstrapCluster: ptr(harness.AgentNetworkCluster), + Name: tc.name, + ProviderId: tc.catalogID, + UpstreamUrl: vllm.URL, + ApiKey: &staticKey, + Enabled: ptr(true), }) require.NoError(t, err, "create %s provider", tc.name) t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) diff --git a/e2e/agentnetwork/guardrail_groupswitch_test.go b/e2e/agentnetwork/guardrail_groupswitch_test.go index 5f2dce52e..7108bb587 100644 --- a/e2e/agentnetwork/guardrail_groupswitch_test.go +++ b/e2e/agentnetwork/guardrail_groupswitch_test.go @@ -73,7 +73,6 @@ func TestGuardrailGroupSwitchTakesEffectAfterTTL(t *testing.T) { {Id: modelA, InputPer1k: 0.001, OutputPer1k: 0.001}, {Id: modelB, InputPer1k: 0.001, OutputPer1k: 0.001}, }, - BootstrapCluster: ptr(harness.AgentNetworkCluster), }) require.NoError(t, err, "create provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) diff --git a/e2e/agentnetwork/guardrail_multipolicy_test.go b/e2e/agentnetwork/guardrail_multipolicy_test.go index 1d664fb58..2bb56b3bc 100644 --- a/e2e/agentnetwork/guardrail_multipolicy_test.go +++ b/e2e/agentnetwork/guardrail_multipolicy_test.go @@ -61,15 +61,14 @@ func TestGuardrailMultiPolicyModelAllowlist(t *testing.T) { } // pRestricted declares the two guardrailed models so routing is deterministic - // (model -> provider). Created first, so it carries the bootstrap cluster. + // (model -> provider). pRestricted, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: "restricted", - ProviderId: "openai_api", - UpstreamUrl: vllm.URL, - ApiKey: &staticKey, - Enabled: ptr(true), - Models: models(modelSelected, modelOther), - BootstrapCluster: ptr(harness.AgentNetworkCluster), + Name: "restricted", + ProviderId: "openai_api", + UpstreamUrl: vllm.URL, + ApiKey: &staticKey, + Enabled: ptr(true), + Models: models(modelSelected, modelOther), }) require.NoError(t, err, "create restricted provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), pRestricted.Id) }) diff --git a/e2e/agentnetwork/guardrail_pergroup_providers_test.go b/e2e/agentnetwork/guardrail_pergroup_providers_test.go index eddae65c3..0e98330c4 100644 --- a/e2e/agentnetwork/guardrail_pergroup_providers_test.go +++ b/e2e/agentnetwork/guardrail_pergroup_providers_test.go @@ -115,7 +115,7 @@ func TestGuardrailPerGroupAllowlist_AllProviders(t *testing.T) { staticKey := "static-e2e-token" enabled := true - for i, c := range cases { + for _, c := range cases { req := api.AgentNetworkProviderRequest{ Name: "e2e-pergroup-" + c.name, ProviderId: c.catalogID, @@ -124,9 +124,6 @@ func TestGuardrailPerGroupAllowlist_AllProviders(t *testing.T) { Enabled: ptr(true), Models: c.models, } - if i == 0 { - req.BootstrapCluster = ptr(harness.AgentNetworkCluster) - } prov, perr := srv.CreateProvider(ctx, req) require.NoError(t, perr, "create provider %s", c.name) c.providerID = prov.Id @@ -283,13 +280,12 @@ func TestGuardrailMultiGroupUser(t *testing.T) { // P1 — union scenario: two restricting policies, one per group. p1, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: "e2e-mg-union", - ProviderId: "openai_api", - UpstreamUrl: vllm.URL, - ApiKey: &staticKey, - Enabled: ptr(true), - Models: priced(unionA, unionB, unionC), - BootstrapCluster: ptr(harness.AgentNetworkCluster), + Name: "e2e-mg-union", + ProviderId: "openai_api", + UpstreamUrl: vllm.URL, + ApiKey: &staticKey, + Enabled: ptr(true), + Models: priced(unionA, unionB, unionC), }) require.NoError(t, err, "create union provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), p1.Id) }) diff --git a/e2e/agentnetwork/guardrail_test.go b/e2e/agentnetwork/guardrail_test.go index 6a2487a88..ddc95d62d 100644 --- a/e2e/agentnetwork/guardrail_test.go +++ b/e2e/agentnetwork/guardrail_test.go @@ -115,14 +115,11 @@ func TestModelAllowlistEnforced(t *testing.T) { }) require.NoError(t, err, "mint setup key") - // Providers with their configured (allowed) models; the first bootstraps the cluster. + // Providers with their configured (allowed) models ids := make([]string, 0, len(providers)) allowed := make([]string, 0, len(providers)) - for i, pc := range providers { + for _, pc := range providers { req := providerRequest(pc) - if i == 0 { - req.BootstrapCluster = ptr(harness.AgentNetworkCluster) - } prov, perr := srv.CreateProvider(ctx, req) require.NoError(t, perr, "create provider %s", pc.name) id := prov.Id diff --git a/e2e/agentnetwork/main_test.go b/e2e/agentnetwork/main_test.go index 17c5e00be..cc366b3fb 100644 --- a/e2e/agentnetwork/main_test.go +++ b/e2e/agentnetwork/main_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/netbirdio/netbird/e2e/harness" + "github.com/netbirdio/netbird/shared/management/http/api" ) // srv is the shared combined server for the package, ready (PAT-authenticated) @@ -42,5 +43,14 @@ func run(m *testing.M) int { return 1 } + // Bootstrap the account's agent-network endpoint once for the package: + // providers no longer have settings side effects, and every data-plane + // test expects the shared account pinned to the combined proxy cluster. + cluster := harness.AgentNetworkCluster + if _, err := srv.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ProxyAddress: &cluster}); err != nil { + fmt.Fprintf(os.Stderr, "e2e: bootstrap agent-network endpoint: %v\n", err) + return 1 + } + return m.Run() } diff --git a/e2e/agentnetwork/management_test.go b/e2e/agentnetwork/management_test.go index 6962f9796..9e3176d68 100644 --- a/e2e/agentnetwork/management_test.go +++ b/e2e/agentnetwork/management_test.go @@ -21,11 +21,10 @@ func ptr[T any](v T) *T { return &v } func newProvider(t *testing.T, ctx context.Context, name string) api.AgentNetworkProvider { t.Helper() prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: name, - ProviderId: "openai_api", - UpstreamUrl: "https://api.openai.com", - ApiKey: ptr("sk-dummy-e2e-key"), - BootstrapCluster: ptr("eu.proxy.netbird.test"), + Name: name, + ProviderId: "openai_api", + UpstreamUrl: "https://api.openai.com", + ApiKey: ptr("sk-dummy-e2e-key"), }) require.NoError(t, err, "create provider %q", name) t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) @@ -57,17 +56,11 @@ func TestProviderLifecycle(t *testing.T) { }} } - for i, pc := range cases { - i, pc := i, pc + for _, pc := range cases { + pc := pc t.Run(pc.name, func(t *testing.T) { req := providerRequest(pc) req.Name = "lc-" + pc.name - // Bootstrap the cluster on the first create in case the matrix has - // not run (e.g. no provider keys → settings not yet bootstrapped). - if i == 0 { - req.BootstrapCluster = ptr(harness.AgentNetworkCluster) - } - prov, err := srv.CreateProvider(ctx, req) require.NoError(t, err, "create %s provider", pc.name) t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) @@ -137,45 +130,65 @@ func TestProviderValidation(t *testing.T) { requireClientError(t, err) } -// TestSettingsRoundTrip flips the collection toggles and confirms cluster / -// subdomain stay immutable, then restores the original state. +// TestSettingsRoundTrip flips the collection toggles and confirms the +// endpoint and proxy address stay immutable, then restores the original +// state. A second bootstrap attempt must be rejected as a conflict. func TestSettingsRoundTrip(t *testing.T) { ctx := context.Background() - // Settings are bootstrapped on first provider create. - newProvider(t, ctx, "Settings Bootstrap") - + // The package's TestMain bootstrapped the shared account's endpoint. before, err := srv.GetSettings(ctx) require.NoError(t, err, "get settings") - require.NotEmpty(t, before.Cluster, "settings must carry an assigned cluster") + require.NotEmpty(t, before.Endpoint, "settings must carry the bootstrapped endpoint") + require.NotEmpty(t, before.ProxyAddress, "settings must carry the bootstrapped proxy address") + + require.NotNil(t, before.AccessLogRetentionDays, "bootstrapped settings must carry a retention") + beforeRetention := *before.AccessLogRetentionDays flipped, err := srv.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ + Endpoint: before.Endpoint, + ProxyAddress: before.ProxyAddress, EnableLogCollection: !before.EnableLogCollection, EnablePromptCollection: !before.EnablePromptCollection, RedactPii: !before.RedactPii, + AccessLogRetentionDays: beforeRetention, }) require.NoError(t, err, "update settings") assert.Equal(t, !before.EnableLogCollection, flipped.EnableLogCollection, "log collection toggle must flip") assert.Equal(t, !before.EnablePromptCollection, flipped.EnablePromptCollection, "prompt collection toggle must flip") - assert.Equal(t, before.Cluster, flipped.Cluster, "cluster must be immutable across updates") - assert.Equal(t, before.Subdomain, flipped.Subdomain, "subdomain must be immutable across updates") + require.NotNil(t, flipped.AccessLogRetentionDays) + assert.Equal(t, beforeRetention, *flipped.AccessLogRetentionDays, + "retention sent unchanged must round-trip, not reset to the zero value") + assert.Equal(t, before.Endpoint, flipped.Endpoint, "endpoint must be immutable across updates") + assert.Equal(t, before.ProxyAddress, flipped.ProxyAddress, "proxy address must be immutable across updates") - // A cluster different from the pinned one must be rejected; echoing the - // pinned one back is valid. + // The account is already bootstrapped: a second bootstrap is a conflict, + // whatever shape it asks for. + _, err = srv.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + Endpoint: ptr("attacker.cluster.invalid"), + }) + requireClientError(t, err) + + // The identity fields ride along on the PUT as a required echo: a request + // carrying a different endpoint is rejected without applying anything. _, err = srv.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - Cluster: ptr("attacker.cluster.invalid"), + Endpoint: "other.cluster.invalid", + ProxyAddress: before.ProxyAddress, EnableLogCollection: before.EnableLogCollection, EnablePromptCollection: before.EnablePromptCollection, RedactPii: before.RedactPii, + AccessLogRetentionDays: beforeRetention, }) requireClientError(t, err) // Restore the original toggles. _, err = srv.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - Cluster: ptr(before.Cluster), + Endpoint: before.Endpoint, + ProxyAddress: before.ProxyAddress, EnableLogCollection: before.EnableLogCollection, EnablePromptCollection: before.EnablePromptCollection, RedactPii: before.RedactPii, + AccessLogRetentionDays: beforeRetention, }) require.NoError(t, err, "restore settings") } diff --git a/e2e/agentnetwork/settings_bootstrap_test.go b/e2e/agentnetwork/settings_bootstrap_test.go index ea56f7064..806ce82f4 100644 --- a/e2e/agentnetwork/settings_bootstrap_test.go +++ b/e2e/agentnetwork/settings_bootstrap_test.go @@ -4,6 +4,7 @@ package agentnetwork import ( "context" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -14,7 +15,8 @@ import ( ) // harnessStartFresh boots a dedicated combined server with its own fresh -// account and registers its teardown on t. +// account and registers its teardown on t. Unlike the shared srv, the fresh +// account has NOT had its agent-network endpoint bootstrapped. func harnessStartFresh(ctx context.Context, t *testing.T) (*harness.Combined, error) { t.Helper() fresh, err := harness.StartCombined(ctx) @@ -28,16 +30,16 @@ func harnessStartFresh(ctx context.Context, t *testing.T) (*harness.Combined, er return fresh, nil } -// TestSettingsBootstrapViaPut covers the settings-first bootstrap path on an +// TestSettingsBootstrapViaPost covers the explicit bootstrap contract on an // account that has never been bootstrapped: the GET reads as the defaults -// with an empty cluster/subdomain/endpoint, a PUT without a cluster has -// nothing to pin and fails, and a PUT carrying a cluster creates the row and -// pins it immutably. The shared srv cannot provide that starting state (any -// provider-creating test bootstraps it, and test order is deliberately not -// relied on), so this boots a dedicated combined server — the image is -// already built and cached by TestMain's StartCombined, so the extra cost is -// one container start. -func TestSettingsBootstrapViaPut(t *testing.T) { +// with an empty endpoint/proxy_address, a PUT has no row to update and fails, +// and a POST creates the row and assigns the immutable endpoint — labeled +// beneath a proxy address here, with the toggle overrides from the same +// request applied. The shared srv cannot provide that starting state +// (TestMain bootstraps it), so this boots a dedicated combined server — the +// image is already built and cached by TestMain's StartCombined, so the extra +// cost is one container start. +func TestSettingsBootstrapViaPost(t *testing.T) { ctx := context.Background() fresh, err := harnessStartFresh(ctx, t) @@ -47,32 +49,35 @@ func TestSettingsBootstrapViaPut(t *testing.T) { // as an error and not as a null body. before, err := fresh.GetSettings(ctx) require.NoError(t, err, "get settings on a fresh account must succeed") - assert.Empty(t, before.Cluster, "cluster must be empty before bootstrap") - assert.Empty(t, before.Subdomain, "subdomain must be empty before bootstrap") - assert.Empty(t, before.Endpoint, "endpoint must be empty before bootstrap, not a bare dot") + assert.Empty(t, before.Endpoint, "endpoint must be empty before bootstrap") + assert.Empty(t, before.ProxyAddress, "proxy address must be empty before bootstrap") + assert.False(t, before.Dedicated, "an unbootstrapped account has no serving shape") assert.True(t, before.EnableLogCollection, "defaults must show log collection on, matching bootstrap") assert.False(t, before.EnablePromptCollection, "defaults must show prompt collection off") - // A PUT without a cluster has nothing to pin the account to. + // A PUT has no row to update yet — bootstrap is the explicit POST. _, err = fresh.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - EnableLogCollection: true, + EnableLogCollection: true, + AccessLogRetentionDays: 30, }) requireClientError(t, err) - // A PUT carrying a cluster bootstraps the account and applies the - // mutable fields from the same request. Every toggle is set away from - // its bootstrap default so each assertion can actually fail. + // A POST with a proxy address bootstraps a labeled endpoint and applies + // the toggles from the same request. Every toggle is set away from its + // bootstrap default so each assertion can actually fail. const cluster = "e2e.bootstrap.netbird.selfhosted" - bootstrapped, err := fresh.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - Cluster: ptr(cluster), - EnableLogCollection: false, - EnablePromptCollection: true, - RedactPii: true, + bootstrapped, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + ProxyAddress: ptr(cluster), + EnableLogCollection: ptr(false), + EnablePromptCollection: ptr(true), + RedactPii: ptr(true), }) - require.NoError(t, err, "bootstrap settings via PUT must succeed") - assert.Equal(t, cluster, bootstrapped.Cluster, "cluster must be pinned from the request") - require.NotEmpty(t, bootstrapped.Subdomain, "subdomain must be assigned at bootstrap") - assert.Equal(t, bootstrapped.Subdomain+"."+cluster, bootstrapped.Endpoint, "endpoint must combine subdomain and cluster") + require.NoError(t, err, "bootstrap settings via POST must succeed") + assert.Equal(t, cluster, bootstrapped.ProxyAddress, "proxy address must be pinned from the request") + require.NotEmpty(t, bootstrapped.Endpoint, "endpoint must be assigned at bootstrap") + assert.True(t, strings.HasSuffix(bootstrapped.Endpoint, "."+cluster), + "labeled endpoint must hang one label beneath the proxy address: %s", bootstrapped.Endpoint) + assert.False(t, bootstrapped.Dedicated, "a labeled pin is not dedicated") assert.False(t, bootstrapped.EnableLogCollection, "log collection from the bootstrap request must override the default") assert.True(t, bootstrapped.EnablePromptCollection, "prompt collection from the bootstrap request must apply") assert.True(t, bootstrapped.RedactPii, "redact toggle from the bootstrap request must apply") @@ -85,30 +90,90 @@ func TestSettingsBootstrapViaPut(t *testing.T) { assert.Equal(t, bootstrapped.EnablePromptCollection, after.EnablePromptCollection, "prompt collection must persist") assert.Equal(t, bootstrapped.RedactPii, after.RedactPii, "redact toggle must persist") - // Once bootstrapped, later updates may omit the cluster entirely. + // Once bootstrapped, PUT updates the toggles. The identity fields ride + // along as a required echo of the assigned values; a matching echo is + // accepted and never written. persisted, err := fresh.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ + Endpoint: bootstrapped.Endpoint, + ProxyAddress: bootstrapped.ProxyAddress, EnableLogCollection: true, EnablePromptCollection: false, RedactPii: true, + AccessLogRetentionDays: 21, }) - require.NoError(t, err, "post-bootstrap update without cluster must succeed") - assert.Equal(t, cluster, persisted.Cluster, "omitted cluster must keep the pinned value") + require.NoError(t, err, "post-bootstrap update must succeed") + require.NotNil(t, persisted.AccessLogRetentionDays) + assert.Equal(t, 21, *persisted.AccessLogRetentionDays, "retention from the update must apply") + assert.Equal(t, bootstrapped.Endpoint, persisted.Endpoint, "endpoint must survive updates untouched") + assert.Equal(t, cluster, persisted.ProxyAddress, "proxy address must survive updates untouched") assert.True(t, persisted.EnableLogCollection, "post-bootstrap toggle must apply") assert.False(t, persisted.EnablePromptCollection, "post-bootstrap toggle must apply") - // The cluster is immutable: a different value is rejected rather than - // silently ignored, and the rejected update must not disturb anything. + // The endpoint is immutable: a PUT carrying a different endpoint is + // rejected, and a second bootstrap is rejected as a conflict. Neither + // rejected write may disturb anything. _, err = fresh.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ - Cluster: ptr("other.cluster.invalid"), - EnableLogCollection: false, + Endpoint: "other.cluster.invalid", + ProxyAddress: persisted.ProxyAddress, + EnableLogCollection: persisted.EnableLogCollection, + EnablePromptCollection: persisted.EnablePromptCollection, + RedactPii: persisted.RedactPii, + AccessLogRetentionDays: 21, + }) + requireClientError(t, err) + + _, err = fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + Endpoint: ptr("other.cluster.invalid"), }) requireClientError(t, err) final, err := fresh.GetSettings(ctx) - require.NoError(t, err, "get settings after the rejected cluster change must succeed") - assert.Equal(t, persisted.Cluster, final.Cluster, "rejected update must not change the cluster") - assert.Equal(t, persisted.Endpoint, final.Endpoint, "rejected update must not change the endpoint") - assert.Equal(t, persisted.EnableLogCollection, final.EnableLogCollection, "rejected update must not apply its toggles") - assert.Equal(t, persisted.EnablePromptCollection, final.EnablePromptCollection, "rejected update must not apply its toggles") - assert.Equal(t, persisted.RedactPii, final.RedactPii, "rejected update must not apply its toggles") + require.NoError(t, err, "get settings after the rejected bootstrap must succeed") + assert.Equal(t, persisted.Endpoint, final.Endpoint, "rejected bootstrap must not change the endpoint") + assert.Equal(t, persisted.ProxyAddress, final.ProxyAddress, "rejected bootstrap must not change the proxy address") + assert.Equal(t, persisted.EnableLogCollection, final.EnableLogCollection, "rejected bootstrap must not apply its toggles") + assert.Equal(t, persisted.EnablePromptCollection, final.EnablePromptCollection, "rejected bootstrap must not apply its toggles") + assert.Equal(t, persisted.RedactPii, final.RedactPii, "rejected bootstrap must not apply its toggles") +} + +// TestSettingsBootstrapSelfAddressed covers the dedicated shape end to end: +// a POST carrying an endpoint claims the hostname verbatim, the proxy address +// equals it, and the pin reads as dedicated — the address-first flow a +// self-hosted operator uses before deploying the proxy that will declare it. +// The tail covers the recovery path the guarded DELETE exists for: with no +// providers and no proxy at the address, the claim can be released and a +// fresh bootstrap succeeds — the fix for a typo'd immutable endpoint. +func TestSettingsBootstrapSelfAddressed(t *testing.T) { + ctx := context.Background() + + fresh, err := harnessStartFresh(ctx, t) + require.NoError(t, err, "start dedicated combined server") + + created, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + Endpoint: ptr("gw.e2e.netbird.selfhosted"), + }) + require.NoError(t, err, "self-addressed bootstrap must succeed") + assert.Equal(t, "gw.e2e.netbird.selfhosted", created.Endpoint, "endpoint must be claimed verbatim") + assert.Equal(t, created.Endpoint, created.ProxyAddress, "self-addressed: proxy address is the endpoint") + assert.True(t, created.Dedicated, "a self-addressed pin is dedicated") + + // No providers exist and no proxy declares the address, so both delete + // guards are clear: the delete releases the claim and the account reads + // as unbootstrapped defaults again. + require.NoError(t, fresh.DeleteSettings(ctx), "guarded delete with both guards clear must succeed") + + after, err := fresh.GetSettings(ctx) + require.NoError(t, err, "get settings after delete must succeed") + assert.Empty(t, after.Endpoint, "a deleted account must read as unbootstrapped") + + // A second delete has nothing to remove. + requireClientError(t, fresh.DeleteSettings(ctx)) + + // Re-creating is a fresh bootstrap — the released hostname is free to be + // claimed again, or a different one chosen. + recreated, err := fresh.CreateSettings(ctx, api.AgentNetworkSettingsCreateRequest{ + Endpoint: ptr("gw2.e2e.netbird.selfhosted"), + }) + require.NoError(t, err, "bootstrap after delete must succeed") + assert.Equal(t, "gw2.e2e.netbird.selfhosted", recreated.Endpoint, "the fresh bootstrap claims the new hostname") } diff --git a/e2e/agentnetwork/skiptls_test.go b/e2e/agentnetwork/skiptls_test.go index 1f57605e3..85af8c604 100644 --- a/e2e/agentnetwork/skiptls_test.go +++ b/e2e/agentnetwork/skiptls_test.go @@ -66,9 +66,7 @@ func TestProviderSkipTLSVerification(t *testing.T) { } } - // First create bootstraps the account cluster. insecureReq := newReq("skip-tls", insecureModel, true) - insecureReq.BootstrapCluster = ptr(harness.AgentNetworkCluster) insecureProv, err := srv.CreateProvider(ctx, insecureReq) require.NoError(t, err, "create skip-tls provider") t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), insecureProv.Id) }) diff --git a/e2e/agentnetwork/vllm_test.go b/e2e/agentnetwork/vllm_test.go index cd598f1ed..6f5117b2f 100644 --- a/e2e/agentnetwork/vllm_test.go +++ b/e2e/agentnetwork/vllm_test.go @@ -57,12 +57,11 @@ func TestVLLMProvider(t *testing.T) { // is enumerated so the router dispatches this model string to this provider. dummyKey := "sk-vllm-e2e" prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ - Name: "vllm", - ProviderId: "vllm", - UpstreamUrl: vllm.URL, - ApiKey: &dummyKey, - Enabled: ptr(true), - BootstrapCluster: ptr(harness.AgentNetworkCluster), + Name: "vllm", + ProviderId: "vllm", + UpstreamUrl: vllm.URL, + ApiKey: &dummyKey, + Enabled: ptr(true), Models: &[]api.AgentNetworkProviderModel{ {Id: harness.VLLMModel, InputPer1k: 0.001, OutputPer1k: 0.002}, }, diff --git a/e2e/harness/Dockerfile.client b/e2e/harness/Dockerfile.client index 114577d60..74a3ec245 100644 --- a/e2e/harness/Dockerfile.client +++ b/e2e/harness/Dockerfile.client @@ -20,5 +20,9 @@ ENV NETBIRD_BIN="/usr/local/bin/netbird" \ NB_ENABLE_CAPTURE="false" \ NB_ENTRYPOINT_SERVICE_TIMEOUT="30" ENTRYPOINT [ "/usr/local/bin/netbird-entrypoint.sh" ] -COPY client/netbird-entrypoint.sh /usr/local/bin/netbird-entrypoint.sh +# --chmod because the build context is not always a git checkout. A suite in +# another module builds from this module's extracted copy in the module cache, +# where every file is 0444 — the cache drops the executable bit git records — and +# a bare COPY then produces an entrypoint the runtime cannot exec. +COPY --chmod=0755 client/netbird-entrypoint.sh /usr/local/bin/netbird-entrypoint.sh COPY --from=builder /out/netbird /usr/local/bin/netbird diff --git a/e2e/harness/agentnetwork.go b/e2e/harness/agentnetwork.go index 078e697af..e85475dff 100644 --- a/e2e/harness/agentnetwork.go +++ b/e2e/harness/agentnetwork.go @@ -126,17 +126,33 @@ func (c *Combined) DeleteGuardrail(ctx context.Context, id string) error { return anDelete(ctx, c, "/api/agent-network/guardrails/"+id) } -// GetSettings returns the account's agent-network settings row. It exists only -// after the first provider create bootstraps it. +// CreateSettings bootstraps the account's agent-network settings row, +// assigning the immutable endpoint. Exactly one of req.ProxyAddress (labeled +// endpoint beneath that cluster) and req.Endpoint (self-addressed dedicated +// endpoint) must be set; a second bootstrap returns a conflict. +func (c *Combined) CreateSettings(ctx context.Context, req api.AgentNetworkSettingsCreateRequest) (api.AgentNetworkSettings, error) { + return anRequest[api.AgentNetworkSettings](ctx, c, http.MethodPost, "/api/agent-network/settings", req) +} + +// GetSettings returns the account's agent-network settings row. Before the +// CreateSettings bootstrap it reads as the defaults with an empty endpoint. func (c *Combined) GetSettings(ctx context.Context) (api.AgentNetworkSettings, error) { return anRequest[api.AgentNetworkSettings](ctx, c, http.MethodGet, "/api/agent-network/settings", nil) } -// UpdateSettings applies the mutable collection toggles. +// UpdateSettings applies the mutable collection toggles. The request must +// echo the assigned endpoint and proxy address unchanged — the server rejects +// a PUT that tries to change them. func (c *Combined) UpdateSettings(ctx context.Context, req api.AgentNetworkSettingsRequest) (api.AgentNetworkSettings, error) { return anRequest[api.AgentNetworkSettings](ctx, c, http.MethodPut, "/api/agent-network/settings", req) } +// DeleteSettings removes the account's settings row, releasing the endpoint. +// Refused while providers exist or a proxy is actively serving the endpoint. +func (c *Combined) DeleteSettings(ctx context.Context) error { + return anDelete(ctx, c, "/api/agent-network/settings") +} + // ListConsumption returns the account's consumption rows (possibly empty). func (c *Combined) ListConsumption(ctx context.Context) ([]api.AgentNetworkConsumption, error) { return anRequest[[]api.AgentNetworkConsumption](ctx, c, http.MethodGet, "/api/agent-network/consumption", nil) diff --git a/e2e/harness/client.go b/e2e/harness/client.go index f53d0ea64..0d7f016a6 100644 --- a/e2e/harness/client.go +++ b/e2e/harness/client.go @@ -32,12 +32,36 @@ type Client struct { container testcontainers.Container } +// clientOptions is what the ClientOption values assemble. +type clientOptions struct { + name string +} + +// ClientOption adjusts how StartClient runs the agent. +type ClientOption func(*clientOptions) + +// WithClientName names the agent, which sets both its network alias and its +// container hostname. The hostname matters beyond addressing: the agent reports +// it to management at registration, so it is the name the peer appears under in +// the API. +// +// Required to run more than one agent against the same server — the default name +// is shared, and two containers cannot hold the same alias on one network. +func WithClientName(name string) ClientOption { + return func(o *clientOptions) { o.name = name } +} + // StartClient builds the client image and runs it on the combined server's // network, joining via the given setup key. The image entrypoint brings the // daemon up automatically; callers wait for connectivity with WaitConnected / // WaitProxyPeer. -func StartClient(ctx context.Context, c *Combined, setupKey string) (*Client, error) { - root, err := repoRoot() +func StartClient(ctx context.Context, c *Combined, setupKey string, opts ...ClientOption) (*Client, error) { + o := clientOptions{name: clientAlias} + for _, opt := range opts { + opt(&o) + } + + root, err := repoRoot(ctx) if err != nil { return nil, err } @@ -47,9 +71,13 @@ func StartClient(ctx context.Context, c *Combined, setupKey string) (*Client, er } req := testcontainers.ContainerRequest{ - Image: clientImage, + Image: clientImage, + // The agent reports the container's hostname to management, so this is + // the name the peer is addressable by in the API as well as on the + // network. The entrypoint takes no hostname flag of its own. + Hostname: o.name, Networks: []string{c.network.Name}, - NetworkAliases: map[string][]string{c.network.Name: {clientAlias}}, + NetworkAliases: map[string][]string{c.network.Name: {o.name}}, Env: map[string]string{ "NB_MANAGEMENT_URL": combinedExposedURL, "NB_SETUP_KEY": setupKey, diff --git a/e2e/harness/combined.go b/e2e/harness/combined.go index b2f0d89d2..e03f9f256 100644 --- a/e2e/harness/combined.go +++ b/e2e/harness/combined.go @@ -61,11 +61,68 @@ type Combined struct { workDir string } +// combinedOptions is what the CombinedOption values assemble. +type combinedOptions struct { + geolocation bool + env map[string]string +} + +// CombinedOption adjusts how StartCombined boots the server. The defaults suit a +// suite that only drives the API; the options exist for the ones that need more +// of the product than that. +type CombinedOption func(*combinedOptions) + +// WithGeolocation leaves the GeoLite database download enabled. It is off by +// default because the download adds startup latency that most suites get nothing +// for. A suite asserting on location-based posture checks needs it: management +// evaluates those rules against the database, and without it the rule fails +// instead of passing without having been checked. +func WithGeolocation() CombinedOption { + return func(o *combinedOptions) { o.geolocation = true } +} + +// WithServerEnv adds environment variables to the combined container, overriding +// the defaults on a key collision. For settings this harness does not model +// directly, so a suite needing one does not have to fork the harness to get it. +func WithServerEnv(env map[string]string) CombinedOption { + return func(o *combinedOptions) { + if o.env == nil { + o.env = map[string]string{} + } + for k, v := range env { + o.env[k] = v + } + } +} + +// combinedEnv is the combined container's environment: setup-PAT enabled so the +// caller can mint an admin token through /api/setup, geolocation off unless the +// suite asked for it, and whatever the suite added on top. +func combinedEnv(o combinedOptions) map[string]string { + env := map[string]string{ + "NB_SETUP_PAT_ENABLED": "true", + } + if !o.geolocation { + // Skip the GeoLite DB download — it blocks startup and agent-network + // ingest doesn't use geolocation. + env["NB_DISABLE_GEOLOCATION"] = "true" + } + for k, v := range o.env { + env[k] = v + } + return env +} + // StartCombined builds the combined server from its multistage Dockerfile and // boots it with setup-PAT enabled on a fresh shared network, returning once the // API is serving. The caller still owns minting the admin PAT via Bootstrap. -func StartCombined(ctx context.Context) (*Combined, error) { - root, err := repoRoot() +func StartCombined(ctx context.Context, opts ...CombinedOption) (*Combined, error) { + var o combinedOptions + for _, opt := range opts { + opt(&o) + } + + root, err := repoRoot(ctx) if err != nil { return nil, err } @@ -88,7 +145,7 @@ func StartCombined(ctx context.Context) (*Combined, error) { return nil, fmt.Errorf("create work dir: %w", err) } - cfg := fmt.Sprintf(combinedConfigYAML, combinedExposedURL, containerIssuer) + cfg := fmt.Sprintf(combinedConfigYAML, combinedExposedURL, !o.geolocation, containerIssuer) if err := os.WriteFile(filepath.Join(workDir, "config.yaml"), []byte(cfg), 0o644); err != nil { //nolint:gosec // non-secret config, bind-mounted and read by the container _ = net.Remove(ctx) return nil, fmt.Errorf("write combined config: %w", err) @@ -112,13 +169,8 @@ func StartCombined(ctx context.Context) (*Combined, error) { ExposedPorts: []string{combinedHTTPPort}, Networks: []string{net.Name}, NetworkAliases: map[string][]string{net.Name: {combinedAlias}}, - Env: map[string]string{ - "NB_SETUP_PAT_ENABLED": "true", - // Skip the GeoLite DB download — it blocks startup and agent-network - // ingest doesn't use geolocation. - "NB_DISABLE_GEOLOCATION": "true", - }, - Cmd: []string{"--config", "/nb/config.yaml"}, + Env: combinedEnv(o), + Cmd: []string{"--config", "/nb/config.yaml"}, HostConfigModifier: func(hc *container.HostConfig) { hc.Binds = append(hc.Binds, workDir+":/nb") }, diff --git a/e2e/harness/config.go b/e2e/harness/config.go index 71b3656c5..f0952b18c 100644 --- a/e2e/harness/config.go +++ b/e2e/harness/config.go @@ -15,6 +15,11 @@ package harness // server is required to load it — a broken path or malformed file fails startup // rather than silently falling back to the compiled-in rates, and TestMain then // fails with the container logs. +// +// disableGeoliteUpdate is a parameter rather than a fixed true because a suite +// that exercises geolocation needs the database: management can only evaluate a +// location rule with GeoLite loaded, and a rule it cannot evaluate fails rather +// than passing vacuously. See WithGeolocation. const combinedConfigYAML = `server: listenAddress: ":8080" exposedAddress: "%s" @@ -25,7 +30,7 @@ const combinedConfigYAML = `server: authSecret: "e2e-relay-secret" dataDir: "/nb/data" disableAnonymousMetrics: true - disableGeoliteUpdate: true + disableGeoliteUpdate: %t auth: issuer: "%s" store: diff --git a/e2e/harness/options_test.go b/e2e/harness/options_test.go new file mode 100644 index 000000000..8a5557a83 --- /dev/null +++ b/e2e/harness/options_test.go @@ -0,0 +1,161 @@ +//go:build e2e + +package harness + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The options exist so a suite can ask for a deployment this harness would not +// otherwise give it. What they configure is a container environment and a config +// file, both assembled before anything is started, so they are checkable without +// Docker — which is the point: a wiring mistake here would otherwise only show up +// as a puzzling failure minutes into a container run. + +func TestCombinedEnvGeolocation(t *testing.T) { + var off combinedOptions + assert.Equal(t, "true", combinedEnv(off)["NB_DISABLE_GEOLOCATION"], + "geolocation should be off by default") + + var on combinedOptions + WithGeolocation()(&on) + assert.NotContains(t, combinedEnv(on), "NB_DISABLE_GEOLOCATION", + "WithGeolocation must leave NB_DISABLE_GEOLOCATION unset, so the server downloads the database") + assert.Equal(t, "true", combinedEnv(on)["NB_SETUP_PAT_ENABLED"], + "the setup PAT must stay enabled whatever else is configured; Bootstrap depends on it") +} + +// The config file carries the same decision as the environment variable, and the +// server needs both to agree: disableGeoliteUpdate suppresses the download even +// when geolocation itself is enabled. +func TestCombinedConfigGeolocation(t *testing.T) { + for _, tc := range []struct { + name string + opts []CombinedOption + want string + }{ + {name: "default", want: "disableGeoliteUpdate: true"}, + {name: "with geolocation", opts: []CombinedOption{WithGeolocation()}, want: "disableGeoliteUpdate: false"}, + } { + t.Run(tc.name, func(t *testing.T) { + var o combinedOptions + for _, opt := range tc.opts { + opt(&o) + } + cfg := fmt.Sprintf(combinedConfigYAML, combinedExposedURL, !o.geolocation, containerIssuer) + assert.Contains(t, cfg, tc.want, "geolocation not rendered as expected") + // The issuer is the last verb; a mis-ordered argument list would put + // the boolean here instead and the server would fail to start. + assert.Contains(t, cfg, `issuer: "`+containerIssuer+`"`, "issuer not rendered") + }) + } +} + +func TestWithServerEnvOverrides(t *testing.T) { + var o combinedOptions + WithServerEnv(map[string]string{"NB_LOG_LEVEL": "debug"})(&o) + WithServerEnv(map[string]string{"NB_SETUP_PAT_ENABLED": "false"})(&o) + + env := combinedEnv(o) + assert.Equal(t, "debug", env["NB_LOG_LEVEL"], "added variable missing") + assert.Equal(t, "false", env["NB_SETUP_PAT_ENABLED"], "a suite must be able to override a default") +} + +// Two agents on one network cannot share an alias, so the name has to reach both +// the alias and the hostname. The hostname is the one management records, so it is +// also what the peer is addressable by through the API. +func TestWithClientName(t *testing.T) { + o := clientOptions{name: clientAlias} + require.Equal(t, "client", o.name, "unexpected default client name") + + WithClientName("peer2")(&o) + assert.Equal(t, "peer2", o.name, "WithClientName did not take") +} + +// repoRoot has to recognise this module rather than merely finding a go.mod, or a +// suite in another module gets its own root and a build context without the +// component Dockerfiles in it. +func TestIsModule(t *testing.T) { + dir := t.TempDir() + + other := filepath.Join(dir, "go.mod") + require.NoError(t, os.WriteFile(other, []byte("module example.com/other\n\ngo 1.25\n"), 0o600)) + assert.False(t, isModule(other, modulePath), "another module's go.mod must not be taken for this repo") + + ours := filepath.Join(dir, "ours.mod") + require.NoError(t, os.WriteFile(ours, []byte("// a comment\n\nmodule "+modulePath+"\n\ngo 1.25\n"), 0o600)) + assert.True(t, isModule(ours, modulePath), "this repo's go.mod was not recognised") + + assert.False(t, isModule(filepath.Join(dir, "absent.mod"), modulePath), + "a missing go.mod must not report a match") +} + +// Running from inside the repo, repoRoot finds it by walking up — the module +// lookup is only the fallback, and this asserts the walk still wins so an in-repo +// run never depends on the module cache. +func TestRepoRootFindsThisRepo(t *testing.T) { + root, err := repoRoot(context.Background()) + require.NoError(t, err) + assert.True(t, isModule(filepath.Join(root, "go.mod"), modulePath), + "repoRoot returned %s, which is not this module", root) + + for _, f := range []string{combinedDockerfile, clientDockerfile} { + _, err := os.Stat(filepath.Join(root, f)) + assert.NoError(t, err, "%s is not present under the reported root %s", f, root) + } +} + +// A caller that vendors its dependencies puts the go command in automatic vendor +// mode, where `go list -m -f {{.Dir}}` succeeds and reports an EMPTY directory: +// vendor/ holds packages, not module source. Without -mod=readonly the lookup +// would come back empty and the harness would report a missing module for a +// dependency that is present. +func TestModuleDirResolvesUnderVendorMode(t *testing.T) { + if _, err := exec.LookPath("go"); err != nil { + t.Skip("no go tool on PATH") + } + ctx := context.Background() + + base := t.TempDir() + dep := filepath.Join(base, "dep") + main := filepath.Join(base, "main") + require.NoError(t, os.MkdirAll(dep, 0o750)) + require.NoError(t, os.MkdirAll(main, 0o750)) + + // A local replacement rather than a real dependency, so this needs no network. + require.NoError(t, os.WriteFile(filepath.Join(dep, "go.mod"), + []byte("module example.com/dep\n\ngo 1.25\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dep, "dep.go"), + []byte("package dep\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(main, "go.mod"), + []byte("module example.com/main\n\ngo 1.25\n\nrequire example.com/dep v0.0.0\n\nreplace example.com/dep v0.0.0 => ../dep\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(main, "main.go"), + []byte("package main\n\nimport _ \"example.com/dep\"\n\nfunc main() {}\n"), 0o600)) + + t.Chdir(main) + vendor := exec.CommandContext(ctx, "go", "mod", "vendor") + out, err := vendor.CombinedOutput() + require.NoError(t, err, "go mod vendor: %s", out) + + dir, err := moduleDir(ctx, "example.com/dep") + require.NoError(t, err, "the module must still resolve with a vendor directory present") + assert.Equal(t, dep, dir, "resolved the wrong directory") +} + +// A cancelled context has to stop the lookup rather than leaving the caller +// waiting on a subprocess it has already given up on. +func TestModuleDirHonoursContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := moduleDir(ctx, modulePath) + assert.ErrorIs(t, err, context.Canceled, "a cancelled context must stop the lookup") +} diff --git a/e2e/harness/paths.go b/e2e/harness/paths.go index d7df6bbfa..569c32efc 100644 --- a/e2e/harness/paths.go +++ b/e2e/harness/paths.go @@ -3,27 +3,82 @@ package harness import ( + "context" "fmt" "os" + "os/exec" "path/filepath" + "strings" ) -// repoRoot walks up from the working directory to the module root (the -// directory holding go.mod), so the Docker build context is correct no matter -// which package the test runs from. -func repoRoot() (string, error) { +// modulePath is this module, used both to recognise the repo when walking up +// from the working directory and to locate it when the suite lives elsewhere. +const modulePath = "github.com/netbirdio/netbird" + +// repoRoot returns the directory the component Dockerfiles are built from. +// +// Walking up from the working directory finds it for any test inside this repo, +// no matter which package it runs from. A suite in another module gets a +// different answer that way — its own module root, where combined/Dockerfile +// does not exist — so the ancestor has to be this module and not merely some +// module. When it is not, the build context is the extracted module directory of +// whichever version that suite depends on, which is the right one: the server it +// tests against is then built from the same revision as the client library it +// was compiled with. +func repoRoot(ctx context.Context) (string, error) { dir, err := os.Getwd() if err != nil { return "", err } for { - if _, statErr := os.Stat(filepath.Join(dir, "go.mod")); statErr == nil { + if isModule(filepath.Join(dir, "go.mod"), modulePath) { return dir, nil } parent := filepath.Dir(dir) if parent == dir { - return "", fmt.Errorf("go.mod not found above %s", dir) + break } dir = parent } + return moduleDir(ctx, modulePath) +} + +// isModule reports whether the go.mod at path declares the given module. +func isModule(path, want string) bool { + b, err := os.ReadFile(path) + if err != nil { + return false + } + for _, line := range strings.Split(string(b), "\n") { + if rest, ok := strings.CutPrefix(strings.TrimSpace(line), "module "); ok { + return strings.TrimSpace(rest) == want + } + } + return false +} + +// moduleDir asks the go tool where a module's source is, which for a dependent +// module is its extracted copy in the module cache. The cache is read-only, and +// a Docker build context is only ever read. +// +// -mod=readonly is required rather than cosmetic. A caller that vendors its +// dependencies puts the go command in automatic vendor mode, where this lookup +// succeeds with an EMPTY directory — vendor/ holds packages, not module source, +// so there is nothing to report. Asking in readonly mode resolves against the +// module graph instead, which answers for both a cached module and a local +// replacement, and neither writes to go.mod. +func moduleDir(ctx context.Context, module string) (string, error) { + cmd := exec.CommandContext(ctx, "go", "list", "-mod=readonly", "-m", "-f", "{{.Dir}}", module) + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("locate %s: %w", module, err) + } + dir := strings.TrimSpace(string(out)) + if dir == "" { + return "", fmt.Errorf("locate %s: the go tool reported no directory; run `go mod download %s`", module, module) + } + if _, err := os.Stat(dir); err != nil { + return "", fmt.Errorf("locate %s: %w", module, err) + } + return dir, nil } diff --git a/e2e/harness/proxy.go b/e2e/harness/proxy.go index 85f3518d4..3d709b439 100644 --- a/e2e/harness/proxy.go +++ b/e2e/harness/proxy.go @@ -43,7 +43,7 @@ type Proxy struct { // or override any NB_PROXY_* var (e.g. NB_PROXY_TUNNEL_CACHE_TTL for tests that // need a short authorization-cache window). func StartProxy(ctx context.Context, c *Combined, proxyToken string, envOverrides ...map[string]string) (*Proxy, error) { - root, err := repoRoot() + root, err := repoRoot(ctx) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index f8a1a84b0..f98073417 100644 --- a/go.mod +++ b/go.mod @@ -99,7 +99,7 @@ require ( github.com/pires/go-proxyproto v0.11.0 github.com/pkg/sftp v1.13.9 github.com/prometheus/client_golang v1.23.2 - github.com/quic-go/quic-go v0.55.0 + github.com/quic-go/quic-go v0.59.1 github.com/redis/go-redis/v9 v9.7.3 github.com/rs/xid v1.3.0 github.com/shirou/gopsutil/v4 v4.25.8 @@ -239,7 +239,6 @@ require ( github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect - github.com/jchv/go-winloader v0.0.0-20250406163304-c1995be93bd1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -340,4 +339,4 @@ replace github.com/dexidp/dex/api/v2 => github.com/netbirdio/dex/api/v2 v2.0.0-2 replace github.com/mailru/easyjson => github.com/netbirdio/easyjson v0.9.0 -replace github.com/wailsapp/wails/v3 => github.com/netbirdio/wails/v3 v3.0.0-beta.3.0.20260807055527-fc03f984d701 +replace github.com/wailsapp/wails/v3 => github.com/netbirdio/wails/v3 v3.0.0-beta.3.0.20260810103952-24e716aea4db diff --git a/go.sum b/go.sum index f6b32e32a..31e8b5454 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,6 @@ github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jchv/go-winloader v0.0.0-20250406163304-c1995be93bd1 h1:njuLRcjAuMKr7kI3D85AXWkw6/+v9PwtV6M6o11sWHQ= -github.com/jchv/go-winloader v0.0.0-20250406163304-c1995be93bd1/go.mod h1:alcuEEnZsY1WQsagKhZDsoPCRoOijYqhZvPwLG0kzVs= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -490,8 +488,8 @@ github.com/netbirdio/service v0.0.0-20240911161631-f62744f42502 h1:3tHlFmhTdX9ax github.com/netbirdio/service v0.0.0-20240911161631-f62744f42502/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM= github.com/netbirdio/signal-dispatcher/dispatcher v0.0.0-20250805121659-6b4ac470ca45 h1:ujgviVYmx243Ksy7NdSwrdGPSRNE3pb8kEDSpH0QuAQ= github.com/netbirdio/signal-dispatcher/dispatcher v0.0.0-20250805121659-6b4ac470ca45/go.mod h1:5/sjFmLb8O96B5737VCqhHyGRzNFIaN/Bu7ZodXc3qQ= -github.com/netbirdio/wails/v3 v3.0.0-beta.3.0.20260807055527-fc03f984d701 h1:QL9nupfRom0L9jcY7N9l/Bc6QK2PtC6pHzC+ftpTqpw= -github.com/netbirdio/wails/v3 v3.0.0-beta.3.0.20260807055527-fc03f984d701/go.mod h1:BzATbK71VFikMMMCo434wAi0QcaI03P+xeaWgDvQvjw= +github.com/netbirdio/wails/v3 v3.0.0-beta.3.0.20260810103952-24e716aea4db h1:gBOE2r4AW1soSmpYJC5/n9/1L8UQ8+HLjed8CY/TzZY= +github.com/netbirdio/wails/v3 v3.0.0-beta.3.0.20260810103952-24e716aea4db/go.mod h1:bsdahLwBQxXjlmdPPeQyrTcDJfcqAr/ymFj0RXhwtWI= github.com/netbirdio/wireguard-go v0.0.0-20260628102922-2834bebf6c1a h1:3CWK+yTvRKOcC0Q8VCTGy4l60TEb27CQVS7LkMxwjmw= github.com/netbirdio/wireguard-go v0.0.0-20260628102922-2834bebf6c1a/go.mod h1:rpwXGsirqLqN2L0JDJQlwOboGHmptD5ZD6T2VmcqhTw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -582,8 +580,8 @@ github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEo github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= -github.com/quic-go/quic-go v0.55.0 h1:zccPQIqYCXDt5NmcEabyYvOnomjs8Tlwl7tISjJh9Mk= -github.com/quic-go/quic-go v0.55.0/go.mod h1:DR51ilwU1uE164KuWXhinFcKWGlEjzys2l8zUl5Ss1U= +github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic= +github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -793,7 +791,6 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/infrastructure_files/migrate-to-enterprise.sh b/infrastructure_files/migrate-to-enterprise.sh index 8e1fcf521..5f69b4a90 100755 --- a/infrastructure_files/migrate-to-enterprise.sh +++ b/infrastructure_files/migrate-to-enterprise.sh @@ -173,11 +173,11 @@ EOF # --------------------------------------------------------------------------- detect_combined_service() { - yq eval '.services | to_entries | map(select(.value.image | test("^netbirdio/netbird-server"))) | .[0].key // ""' "$COMPOSE_FILE" + yq eval '.services | to_entries | map(select(.value.image | test("^(ghcr\\.io/)?netbirdio/netbird-server([:@]|$)"))) | .[0].key // ""' "$COMPOSE_FILE" } detect_dashboard_service() { - yq eval '.services | to_entries | map(select(.value.image | test("^netbirdio/dashboard"))) | .[0].key // ""' "$COMPOSE_FILE" + yq eval '.services | to_entries | map(select(.value.image | test("^(ghcr\\.io/)?netbirdio/dashboard([:@]|$)"))) | .[0].key // ""' "$COMPOSE_FILE" } detect_config_yaml_host_path() { @@ -661,12 +661,12 @@ init_migration() { COMPOSE_NETWORK=$(detect_compose_network) if [[ -z "$COMBINED_SERVICE" ]]; then - echo "Could not find a service running netbirdio/netbird-server* in $COMPOSE_FILE." > /dev/stderr + echo "Could not find a service running netbirdio/netbird-server or ghcr.io/netbirdio/netbird-server in $COMPOSE_FILE." > /dev/stderr echo "This script targets the community combined-server deployment." > /dev/stderr exit 1 fi if [[ -z "$DASHBOARD_SERVICE" ]]; then - echo "Could not find a service running netbirdio/dashboard* in $COMPOSE_FILE." > /dev/stderr + echo "Could not find a service running netbirdio/dashboard or ghcr.io/netbirdio/dashboard in $COMPOSE_FILE." > /dev/stderr exit 1 fi if [[ -z "$CONFIG_YAML_HOST" ]]; then diff --git a/infrastructure_files/observability/grafana/dashboards/management-enterprise.json b/infrastructure_files/observability/grafana/dashboards/management-enterprise.json new file mode 100644 index 000000000..481050a02 --- /dev/null +++ b/infrastructure_files/observability/grafana/dashboards/management-enterprise.json @@ -0,0 +1,8857 @@ +{ + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "builtIn": true, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "query": { + "datasource": { + "name": "-- Grafana --" + }, + "group": "grafana", + "kind": "DataQuery", + "spec": {}, + "version": "v0" + } + } + }, + { + "kind": "AnnotationQuery", + "spec": { + "enable": true, + "hide": false, + "iconColor": "red", + "name": "Deployments", + "query": { + "datasource": { + "name": "grafana" + }, + "group": "datasource", + "kind": "DataQuery", + "spec": { + "expr": "", + "interval": "", + "refId": "Anno", + "tags": [], + "type": "tags" + }, + "version": "v0" + } + } + } + ], + "cursorSync": "Crosshair", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, sum by(le) (rate(management_grpc_updatechannel_queue_length_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Connected grpc streams", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p99 of the update-channel queue length: how many network map updates are waiting to be delivered to a peer. The channel buffer is 100 messages - approaching it means updates start getting dropped and peers fall behind.", + "id": 1, + "links": [], + "title": "UpdateChannel Queue length max", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-10": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_total_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\", type=\"read\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HTTP read duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of read API requests (GET/OPTIONS), from the total request duration histogram tagged type=read.", + "id": 10, + "links": [], + "title": "Read HTTP API Requests p95 Duration (GET/OPTIONS)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-11": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_updatechannel_close_one_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "CloseOne", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(rate(management_updatechannel_send_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "Send", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(rate(management_updatechannel_create_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "Create", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_updatechannel_get_all_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GetAll", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_updatechannel_haschannel_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HasChannel", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_updatechannel_close_multiple_channels_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "CloseMultiple", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "F" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate per minute of update-channel operations: close one, send update, create, get all connected peers, has-channel and close multiple. Together they show how much channel churn an instance is handling.", + "id": 11, + "links": [], + "title": "Update Channel operations", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 69, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-12": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_updatechannel_create_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\", closed=\"true\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_updatechannel_create_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "__expr__" + }, + "group": "__expr__", + "kind": "DataQuery", + "spec": { + "expression": "100-$A/$B*100", + "type": "math" + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Tracks how often a new peer update channel replaces a channel that was still open (closed=true), i.e. peers reconnecting before management noticed the previous stream was gone. The plotted expression is 100 - recreated/total, so the line drops as recreations rise.", + "id": 12, + "links": [], + "title": "Percentage of Recreated channels", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "displayName", + "value": "Recreation" + } + ] + } + ] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-13": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_close_one_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "CloseOne", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_send_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "Send", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_create_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "Create", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_get_all_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GetAll", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_haschannel_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "CloseMultiple", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_close_multiple_channels_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HasChannel", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "F" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of each update-channel operation, in microseconds. These are in-memory operations behind a lock, so growth here means lock contention rather than I/O.", + "id": 13, + "links": [], + "title": "Update Channel methods p95 Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-14": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum(management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Connected grpc streams", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Total number of peers currently holding an open gRPC Sync stream across all management instances (management.grpc.connected.streams). A sudden drop means peers were disconnected and will come back as a Login/Sync reconnect storm.", + "id": 14, + "links": [], + "title": "Connected peers", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + }, + { + "color": "red", + "value": 0 + }, + { + "color": "#EAB839", + "value": 8000 + }, + { + "color": "green", + "value": 11000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": true, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-15": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "sum by(instance) (management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"})", + "instant": false, + "legendFormat": "Total", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Open peer Sync streams over time, in total per instance (A) and as raw series (B). Steps down mark instance restarts or LB rebalancing; the ramp back up is the reconnect wave.", + "id": 15, + "links": [], + "title": "Connected peers historical", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-16": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_sync_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Sync duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of the Sync gRPC request - the time management needs to authenticate the peer, build its first network map and open the update channel. Primary latency SLI for peer connects.", + "id": 16, + "links": [], + "title": "gRPC Sync Request p95 Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-17": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_login_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Login duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of the Login gRPC request (peer authentication plus initial configuration and relay credentials). Rises when the IdP, the store or account locking is slow.", + "id": 17, + "links": [], + "title": "gRPC Login Request p95 Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-18": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_account_get_peer_network_map_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken to build the network map returned to a single peer, in milliseconds.", + "id": 18, + "links": [], + "title": "GetPeerNetworkMap Latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-19": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(le) (increase(management_account_network_map_object_count_objects_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the number of objects (peers, routes, firewall rules, DNS entries and so on) in the network maps sent to clients. Large maps make both calculation and serialization more expensive.", + "id": 19, + "links": [], + "title": "NetworkMap Objects", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 0, + "reverse": false, + "unit": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "login", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "sync", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (rate(management_grpc_key_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "get key", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of the three peer-facing gRPC calls: Login (authenticate, return initial config and relay credentials), Sync (open the update channel and receive network map updates) and GetServerKey (fetch the server WireGuard public key). Elevated Login/Sync rates usually mean peers are reconnecting in a loop.", + "id": 2, + "links": [], + "title": "gRPC Requests", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 51, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": true, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": {}, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-20": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_updatechannel_tosyncresponse_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken to convert a calculated network map into the gRPC SyncResponse sent to the peer, in microseconds.", + "id": 20, + "links": [], + "title": "To Sync Response", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "µs" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-23": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(le) (increase(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of per-account p95 Sync durations - each sample is one account's p95, flushed every 60s. Shows whether slow Syncs are spread across the fleet or concentrated in a few accounts.", + "id": 23, + "links": [], + "title": "GRPC Sync Latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.6, + "fill": "dark-red", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-24": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_grpc_login_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of Login gRPC request durations across all peers. The right tail is what a peer experiences on startup; samples beyond 7s are what the high-latency counters in the Debugging row track.", + "id": 24, + "links": [], + "title": "GRPC Login Latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-25": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by(environment) (rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Total", + "range": true + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Login requests, per instance (A) and in total (B). Login is the most expensive peer call (IdP plus store writes); a sustained high rate means clients are restarting or failing to hold their Sync stream.", + "id": 25, + "links": [], + "title": "gRPC Login Request rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-26": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by(environment) (rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Total", + "range": true + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Sync requests, per instance (A) and in total (B). Every Sync opens or re-opens a peer update channel, so a spike here is a reconnect storm and predicts CPU spent on network map calculation.", + "id": 26, + "links": [], + "title": "gRPC Sync Request rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-27": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_grpc_updatechannel_queue_length_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of update-channel queue lengths across all peers. Mass moving to the right shows a growing backlog of undelivered network map updates.", + "id": 27, + "links": [], + "title": "Update Channel heat map", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Greens", + "steps": 73 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-29": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(endpoint) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval])*60) > 10", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{endpoint}} > 10", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Total", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Successful (2xx) GET requests per minute broken down by endpoint, limited to endpoints above 10 rpm. Shows which read endpoints dashboard and API clients hit hardest.", + "id": 29, + "links": [], + "title": "HTTP Read request counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-30": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(endpoint) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60) > 1", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{endpoint}} > 1", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Total", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Successful (2xx) non-GET/OPTIONS requests per minute by endpoint, limited to endpoints above 1 rpm. Writes take account locks and fan out network map updates, so read this together with the Network Map row.", + "id": 30, + "links": [], + "title": "HTTP Write request counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-31": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_store_persistence_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of how long it takes to save or delete an account in the store (management.store.persistence.duration.ms). Widening buckets point at database write pressure.", + "id": 31, + "links": [], + "title": "Store Persistence latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.6, + "fill": "dark-red", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Reds", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-32": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_store_transaction_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of store transaction execution time. Nearly every API and gRPC call runs through a transaction, so this is the first place to look when latency rises everywhere at once.", + "id": 32, + "links": [], + "title": "Store Transaction latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.6, + "fill": "dark-red", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Reds", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-33": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_account_peer_meta_update_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{endpoint}} > 10", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate per minute of peers reporting changed metadata (OS, version, hostname, addresses). Each change persists the peer and can trigger an account peers update.", + "id": 33, + "links": [], + "title": "Peer meta updates counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-34": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_grpc_sync_request_blocked_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Sync request rate", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Sync requests rejected because the peer is blocked (login expired, peer disabled or account restricted). A rising line means clients retrying without valid access.", + "id": 34, + "links": [], + "title": "gRPC Sync Request Blocked rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-35": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_grpc_login_request_blocked_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Login request rate", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Login requests rejected because the peer is blocked. Persistent volume here is usually expired peer logins retrying in a loop.", + "id": 35, + "links": [], + "title": "gRPC Login Request Blocked rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-36": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_account_update_account_peers_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken by an account peers update: preparing the data and pushing a fresh network map to every peer in the account, in milliseconds. Scales with account size.", + "id": 36, + "links": [], + "title": "UpdateAccountPeers Latency", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-37": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_updatechannel_calc_networkmap_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken to calculate one peer's network map, in milliseconds. This is the dominant CPU cost when many peers reconnect at once.", + "id": 37, + "links": [], + "title": "Network Map Calculation", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-39": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_updatechannel_calc_posturechecks_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of the time taken to evaluate a peer's posture checks while building its network map, in microseconds.", + "id": 39, + "links": [], + "title": "Posture Check Calculation", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "µs" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-45": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "sort_desc(\n topk(\n 10,\n sum by (account_id) (\n increase(management_grpc_login_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n )\n )\n)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [ + { + "group": "organize", + "kind": "Transformation", + "spec": { + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Time": "" + } + } + } + } + ] + } + }, + "description": "Top 10 accounts by number of Login requests that exceeded the 7s high-latency threshold.", + "id": 45, + "links": [], + "title": "High Login Latency Ranking (login count per account)", + "vizConfig": { + "group": "table", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "footer": { + "reducers": [] + }, + "inspect": false, + "tooltip": { + "placement": "auto" + }, + "wrapHeaderText": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Time" + } + ] + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-46": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "sort_desc(\n topk(\n 10,\n sum by (account_id) (\n increase(management_grpc_sync_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n )\n )\n)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [ + { + "group": "organize", + "kind": "Transformation", + "spec": { + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Time": "" + } + } + } + } + ] + } + }, + "description": "Top 10 accounts by number of Sync requests that exceeded the 7s high-latency threshold. Use it to find which tenant is behind a latency spike.", + "id": 46, + "links": [], + "title": "High Sync Latency Ranking (sync count per account)", + "vizConfig": { + "group": "table", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "footer": { + "reducers": [] + }, + "inspect": false, + "tooltip": { + "placement": "auto" + }, + "wrapHeaderText": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Time" + } + ] + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-47": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "count(\n sum by (account_id) (\n increase(management_grpc_sync_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n ) > 1\n)", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of distinct accounts with more than one slow (>7s) Sync in the interval - tells you whether a latency spike is fleet-wide or limited to a few tenants.", + "id": 47, + "links": [], + "title": "Accounts with high Sync latency", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-48": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "count(\n sum by (account_id) (\n increase(management_grpc_login_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n ) > 1\n)", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of distinct accounts with more than one slow (>7s) Login in the interval.", + "id": 48, + "links": [], + "title": "Accounts with high Login latency", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-50": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${logs_datasource}" + }, + "group": "loki", + "kind": "DataQuery", + "spec": { + "direction": "backward", + "editorMode": "code", + "expr": "sort_desc(\ntopk(\n 20,\n sum by (peerID) (\n count_over_time(\n {job=\"$log_job\"} |= \"Sync took\" | regexp `peerID:\\s(?P[^,\\]]+)` [$__range]\n )\n )\n)\n)", + "queryType": "instant" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [ + { + "group": "organize", + "kind": "Transformation", + "spec": { + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value #A": "number of times ", + "peerID": "peerPubKey" + } + } + } + } + ] + } + }, + "description": "Top 20 peers by number of Sync log lines in the selected range, parsed out of the management logs to find individual peers stuck in a reconnect loop. Pick the log stream with the `log stream` variable; the query matches the `Sync took` debug line, so management must run at debug level for this panel to have data.", + "id": 50, + "links": [], + "title": "Highest Syncs by peer key", + "vizConfig": { + "group": "table", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "footer": { + "reducers": [] + }, + "inspect": false, + "tooltip": { + "placement": "auto" + }, + "wrapHeaderText": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "labels" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "peerPubKey" + }, + "properties": [ + { + "id": "custom.width", + "value": 455 + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "number of times " + } + ] + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-52": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "wsproxy_active_connections{job=~\"$job\", instance=~\"$instance\"}", + "legendFormat": "{{__name__}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Currently open WebSocket proxy connections. The wsproxy fronts gRPC for browser-based clients, so this is the number of connected web clients.", + "id": 52, + "links": [], + "title": "Proxy Active Connections", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-53": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "delta(wsproxy_bytes_transferred_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])", + "legendFormat": "{{direction}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Bytes transferred through the WebSocket proxy per interval, split by direction.", + "id": 53, + "links": [], + "title": "Proxy Traffic by Direction", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-54": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "increase(wsproxy_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{error_type}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "WebSocket proxy errors grouped by error type - dial failures, read/write errors and protocol issues on browser client connections.", + "id": 54, + "links": [], + "title": "Errors by Type", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-65": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "exemplar": false, + "expr": "management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "legendFormat": "{{host}}", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Open peer Sync streams per management instance. Use it to confirm the load balancer spreads streams evenly - a skewed instance carries proportionally more update-channel and network map work.", + "id": 65, + "links": [], + "title": "Connected Peers per Server", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-66": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(host) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval]) * 60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Successful GET requests per minute grouped by management instance - read load distribution across the fleet.", + "id": 66, + "links": [], + "title": "HTTP Read request counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-67": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(host) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Successful write requests per minute grouped by management instance - write load distribution across the fleet.", + "id": 67, + "links": [], + "title": "HTTP Write request counter", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_total_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\", type=\"write\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HTTP write duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of write API requests (PUT/POST/DELETE), tagged type=write. Writes acquire account locks and trigger peer updates, so they are normally slower than reads.", + "id": 7, + "links": [], + "title": "Write HTTP API Requests p95 Duration (PUT/POST/DELETE)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-74": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (increase(management_pat_usage_distribution_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Distribution of personal access token usage counts, sampled once per minute per token. Mass in the high buckets means a single PAT is hammering the API.", + "id": 74, + "links": [], + "title": "PAT usage counter", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-76": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "gnatsd_connz_subscriptions{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "{{host}}", + "range": false, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Current subscriptions held by each client connected to NATS (connz). Signal instances subscribe per connected peer, so this tracks how peers are spread across signal nodes.", + "id": 76, + "links": [], + "title": "Subscriptions per node", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-77": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "gnatsd_connz_subscriptions{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Subscriptions per connected NATS client over time. Steps mark signal instances restarting and re-subscribing their peers.", + "id": 77, + "links": [], + "title": "Subscriptions per node", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-78": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "gnatsd_connz_num_connections{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of client connections on each NATS node - effectively how many signal instances are attached to that node.", + "id": 78, + "links": [], + "title": "Signal nodes connected per NATs", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-79": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(environment) (rate(gnatsd_connz_out_msgs{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of messages NATS delivers out to its clients: the peer signalling throughput flowing through the cluster.", + "id": 79, + "links": [], + "title": "Messages per Second", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "HTTP request duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 duration of incoming REST API requests by endpoint and method (management.http.request.duration.ms). Covers dashboard and API clients only - peer traffic is on the gRPC panels.", + "id": 8, + "links": [], + "title": "HTTP API Requests p95 Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-80": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(environment) (rate(gnatsd_connz_out_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of bytes NATS delivers out to its clients - signalling bandwidth, useful next to the message rate to spot unusually large payloads.", + "id": 80, + "links": [], + "title": "Data per Second", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-81": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(environment) (increase(gnatsd_varz_slow_consumers{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Increase of the NATS slow-consumer counter. Anything above zero means a signal node could not read fast enough and NATS dropped its connection, so peers lose signalling until it reconnects.", + "id": 81, + "links": [], + "title": "Slow consumers detected", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-82": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "gnatsd_varz_subscriptions{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{host}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Total subscriptions registered on each NATS server (varz). Grows roughly with the number of connected peers.", + "id": 82, + "links": [], + "title": "Total number of NATS subscriptions", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-83": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{instance}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Resident memory of the scraped processes. process_resident_memory_bytes is exposed by every Go/Prometheus process, so narrow the job and instance variables to the NATS nodes to read this panel as NATS memory.", + "id": 83, + "links": [], + "title": "NATS memory consumption", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-84": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "idelta(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[2m])", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Per-account Sync p95 samples as a heatmap (2m deltas). Use it to spot a handful of large or slow accounts pulling the overall Sync latency up.", + "id": 84, + "links": [], + "title": "GRPC Sync Latency By Account", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.6, + "fill": "dark-red", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-85": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "idelta(management_grpc_login_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[2m])", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Per-account Login p95 samples as a heatmap (2m deltas). Highlights accounts whose logins are much slower than the fleet average.", + "id": 85, + "links": [], + "title": "GRPC Login Latency By Account", + "vizConfig": { + "group": "heatmap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Purples", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "placement": "bottom", + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "axisWidth": 60, + "decimals": 1, + "reverse": false, + "unit": "ms" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-86": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Sync duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 of the per-account Sync p95 distribution - the latency seen by the worst-served accounts rather than by the average request. Diverges from the global p95 when only a few accounts are hurting.", + "id": 86, + "links": [], + "title": "gRPC Sync Request p95 Duration (by Account)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-87": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_login_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "GRPC Login duration", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "p95 of the per-account Login p95 distribution - login latency for the worst-served accounts rather than the average request.", + "id": 87, + "links": [], + "title": "gRPC Login Request p95 Duration (by Account)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-88": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{environment}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Account peers updates triggered per minute (2m increase halved). Each trigger recalculates and pushes network maps to every peer in the account.", + "id": 88, + "links": [], + "title": "NetworkMap Triggers Total", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 19, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-89": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(resource) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Account peers updates over 2m intervals, grouped by the resource that triggered them - the time series behind the pie chart above.", + "id": 89, + "links": [], + "title": "NetworkMap Triggers By Source", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.9, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(environment) (rate(management_idp_authenticate_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "authenticate (mgtm->auth0)", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_idp_update_user_meta_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "update user meta", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(management_idp_get_account_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "get account", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Requests per minute that management sends to the configured identity provider: authenticate (service token refresh), user metadata updates and account lookups. IdP throttling shows up here before it shows up as Login latency.", + "id": 9, + "links": [], + "title": "IdP Requests", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 69, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-90": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "sum by(resource) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[1h]))", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Account peers updates over the last hour, grouped by the resource that triggered them (policy, group, peer, route, user, network, ...). Shows which kind of configuration change generates the most fan-out work.", + "id": 90, + "links": [], + "title": "NetworkMap Triggers by source in last hour", + "vizConfig": { + "group": "piechart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#73BF69", + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + } + }, + "overrides": [] + }, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-91": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(resource) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[1h]))", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Network maps computed in the last hour, grouped by the resource whose change triggered the computation.", + "id": 91, + "links": [], + "title": "NetworkMaps by source in last hour", + "vizConfig": { + "group": "piechart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#73BF69", + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + } + }, + "overrides": [] + }, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-92": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(environment) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{environment}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Network maps computed per minute (2m increase halved). One account peers update fans out into one map per peer, so this - not the trigger count - is the real CPU driver.", + "id": 92, + "links": [], + "title": "NetworkMaps Total", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 19, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + }, + "panel-93": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(resource) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{resource}}", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Network maps computed per minute, grouped by the resource that triggered the computation.", + "id": 93, + "links": [], + "title": "NetworkMaps Total By Source", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.9, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.2.0-30616302309" + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-14" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-65" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-15" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-2" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-23" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-24" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-16" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-17" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-84" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-85" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-86" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-87" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-26" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 48 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-25" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 48 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-34" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 56 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-35" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 56 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-31" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 64 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-32" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 64 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-9" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 72 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-8" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 72 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-29" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 80 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-30" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 80 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-66" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 88 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-67" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 88 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-10" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 96 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-7" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 96 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-33" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 104 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-74" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 104 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-11" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 112 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-13" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 112 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-12" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 120 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-1" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 120 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-27" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 128 + } + } + ] + } + }, + "title": "General" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-76" + }, + "height": 8, + "width": 24, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-77" + }, + "height": 8, + "width": 24, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-78" + }, + "height": 8, + "width": 24, + "x": 0, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-79" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-80" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-81" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-82" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-83" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 40 + } + } + ] + } + }, + "title": "NATS" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-46" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-45" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-47" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-48" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-50" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 16 + } + } + ] + } + }, + "title": "Debugging" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-37" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-20" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-39" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-36" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-18" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-19" + }, + "height": 8, + "width": 23, + "x": 0, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-90" + }, + "height": 19, + "width": 6, + "x": 0, + "y": 48 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-88" + }, + "height": 10, + "width": 17, + "x": 6, + "y": 48 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-89" + }, + "height": 9, + "width": 17, + "x": 6, + "y": 58 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-91" + }, + "height": 19, + "width": 6, + "x": 0, + "y": 67 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-92" + }, + "height": 10, + "width": 17, + "x": 6, + "y": 67 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-93" + }, + "height": 9, + "width": 17, + "x": 6, + "y": 77 + } + } + ] + } + }, + "title": "Network Map" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-52" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-53" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-54" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + } + ] + } + }, + "title": "Browser Client" + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [], + "timeSettings": { + "autoRefresh": "auto", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "fiscalYearStartMonth": 0, + "from": "now-1h", + "hideTimepicker": false, + "timezone": "browser", + "to": "now" + }, + "title": "Management generic", + "variables": [ + { + "kind": "DatasourceVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "description": "Prometheus data source scraping the management metrics.", + "hide": "dontHide", + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "pluginId": "prometheus", + "refresh": "onDashboardLoad", + "regex": "", + "skipUrlSync": false + } + }, + { + "kind": "DatasourceVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "description": "Loki data source holding the management logs.", + "hide": "dontHide", + "includeAll": false, + "label": "Logs data source", + "multi": false, + "name": "logs_datasource", + "options": [], + "pluginId": "loki", + "refresh": "onDashboardLoad", + "regex": "", + "skipUrlSync": false + } + }, + { + "kind": "QueryVariable", + "spec": { + "allValue": ".*", + "allowCustomValue": true, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "definition": "label_values(job)", + "description": "Prometheus scrape job(s) exposing the management metrics.", + "hide": "dontHide", + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [], + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allValue": ".*", + "allowCustomValue": true, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "definition": "label_values(up{job=~\"$job\"},instance)", + "description": "Instance(s) of the selected job(s). Depends on the job variable.", + "hide": "dontHide", + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(up{job=~\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "definition": "label_values(job)", + "description": "Loki stream carrying the management logs (adjust the label if your setup does not use `job`).", + "hide": "dontHide", + "includeAll": false, + "label": "log stream", + "multi": false, + "name": "log_job", + "options": [], + "query": { + "datasource": { + "name": "${logs_datasource}" + }, + "group": "loki", + "kind": "DataQuery", + "spec": { + "label": "job", + "refId": "LokiVariableQueryEditor-VariableQuery", + "type": 1 + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + }, + { + "kind": "CustomVariable", + "spec": { + "allValue": "5m", + "allowCustomValue": true, + "current": { + "text": "60s", + "value": "60s" + }, + "hide": "dontHide", + "includeAll": false, + "label": "interval", + "multi": false, + "name": "interval", + "options": [], + "query": "5m,60s", + "skipUrlSync": false, + "valuesFormat": "csv" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(management_store_query_count_total,method)", + "hide": "hideVariable", + "includeAll": true, + "multi": true, + "name": "method", + "options": [], + "query": { + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(management_store_query_count_total,method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "disabled" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "definition": "label_values(aws_rds_cpuutilization_average,dimension_DBClusterIdentifier)", + "hide": "hideVariable", + "includeAll": true, + "multi": true, + "name": "dimension_DBClusterIdentifier", + "options": [], + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(aws_rds_cpuutilization_average,dimension_DBClusterIdentifier)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "disabled" + } + } + ] +} diff --git a/management/internals/controllers/network_map/controller/controller.go b/management/internals/controllers/network_map/controller/controller.go index 5785004db..356dc9f67 100644 --- a/management/internals/controllers/network_map/controller/controller.go +++ b/management/internals/controllers/network_map/controller/controller.go @@ -176,6 +176,7 @@ func (c *Controller) sendUpdateAccountPeers(ctx context.Context, accountID strin semaphore := make(chan struct{}, 10) c.injectAllProxyPolicies(ctx, account) + account.PrecomputePostureValidation(ctx) dnsCache := &cache.DNSConfigCache{} dnsDomain := c.GetDNSDomain(account.Settings) peersCustomZone := account.GetPeersCustomZone(ctx, dnsDomain) @@ -357,6 +358,7 @@ func (c *Controller) sendUpdateForAffectedPeers(ctx context.Context, accountID s // network map that omitted the synth DNS zone, and the agent kept // resolving against the stale or absent record. c.injectAllProxyPolicies(ctx, account) + account.PrecomputePostureValidation(ctx) dnsCache := &cache.DNSConfigCache{} dnsDomain := c.GetDNSDomain(account.Settings) peersCustomZone := account.GetPeersCustomZone(ctx, dnsDomain) diff --git a/management/internals/modules/agentnetwork/handlers/budget_handler_test.go b/management/internals/modules/agentnetwork/handlers/budget_handler_test.go index 4038761c5..3a7709461 100644 --- a/management/internals/modules/agentnetwork/handlers/budget_handler_test.go +++ b/management/internals/modules/agentnetwork/handlers/budget_handler_test.go @@ -102,8 +102,8 @@ func TestSettingsHandler_GetExposesCollectionToggles(t *testing.T) { require.NoError(t, f.store.SaveAgentNetworkSettings(context.Background(), &agentNetworkTypes.Settings{ AccountID: testAccountID, - Cluster: "eu.proxy.netbird.io", - Subdomain: "violet", + Domain: "violet.eu.proxy.netbird.io", + ProxyAddress: "eu.proxy.netbird.io", EnableLogCollection: true, EnablePromptCollection: true, RedactPii: false, diff --git a/management/internals/modules/agentnetwork/handlers/providers_handler.go b/management/internals/modules/agentnetwork/handlers/providers_handler.go index c05363101..0d8a44ca3 100644 --- a/management/internals/modules/agentnetwork/handlers/providers_handler.go +++ b/management/internals/modules/agentnetwork/handlers/providers_handler.go @@ -155,12 +155,7 @@ func (h *handler) createProvider(w http.ResponseWriter, r *http.Request) { provider := types.NewProvider(userAuth.AccountId) provider.FromAPIRequest(&req) - bootstrapCluster := "" - if req.BootstrapCluster != nil { - bootstrapCluster = *req.BootstrapCluster - } - - created, err := h.manager.CreateProvider(r.Context(), userAuth.UserId, provider, bootstrapCluster) + created, err := h.manager.CreateProvider(r.Context(), userAuth.UserId, provider) if err != nil { util.WriteError(r.Context(), err, w) return diff --git a/management/internals/modules/agentnetwork/handlers/settings_handler.go b/management/internals/modules/agentnetwork/handlers/settings_handler.go index 171750838..0a365f9ce 100644 --- a/management/internals/modules/agentnetwork/handlers/settings_handler.go +++ b/management/internals/modules/agentnetwork/handlers/settings_handler.go @@ -12,13 +12,55 @@ import ( "github.com/netbirdio/netbird/shared/management/http/util" ) -// addSettingsEndpoints registers the Agent Network settings routes. The -// settings row is bootstrapped server-side on first provider create or on the -// first PUT carrying a cluster; GET reads it and PUT applies a partial update -// of the mutable collection toggles (cluster/subdomain stay immutable). +// addSettingsEndpoints registers the Agent Network settings routes. POST +// bootstraps the settings row, assigning the account's immutable endpoint; +// GET reads it (defaults with an empty endpoint before bootstrap); PUT +// carries every field, replacing the mutable collection toggles and rejecting +// any change to the identity fields; DELETE removes the row — guarded so it +// stays a bootstrap-repair operation — releasing the endpoint for a fresh +// bootstrap. func (h *handler) addSettingsEndpoints(router *mux.Router) { router.HandleFunc("/agent-network/settings", h.getSettings).Methods("GET", "OPTIONS") + router.HandleFunc("/agent-network/settings", h.createSettings).Methods("POST", "OPTIONS") router.HandleFunc("/agent-network/settings", h.updateSettings).Methods("PUT", "OPTIONS") + router.HandleFunc("/agent-network/settings", h.deleteSettings).Methods("DELETE", "OPTIONS") +} + +// createSettings bootstraps the account's settings row. Exactly one of +// proxy_address (labeled endpoint; the server allocates the label) and +// endpoint (self-addressed, claimed verbatim) must be provided; optional +// collection toggles ride along with defaults for omitted fields. +func (h *handler) createSettings(w http.ResponseWriter, r *http.Request) { + userAuth, err := nbcontext.GetUserAuthFromContext(r.Context()) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + + var req api.AgentNetworkSettingsCreateRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + util.WriteErrorResponse("couldn't parse JSON request", http.StatusBadRequest, w) + return + } + + settings := types.DefaultSettings(userAuth.AccountId) + settings.FromAPICreateRequest(&req) + + proxyAddress := "" + if req.ProxyAddress != nil { + proxyAddress = *req.ProxyAddress + } + endpoint := "" + if req.Endpoint != nil { + endpoint = *req.Endpoint + } + + created, err := h.manager.CreateSettings(r.Context(), userAuth.UserId, settings, proxyAddress, endpoint) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + util.WriteJSONObject(r.Context(), w, created.ToAPIResponse()) } // updateSettings replaces the mutable settings fields on the account's row. @@ -48,6 +90,24 @@ func (h *handler) updateSettings(w http.ResponseWriter, r *http.Request) { util.WriteJSONObject(r.Context(), w, updated.ToAPIResponse()) } +// deleteSettings removes the account's settings row, releasing the endpoint. +// The manager refuses (412) while providers exist or a proxy is actively +// serving the endpoint; a later POST bootstraps fresh, allocating a new +// endpoint. +func (h *handler) deleteSettings(w http.ResponseWriter, r *http.Request) { + userAuth, err := nbcontext.GetUserAuthFromContext(r.Context()) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + + if err := h.manager.DeleteSettings(r.Context(), userAuth.AccountId, userAuth.UserId); err != nil { + util.WriteError(r.Context(), err, w) + return + } + util.WriteJSONObject(r.Context(), w, util.EmptyObject{}) +} + // getSettings returns the account's agent-network settings. Accounts that // haven't been bootstrapped yet read as the defaults with an empty cluster, // subdomain and endpoint; the manager synthesises that view. diff --git a/management/internals/modules/agentnetwork/handlers/settings_handler_test.go b/management/internals/modules/agentnetwork/handlers/settings_handler_test.go index 636ec5b26..400208e1c 100644 --- a/management/internals/modules/agentnetwork/handlers/settings_handler_test.go +++ b/management/internals/modules/agentnetwork/handlers/settings_handler_test.go @@ -1,20 +1,25 @@ package handlers import ( + "context" "encoding/json" + "fmt" "net/http" + "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + rpproxy "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy" "github.com/netbirdio/netbird/shared/management/http/api" ) // TestSettingsHandler_GetUnbootstrappedReturnsDefaults pins the settings-read // convention shared with the account and DNS settings endpoints: settings // always read as a JSON object. Before bootstrap that object carries the -// defaults with an empty cluster/subdomain/endpoint (the "not bootstrapped" +// defaults with an empty endpoint/proxy_address (the "not bootstrapped" // signal) and no timestamps — never a 404 and never the legacy null body. func TestSettingsHandler_GetUnbootstrappedReturnsDefaults(t *testing.T) { f := newAgentNetworkHandlerFixture(t) @@ -27,9 +32,9 @@ func TestSettingsHandler_GetUnbootstrappedReturnsDefaults(t *testing.T) { var got api.AgentNetworkSettings require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) - assert.Empty(t, got.Cluster, "cluster must be empty until bootstrapped") - assert.Empty(t, got.Subdomain, "subdomain must be empty until bootstrapped") - assert.Empty(t, got.Endpoint, "endpoint must be empty until bootstrapped, not a bare dot") + assert.Empty(t, got.Endpoint, "endpoint must be empty until bootstrapped") + assert.Empty(t, got.ProxyAddress, "proxy address must be empty until bootstrapped") + assert.False(t, got.Dedicated, "an unbootstrapped account has no serving shape") assert.True(t, got.EnableLogCollection, "defaults must show log collection on, matching bootstrap") assert.False(t, got.EnablePromptCollection, "defaults must show prompt collection off") assert.False(t, got.RedactPii, "defaults must show redaction off") @@ -39,62 +44,149 @@ func TestSettingsHandler_GetUnbootstrappedReturnsDefaults(t *testing.T) { assert.Nil(t, got.UpdatedAt, "no timestamps before a row exists") } -// TestSettingsHandler_PutBootstrapsWithCluster covers the settings-first -// bootstrap path: a PUT carrying a cluster on an unbootstrapped account -// creates the row (cluster pinned, subdomain assigned) and applies the -// mutable fields from the same request. -func TestSettingsHandler_PutBootstrapsWithCluster(t *testing.T) { +// TestSettingsHandler_PostBootstrapsLabeled covers the labeled bootstrap +// shape: a POST carrying a proxy_address allocates a label beneath it, so the +// endpoint hangs one label under the shared cluster's address and the pin is +// not dedicated. Toggles riding along apply; omitted ones keep defaults. +func TestSettingsHandler_PostBootstrapsLabeled(t *testing.T) { f := newAgentNetworkHandlerFixture(t) - rec := f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "eu.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": true, "redact_pii": false, "access_log_retention_days": 30}`) - require.Equal(t, http.StatusOK, rec.Code, "bootstrap PUT must succeed: %s", rec.Body.String()) + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "enable_prompt_collection": true, "access_log_retention_days": 14}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) var got api.AgentNetworkSettings require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) - assert.Equal(t, "eu.proxy.netbird.io", got.Cluster, "cluster must be pinned from the request") - assert.NotEmpty(t, got.Subdomain, "subdomain must be assigned at bootstrap") - assert.Equal(t, got.Subdomain+".eu.proxy.netbird.io", got.Endpoint, "endpoint must combine subdomain and cluster") - assert.True(t, got.EnableLogCollection, "toggle from the bootstrap request must apply") + assert.Equal(t, "eu.proxy.netbird.io", got.ProxyAddress, "proxy address must be pinned from the request") + require.NotEmpty(t, got.Endpoint, "endpoint must be allocated at bootstrap") + assert.True(t, strings.HasSuffix(got.Endpoint, ".eu.proxy.netbird.io"), + "labeled endpoint must hang off the proxy address: %s", got.Endpoint) + label := strings.TrimSuffix(got.Endpoint, ".eu.proxy.netbird.io") + assert.NotContains(t, label, ".", "the allocated label must be a single DNS label: %s", label) + assert.False(t, got.Dedicated, "a labeled pin is not dedicated") + assert.True(t, got.EnableLogCollection, "omitted toggle must keep its default") assert.True(t, got.EnablePromptCollection, "toggle from the bootstrap request must apply") require.NotNil(t, got.AccessLogRetentionDays) - assert.Equal(t, 30, *got.AccessLogRetentionDays, "retention from the bootstrap request must apply") + assert.Equal(t, 14, *got.AccessLogRetentionDays, "retention from the bootstrap request must apply") + assert.NotNil(t, got.CreatedAt, "a persisted row carries timestamps") // The row is now readable via GET. rec = f.do(t, http.MethodGet, "/agent-network/settings", "") require.Equal(t, http.StatusOK, rec.Code, "GET after bootstrap must succeed") + var read api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &read)) + assert.Equal(t, got.Endpoint, read.Endpoint, "GET must return the bootstrapped endpoint") } -// TestSettingsHandler_PutWithoutClusterOnUnbootstrapped pins that a PUT -// without a cluster cannot conjure a settings row out of nothing — there is -// no cluster to pin — and surfaces as 404 like the GET. -func TestSettingsHandler_PutWithoutClusterOnUnbootstrapped(t *testing.T) { +// TestSettingsHandler_PostBootstrapsSelfAddressed covers the dedicated shape: +// a POST carrying an endpoint claims the hostname verbatim, the proxy address +// equals it, and the pin reads as dedicated. The claim is legitimate before +// any proxy declares the address (address-first). +func TestSettingsHandler_PostBootstrapsSelfAddressed(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"endpoint": "Brave-Otter.Gateway.Example.com"}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + + var got api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, "brave-otter.gateway.example.com", got.Endpoint, + "endpoint must be claimed verbatim, lowercased") + assert.Equal(t, got.Endpoint, got.ProxyAddress, "self-addressed: the proxy address is the endpoint") + assert.True(t, got.Dedicated, "a self-addressed pin is dedicated") + assert.True(t, got.EnableLogCollection, "omitted toggles must keep their defaults") +} + +// TestSettingsHandler_PostRequiresExactlyOneIdentityField pins the request +// contract: proxy_address and endpoint are mutually exclusive and one is +// required — both or neither is a validation error, not a guess. +func TestSettingsHandler_PostRequiresExactlyOneIdentityField(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", `{}`) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, + "empty POST must be rejected: got %d body=%s", rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "endpoint": "brave-otter.gateway.example.com"}`) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, + "POST with both identity fields must be rejected: got %d body=%s", rec.Code, rec.Body.String()) +} + +// TestSettingsHandler_PostRejectsMalformedHostnames pins per-write input +// validation: shapes canonicalization cannot repair — trailing dots, embedded +// whitespace, empty labels — are rejected with a validation error instead of +// landing in an immutable column. +func TestSettingsHandler_PostRejectsMalformedHostnames(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + for name, body := range map[string]string{ + "trailing dot": `{"endpoint": "gateway.example.com."}`, + "leading dot": `{"endpoint": ".gateway.example.com"}`, + "inner whitespace": `{"endpoint": "gate way.example.com"}`, + "empty label": `{"proxy_address": "eu..proxy.netbird.io"}`, + } { + rec := f.do(t, http.MethodPost, "/agent-network/settings", body) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, + "%s must be rejected: got %d body=%s", name, rec.Code, rec.Body.String()) + } +} + +// TestSettingsHandler_PostConflictsOnSecondBootstrap pins that bootstrap is a +// one-time create: a second POST returns 409 and leaves the row untouched. +func TestSettingsHandler_PostConflictsOnSecondBootstrap(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", `{"proxy_address": "eu.proxy.netbird.io"}`) + require.Equal(t, http.StatusOK, rec.Code, "first bootstrap must succeed: %s", rec.Body.String()) + var first api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &first)) + + rec = f.do(t, http.MethodPost, "/agent-network/settings", `{"proxy_address": "us.proxy.netbird.io"}`) + assert.Equal(t, http.StatusConflict, rec.Code, + "second bootstrap must 409: got %d body=%s", rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodGet, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code) + var got api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, first.Endpoint, got.Endpoint, "the original endpoint must survive the rejected bootstrap") + assert.Equal(t, first.ProxyAddress, got.ProxyAddress, "the original proxy address must survive") +} + +// TestSettingsHandler_PutBeforeBootstrapIs404 pins that a PUT cannot conjure a +// settings row out of nothing — bootstrap is the explicit POST — and the +// error points the caller there. +func TestSettingsHandler_PutBeforeBootstrapIs404(t *testing.T) { f := newAgentNetworkHandlerFixture(t) rec := f.do(t, http.MethodPut, "/agent-network/settings", `{"enable_log_collection": false, "enable_prompt_collection": false, "redact_pii": false}`) assert.Equal(t, http.StatusNotFound, rec.Code, - "cluster-less PUT on an unbootstrapped account must 404: got %d body=%s", rec.Code, rec.Body.String()) - assert.Contains(t, rec.Body.String(), "cluster", - "the error must point the caller at the bootstrap paths: %s", rec.Body.String()) + "PUT on an unbootstrapped account must 404: got %d body=%s", rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), "/api/agent-network/settings", + "the error must point the caller at the bootstrap POST: %s", rec.Body.String()) } // TestSettingsHandler_PutReplacesMutableFields pins the update contract shared -// with the other PUT endpoints: the request replaces every mutable field, so a -// toggle absent from the JSON lands as its zero value rather than being -// preserved. Cluster and subdomain survive untouched. +// with the other PUT endpoints: the request carries every field, replacing the +// mutable ones. The identity fields ride along as a required echo of the +// assigned values — compared, never written — so the endpoint and proxy +// address survive every accepted update. func TestSettingsHandler_PutReplacesMutableFields(t *testing.T) { f := newAgentNetworkHandlerFixture(t) - rec := f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "eu.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": true, "redact_pii": true, "access_log_retention_days": 14}`) - require.Equal(t, http.StatusOK, rec.Code, "bootstrap PUT must succeed: %s", rec.Body.String()) + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "enable_prompt_collection": true, "redact_pii": true, "access_log_retention_days": 14}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) var before api.AgentNetworkSettings require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &before)) - rec = f.do(t, http.MethodPut, "/agent-network/settings", - `{"enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false}`) + rec = f.do(t, http.MethodPut, "/agent-network/settings", fmt.Sprintf( + `{"endpoint": %q, "proxy_address": %q, "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false, "access_log_retention_days": 7}`, + before.Endpoint, before.ProxyAddress)) require.Equal(t, http.StatusOK, rec.Code, "update PUT must succeed: %s", rec.Body.String()) var got api.AgentNetworkSettings @@ -103,35 +195,201 @@ func TestSettingsHandler_PutReplacesMutableFields(t *testing.T) { assert.False(t, got.EnablePromptCollection, "sent toggle must apply") assert.False(t, got.RedactPii, "sent toggle must apply") require.NotNil(t, got.AccessLogRetentionDays) - assert.Equal(t, 0, *got.AccessLogRetentionDays, - "retention absent from the request must land as the zero value — PUT replaces all mutable fields") - assert.Equal(t, before.Cluster, got.Cluster, "cluster must survive updates untouched") - assert.Equal(t, before.Subdomain, got.Subdomain, "subdomain must survive updates untouched") + assert.Equal(t, 7, *got.AccessLogRetentionDays, "sent retention must apply") + assert.Equal(t, before.Endpoint, got.Endpoint, "endpoint must survive updates untouched") + assert.Equal(t, before.ProxyAddress, got.ProxyAddress, "proxy address must survive updates untouched") } -// TestSettingsHandler_PutRejectsClusterChange pins cluster immutability: once -// assigned, a differing cluster is rejected as a validation error instead of -// being silently ignored, so callers never observe a value other than the one -// they sent. Echoing the assigned cluster back stays valid, which lets -// declarative clients send their full desired state idempotently. -func TestSettingsHandler_PutRejectsClusterChange(t *testing.T) { +// TestSettingsHandler_PutRejectsChangedIdentity pins the immutability contract: +// the PUT carries the identity fields like every other field, but they are an +// echo — a request carrying a different endpoint or proxy address is rejected +// as a validation error and the row is left untouched. The comparison is +// lenient about casing (the stored values are normalized lowercase), so a +// client replaying a GET response with different casing is not rejected. +func TestSettingsHandler_PutRejectsChangedIdentity(t *testing.T) { f := newAgentNetworkHandlerFixture(t) - rec := f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "eu.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false}`) - require.Equal(t, http.StatusOK, rec.Code, "bootstrap PUT must succeed: %s", rec.Body.String()) + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "enable_prompt_collection": true}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + var before api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &before)) - rec = f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "us.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false}`) - assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, - "cluster change must be rejected as a validation error: got %d body=%s", rec.Code, rec.Body.String()) + for name, body := range map[string]string{ + "changed endpoint": fmt.Sprintf( + `{"endpoint": "other.gateway.example.com", "proxy_address": %q, "enable_log_collection": false, "enable_prompt_collection": false, "redact_pii": false, "access_log_retention_days": 30}`, + before.ProxyAddress), + "changed proxy_address": fmt.Sprintf( + `{"endpoint": %q, "proxy_address": "us.proxy.netbird.io", "enable_log_collection": false, "enable_prompt_collection": false, "redact_pii": false, "access_log_retention_days": 30}`, + before.Endpoint), + "omitted identity": `{"enable_log_collection": false, "enable_prompt_collection": false, "redact_pii": false, "access_log_retention_days": 30}`, + } { + rec = f.do(t, http.MethodPut, "/agent-network/settings", body) + assert.Equal(t, http.StatusUnprocessableEntity, rec.Code, + "%s must be rejected: got %d body=%s", name, rec.Code, rec.Body.String()) + } - rec = f.do(t, http.MethodPut, "/agent-network/settings", - `{"cluster": "eu.proxy.netbird.io", "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": true}`) - require.Equal(t, http.StatusOK, rec.Code, "echoing the assigned cluster must stay valid: %s", rec.Body.String()) + // The rejected updates must not have applied anything — toggles included. + rec = f.do(t, http.MethodGet, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code) + var got api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, before.Endpoint, got.Endpoint, "rejected PUT must not change the endpoint") + assert.Equal(t, before.ProxyAddress, got.ProxyAddress, "rejected PUT must not change the proxy address") + assert.True(t, got.EnablePromptCollection, "rejected PUT must not apply its toggles") + + // An uppercased echo of the assigned values still names the same host and + // must be accepted. + rec = f.do(t, http.MethodPut, "/agent-network/settings", fmt.Sprintf( + `{"endpoint": %q, "proxy_address": %q, "enable_log_collection": true, "enable_prompt_collection": true, "redact_pii": false, "access_log_retention_days": 30}`, + strings.ToUpper(before.Endpoint), strings.ToUpper(before.ProxyAddress))) + assert.Equal(t, http.StatusOK, rec.Code, + "an uppercased identity echo must be accepted: got %d body=%s", rec.Code, rec.Body.String()) +} + +// TestSettingsHandler_PutOmittedRetentionLandsAsZero documents a residual the +// required-ness of access_log_retention_days does not remove. Marking the field +// required changes the generated client type from *int to int, so a generated +// client cannot omit it — but nothing validates OpenAPI required-ness at +// runtime, so a hand-rolled body without the field still decodes as 0, which +// the API documents as "keep indefinitely". +// +// That is the same latitude the three booleans already have, so it is left +// consistent rather than special-cased. This test exists to make the gap +// explicit: if request validation is ever added, this expectation is what +// changes. +func TestSettingsHandler_PutOmittedRetentionLandsAsZero(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "access_log_retention_days": 14}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + var before api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &before)) + + rec = f.do(t, http.MethodPut, "/agent-network/settings", fmt.Sprintf( + `{"endpoint": %q, "proxy_address": %q, "enable_log_collection": true, "enable_prompt_collection": false, "redact_pii": false}`, + before.Endpoint, before.ProxyAddress)) + require.Equal(t, http.StatusOK, rec.Code, "update PUT must succeed: %s", rec.Body.String()) var got api.AgentNetworkSettings require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) - assert.Equal(t, "eu.proxy.netbird.io", got.Cluster, "cluster must be unchanged") - assert.True(t, got.RedactPii, "toggle sent alongside the echoed cluster must apply") + require.NotNil(t, got.AccessLogRetentionDays) + assert.Equal(t, 0, *got.AccessLogRetentionDays, + "a non-conforming body that omits retention still replaces it with the zero value") +} + +// TestSettingsHandler_DeleteBeforeBootstrapIs404 pins that DELETE on an +// account with no settings row is a 404, mirroring the PUT. +func TestSettingsHandler_DeleteBeforeBootstrapIs404(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodDelete, "/agent-network/settings", "") + assert.Equal(t, http.StatusNotFound, rec.Code, + "DELETE on an unbootstrapped account must 404: got %d body=%s", rec.Code, rec.Body.String()) +} + +// TestSettingsHandler_DeleteBlockedByProviders pins the first delete guard: +// while any provider exists for the account, the delete is refused with 412 +// and the row survives. Providers route through the endpoint — the guard +// keeps DELETE a bootstrap-repair operation rather than a way to abandon a +// configured gateway. +func TestSettingsHandler_DeleteBlockedByProviders(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", `{"proxy_address": "eu.proxy.netbird.io"}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + var before api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &before)) + + f.seedProvider(t, "prov-guard") + + rec = f.do(t, http.MethodDelete, "/agent-network/settings", "") + assert.Equal(t, http.StatusPreconditionFailed, rec.Code, + "delete with a provider present must be refused: got %d body=%s", rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodGet, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code) + var got api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, before.Endpoint, got.Endpoint, "the refused delete must leave the row intact") +} + +// TestSettingsHandler_DeleteBlockedByActiveProxy pins the second delete +// guard: while a proxy is actively serving the endpoint — an active proxy +// row declaring the endpoint hostname as its cluster address, the dedicated +// shape — the delete is refused with 412. A proxy that has disconnected no +// longer blocks: the guard is about a live serving path, not history. +// +// The proxy declares its address with mixed casing on purpose: Connect +// stores the declared address verbatim while the settings row is normalized +// lowercase, and hostnames are case-insensitive, so the guard must match +// across the casing difference rather than be sidestepped by it. +func TestSettingsHandler_DeleteBlockedByActiveProxy(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + const endpoint = "gw.dedicated.example.com" + rec := f.do(t, http.MethodPost, "/agent-network/settings", fmt.Sprintf(`{"endpoint": %q}`, endpoint)) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + + now := time.Now() + accountID := testAccountID + proxyRow := &rpproxy.Proxy{ + ID: "proxy-guard", + SessionID: "sess-1", + ClusterAddress: "GW.Dedicated.Example.Com", + AccountID: &accountID, + LastSeen: now, + ConnectedAt: &now, + Status: rpproxy.StatusConnected, + } + require.NoError(t, f.store.SaveProxy(context.Background(), proxyRow)) + + rec = f.do(t, http.MethodDelete, "/agent-network/settings", "") + assert.Equal(t, http.StatusPreconditionFailed, rec.Code, + "delete with an active proxy at the endpoint must be refused: got %d body=%s", rec.Code, rec.Body.String()) + + // Once the proxy disconnects it no longer serves the endpoint, so the + // delete goes through. + require.NoError(t, f.store.DisconnectProxy(context.Background(), proxyRow.ID, proxyRow.SessionID)) + rec = f.do(t, http.MethodDelete, "/agent-network/settings", "") + assert.Equal(t, http.StatusOK, rec.Code, + "delete after the proxy disconnected must succeed: got %d body=%s", rec.Code, rec.Body.String()) +} + +// TestSettingsHandler_DeleteReleasesEndpointForFreshBootstrap pins the +// full-reset semantic that gives replace-on-change clients (e.g. Terraform's +// RequiresReplace) a real path: with both guards clear the delete succeeds, +// the account reads as the defaults again, and a fresh bootstrap draws a +// fresh label. The released hostname is not reserved — a fresh draw may even +// legitimately re-pick it — so the assertions check the new row's shape, not +// that the label differs. +func TestSettingsHandler_DeleteReleasesEndpointForFreshBootstrap(t *testing.T) { + f := newAgentNetworkHandlerFixture(t) + + rec := f.do(t, http.MethodPost, "/agent-network/settings", + `{"proxy_address": "eu.proxy.netbird.io", "enable_prompt_collection": true}`) + require.Equal(t, http.StatusOK, rec.Code, "bootstrap POST must succeed: %s", rec.Body.String()) + + rec = f.do(t, http.MethodDelete, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code, + "delete with both guards clear must succeed: got %d body=%s", rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodGet, "/agent-network/settings", "") + require.Equal(t, http.StatusOK, rec.Code) + var after api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &after)) + assert.Empty(t, after.Endpoint, "a deleted account must read as unbootstrapped defaults") + assert.False(t, after.EnablePromptCollection, "the deleted row's toggles must not linger") + + rec = f.do(t, http.MethodPost, "/agent-network/settings", `{"proxy_address": "eu.proxy.netbird.io"}`) + require.Equal(t, http.StatusOK, rec.Code, "re-bootstrap after delete must succeed: %s", rec.Body.String()) + var second api.AgentNetworkSettings + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &second)) + require.NotEmpty(t, second.Endpoint, "the fresh bootstrap must allocate an endpoint") + assert.True(t, strings.HasSuffix(second.Endpoint, ".eu.proxy.netbird.io"), + "the fresh endpoint must hang beneath the requested proxy address: %s", second.Endpoint) + assert.False(t, second.EnablePromptCollection, + "the fresh row must carry bootstrap defaults, not the deleted row's toggles") + assert.NotNil(t, second.CreatedAt, "the fresh row is persisted and carries timestamps") } diff --git a/management/internals/modules/agentnetwork/labelgen/adjectives.go b/management/internals/modules/agentnetwork/labelgen/adjectives.go new file mode 100644 index 000000000..5058c2f1c --- /dev/null +++ b/management/internals/modules/agentnetwork/labelgen/adjectives.go @@ -0,0 +1,37 @@ +package labelgen + +// adjectives is the descriptor half of a generated label. It pairs with the +// noun pool in words.go to form `-` labels, and is kept +// separate because words.go is almost entirely nouns — drawing both halves +// from it produced unreadable pairs like "millet-hammock". Entries are +// lowercase ASCII, 4-12 chars, free of hyphens and digits, screened for +// offensive/brand/region-specific terms, and disjoint from the noun pool +// (enforced by TestAdjectives_AreDisjointFromNouns). +var adjectives = []string{ + "able", "active", "adept", "agile", "airy", "alert", "amiable", "ample", + "ancient", "ardent", "artful", "astute", "balmy", "blithe", "bold", "bonny", + "brave", "breezy", "brisk", "bubbly", "buoyant", "bushy", "candid", "canny", + "cheery", "chilly", "chipper", "chunky", "civil", "classic", "clever", "comely", + "compact", "cordial", "cosmic", "courtly", "crafty", "creamy", "crisp", "cuddly", + "curious", "dainty", "dapper", "daring", "dashing", "deft", "dewy", "diligent", + "downy", "dreamy", "dulcet", "durable", "dusky", "eager", "earnest", "earthy", + "easy", "elated", "elegant", "epic", "fabled", "faithful", "fancy", "fearless", + "feisty", "fervent", "fleet", "fluffy", "fond", "frisky", "frosty", "gallant", + "genial", "genteel", "gentle", "giddy", "gilded", "glad", "glassy", "gleaming", + "glossy", "graceful", "grand", "grainy", "hale", "hardy", "hearty", "hefty", + "honest", "hopeful", "humble", "hushed", "immense", "jaunty", "jolly", "jovial", + "joyful", "jubilant", "keen", "kindly", "kindred", "lanky", "leafy", "limber", + "lively", "lofty", "loyal", "lucent", "lucid", "luminous", "lush", "maroon", + "mellow", "merry", "mighty", "mindful", "mirthful", "misty", "modest", "muted", + "nifty", "nimble", "noble", "patient", "peaceful", "pearly", "peppy", "perky", + "petite", "placid", "playful", "pleasant", "plucky", "plush", "polite", "posh", + "prancing", "pristine", "prompt", "proud", "prudent", "quaint", "quick", "quirky", + "radiant", "ready", "regal", "restful", "robust", "rosy", "ruddy", "rugged", + "sandy", "satin", "saucy", "savvy", "sedate", "serene", "shady", "shiny", + "silken", "silky", "sincere", "sleek", "slender", "smart", "smooth", "snappy", + "snug", "soaring", "sparkly", "spiffy", "spirited", "sprightly", "spry", "stalwart", + "stately", "steady", "sterling", "stoic", "stormy", "stout", "sturdy", "sunlit", + "supple", "svelte", "tawny", "tender", "tidy", "timeless", "trusty", "upbeat", + "urbane", "valiant", "vast", "vernal", "vibrant", "vintage", "whimsy", "willing", + "windy", "winsome", "wintry", "witty", "worthy", "zesty", "zippy", +} diff --git a/management/internals/modules/agentnetwork/labelgen/labelgen.go b/management/internals/modules/agentnetwork/labelgen/labelgen.go index b45ff4ea8..549767096 100644 --- a/management/internals/modules/agentnetwork/labelgen/labelgen.go +++ b/management/internals/modules/agentnetwork/labelgen/labelgen.go @@ -64,3 +64,20 @@ func PickUnique(rng *rand.Rand, taken map[string]struct{}, fallbackSuffix string w := pool[rng.Intn(len(pool))] return fmt.Sprintf("%s-%s", w, fallbackSuffix) } + +// PickTuple returns an adjective-noun label such as "brave-otter". It is still +// a single DNS label. +// +// Unlike PickUnique it takes no `taken` set and has no fallback suffix. The +// noun pool holds 857 entries, which is ample per cluster but a hard ceiling +// once labels must be unique across one shared zone; pairing an adjective with +// a noun spans len(adjectives) * 857 instead. Uniqueness is enforced by a +// database constraint and retried by the caller, rather than guessed from a +// pre-read set that a concurrent allocation can invalidate. +func PickTuple(rng *rand.Rand) string { + nouns := uniqueWords() + if len(nouns) == 0 || len(adjectives) == 0 { + return "" + } + return adjectives[rng.Intn(len(adjectives))] + "-" + nouns[rng.Intn(len(nouns))] +} diff --git a/management/internals/modules/agentnetwork/labelgen/labelgen_test.go b/management/internals/modules/agentnetwork/labelgen/labelgen_test.go index f03a3501d..7e12fc133 100644 --- a/management/internals/modules/agentnetwork/labelgen/labelgen_test.go +++ b/management/internals/modules/agentnetwork/labelgen/labelgen_test.go @@ -99,3 +99,82 @@ func TestUniqueWords_DropsDuplicates(t *testing.T) { } assert.GreaterOrEqual(t, len(pool), 500, "Pool must contain at least 500 unique words") } + +// TestPickTuple_ShapeAndPoolMembership locks the wire-visible shape: an +// adjective and a noun, each from its own pool, joined by a single hyphen so +// the result stays one DNS label. +func TestPickTuple_ShapeAndPoolMembership(t *testing.T) { + nouns := uniqueWords() + inNouns := make(map[string]struct{}, len(nouns)) + for _, w := range nouns { + inNouns[w] = struct{}{} + } + inAdjectives := make(map[string]struct{}, len(adjectives)) + for _, a := range adjectives { + inAdjectives[a] = struct{}{} + } + + rng := rand.New(rand.NewSource(7)) + for i := 0; i < 200; i++ { + got := PickTuple(rng) + + parts := strings.Split(got, "-") + require.Len(t, parts, 2, "PickTuple must produce exactly two hyphen-joined words; got %q", got) + + _, adjOK := inAdjectives[parts[0]] + assert.True(t, adjOK, "First half must be an adjective; %q not in adjectives (from %q)", parts[0], got) + _, nounOK := inNouns[parts[1]] + assert.True(t, nounOK, "Second half must be a noun; %q not in words (from %q)", parts[1], got) + + assert.LessOrEqual(t, len(got), 63, "Label must fit a DNS label; got %q (%d chars)", got, len(got)) + } +} + +// TestAdjectives_AreDisjointFromNouns keeps the namespace a clean product and +// prevents nonsense like "azure-azure": a handful of the noun pool's entries +// are adjectival, and any overlap would let the same word land on both sides. +func TestAdjectives_AreDisjointFromNouns(t *testing.T) { + nouns := make(map[string]struct{}, len(uniqueWords())) + for _, w := range uniqueWords() { + nouns[w] = struct{}{} + } + for _, a := range adjectives { + _, clash := nouns[a] + assert.False(t, clash, "Adjective %q also appears in the noun pool; remove it from one list", a) + } +} + +// TestAdjectives_AreDNSSafeAndDeduplicated mirrors the curation contract stated +// in words.go: lowercase ASCII, 4-12 chars, no digits or hyphens, no repeats. +func TestAdjectives_AreDNSSafeAndDeduplicated(t *testing.T) { + seen := make(map[string]struct{}, len(adjectives)) + for _, a := range adjectives { + _, dup := seen[a] + assert.False(t, dup, "Duplicate adjective %q", a) + seen[a] = struct{}{} + + assert.Regexp(t, `^[a-z]{4,12}$`, a, "Adjective %q must be 4-12 lowercase ASCII letters", a) + } + assert.Greater(t, len(adjectives), 150, "Adjective pool too small to give a useful namespace") +} + +// TestPickTuple_DeterministicWithSeededRng documents that generation is a pure +// function of the rng, which is what makes allocation retries reproducible in tests. +func TestPickTuple_DeterministicWithSeededRng(t *testing.T) { + a := PickTuple(rand.New(rand.NewSource(42))) + b := PickTuple(rand.New(rand.NewSource(42))) + assert.Equal(t, a, b, "Same seed must yield the same tuple") +} + +// TestPickTuple_SpansALargeNamespace guards the reason we moved to tuples: a +// single-word pool caps the GLOBAL namespace at 857. Drawing many tuples must +// yield overwhelmingly distinct values. +func TestPickTuple_SpansALargeNamespace(t *testing.T) { + rng := rand.New(rand.NewSource(11)) + seen := make(map[string]struct{}, 2000) + for i := 0; i < 2000; i++ { + seen[PickTuple(rng)] = struct{}{} + } + assert.Greater(t, len(seen), 1900, + "2000 draws should be nearly all distinct across a ~200k namespace; got %d unique", len(seen)) +} diff --git a/management/internals/modules/agentnetwork/manager.go b/management/internals/modules/agentnetwork/manager.go index ba2c06826..379672989 100644 --- a/management/internals/modules/agentnetwork/manager.go +++ b/management/internals/modules/agentnetwork/manager.go @@ -22,7 +22,6 @@ import ( "github.com/netbirdio/netbird/management/server/permissions/modules" "github.com/netbirdio/netbird/management/server/permissions/operations" "github.com/netbirdio/netbird/management/server/store" - "github.com/netbirdio/netbird/shared/management/proto" "github.com/netbirdio/netbird/shared/management/status" ) @@ -48,7 +47,7 @@ func ensureSessionKeys(p *types.Provider) error { type Manager interface { GetAllProviders(ctx context.Context, accountID, userID string) ([]*types.Provider, error) GetProvider(ctx context.Context, accountID, userID, providerID string) (*types.Provider, error) - CreateProvider(ctx context.Context, userID string, provider *types.Provider, bootstrapCluster string) (*types.Provider, error) + CreateProvider(ctx context.Context, userID string, provider *types.Provider) (*types.Provider, error) UpdateProvider(ctx context.Context, userID string, provider *types.Provider) (*types.Provider, error) DeleteProvider(ctx context.Context, accountID, userID, providerID string) error @@ -71,7 +70,9 @@ type Manager interface { DeleteBudgetRule(ctx context.Context, accountID, userID, ruleID string) error GetSettings(ctx context.Context, accountID, userID string) (*types.Settings, error) + CreateSettings(ctx context.Context, userID string, settings *types.Settings, proxyAddress, endpoint string) (*types.Settings, error) UpdateSettings(ctx context.Context, userID string, settings *types.Settings) (*types.Settings, error) + DeleteSettings(ctx context.Context, accountID, userID string) error ListConsumption(ctx context.Context, accountID, userID string) ([]*types.Consumption, error) ListAccessLogs(ctx context.Context, accountID, userID string, filter types.AgentNetworkAccessLogFilter) ([]*types.AgentNetworkAccessLog, int64, error) @@ -123,11 +124,10 @@ type managerImpl struct { proxyController proxy.Controller // reconcileCache holds the last set of synthesised proxy mappings - // per account so reconcile can emit precise Create/Update/Delete - // updates instead of a full re-push on every mutation. Keyed by - // accountID, then by synthesised service ID. + // per account, each paired with the proxy that served it, so a change + // of serving proxy can be diffed without re-deriving it. reconcileMu sync.Mutex - reconcileCache map[string]map[string]*proto.ProxyMapping + reconcileCache map[string]map[string]syntheticMapping // labelRngMu guards labelRng. PickUnique consumes math/rand.Source // state; concurrent provider creates would otherwise race. @@ -151,7 +151,7 @@ func NewManager( accountManager: accountManager, permissionsManager: permissionsManager, proxyController: proxyController, - reconcileCache: make(map[string]map[string]*proto.ProxyMapping), + reconcileCache: make(map[string]map[string]syntheticMapping), labelRng: rand.New(rand.NewSource(time.Now().UnixNano())), } } @@ -170,19 +170,14 @@ func (m *managerImpl) GetProvider(ctx context.Context, accountID, userID, provid return m.store.GetAgentNetworkProviderByID(ctx, store.LockingStrengthNone, accountID, providerID) } -// CreateProvider persists a new provider for the account. bootstrapCluster -// is used only when the per-account agent-network Settings row hasn't -// been created yet; otherwise it is ignored (the cluster is pinned on -// Settings and every provider in the account routes through it). -func (m *managerImpl) CreateProvider(ctx context.Context, userID string, provider *types.Provider, bootstrapCluster string) (*types.Provider, error) { +// CreateProvider persists a new provider for the account. Providers have no +// settings side effects: the account's endpoint is bootstrapped separately and +// explicitly via CreateSettings, and every provider in the account routes +// through it. +func (m *managerImpl) CreateProvider(ctx context.Context, userID string, provider *types.Provider) (*types.Provider, error) { if err := m.requirePermission(ctx, provider.AccountID, userID, modules.AgentNetworkProviders, operations.Create); err != nil { return nil, err } - if strings.TrimSpace(bootstrapCluster) != "" { - if err := m.requireSettingsBootstrapPermission(ctx, provider.AccountID, userID); err != nil { - return nil, err - } - } // An empty api_key would silently produce a synthesised service // that 401s on every upstream request. Surface the misconfiguration @@ -206,16 +201,6 @@ func (m *managerImpl) CreateProvider(ctx context.Context, userID string, provide return nil, fmt.Errorf("save agent network provider: %w", err) } - if strings.TrimSpace(bootstrapCluster) != "" { - if _, err := m.bootstrapSettingsIfNeeded(ctx, m.store, provider.AccountID, bootstrapCluster); err != nil { - // The provider create has already succeeded; logging the - // bootstrap miss matches the plan's PoC behaviour. The synth - // path treats a missing settings row as a no-op, and the next - // provider create retries the bootstrap. - log.WithContext(ctx).Debugf("agent-network bootstrap settings for account %s on cluster %s: %v", provider.AccountID, bootstrapCluster, err) - } - } - m.accountManager.StoreEvent(ctx, userID, provider.ID, provider.AccountID, activity.AgentNetworkProviderCreated, provider.EventMeta()) m.reconcile(ctx, provider.AccountID) @@ -560,52 +545,44 @@ func (m *managerImpl) DeleteBudgetRule(ctx context.Context, accountID, userID, r } // UpdateSettings replaces the mutable account-level settings — the collection -// toggles and retention — on the account's row. When the account has no -// settings row yet, a non-empty settings.Cluster bootstraps one (same path as -// first provider create); without it the update fails with NotFound. On an -// existing row the cluster and subdomain are immutable: a differing -// settings.Cluster is rejected rather than silently ignored so callers never -// observe a value other than what they sent. Because the collection toggles -// change the synthesised service config (prompt-capture gating, access-log -// emission), a reconcile is triggered so the proxy and peer network maps -// converge on the new state. +// toggles and retention — on the account's row. The identity fields (Domain, +// ProxyAddress) are assigned at bootstrap (CreateSettings) and immutable: the +// request carries them, matching the PUT convention of every other endpoint, +// but they are only compared against the stored row — a request carrying +// different values is rejected, and the stored values are never overwritten. +// When the account has no settings row yet the update fails with NotFound. +// Because the collection toggles change the synthesised service config +// (prompt-capture gating, access-log emission), a reconcile is triggered so +// the proxy and peer network maps converge on the new state. func (m *managerImpl) UpdateSettings(ctx context.Context, userID string, settings *types.Settings) (*types.Settings, error) { if err := m.requirePermission(ctx, settings.AccountID, userID, modules.AgentNetworkSettings, operations.Update); err != nil { return nil, err } - requestedCluster := strings.TrimSpace(settings.Cluster) - // The row lock from LockingStrengthUpdate only holds for the duration of - // the surrounding transaction, so the read, the cluster-immutability - // check, and the save must share one — otherwise concurrent PUTs could - // interleave between them. + // the surrounding transaction, so the read and the save must share one — + // otherwise concurrent PUTs could interleave between them. var updated *types.Settings err := m.store.ExecuteInTransaction(ctx, func(tx store.Store) error { existing, err := tx.GetAgentNetworkSettings(ctx, store.LockingStrengthUpdate, settings.AccountID) switch { case err == nil: - if requestedCluster != "" && requestedCluster != existing.Cluster { - return status.Errorf(status.InvalidArgument, "cluster is immutable once assigned (current: %s)", existing.Cluster) - } case isNotFound(err): - if requestedCluster == "" { - return status.Errorf(status.NotFound, "agent network settings have not been bootstrapped yet; pass cluster to bootstrap them, or create a provider with bootstrap_cluster set") - } - // Bootstrapping pins the cluster and subdomain — a settings - // create on top of the update the caller already passed, matching - // the gate on the provider-create bootstrap path. - if err := m.requirePermission(ctx, settings.AccountID, userID, modules.AgentNetworkSettings, operations.Create); err != nil { - return err - } - existing, err = m.bootstrapSettingsIfNeeded(ctx, tx, settings.AccountID, requestedCluster) - if err != nil { - return err - } + return status.Errorf(status.NotFound, "agent network settings have not been bootstrapped yet; POST /api/agent-network/settings to bootstrap them") default: return fmt.Errorf("get agent network settings: %w", err) } + // The identity echo is compared leniently (trimmed, case-insensitive): + // the stored values are normalized lowercase, and a client replaying a + // GET response must never be rejected over casing it didn't choose. + if !hostnamesEquivalent(settings.Domain, existing.Domain) { + return status.Errorf(status.InvalidArgument, "endpoint is immutable: it must match the assigned endpoint %q; delete the settings to release it and bootstrap again", existing.Domain) + } + if !hostnamesEquivalent(settings.ProxyAddress, existing.ProxyAddress) { + return status.Errorf(status.InvalidArgument, "proxy_address is immutable: it must match the assigned proxy address %q; delete the settings to release it and bootstrap again", existing.ProxyAddress) + } + existing.EnableLogCollection = settings.EnableLogCollection existing.EnablePromptCollection = settings.EnablePromptCollection existing.RedactPii = settings.RedactPii @@ -632,6 +609,83 @@ func (m *managerImpl) UpdateSettings(ctx context.Context, userID string, setting return updated, nil } +// hostnamesEquivalent reports whether a caller-supplied hostname names the +// same host as a stored (normalized, lowercase) one: equal after trimming and +// case folding. No structural validation — an arbitrary mismatch and a +// malformed value are both simply "not the assigned value". +func hostnamesEquivalent(supplied, stored string) bool { + return strings.EqualFold(strings.TrimSpace(supplied), stored) +} + +// DeleteSettings removes the account's settings row, releasing the endpoint. +// Two guards make this a bootstrap-repair operation rather than a way to tear +// down a serving gateway, both re-checked under the row lock: +// +// - No Agent Network providers may exist for the account. Providers route +// through the endpoint; delete them first. +// - No proxy may be actively serving the endpoint — that is, no active proxy +// declares the endpoint hostname as its cluster address. This is the +// dedicated (self-addressed) shape's guard: the proxy at the address IS +// this account's gateway. A labeled endpoint hangs beneath a shared +// cluster's address, and with the account's providers already gone the +// shared proxy serves nothing of the account's, so the parent cluster +// being up does not block the delete. +// +// Bootstrapping again after a delete allocates fresh — the released hostname +// is not reserved. That full-reset semantic is what gives clients that model +// immutability as replace-on-change (e.g. Terraform's RequiresReplace) a real +// path: tear down providers, delete, re-create. +func (m *managerImpl) DeleteSettings(ctx context.Context, accountID, userID string) error { + if err := m.requirePermission(ctx, accountID, userID, modules.AgentNetworkSettings, operations.Delete); err != nil { + return err + } + + var deleted *types.Settings + err := m.store.ExecuteInTransaction(ctx, func(tx store.Store) error { + existing, err := tx.GetAgentNetworkSettings(ctx, store.LockingStrengthUpdate, accountID) + switch { + case err == nil: + case isNotFound(err): + return status.Errorf(status.NotFound, "agent network settings have not been bootstrapped yet; there is nothing to delete") + default: + return fmt.Errorf("get agent network settings: %w", err) + } + + providers, err := tx.GetAccountAgentNetworkProviders(ctx, store.LockingStrengthNone, accountID) + if err != nil { + return fmt.Errorf("get agent network providers: %w", err) + } + if len(providers) > 0 { + return status.Errorf(status.PreconditionFailed, "agent network settings cannot be deleted while %d provider(s) exist; delete the providers first", len(providers)) + } + + serving, err := tx.HasActiveProxyAtClusterAddress(ctx, existing.Domain) + if err != nil { + return fmt.Errorf("check for a proxy serving the endpoint: %w", err) + } + if serving { + return status.Errorf(status.PreconditionFailed, "agent network settings cannot be deleted while a proxy is actively serving the endpoint %q", existing.Domain) + } + + if err := tx.DeleteAgentNetworkSettings(ctx, accountID); err != nil { + return fmt.Errorf("delete agent network settings: %w", err) + } + deleted = existing + return nil + }) + if err != nil { + return err + } + + m.accountManager.StoreEvent(ctx, userID, accountID, accountID, activity.AgentNetworkSettingsDeleted, map[string]any{ + "endpoint": deleted.Domain, + "proxy_address": deleted.ProxyAddress, + }) + m.reconcile(ctx, accountID) + + return nil +} + // isNotFound reports whether err is a status.NotFound error. func isNotFound(err error) bool { var sErr *status.Error @@ -678,74 +732,162 @@ func (m *managerImpl) GetSettings(ctx context.Context, accountID, userID string) } } -// requireSettingsBootstrapPermission gates the one-time settings bootstrap a -// first provider create performs. Pinning the account's cluster and subdomain -// is a settings write, so it needs the settings permission on top of the -// provider one. No-op once the settings row exists. -func (m *managerImpl) requireSettingsBootstrapPermission(ctx context.Context, accountID, userID string) error { - _, err := m.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, accountID) - if err == nil { - return nil - } - if !isNotFound(err) { - return fmt.Errorf("get agent network settings: %w", err) - } - return m.requirePermission(ctx, accountID, userID, modules.AgentNetworkSettings, operations.Create) -} +// maxDomainAllocationAttempts bounds the label search when bootstrapping a +// labeled endpoint. Package-level (rather than function-local) so tests can +// assert on the exhaustion path without duplicating the literal. +const maxDomainAllocationAttempts = 10 -// bootstrapSettingsIfNeeded creates the per-account agent-network -// settings row when missing. The cluster comes from the create-time -// hint the dashboard sends (auto-picked from the active cluster list); -// the subdomain is picked from the curated wordlist avoiding -// collisions on the same cluster. Idempotent: if a row already exists -// it is returned untouched and the hint is ignored. st is the store to -// operate on — pass the transaction store when calling from within one. -func (m *managerImpl) bootstrapSettingsIfNeeded(ctx context.Context, st store.Store, accountID, providerCluster string) (*types.Settings, error) { - if accountID == "" { - return nil, fmt.Errorf("bootstrap settings: account id is required") +// CreateSettings bootstraps the per-account settings row, assigning the +// account's immutable endpoint. Exactly one of proxyAddress and endpoint must +// be non-empty: proxyAddress allocates a labeled endpoint one label beneath +// the given cluster address; endpoint claims the given hostname verbatim as a +// self-addressed (dedicated) endpoint — a legitimate claim before any proxy +// declares the address (address-first). settings carries the account ID and +// the initial collection toggles; its identity fields are assigned here. +func (m *managerImpl) CreateSettings(ctx context.Context, userID string, settings *types.Settings, proxyAddress, endpoint string) (*types.Settings, error) { + if settings == nil || settings.AccountID == "" { + return nil, status.Errorf(status.InvalidArgument, "account id is required") } - if strings.TrimSpace(providerCluster) == "" { - return nil, fmt.Errorf("bootstrap settings: provider cluster is required") + if err := m.requirePermission(ctx, settings.AccountID, userID, modules.AgentNetworkSettings, operations.Create); err != nil { + return nil, err } - existing, err := st.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, accountID) - if err == nil { - return existing, nil + hasProxyAddress := strings.TrimSpace(proxyAddress) != "" + hasEndpoint := strings.TrimSpace(endpoint) != "" + if hasProxyAddress == hasEndpoint { + return nil, status.Errorf(status.InvalidArgument, "exactly one of proxy_address and endpoint is required") } - if !isNotFound(err) { + + // Fail fast on an existing row for a clean 409; the insert below stays + // the authority against concurrent bootstraps (the primary key wins). + if _, err := m.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, settings.AccountID); err == nil { + return nil, status.Errorf(status.AlreadyExists, "agent network settings already bootstrapped for account %s", settings.AccountID) + } else if !isNotFound(err) { return nil, fmt.Errorf("get agent network settings: %w", err) } - siblings, err := st.GetAgentNetworkSettingsByCluster(ctx, store.LockingStrengthNone, providerCluster) - if err != nil { - return nil, fmt.Errorf("list agent network settings on cluster: %w", err) - } - taken := make(map[string]struct{}, len(siblings)) - for _, s := range siblings { - taken[s.Subdomain] = struct{}{} - } - - suffix := accountID - if len(suffix) > 4 { - suffix = suffix[:4] - } - - m.labelRngMu.Lock() - subdomain := labelgen.PickUnique(m.labelRng, taken, suffix) - m.labelRngMu.Unlock() - now := time.Now().UTC() - settings := types.DefaultSettings(accountID) - settings.Cluster = providerCluster - settings.Subdomain = subdomain settings.CreatedAt = now settings.UpdatedAt = now - if err := st.SaveAgentNetworkSettings(ctx, settings); err != nil { - return nil, fmt.Errorf("save agent network settings: %w", err) + + var err error + if hasEndpoint { + err = m.bootstrapSelfAddressed(ctx, settings, endpoint) + } else { + err = m.bootstrapLabeled(ctx, settings, proxyAddress) } + if err != nil { + return nil, err + } + + m.accountManager.StoreEvent(ctx, userID, settings.AccountID, settings.AccountID, activity.AgentNetworkSettingsUpdated, map[string]any{ + "bootstrapped": true, + "endpoint": settings.Domain, + "dedicated": settings.Dedicated(), + }) + m.reconcile(ctx, settings.AccountID) + return settings, nil } +// bootstrapSelfAddressed claims the given hostname as the account's endpoint, +// served only by a proxy declaring exactly that address (Domain == +// ProxyAddress). The domain unique index is the arbiter of availability. +func (m *managerImpl) bootstrapSelfAddressed(ctx context.Context, settings *types.Settings, endpoint string) error { + hostname, err := types.NormalizeHostname(endpoint) + if err != nil { + return status.Errorf(status.InvalidArgument, "invalid endpoint: %s", err) + } + + settings.Domain = hostname + settings.ProxyAddress = hostname + if err := m.store.CreateAgentNetworkSettings(ctx, settings); err != nil { + if isUniqueConstraintError(err) { + // The violation is either the account primary key (a concurrent + // bootstrap for the same account won) or the domain index + // (another account holds the hostname). Distinguish by re-read. + if _, getErr := m.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, settings.AccountID); getErr == nil { + return status.Errorf(status.AlreadyExists, "agent network settings already bootstrapped for account %s", settings.AccountID) + } + return status.Errorf(status.AlreadyExists, "endpoint %s is already taken", hostname) + } + return fmt.Errorf("create agent network settings: %w", err) + } + return nil +} + +// bootstrapLabeled allocates a labeled endpoint one label beneath the given +// cluster address: Domain =