From e89aad09f5c2ae2205720ec42f665ad948af8f66 Mon Sep 17 00:00:00 2001 From: Nicolas Frati Date: Fri, 8 May 2026 16:31:20 +0200 Subject: [PATCH 01/29] [management] Enable MFA for local users (#5804) * wip: totp for local users * fix providers not getting populated * polished UI and fix post_login_redirect_uri * fix: make sure logout is only prompted from oidc flow Signed-off-by: jnfrati * update templates Signed-off-by: jnfrati * deps: update dex dependency Signed-off-by: jnfrati * fix qube issues Signed-off-by: jnfrati * replace window with globalThis on home html Signed-off-by: jnfrati * fixed coderabbit comments Signed-off-by: jnfrati * debug * remove unused config and rename totp issuer * deps: update dex reference to latest * add dashboard post logout redirect uri to embedded config * implemented api for mfa configuration * update docs and config parsing * catch error on idp manager init mfa * fix tests * Add remember me for MFA * Add cookie encryption and session share between tabs * fixed logout showing non actionable error and session cookie encription key * fixed missing mfa settings on sql query for account * fix code index for mfa activity --------- Signed-off-by: jnfrati Co-authored-by: braginini --- combined/cmd/config.go | 36 ++-- combined/config.yaml.example | 10 + go.mod | 44 +++-- go.sum | 97 +++++++--- idp/dex/config.go | 161 ++++++++++++++++- idp/dex/provider.go | 79 +++++++- idp/dex/provider_test.go | 26 +++ idp/dex/web/templates/home.html | 12 ++ idp/dex/web/templates/logout.html | 14 ++ idp/dex/web/templates/password.html | 2 + idp/dex/web/templates/totp_verify.html | 44 +++++ idp/dex/web/templates/webauthn_verify.html | 12 ++ management/internals/server/modules.go | 32 +++- management/server/account.go | 26 +++ management/server/activity/codes.go | 8 + .../handlers/accounts/accounts_handler.go | 4 + .../accounts/accounts_handler_test.go | 6 + management/server/idp/embedded.go | 171 +++++++++++++++++- management/server/idp/embedded_test.go | 67 +++++++ management/server/store/sql_store.go | 15 +- management/server/types/settings.go | 5 + shared/management/http/api/openapi.yml | 4 + shared/management/http/api/types.gen.go | 3 + 23 files changed, 791 insertions(+), 87 deletions(-) create mode 100644 idp/dex/web/templates/home.html create mode 100644 idp/dex/web/templates/logout.html create mode 100644 idp/dex/web/templates/totp_verify.html create mode 100644 idp/dex/web/templates/webauthn_verify.html diff --git a/combined/cmd/config.go b/combined/cmd/config.go index 9959f7a56..fe350e52a 100644 --- a/combined/cmd/config.go +++ b/combined/cmd/config.go @@ -133,13 +133,18 @@ type ManagementConfig struct { // AuthConfig contains authentication/identity provider settings type AuthConfig struct { - Issuer string `yaml:"issuer"` - LocalAuthDisabled bool `yaml:"localAuthDisabled"` - SignKeyRefreshEnabled bool `yaml:"signKeyRefreshEnabled"` - Storage AuthStorageConfig `yaml:"storage"` - DashboardRedirectURIs []string `yaml:"dashboardRedirectURIs"` - CLIRedirectURIs []string `yaml:"cliRedirectURIs"` - Owner *AuthOwnerConfig `yaml:"owner,omitempty"` + Issuer string `yaml:"issuer"` + LocalAuthDisabled bool `yaml:"localAuthDisabled"` + SignKeyRefreshEnabled bool `yaml:"signKeyRefreshEnabled"` + MfaSessionMaxLifetime string `yaml:"mfaSessionMaxLifetime"` + MfaSessionIdleTimeout string `yaml:"mfaSessionIdleTimeout"` + MfaSessionRememberMe bool `yaml:"mfaSessionRememberMe"` + SessionCookieEncryptionKey string `yaml:"sessionCookieEncryptionKey"` + Storage AuthStorageConfig `yaml:"storage"` + DashboardRedirectURIs []string `yaml:"dashboardRedirectURIs"` + CLIRedirectURIs []string `yaml:"cliRedirectURIs"` + Owner *AuthOwnerConfig `yaml:"owner,omitempty"` + DashboardPostLogoutRedirectURIs []string `yaml:"dashboardPostLogoutRedirectURIs"` } // AuthStorageConfig contains auth storage settings @@ -581,10 +586,14 @@ func (c *CombinedConfig) buildEmbeddedIdPConfig(mgmt ManagementConfig) (*idp.Emb } cfg := &idp.EmbeddedIdPConfig{ - Enabled: true, - Issuer: mgmt.Auth.Issuer, - LocalAuthDisabled: mgmt.Auth.LocalAuthDisabled, - SignKeyRefreshEnabled: mgmt.Auth.SignKeyRefreshEnabled, + Enabled: true, + Issuer: mgmt.Auth.Issuer, + LocalAuthDisabled: mgmt.Auth.LocalAuthDisabled, + SignKeyRefreshEnabled: mgmt.Auth.SignKeyRefreshEnabled, + MfaSessionMaxLifetime: mgmt.Auth.MfaSessionMaxLifetime, + MfaSessionIdleTimeout: mgmt.Auth.MfaSessionIdleTimeout, + MfaSessionRememberMe: mgmt.Auth.MfaSessionRememberMe, + SessionCookieEncryptionKey: mgmt.Auth.SessionCookieEncryptionKey, Storage: idp.EmbeddedStorageConfig{ Type: authStorageType, Config: idp.EmbeddedStorageTypeConfig{ @@ -592,8 +601,9 @@ func (c *CombinedConfig) buildEmbeddedIdPConfig(mgmt ManagementConfig) (*idp.Emb DSN: authStorageDSN, }, }, - DashboardRedirectURIs: mgmt.Auth.DashboardRedirectURIs, - CLIRedirectURIs: mgmt.Auth.CLIRedirectURIs, + DashboardRedirectURIs: mgmt.Auth.DashboardRedirectURIs, + CLIRedirectURIs: mgmt.Auth.CLIRedirectURIs, + DashboardPostLogoutRedirectURIs: mgmt.Auth.DashboardPostLogoutRedirectURIs, } if mgmt.Auth.Owner != nil && mgmt.Auth.Owner.Email != "" { diff --git a/combined/config.yaml.example b/combined/config.yaml.example index af85b0477..66bc71703 100644 --- a/combined/config.yaml.example +++ b/combined/config.yaml.example @@ -86,6 +86,13 @@ server: issuer: "https://example.com/oauth2" localAuthDisabled: false signKeyRefreshEnabled: false + # MFA session settings (applies when TOTP is enabled for an account) + # mfaSessionMaxLifetime: "24h" # Max duration for an MFA session from creation + # mfaSessionIdleTimeout: "1h" # MFA session expires after this idle period + # mfaSessionRememberMe: false # Pre-check "remember me" on login so the MFA session persists across tabs/restarts + # Optional AES key for encrypting embedded IdP session cookies. Can also be set via NB_IDP_SESSION_COOKIE_ENCRYPTION_KEY. + # Must be 16/24/32 raw bytes or base64-encoded to one of those lengths (for example: openssl rand -hex 16). + # sessionCookieEncryptionKey: "" # OAuth2 redirect URIs for dashboard dashboardRedirectURIs: - "https://app.example.com/nb-auth" @@ -93,6 +100,9 @@ server: # OAuth2 redirect URIs for CLI cliRedirectURIs: - "http://localhost:53000/" + # OAuth2 post-logout redirect URIs for dashboard (RP-initiated logout) + # dashboardPostLogoutRedirectURIs: + # - "https://app.example.com/" # Optional initial admin user # owner: # email: "admin@example.com" diff --git a/go.mod b/go.mod index bc4e8af15..84aeab941 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/onsi/gomega v1.27.6 github.com/rs/cors v1.8.0 github.com/sirupsen/logrus v1.9.4 - github.com/spf13/cobra v1.10.1 + github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.9 github.com/vishvananda/netlink v1.3.1 golang.org/x/crypto v0.50.0 @@ -41,11 +41,11 @@ require ( github.com/cilium/ebpf v0.15.0 github.com/coder/websocket v1.8.14 github.com/coreos/go-iptables v0.7.0 - github.com/coreos/go-oidc/v3 v3.14.1 + github.com/coreos/go-oidc/v3 v3.18.0 github.com/creack/pty v1.1.24 github.com/crowdsecurity/crowdsec v1.7.7 github.com/crowdsecurity/go-cs-bouncer v0.0.21 - github.com/dexidp/dex v0.0.0-00010101000000-000000000000 + github.com/dexidp/dex v2.13.0+incompatible github.com/dexidp/dex/api/v2 v2.4.0 github.com/ebitengine/purego v0.8.4 github.com/eko/gocache/lib/v4 v4.2.0 @@ -53,9 +53,9 @@ require ( github.com/eko/gocache/store/redis/v4 v4.2.2 github.com/fsnotify/fsnotify v1.9.0 github.com/gliderlabs/ssh v0.3.8 - github.com/go-jose/go-jose/v4 v4.1.3 + github.com/go-jose/go-jose/v4 v4.1.4 github.com/godbus/dbus/v5 v5.1.0 - github.com/golang-jwt/jwt/v5 v5.3.0 + github.com/golang-jwt/jwt/v5 v5.3.1 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.7.0 github.com/google/gopacket v1.1.19 @@ -113,7 +113,7 @@ require ( go.opentelemetry.io/otel/exporters/prometheus v0.64.0 go.opentelemetry.io/otel/metric v1.43.0 go.opentelemetry.io/otel/sdk/metric v1.43.0 - go.uber.org/mock v0.5.2 + go.uber.org/mock v0.6.0 go.uber.org/zap v1.27.0 goauthentik.io/api/v3 v3.2023051.3 golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b @@ -141,7 +141,7 @@ require ( filippo.io/edwards25519 v1.1.1 // indirect github.com/AppsFlyer/go-sundheit v0.6.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect + github.com/Azure/go-ntlmssp v0.1.0 // indirect github.com/BurntSushi/toml v1.5.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.3.0 // indirect @@ -168,6 +168,7 @@ require ( github.com/aws/smithy-go v1.23.0 // indirect github.com/beevik/etree v1.6.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/caddyserver/zerossl v0.1.3 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -183,6 +184,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fredbi/uri v1.1.1 // indirect + github.com/fxamacker/cbor/v2 v2.9.1 // indirect github.com/fyne-io/gl-js v0.2.0 // indirect github.com/fyne-io/glfw-js v0.3.0 // indirect github.com/fyne-io/image v0.1.1 // indirect @@ -190,7 +192,7 @@ require ( github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect github.com/go-gl/gl v0.0.0-20231021071112-07e5d0ea2e71 // indirect github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a // indirect - github.com/go-ldap/ldap/v3 v3.4.12 // indirect + github.com/go-ldap/ldap/v3 v3.4.13 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect @@ -206,11 +208,15 @@ require ( github.com/go-sql-driver/mysql v1.9.3 // indirect github.com/go-text/render v0.2.0 // indirect github.com/go-text/typesetting v0.2.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/go-webauthn/webauthn v0.16.4 // indirect + github.com/go-webauthn/x v0.2.3 // indirect github.com/goccy/go-yaml v1.18.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/go-tpm v0.9.8 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect github.com/googleapis/gax-go/v2 v2.21.0 // indirect @@ -218,7 +224,13 @@ require ( github.com/hack-pad/go-indexeddb v0.3.2 // indirect github.com/hack-pad/safejs v0.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/hcl v1.0.1-vault-7 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/huin/goupnp v1.2.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -238,13 +250,13 @@ require ( github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/koron/go-ssdp v0.0.4 // indirect github.com/kr/fs v0.1.0 // indirect - github.com/lib/pq v1.10.9 // indirect + github.com/lib/pq v1.12.3 // indirect github.com/libdns/libdns v0.2.2 // indirect github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect - github.com/mattn/go-sqlite3 v1.14.32 // indirect + github.com/mattn/go-sqlite3 v1.14.42 // indirect github.com/mdelapenya/tlscert v0.2.0 // indirect github.com/mdlayher/genetlink v1.3.2 // indirect github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect @@ -265,8 +277,10 @@ require ( github.com/nxadm/tail v1.4.11 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/onsi/ginkgo/v2 v2.9.5 // indirect + github.com/openbao/openbao/api/v2 v2.5.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pion/dtls/v2 v2.2.10 // indirect github.com/pion/dtls/v3 v3.0.9 // indirect github.com/pion/mdns/v2 v2.0.7 // indirect @@ -275,11 +289,13 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/pquerna/otp v1.5.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.19.2 // indirect github.com/russellhaering/goxmldsig v1.6.0 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect github.com/rymdport/portal v0.4.2 // indirect github.com/shirou/gopsutil/v4 v4.25.8 // indirect github.com/shoenig/go-m1cpu v0.2.1 // indirect @@ -288,11 +304,13 @@ require ( github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c // indirect github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef // indirect github.com/stretchr/objx v0.5.2 // indirect + github.com/tinylib/msgp v1.6.3 // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect github.com/tklauser/numcpus v0.10.0 // indirect github.com/vishvananda/netns v0.0.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/wlynxg/anet v0.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/yuin/goldmark v1.7.8 // indirect github.com/zeebo/blake3 v0.2.3 // indirect go.mongodb.org/mongo-driver v1.17.9 // indirect @@ -319,10 +337,12 @@ replace github.com/getlantern/systray => github.com/netbirdio/systray v0.0.0-202 replace golang.zx2c4.com/wireguard => github.com/netbirdio/wireguard-go v0.0.0-20260107100953-33b7c9d03db0 -replace github.com/cloudflare/circl => github.com/cunicu/circl v0.0.0-20230801113412-fec58fc7b5f6 +replace github.com/cloudflare/circl => codeberg.org/cunicu/circl v0.0.0-20230801113412-fec58fc7b5f6 replace github.com/pion/ice/v4 => github.com/netbirdio/ice/v4 v4.0.0-20250908184934-6202be846b51 -replace github.com/dexidp/dex => github.com/netbirdio/dex v0.244.0 +replace github.com/dexidp/dex => github.com/netbirdio/dex v0.244.1-0.20260415145816-a0c6b40ff9f2 + +replace github.com/dexidp/dex/api/v2 => github.com/netbirdio/dex/api/v2 v2.0.0-20260415145816-a0c6b40ff9f2 replace github.com/mailru/easyjson => github.com/netbirdio/easyjson v0.9.0 diff --git a/go.sum b/go.sum index d54dc01e6..851d1ce66 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3R cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +codeberg.org/cunicu/circl v0.0.0-20230801113412-fec58fc7b5f6 h1:b8xUw3004wk+3ipBhu0VU4RtUJsegMIiqjxSK4++lzA= +codeberg.org/cunicu/circl v0.0.0-20230801113412-fec58fc7b5f6/go.mod h1:+CauBF6R70Jqcyl8N2hC8pAXYbWkGIezuSbuGLtRhnw= cunicu.li/go-rosenpass v0.4.0 h1:LtPtBgFWY/9emfgC4glKLEqS0MJTylzV6+ChRhiZERw= cunicu.li/go-rosenpass v0.4.0/go.mod h1:MPbjH9nxV4l3vEagKVdFNwHOketqgS5/To1VYJplf/M= dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= @@ -23,8 +25,8 @@ github.com/AppsFlyer/go-sundheit v0.6.0 h1:d2hBvCjBSb2lUsEWGfPigr4MCOt04sxB+Rppl github.com/AppsFlyer/go-sundheit v0.6.0/go.mod h1:LDdBHD6tQBtmHsdW+i1GwdTt6Wqc0qazf5ZEJVTbTME= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= -github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A= +github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -91,6 +93,8 @@ github.com/beevik/etree v1.6.0/go.mod h1:bh4zJxiIr62SOf9pRzN7UUYaEDa9HEKafK25+sL github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -117,8 +121,8 @@ github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpS github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/coreos/go-iptables v0.7.0 h1:XWM3V+MPRr5/q51NuWSgU0fqMad64Zyxs8ZUoMsamr8= github.com/coreos/go-iptables v0.7.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= -github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk= -github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= +github.com/coreos/go-oidc/v3 v3.18.0 h1:V9orjXynvu5wiC9SemFTWnG4F45v403aIcjWo0d41+A= +github.com/coreos/go-oidc/v3 v3.18.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -130,14 +134,10 @@ github.com/crowdsecurity/go-cs-bouncer v0.0.21 h1:arPz0VtdVSaz+auOSfHythzkZVLyy1 github.com/crowdsecurity/go-cs-bouncer v0.0.21/go.mod h1:4JiH0XXA4KKnnWThItUpe5+heJHWzsLOSA2IWJqUDBA= github.com/crowdsecurity/go-cs-lib v0.0.25 h1:Ov6VPW9yV+OPsbAIQk1iTkEWhwkpaG0v3lrBzeqjzj4= github.com/crowdsecurity/go-cs-lib v0.0.25/go.mod h1:X0GMJY2CxdA1S09SpuqIKaWQsvRGxXmecUp9cP599dE= -github.com/cunicu/circl v0.0.0-20230801113412-fec58fc7b5f6 h1:/DS5cDX3FJdl+XaN2D7XAwFpuanTxnp52DBLZAaJKx0= -github.com/cunicu/circl v0.0.0-20230801113412-fec58fc7b5f6/go.mod h1:+CauBF6R70Jqcyl8N2hC8pAXYbWkGIezuSbuGLtRhnw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dexidp/dex/api/v2 v2.4.0 h1:gNba7n6BKVp8X4Jp24cxYn5rIIGhM6kDOXcZoL6tr9A= -github.com/dexidp/dex/api/v2 v2.4.0/go.mod h1:/p550ADvFFh7K95VmhUD+jgm15VdaNnab9td8DHOpyI= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -156,6 +156,8 @@ github.com/eko/gocache/store/go_cache/v4 v4.2.2 h1:tAI9nl6TLoJyKG1ujF0CS0n/IgTEM github.com/eko/gocache/store/go_cache/v4 v4.2.2/go.mod h1:T9zkHokzr8K9EiC7RfMbDg6HSwaV6rv3UdcNu13SGcA= github.com/eko/gocache/store/redis/v4 v4.2.2 h1:Thw31fzGuH3WzJywsdbMivOmP550D6JS7GDHhvCJPA0= github.com/eko/gocache/store/redis/v4 v4.2.2/go.mod h1:LaTxLKx9TG/YUEybQvPMij++D7PBTIJ4+pzvk0ykz0w= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -171,6 +173,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= +github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/fyne-io/gl-js v0.2.0 h1:+EXMLVEa18EfkXBVKhifYB6OGs3HwKO3lUElA0LlAjs= github.com/fyne-io/gl-js v0.2.0/go.mod h1:ZcepK8vmOYLu96JoxbCKJy2ybr+g1pTnaBDdl7c3ajI= github.com/fyne-io/glfw-js v0.3.0 h1:d8k2+Y7l+zy2pc7wlGRyPfTgZoqDf3AI4G+2zOWhWUk= @@ -189,10 +193,10 @@ github.com/go-gl/gl v0.0.0-20231021071112-07e5d0ea2e71 h1:5BVwOaUSBTlVZowGO6VZGw github.com/go-gl/gl v0.0.0-20231021071112-07e5d0ea2e71/go.mod h1:9YTyiznxEY1fVinfM7RvRcjRHbw2xLBJ3AAGIT0I4Nw= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a h1:vxnBhFDDT+xzxf1jTJKMKZw3H0swfWk9RpWbBbDK5+0= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= -github.com/go-ldap/ldap/v3 v3.4.12 h1:1b81mv7MagXZ7+1r7cLTWmyuTqVqdwbtJSjC0DAp9s4= -github.com/go-ldap/ldap/v3 v3.4.12/go.mod h1:+SPAGcTtOfmGsCb3h1RFiq4xpp4N636G75OEace8lNo= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-ldap/ldap/v3 v3.4.13 h1:+x1nG9h+MZN7h/lUi5Q3UZ0fJ1GyDQYbPvbuH38baDQ= +github.com/go-ldap/ldap/v3 v3.4.13/go.mod h1:LxsGZV6vbaK0sIvYfsv47rfh4ca0JXokCoKjZxsszv0= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -229,12 +233,20 @@ github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI6 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= +github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-text/render v0.2.0 h1:LBYoTmp5jYiJ4NPqDc2pz17MLmA3wHw1dZSVGcOdeAc= github.com/go-text/render v0.2.0/go.mod h1:CkiqfukRGKJA5vZZISkjSYrcdtgKQWRa2HIzvwNN5SU= github.com/go-text/typesetting v0.2.1 h1:x0jMOGyO3d1qFAPI0j4GSsh7M0Q3Ypjzr4+CEVg82V8= github.com/go-text/typesetting v0.2.1/go.mod h1:mTOxEwasOFpAMBjEQDhdWRckoLLeI/+qrQeBCTGEt6M= github.com/go-text/typesetting-utils v0.0.0-20241103174707-87a29e9e6066 h1:qCuYC+94v2xrb1PoS4NIDe7DGYtLnU2wWiQe9a1B1c0= github.com/go-text/typesetting-utils v0.0.0-20241103174707-87a29e9e6066/go.mod h1:DDxDdQEnB70R8owOx3LVpEFvpMK9eeH1o2r0yZhFI9o= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-webauthn/webauthn v0.16.4 h1:R9jqR/cYZa7hRquFF7Za/8qoH/K/TIs1/Q/4CyGN+1Q= +github.com/go-webauthn/webauthn v0.16.4/go.mod h1:SU2ljAgToTV/YLPI0C05QS4qn+e04WpB5g1RMfcZfS4= +github.com/go-webauthn/x v0.2.3 h1:8oArS+Rc1SWFLXhE17KZNx258Z4kUSyaDgsSncCO5RA= +github.com/go-webauthn/x v0.2.3/go.mod h1:tM04GF3V6VYq79AZMl7vbj4q6pz9r7L2criWRzbWhPk= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= @@ -243,8 +255,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -276,6 +288,10 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba h1:qJEJcuLzH5KDR0gKc0zcktin6KSAwL7+jWKBYceddTc= +github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba/go.mod h1:EFYHy8/1y2KfgTAsx7Luu7NGhoxtuVHnNo8jE7FikKc= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= @@ -308,15 +324,29 @@ github.com/hack-pad/safejs v0.1.0/go.mod h1:HdS+bKF1NrE72VoXZeWzxFOVQVUSqZJAG0xN github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= github.com/hashicorp/go-secure-stdlib/base62 v0.1.2 h1:ET4pqyjiGmY09R5y+rSd70J2w45CtbWDNvGqWp/R3Ng= github.com/hashicorp/go-secure-stdlib/base62 v0.1.2/go.mod h1:EdWO6czbmthiwZ3/PUsDV+UD1D5IRU4ActiaWGwt0Yw= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= +github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= +github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -387,8 +417,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= +github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s= github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= github.com/libdns/route53 v1.5.0 h1:2SKdpPFl/qgWsXQvsLNJJAoX7rSxlk7zgoL4jnWdXVA= @@ -406,9 +436,13 @@ github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8S github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattermost/xml-roundtrip-validator v0.1.0 h1:RXbVD2UAl7A7nOTR4u7E3ILa4IbtvKBHw64LDsmu9hU= github.com/mattermost/xml-roundtrip-validator v0.1.0/go.mod h1:qccnGMcpgwcNaBnxqpJpWWUiPNr5H3O8eDgGV9gT5To= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= -github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo= +github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw= @@ -451,8 +485,10 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/netbirdio/dex v0.244.0 h1:1GOvi8wnXYassnKGildzNqRHq0RbcfEUw7LKYpKIN7U= -github.com/netbirdio/dex v0.244.0/go.mod h1:STGInJhPcAflrHmDO7vyit2kSq03PdL+8zQPoGALtcU= +github.com/netbirdio/dex v0.244.1-0.20260415145816-a0c6b40ff9f2 h1:AP7OM/JnTogod3rVcLsMuilSG94kWQCr3z6R4rfVXnc= +github.com/netbirdio/dex v0.244.1-0.20260415145816-a0c6b40ff9f2/go.mod h1:+trSlzHNmdJGvz0oLEyyiuaPstUeD7YO6B3Fx9nyziY= +github.com/netbirdio/dex/api/v2 v2.0.0-20260415145816-a0c6b40ff9f2 h1:HEEGJPsVw7/p7SEL3HWP4vaInxHo8OJSEaOkHpUAk+M= +github.com/netbirdio/dex/api/v2 v2.0.0-20260415145816-a0c6b40ff9f2/go.mod h1:awuTyT29CYALpEyET0S307EgNlPWrc7fFKRAyhsO45M= github.com/netbirdio/easyjson v0.9.0 h1:6Nw2lghSVuy8RSkAYDhDv1thBVEmfVbKZnV7T7Z6Aus= github.com/netbirdio/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/netbirdio/ice/v4 v4.0.0-20250908184934-6202be846b51 h1:Ov4qdafATOgGMB1wbSuh+0aAHcwz9hdvB6VZjh1mVMI= @@ -489,6 +525,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/openbao/openbao/api/v2 v2.5.1 h1:Br79D6L20SbAa5P7xqENxmvv8LyI4HoKosPy7klhn4o= +github.com/openbao/openbao/api/v2 v2.5.1/go.mod h1:Dh5un77tqGgMbmlVEqjqN+8/dMyUohnkaQVg/wXW0Ig= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -501,6 +539,8 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/petermattis/goid v0.0.0-20250303134427-723919f7f203 h1:E7Kmf11E4K7B5hDti2K2NqPb1nlYlGYsu02S1JNd/Bs= github.com/petermattis/goid v0.0.0-20250303134427-723919f7f203/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.10 h1:u2Axk+FyIR1VFTPurktB+1zoEPGIW3bmyj3LEFrXjAA= github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= @@ -542,6 +582,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= +github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -565,6 +607,8 @@ github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russellhaering/goxmldsig v1.6.0 h1:8fdWXEPh2k/NZNQBPFNoVfS3JmzS4ZprY/sAOpKQLks= github.com/russellhaering/goxmldsig v1.6.0/go.mod h1:TrnaquDcYxWXfJrOjeMBTX4mLBeYAqaHEyUeWPxZlBM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/rymdport/portal v0.4.2 h1:7jKRSemwlTyVHHrTGgQg7gmNPJs88xkbKcIL3NlcmSU= github.com/rymdport/portal v0.4.2/go.mod h1:kFF4jslnJ8pD5uCi17brj/ODlfIidOxlgUDTO5ncnC4= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= @@ -587,8 +631,8 @@ github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= -github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= @@ -628,6 +672,8 @@ github.com/ti-mo/conntrack v0.5.1 h1:opEwkFICnDbQc0BUXl73PHBK0h23jEIFVjXsqvF4GY0 github.com/ti-mo/conntrack v0.5.1/go.mod h1:T6NCbkMdVU4qEIgwL0njA6lw/iCAbzchlnwm1Sa314o= github.com/ti-mo/netfilter v0.5.2 h1:CTjOwFuNNeZ9QPdRXt1MZFLFUf84cKtiQutNauHWd40= github.com/ti-mo/netfilter v0.5.2/go.mod h1:Btx3AtFiOVdHReTDmP9AE+hlkOcvIy403u7BXXbWZKo= +github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s= +github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= @@ -646,6 +692,8 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -690,14 +738,15 @@ go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lI go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= -go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= goauthentik.io/api/v3 v3.2023051.3 h1:NebAhD/TeTWNo/9X3/Uj+rM5fG1HaiLOlKTNLQv9Qq4= goauthentik.io/api/v3 v3.2023051.3/go.mod h1:nYECml4jGbp/541hj8GcylKQG1gVBsKppHy4+7G8u4U= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/idp/dex/config.go b/idp/dex/config.go index e686233ad..56ed998c2 100644 --- a/idp/dex/config.go +++ b/idp/dex/config.go @@ -51,6 +51,70 @@ type YAMLConfig struct { // StaticPasswords cause the server use this list of passwords rather than // querying the storage. StaticPasswords []Password `yaml:"staticPasswords" json:"staticPasswords"` + + // Sessions holds authentication session configuration. + // Requires DEX_SESSIONS_ENABLED=true feature flag. + Sessions *Sessions `yaml:"sessions" json:"sessions"` + + // MFA holds multi-factor authentication configuration. + MFA MFAConfig `yaml:"mfa" json:"mfa"` +} + +type Sessions struct { + // CookieName is the name of the session cookie. Defaults to "dex_session". + CookieName string `yaml:"cookieName" json:"cookieName"` + // AbsoluteLifetime is the maximum session lifetime from creation. Defaults to "24h". + AbsoluteLifetime string `yaml:"absoluteLifetime" json:"absoluteLifetime"` + // ValidIfNotUsedFor is the idle timeout. Defaults to "1h". + ValidIfNotUsedFor string `yaml:"validIfNotUsedFor" json:"validIfNotUsedFor"` + // RememberMeCheckedByDefault controls the default state of the "remember me" checkbox. + RememberMeCheckedByDefault *bool `yaml:"rememberMeCheckedByDefault" json:"rememberMeCheckedByDefault"` + // CookieEncryptionKey is the AES key for encrypting session cookies. + // Must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256. + // If empty, cookies are not encrypted. + CookieEncryptionKey string `yaml:"cookieEncryptionKey" json:"cookieEncryptionKey"` + // SSOSharedWithDefault is the default SSO sharing policy for clients without explicit ssoSharedWith. + // "all" = share with all clients, "none" = share with no one (default: "none"). + SSOSharedWithDefault string `yaml:"ssoSharedWithDefault" json:"ssoSharedWithDefault"` +} + +type MFAConfig struct { + Authenticators []MFAAuthenticator `yaml:"authenticators" json:"authenticators"` +} + +type MFAAuthenticator struct { + ID string `yaml:"id" json:"id"` + Type string `yaml:"type" json:"type"` + Config map[string]interface{} `yaml:"config" json:"config"` + + ConnectorTypes []string `yaml:"connectorTypes" json:"connectorTypes"` +} + +type TOTPConfig struct { + Issuer string `yaml:"issuer" json:"issuer"` +} + +// WebAuthnConfig holds configuration for a WebAuthn authenticator. +type WebAuthnConfig struct { + // RPDisplayName is the human-readable relying party name shown in the browser + // dialog during key registration and authentication (e.g., "My Company SSO"). + RPDisplayName string `yaml:"rpDisplayName" json:"rpDisplayName"` + // RPID is the relying party identifier — must match the domain in the browser + // address bar. If empty, derived from the issuer URL hostname. + // Example: "auth.example.com" + RPID string `yaml:"rpID" json:"rpID"` + // RPOrigins is the list of allowed origins for WebAuthn ceremonies. + // If empty, derived from the issuer URL (scheme + host). + // Example: ["https://auth.example.com"] + RPOrigins []string `yaml:"rpOrigins" json:"rpOrigins"` + // AttestationPreference controls what attestation data the authenticator should provide: + // "none" — don't request attestation (simpler, more private) + // "indirect" — authenticator may anonymize attestation (default) + // "direct" — request full attestation (for enterprise key model verification) + AttestationPreference string `yaml:"attestationPreference" json:"attestationPreference"` + // Timeout is the duration allowed for the browser WebAuthn ceremony + // (registration or login). Defaults to "60s". + Timeout string `yaml:"timeout" json:"timeout"` } // Web is the config format for the HTTP server. @@ -116,7 +180,6 @@ type Storage struct { Config map[string]interface{} `yaml:"config" json:"config"` } -// Password represents a static user configuration type Password storage.Password func (p *Password) UnmarshalYAML(node *yaml.Node) error { @@ -429,9 +492,98 @@ func (c *YAMLConfig) Validate() error { if !c.EnablePasswordDB && len(c.StaticPasswords) != 0 { return fmt.Errorf("cannot specify static passwords without enabling password db") } + return nil } +func buildTotpConfig(auth MFAAuthenticator) (*server.TOTPProvider, error) { + data, err := json.Marshal(auth.Config) + if err != nil { + return nil, fmt.Errorf("failed to marshal TOTP config id: %s - %w", auth.ID, err) + } + + var cfg TOTPConfig + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("failed to parse TOTP config id: %s - %w", auth.ID, err) + } + + return server.NewTOTPProvider(cfg.Issuer, auth.ConnectorTypes), nil +} + +func buildWebAuthnConfig(auth MFAAuthenticator, issuerURL string) (*server.WebAuthnProvider, error) { + data, err := json.Marshal(auth.Config) + if err != nil { + return nil, fmt.Errorf("failed to marshal WebAuthn config id: %s - %w", auth.ID, err) + } + + var cfg WebAuthnConfig + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("failed to parse WebAuthn config id: %s - %w", auth.ID, err) + } + + provider, err := server.NewWebAuthnProvider(cfg.RPDisplayName, cfg.RPID, cfg.RPOrigins, + cfg.AttestationPreference, cfg.Timeout, issuerURL, auth.ConnectorTypes) + if err != nil { + return nil, fmt.Errorf("failed to create WebAuthn provider id: %s - err: %w", auth.ID, err) + } + + return provider, nil +} + +func buildMFAProviders(authenticators []MFAAuthenticator, issuerURL string, logger *slog.Logger) map[string]server.MFAProvider { + if len(authenticators) == 0 { + return nil + } + + providers := make(map[string]server.MFAProvider, len(authenticators)) + for _, auth := range authenticators { + switch auth.Type { + case "TOTP": + provider, err := buildTotpConfig(auth) + if err != nil { + logger.Error("failed to parse TOTP config", "id", auth.ID, "err", err) + continue + } + providers[auth.ID] = provider + logger.Info("MFA authenticator configured", "id", auth.ID, "type", auth.Type) + case "WebAuthn": + provider, err := buildWebAuthnConfig(auth, issuerURL) + if err != nil { + logger.Error("failed to parse WebAuthn config", "id", auth.ID, "err", err) + continue + } + providers[auth.ID] = provider + logger.Info("MFA authenticator configured", "id", auth.ID, "type", auth.Type) + default: + logger.Error("unknown MFA authenticator type, skipping", "id", auth.ID, "type", auth.Type) + } + } + return providers +} + +func buildSessionsConfig(sessions *Sessions) *server.SessionConfig { + if sessions == nil { + return nil + } + + if sessions.RememberMeCheckedByDefault == nil { + defaultRememberMeCheckedByDefault := false + sessions.RememberMeCheckedByDefault = &defaultRememberMeCheckedByDefault + } + + absoluteLifetime, _ := parseDuration(sessions.AbsoluteLifetime) + validIfNotUsedFor, _ := parseDuration(sessions.ValidIfNotUsedFor) + + return &server.SessionConfig{ + CookieEncryptionKey: []byte(sessions.CookieEncryptionKey), + CookieName: sessions.CookieName, + AbsoluteLifetime: absoluteLifetime, + ValidIfNotUsedFor: validIfNotUsedFor, + RememberMeCheckedByDefault: *sessions.RememberMeCheckedByDefault, + SSOSharedWithDefault: sessions.SSOSharedWithDefault, + } +} + // ToServerConfig converts YAMLConfig to dex server.Config func (c *YAMLConfig) ToServerConfig(stor storage.Storage, logger *slog.Logger) server.Config { cfg := server.Config{ @@ -448,6 +600,8 @@ func (c *YAMLConfig) ToServerConfig(stor storage.Storage, logger *slog.Logger) s Dir: c.Frontend.Dir, Extra: c.Frontend.Extra, }, + SessionConfig: buildSessionsConfig(c.Sessions), + MFAProviders: buildMFAProviders(c.MFA.Authenticators, c.Issuer, logger), } // Use embedded NetBird-styled templates if no custom dir specified @@ -460,11 +614,6 @@ func (c *YAMLConfig) ToServerConfig(stor storage.Storage, logger *slog.Logger) s } // Apply expiry settings - if c.Expiry.SigningKeys != "" { - if d, err := parseDuration(c.Expiry.SigningKeys); err == nil { - cfg.RotateKeysAfter = d - } - } if c.Expiry.IDTokens != "" { if d, err := parseDuration(c.Expiry.IDTokens); err == nil { cfg.IDTokensValidFor = d diff --git a/idp/dex/provider.go b/idp/dex/provider.go index 24aed1b99..526d6a17a 100644 --- a/idp/dex/provider.go +++ b/idp/dex/provider.go @@ -18,6 +18,7 @@ import ( dexapi "github.com/dexidp/dex/api/v2" "github.com/dexidp/dex/server" + "github.com/dexidp/dex/server/signer" "github.com/dexidp/dex/storage" "github.com/dexidp/dex/storage/sql" jose "github.com/go-jose/go-jose/v4" @@ -70,7 +71,7 @@ func NewProvider(ctx context.Context, config *Config) (*Provider, error) { logger := slog.New(slog.NewTextHandler(os.Stderr, nil)) // Ensure data directory exists - if err := os.MkdirAll(config.DataDir, 0700); err != nil { + if err := os.MkdirAll(config.DataDir, 0o700); err != nil { return nil, fmt.Errorf("failed to create data directory: %w", err) } @@ -101,6 +102,15 @@ func NewProvider(ctx context.Context, config *Config) (*Provider, error) { return nil, fmt.Errorf("failed to create refresh token policy: %w", err) } + localSignerConfig := signer.LocalConfig{ + KeysRotationPeriod: "6h", + } + + localSigner, err := localSignerConfig.Open(ctx, stor, 24*time.Hour, time.Now, logger) + if err != nil { + return nil, fmt.Errorf("failed to create local signer: %w", err) + } + // Build Dex server config - use Dex's types directly dexConfig := server.Config{ Issuer: issuer, @@ -110,12 +120,12 @@ func NewProvider(ctx context.Context, config *Config) (*Provider, error) { ContinueOnConnectorFailure: true, Logger: logger, PrometheusRegistry: prometheus.NewRegistry(), - RotateKeysAfter: 6 * time.Hour, IDTokensValidFor: 24 * time.Hour, RefreshTokenPolicy: refreshPolicy, Web: server.WebConfig{ Issuer: "NetBird", }, + Signer: localSigner, } dexSrv, err := server.NewServer(ctx, dexConfig) @@ -167,6 +177,14 @@ func NewProviderFromYAML(ctx context.Context, yamlConfig *YAMLConfig) (*Provider return nil, fmt.Errorf("failed to create refresh token policy: %w", err) } + localSigner, err := getSigner(ctx, stor, yamlConfig, logger) + if err != nil { + stor.Close() + return nil, fmt.Errorf("failed to create local signer: %w", err) + } + + dexConfig.Signer = localSigner + dexSrv, err := server.NewServer(ctx, dexConfig) if err != nil { stor.Close() @@ -182,6 +200,32 @@ func NewProviderFromYAML(ctx context.Context, yamlConfig *YAMLConfig) (*Provider }, nil } +func getSigner(ctx context.Context, stor storage.Storage, yamlConfig *YAMLConfig, logger *slog.Logger) (signer.Signer, error) { + // Parse expiry durations + idTokensValidFor := 24 * time.Hour // default + if yamlConfig.Expiry.IDTokens != "" { + var err error + idTokensValidFor, err = parseDuration(yamlConfig.Expiry.IDTokens) + if err != nil { + return nil, fmt.Errorf("invalid config value %q for id token expiry: %v", yamlConfig.Expiry.IDTokens, err) + } + } + + localSignerConfig := &signer.LocalConfig{ + KeysRotationPeriod: "720h", // 30 Days + } + + if yamlConfig.Expiry.SigningKeys != "" { + if _, err := parseDuration(yamlConfig.Expiry.SigningKeys); err != nil { + return nil, fmt.Errorf("invalid config value %q for signing key expiry: %v", yamlConfig.Expiry.SigningKeys, err) + } + + localSignerConfig.KeysRotationPeriod = yamlConfig.Expiry.SigningKeys + } + + return localSignerConfig.Open(ctx, stor, idTokensValidFor, time.Now, logger) +} + // initializeStorage sets up connectors, passwords, and clients in storage func initializeStorage(ctx context.Context, stor storage.Storage, cfg *YAMLConfig) error { if cfg.EnablePasswordDB { @@ -241,6 +285,8 @@ func ensureStaticClients(ctx context.Context, stor storage.Storage, clients []st old.RedirectURIs = client.RedirectURIs old.Name = client.Name old.Public = client.Public + old.PostLogoutRedirectURIs = client.PostLogoutRedirectURIs + old.MFAChain = client.MFAChain return old, nil }); err != nil { return fmt.Errorf("failed to update client %s: %w", client.ID, err) @@ -253,9 +299,6 @@ func ensureStaticClients(ctx context.Context, stor storage.Storage, clients []st func buildDexConfig(yamlConfig *YAMLConfig, stor storage.Storage, logger *slog.Logger) server.Config { cfg := yamlConfig.ToServerConfig(stor, logger) cfg.PrometheusRegistry = prometheus.NewRegistry() - if cfg.RotateKeysAfter == 0 { - cfg.RotateKeysAfter = 24 * 30 * time.Hour - } if cfg.IDTokensValidFor == 0 { cfg.IDTokensValidFor = 24 * time.Hour } @@ -450,10 +493,34 @@ func (p *Provider) Storage() storage.Storage { return p.storage } +// SetClientsMFAChain updates the MFAChain field on the dashboard and CLI OAuth2 clients. +// Pass a non-empty slice (e.g. []string{"default-totp"}) to enable MFA, or nil to disable it. +func (p *Provider) SetClientsMFAChain(ctx context.Context, clientIDs []string, mfaChain []string) error { + for _, clientID := range clientIDs { + if err := p.storage.UpdateClient(ctx, clientID, func(old storage.Client) (storage.Client, error) { + old.MFAChain = mfaChain + return old, nil + }); err != nil { + return fmt.Errorf("failed to update MFA chain on client %s: %w", clientID, err) + } + } + return nil +} + // Handler returns the Dex server as an http.Handler for embedding in another server. // The handler expects requests with path prefix "/oauth2/". func (p *Provider) Handler() http.Handler { - return p.dexServer + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Dex's /logout endpoint requires id_token_hint for RP-initiated logout with + // post_logout_redirect_uri. If the dashboard calls logout without one, avoid + // rendering Dex's non-actionable Bad Request page and send the user home. + if strings.HasSuffix(r.URL.Path, "/logout") && r.FormValue("id_token_hint") == "" { + http.Redirect(w, r, "/", http.StatusSeeOther) + return + } + + p.dexServer.ServeHTTP(w, r) + }) } // CreateUser creates a new user with the given email, username, and password. diff --git a/idp/dex/provider_test.go b/idp/dex/provider_test.go index 4ed89fd2e..88828fbbb 100644 --- a/idp/dex/provider_test.go +++ b/idp/dex/provider_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "log/slog" + "net/http" + "net/http/httptest" "os" "path/filepath" "testing" @@ -144,6 +146,30 @@ func TestEncodeDexUserID_MatchesDexFormat(t *testing.T) { assert.Equal(t, knownEncodedID, reEncoded) } +func TestHandlerRedirectsLogoutWithoutIDTokenHint(t *testing.T) { + ctx := context.Background() + + tmpDir, err := os.MkdirTemp("", "dex-logout-handler-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + provider, err := NewProvider(ctx, &Config{ + Issuer: "http://localhost:5556/oauth2", + Port: 5556, + DataDir: tmpDir, + }) + require.NoError(t, err) + defer func() { _ = provider.Stop(ctx) }() + + req := httptest.NewRequest(http.MethodGet, "/oauth2/logout?post_logout_redirect_uri=https://example.com", nil) + rec := httptest.NewRecorder() + + provider.Handler().ServeHTTP(rec, req) + + require.Equal(t, http.StatusSeeOther, rec.Code) + require.Equal(t, "/", rec.Header().Get("Location")) +} + func TestCreateUserInTempDB(t *testing.T) { ctx := context.Background() diff --git a/idp/dex/web/templates/home.html b/idp/dex/web/templates/home.html new file mode 100644 index 000000000..be7c938ae --- /dev/null +++ b/idp/dex/web/templates/home.html @@ -0,0 +1,12 @@ +{{ template "header.html" . }} + + + + +{{ template "footer.html" . }} diff --git a/idp/dex/web/templates/logout.html b/idp/dex/web/templates/logout.html new file mode 100644 index 000000000..b623d35af --- /dev/null +++ b/idp/dex/web/templates/logout.html @@ -0,0 +1,14 @@ +{{ template "header.html" . }} + +
+

Logged Out

+

You have been successfully logged out.

+ + {{ if .BackURL }} +
+ ← Back to Application +
+ {{ end }} +
+ +{{ template "footer.html" . }} diff --git a/idp/dex/web/templates/password.html b/idp/dex/web/templates/password.html index 1d1b8282e..e1bfa7258 100755 --- a/idp/dex/web/templates/password.html +++ b/idp/dex/web/templates/password.html @@ -18,6 +18,7 @@ id="login" name="login" class="nb-input" + autocomplete="username" placeholder="Enter your {{ .UsernamePrompt | lower }}" {{ if .Username }}value="{{ .Username }}"{{ else }}autofocus{{ end }} required @@ -31,6 +32,7 @@ id="password" name="password" class="nb-input" + autocomplete="current-password" placeholder="Enter your password" {{ if .Invalid }}autofocus{{ end }} required diff --git a/idp/dex/web/templates/totp_verify.html b/idp/dex/web/templates/totp_verify.html new file mode 100644 index 000000000..8286418f0 --- /dev/null +++ b/idp/dex/web/templates/totp_verify.html @@ -0,0 +1,44 @@ +{{ template "header.html" . }} + +
+

Two-factor authentication

+ {{ if not (eq .QRCode "") }} +

Scan the QR code below using your authenticator app, then enter the code.

+
+ QR code +
+ {{ else }} +

Enter the code from your authenticator app.

+ {{ end }} + +
+ {{ if .Invalid }} +
+ Invalid code. Please try again. +
+ {{ end }} + +
+ + +
+ + +
+
+ +{{ template "footer.html" . }} diff --git a/idp/dex/web/templates/webauthn_verify.html b/idp/dex/web/templates/webauthn_verify.html new file mode 100644 index 000000000..be7c938ae --- /dev/null +++ b/idp/dex/web/templates/webauthn_verify.html @@ -0,0 +1,12 @@ +{{ template "header.html" . }} + + + + +{{ template "footer.html" . }} diff --git a/management/internals/server/modules.go b/management/internals/server/modules.go index 9b2ec2989..ea94245d5 100644 --- a/management/internals/server/modules.go +++ b/management/internals/server/modules.go @@ -26,6 +26,7 @@ import ( "github.com/netbirdio/netbird/management/server/networks" "github.com/netbirdio/netbird/management/server/networks/resources" "github.com/netbirdio/netbird/management/server/networks/routers" + "github.com/netbirdio/netbird/management/server/types" "github.com/netbirdio/netbird/management/server/permissions" "github.com/netbirdio/netbird/management/server/settings" @@ -113,30 +114,47 @@ func (s *BaseServer) AccountManager() account.Manager { }) } +func isMFAEnabledForAccount(accounts []*types.Account) bool { + if len(accounts) != 1 { + return false + } + + settings := accounts[0].Settings + return settings != nil && settings.LocalMfaEnabled +} + func (s *BaseServer) IdpManager() idp.Manager { return Create(s, func() idp.Manager { - var idpManager idp.Manager - var err error - // Use embedded IdP service if embedded Dex is configured and enabled. // Legacy IdpManager won't be used anymore even if configured. embeddedEnabled := s.Config.EmbeddedIdP != nil && s.Config.EmbeddedIdP.Enabled if embeddedEnabled { - idpManager, err = idp.NewEmbeddedIdPManager(context.Background(), s.Config.EmbeddedIdP, s.Metrics()) + embeddedMgr, err := idp.NewEmbeddedIdPManager(context.Background(), s.Config.EmbeddedIdP, s.Metrics()) if err != nil { log.Fatalf("failed to create embedded IDP service: %v", err) } - return idpManager + + if val := isMFAEnabledForAccount(s.Store().GetAllAccounts(context.Background())); val { + if err := embeddedMgr.SetMFAEnabled(context.Background(), val); err != nil { + log.Errorf("failed to set MFA enabled on embedded IDP: %v", err) + } + } + + return embeddedMgr } // Fall back to external IdP service if s.Config.IdpManagerConfig != nil { - idpManager, err = idp.NewManager(context.Background(), *s.Config.IdpManagerConfig, s.Metrics()) + idpManager, err := idp.NewManager(context.Background(), *s.Config.IdpManagerConfig, s.Metrics()) if err != nil { log.Fatalf("failed to create IDP service: %v", err) } + + return idpManager } - return idpManager + + + return nil }) } diff --git a/management/server/account.go b/management/server/account.go index 45b99839f..364c0c37b 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -386,6 +386,9 @@ func (am *DefaultAccountManager) UpdateAccountSettings(ctx context.Context, acco if err = am.handleInactivityExpirationSettings(ctx, oldSettings, newSettings, userID, accountID); err != nil { return nil, err } + if err = am.handleLocalMfaSettings(ctx, oldSettings, newSettings, userID, accountID); err != nil { + return nil, err + } if oldSettings.DNSDomain != newSettings.DNSDomain { eventMeta := map[string]any{ "old_dns_domain": oldSettings.DNSDomain, @@ -602,6 +605,29 @@ func (am *DefaultAccountManager) handleInactivityExpirationSettings(ctx context. return nil } +func (am *DefaultAccountManager) handleLocalMfaSettings(ctx context.Context, oldSettings, newSettings *types.Settings, userID, accountID string) error { + if oldSettings.LocalMfaEnabled == newSettings.LocalMfaEnabled { + return nil + } + + embeddedIdp, ok := am.idpManager.(*idp.EmbeddedIdPManager) + if !ok { + return nil + } + + if err := embeddedIdp.SetMFAEnabled(ctx, newSettings.LocalMfaEnabled); err != nil { + return fmt.Errorf("failed to toggle MFA: %w", err) + } + + if newSettings.LocalMfaEnabled { + am.StoreEvent(ctx, userID, accountID, accountID, activity.AccountLocalMfaEnabled, nil) + } else { + am.StoreEvent(ctx, userID, accountID, accountID, activity.AccountLocalMfaDisabled, nil) + } + + return nil +} + func (am *DefaultAccountManager) peerLoginExpirationJob(ctx context.Context, accountID string) func() (time.Duration, bool) { return func() (time.Duration, bool) { //nolint diff --git a/management/server/activity/codes.go b/management/server/activity/codes.go index 2388115ff..6c781a952 100644 --- a/management/server/activity/codes.go +++ b/management/server/activity/codes.go @@ -236,6 +236,11 @@ const ( // AccountIPv6Disabled indicates that a user disabled IPv6 overlay for the account AccountIPv6Disabled Activity = 122 + // AccountLocalMfaEnabled indicates that a user enabled TOTP MFA for local users + AccountLocalMfaEnabled Activity = 123 + // AccountLocalMfaDisabled indicates that a user disabled TOTP MFA for local users + AccountLocalMfaDisabled Activity = 124 + AccountDeleted Activity = 99999 ) @@ -386,6 +391,9 @@ var activityMap = map[Activity]Code{ AccountPeerExposeEnabled: {"Account peer expose enabled", "account.setting.peer.expose.enable"}, AccountPeerExposeDisabled: {"Account peer expose disabled", "account.setting.peer.expose.disable"}, + AccountLocalMfaEnabled: {"Account local MFA enabled", "account.setting.local.mfa.enable"}, + AccountLocalMfaDisabled: {"Account local MFA disabled", "account.setting.local.mfa.disable"}, + DomainAdded: {"Domain added", "domain.add"}, DomainDeleted: {"Domain deleted", "domain.delete"}, DomainValidated: {"Domain validated", "domain.validate"}, diff --git a/management/server/http/handlers/accounts/accounts_handler.go b/management/server/http/handlers/accounts/accounts_handler.go index 31820b9fb..209d593bd 100644 --- a/management/server/http/handlers/accounts/accounts_handler.go +++ b/management/server/http/handlers/accounts/accounts_handler.go @@ -277,6 +277,9 @@ func (h *handler) updateAccountRequestSettings(req api.PutApiAccountsAccountIdJS if req.Settings.AutoUpdateAlways != nil { returnSettings.AutoUpdateAlways = *req.Settings.AutoUpdateAlways } + if req.Settings.LocalMfaEnabled != nil { + returnSettings.LocalMfaEnabled = *req.Settings.LocalMfaEnabled + } if req.Settings.Ipv6EnabledGroups != nil { returnSettings.IPv6EnabledGroups = *req.Settings.Ipv6EnabledGroups } @@ -412,6 +415,7 @@ func toAccountResponse(accountID string, settings *types.Settings, meta *types.A Ipv6EnabledGroups: &settings.IPv6EnabledGroups, EmbeddedIdpEnabled: &settings.EmbeddedIdpEnabled, LocalAuthDisabled: &settings.LocalAuthDisabled, + LocalMfaEnabled: &settings.LocalMfaEnabled, } if settings.NetworkRange.IsValid() { diff --git a/management/server/http/handlers/accounts/accounts_handler_test.go b/management/server/http/handlers/accounts/accounts_handler_test.go index fc1517a30..8db76719c 100644 --- a/management/server/http/handlers/accounts/accounts_handler_test.go +++ b/management/server/http/handlers/accounts/accounts_handler_test.go @@ -131,6 +131,7 @@ func TestAccounts_AccountsHandler(t *testing.T) { AutoUpdateVersion: sr(""), EmbeddedIdpEnabled: br(false), LocalAuthDisabled: br(false), + LocalMfaEnabled: br(false), }, expectedArray: true, expectedID: accountID, @@ -157,6 +158,7 @@ func TestAccounts_AccountsHandler(t *testing.T) { AutoUpdateVersion: sr(""), EmbeddedIdpEnabled: br(false), LocalAuthDisabled: br(false), + LocalMfaEnabled: br(false), }, expectedArray: false, expectedID: accountID, @@ -183,6 +185,7 @@ func TestAccounts_AccountsHandler(t *testing.T) { AutoUpdateVersion: sr("latest"), EmbeddedIdpEnabled: br(false), LocalAuthDisabled: br(false), + LocalMfaEnabled: br(false), }, expectedArray: false, expectedID: accountID, @@ -209,6 +212,7 @@ func TestAccounts_AccountsHandler(t *testing.T) { AutoUpdateVersion: sr(""), EmbeddedIdpEnabled: br(false), LocalAuthDisabled: br(false), + LocalMfaEnabled: br(false), }, expectedArray: false, expectedID: accountID, @@ -235,6 +239,7 @@ func TestAccounts_AccountsHandler(t *testing.T) { AutoUpdateVersion: sr(""), EmbeddedIdpEnabled: br(false), LocalAuthDisabled: br(false), + LocalMfaEnabled: br(false), }, expectedArray: false, expectedID: accountID, @@ -261,6 +266,7 @@ func TestAccounts_AccountsHandler(t *testing.T) { AutoUpdateVersion: sr(""), EmbeddedIdpEnabled: br(false), LocalAuthDisabled: br(false), + LocalMfaEnabled: br(false), }, expectedArray: false, expectedID: accountID, diff --git a/management/server/idp/embedded.go b/management/server/idp/embedded.go index 48d3221cc..a1852a8bc 100644 --- a/management/server/idp/embedded.go +++ b/management/server/idp/embedded.go @@ -2,9 +2,11 @@ package idp import ( "context" + "encoding/base64" "errors" "fmt" "net/http" + "os" "strings" "github.com/dexidp/dex/storage" @@ -17,12 +19,13 @@ import ( ) const ( - staticClientDashboard = "netbird-dashboard" - staticClientCLI = "netbird-cli" - defaultCLIRedirectURL1 = "http://localhost:53000/" - defaultCLIRedirectURL2 = "http://localhost:54000/" - defaultScopes = "openid profile email groups" - defaultUserIDClaim = "sub" + staticClientDashboard = "netbird-dashboard" + staticClientCLI = "netbird-cli" + defaultCLIRedirectURL1 = "http://localhost:53000/" + defaultCLIRedirectURL2 = "http://localhost:54000/" + defaultScopes = "openid profile email groups" + defaultUserIDClaim = "sub" + sessionCookieEncryptionKeyEnv = "NB_IDP_SESSION_COOKIE_ENCRYPTION_KEY" ) // EmbeddedIdPConfig contains configuration for the embedded Dex OIDC identity provider @@ -49,6 +52,26 @@ type EmbeddedIdPConfig struct { // Existing local users are preserved and will be able to login again if re-enabled. // Cannot be enabled if no external identity provider connectors are configured. LocalAuthDisabled bool + // MfaSessionMaxLifetime is the maximum MFA session duration from creation (e.g. "24h"). + // Defaults to "24h" if empty. + MfaSessionMaxLifetime string + // MfaSessionIdleTimeout is the idle timeout after which the MFA session expires (e.g. "1h"). + // Defaults to "1h" if empty. + MfaSessionIdleTimeout string + // MfaSessionRememberMe controls the default state of the "remember me" checkbox on the + // login screen. When true, the session cookie persists across browser tabs/restarts so + // MFA is not re-prompted until the session expires. Defaults to false. + MfaSessionRememberMe bool + // SessionCookieEncryptionKey is the optional AES key used to encrypt embedded IdP session cookies. + // It can also be set with NB_IDP_SESSION_COOKIE_ENCRYPTION_KEY. The value must be 16, 24, or 32 + // bytes when provided as a raw string, or base64-encoded to one of those lengths. + SessionCookieEncryptionKey string + // Dashboard Post logout redirect URIs, these are required to tell + // Dex what to allow when an RP-Initiated logout is started by the frontend + // at least one of these must match the dashboard base URL or the dashboard + // DASHBOARD_POST_LOGOUT_URL environment variable + // WARNING: Dex only uses exact match, not wildcards + DashboardPostLogoutRedirectURIs []string // StaticConnectors are additional connectors to seed during initialization StaticConnectors []dex.Connector } @@ -126,6 +149,11 @@ func (c *EmbeddedIdPConfig) ToYAMLConfig() (*dex.YAMLConfig, error) { // todo: resolve import cycle dashboardRedirectURIs = append(dashboardRedirectURIs, baseURL+"/api/reverse-proxy/callback") + dashboardPostLogoutRedirectURIs := c.DashboardPostLogoutRedirectURIs + // It is safe to assume that most installations will share the location of the + // MGMT api and the dashboard, adding baseURL means less configuration for the instance admin + dashboardPostLogoutRedirectURIs = append(dashboardPostLogoutRedirectURIs, baseURL) + cfg := &dex.YAMLConfig{ Issuer: c.Issuer, Storage: dex.Storage{ @@ -148,10 +176,11 @@ func (c *EmbeddedIdPConfig) ToYAMLConfig() (*dex.YAMLConfig, error) { EnablePasswordDB: true, StaticClients: []storage.Client{ { - ID: staticClientDashboard, - Name: "NetBird Dashboard", - Public: true, - RedirectURIs: dashboardRedirectURIs, + ID: staticClientDashboard, + Name: "NetBird Dashboard", + Public: true, + RedirectURIs: dashboardRedirectURIs, + PostLogoutRedirectURIs: sanitizePostLogoutRedirectURIs(dashboardPostLogoutRedirectURIs), }, { ID: staticClientCLI, @@ -163,6 +192,12 @@ func (c *EmbeddedIdPConfig) ToYAMLConfig() (*dex.YAMLConfig, error) { StaticConnectors: c.StaticConnectors, } + // Always initialize MFA providers and sessions so TOTP can be toggled at runtime. + // MFAChain on clients is NOT set here — it's synced from the DB setting on startup. + if err := configureMFA(cfg, c.MfaSessionMaxLifetime, c.MfaSessionIdleTimeout, c.MfaSessionRememberMe, c.SessionCookieEncryptionKey); err != nil { + return nil, err + } + // Add owner user if provided if c.Owner != nil && c.Owner.Email != "" && c.Owner.Hash != "" { username := c.Owner.Username @@ -182,6 +217,100 @@ func (c *EmbeddedIdPConfig) ToYAMLConfig() (*dex.YAMLConfig, error) { return cfg, nil } +// Due to how the frontend generates the logout, sometimes it appends a trailing slash +// and because Dex only allows exact matches, we need to make sure we always have both +// versions of each provided uri +func sanitizePostLogoutRedirectURIs(uris []string) []string { + result := make([]string, 0) + for _, uri := range uris { + if strings.HasSuffix(uri, "/") { + result = append(result, uri) + result = append(result, strings.TrimSuffix(uri, "/")) + } else { + result = append(result, uri) + result = append(result, uri+"/") + } + } + + return result +} + +func configureMFA(cfg *dex.YAMLConfig, sessionMaxLifetime, sessionIdleTimeout string, rememberMe bool, sessionCookieEncryptionKey string) error { + cfg.MFA.Authenticators = []dex.MFAAuthenticator{{ + ID: "default-totp", + // Has to be caps otherwise it will fail + Type: "TOTP", + Config: map[string]interface{}{ + "issuer": "NetBird", + }, + ConnectorTypes: []string{"local"}, + }} + + if sessionMaxLifetime == "" { + sessionMaxLifetime = "24h" + } + if sessionIdleTimeout == "" { + sessionIdleTimeout = "1h" + } + + cookieEncryptionKey, err := resolveSessionCookieEncryptionKey(sessionCookieEncryptionKey) + if err != nil { + return err + } + + cfg.Sessions = &dex.Sessions{ + CookieName: "netbird-session", + AbsoluteLifetime: sessionMaxLifetime, + ValidIfNotUsedFor: sessionIdleTimeout, + RememberMeCheckedByDefault: &rememberMe, + SSOSharedWithDefault: "all", + CookieEncryptionKey: cookieEncryptionKey, + } + // Absolutely required, otherwise the dex server will omit the MFA configuration entirely + os.Setenv("DEX_SESSIONS_ENABLED", "true") + + // Note: MFAChain on clients is NOT set here. + // It is toggled at runtime via SetMFAEnabled() based on the account settings DB value. + return nil +} + +func resolveSessionCookieEncryptionKey(configuredKey string) (string, error) { + key := strings.TrimSpace(configuredKey) + if key == "" { + key = strings.TrimSpace(os.Getenv(sessionCookieEncryptionKeyEnv)) + } + if key == "" { + return "", nil + } + + if validSessionCookieEncryptionKeyLength(len([]byte(key))) { + return key, nil + } + + for _, encoding := range []*base64.Encoding{ + base64.StdEncoding, + base64.RawStdEncoding, + base64.URLEncoding, + base64.RawURLEncoding, + } { + decoded, err := encoding.DecodeString(key) + if err == nil && validSessionCookieEncryptionKeyLength(len(decoded)) { + return string(decoded), nil + } + } + + return "", fmt.Errorf("invalid embedded IdP session cookie encryption key: %s (or sessionCookieEncryptionKey) must be 16, 24, or 32 bytes as a raw string or base64-encoded to one of those lengths; got %d raw bytes", sessionCookieEncryptionKeyEnv, len([]byte(key))) +} + +func validSessionCookieEncryptionKeyLength(length int) bool { + switch length { + case 16, 24, 32: + return true + default: + return false + } +} + // Compile-time check that EmbeddedIdPManager implements Manager interface var _ Manager = (*EmbeddedIdPManager)(nil) @@ -215,6 +344,7 @@ type EmbeddedIdPManager struct { provider *dex.Provider appMetrics telemetry.AppMetrics config EmbeddedIdPConfig + mfaEnabled bool } // NewEmbeddedIdPManager creates a new instance of EmbeddedIdPManager from a configuration. @@ -641,6 +771,27 @@ func (m *EmbeddedIdPManager) IsLocalAuthDisabled() bool { return m.config.LocalAuthDisabled } +// SetMFAEnabled enables or disables TOTP MFA for local users by updating the MFAChain on OAuth2 clients. +func (m *EmbeddedIdPManager) SetMFAEnabled(ctx context.Context, enabled bool) error { + var mfaChain []string + if enabled { + mfaChain = []string{"default-totp"} + } + if err := m.provider.SetClientsMFAChain(ctx, []string{ + staticClientCLI, + staticClientDashboard, + }, mfaChain); err != nil { + return fmt.Errorf("failed to set MFA enabled=%v: %w", enabled, err) + } + m.mfaEnabled = enabled + return nil +} + +// IsMFAEnabled returns whether TOTP MFA is currently enabled for local users. +func (m *EmbeddedIdPManager) IsMFAEnabled() bool { + return m.mfaEnabled +} + // HasNonLocalConnectors checks if there are any identity provider connectors other than local. func (m *EmbeddedIdPManager) HasNonLocalConnectors(ctx context.Context) (bool, error) { return m.provider.HasNonLocalConnectors(ctx) diff --git a/management/server/idp/embedded_test.go b/management/server/idp/embedded_test.go index 4dda483fb..09dc67614 100644 --- a/management/server/idp/embedded_test.go +++ b/management/server/idp/embedded_test.go @@ -2,6 +2,7 @@ package idp import ( "context" + "encoding/base64" "os" "path/filepath" "testing" @@ -313,6 +314,72 @@ func TestEmbeddedIdPManager_UpdateUserPassword(t *testing.T) { }) } +func TestEmbeddedIdPConfig_ToYAMLConfig_SessionCookieEncryptionKey(t *testing.T) { + t.Setenv(sessionCookieEncryptionKeyEnv, "") + + rawKey := "0123456789abcdef0123456789abcdef" + config := &EmbeddedIdPConfig{ + Enabled: true, + Issuer: "http://localhost:5556/dex", + SessionCookieEncryptionKey: base64.StdEncoding.EncodeToString([]byte(rawKey)), + Storage: EmbeddedStorageConfig{ + Type: "sqlite3", + Config: EmbeddedStorageTypeConfig{ + File: filepath.Join(t.TempDir(), "dex.db"), + }, + }, + } + + yamlConfig, err := config.ToYAMLConfig() + require.NoError(t, err) + require.NotNil(t, yamlConfig.Sessions) + assert.Equal(t, rawKey, yamlConfig.Sessions.CookieEncryptionKey) +} + +func TestResolveSessionCookieEncryptionKey(t *testing.T) { + rawKey := "0123456789abcdef0123456789abcdef" + + t.Run("uses raw configured key", func(t *testing.T) { + t.Setenv(sessionCookieEncryptionKeyEnv, "") + + key, err := resolveSessionCookieEncryptionKey(rawKey) + require.NoError(t, err) + assert.Equal(t, rawKey, key) + }) + + t.Run("uses base64 configured key", func(t *testing.T) { + t.Setenv(sessionCookieEncryptionKeyEnv, "") + + key, err := resolveSessionCookieEncryptionKey(base64.StdEncoding.EncodeToString([]byte(rawKey))) + require.NoError(t, err) + assert.Equal(t, rawKey, key) + }) + + t.Run("falls back to env var", func(t *testing.T) { + t.Setenv(sessionCookieEncryptionKeyEnv, rawKey) + + key, err := resolveSessionCookieEncryptionKey("") + require.NoError(t, err) + assert.Equal(t, rawKey, key) + }) + + t.Run("empty key disables encryption", func(t *testing.T) { + t.Setenv(sessionCookieEncryptionKeyEnv, "") + + key, err := resolveSessionCookieEncryptionKey("") + require.NoError(t, err) + assert.Empty(t, key) + }) + + t.Run("rejects invalid key length", func(t *testing.T) { + t.Setenv(sessionCookieEncryptionKeyEnv, "") + + _, err := resolveSessionCookieEncryptionKey("32") + require.Error(t, err) + assert.Contains(t, err.Error(), sessionCookieEncryptionKeyEnv) + }) +} + func TestEmbeddedIdPManager_GetLocalKeysLocation(t *testing.T) { ctx := context.Background() diff --git a/management/server/store/sql_store.go b/management/server/store/sql_store.go index 973101ce3..065a0d306 100644 --- a/management/server/store/sql_store.go +++ b/management/server/store/sql_store.go @@ -236,7 +236,6 @@ func (s *SqlStore) GetPeerJobs(ctx context.Context, accountID, peerID string) ([ Where(accountAndPeerIDQueryCondition, accountID, peerID). Order("created_at DESC"). Find(&jobs).Error - if err != nil { log.WithContext(ctx).Errorf("failed to fetch jobs from store: %s", err) return nil, err @@ -463,7 +462,6 @@ func (s *SqlStore) SavePeer(ctx context.Context, accountID string, peer *nbpeer. return nil }) - if err != nil { return err } @@ -1514,6 +1512,7 @@ func (s *SqlStore) getAccount(ctx context.Context, accountID string) (*types.Acc settings_jwt_groups_enabled, settings_jwt_groups_claim_name, settings_jwt_allow_groups, settings_routing_peer_dns_resolution_enabled, settings_dns_domain, settings_network_range, settings_network_range_v6, settings_ipv6_enabled_groups, settings_lazy_connection_enabled, + settings_local_mfa_enabled, -- Embedded ExtraSettings settings_extra_peer_approval_enabled, settings_extra_user_approval_required, settings_extra_integrated_validator, settings_extra_integrated_validator_groups @@ -1535,6 +1534,7 @@ func (s *SqlStore) getAccount(ctx context.Context, accountID string) (*types.Acc sNetworkRangeV6 sql.NullString sIPv6EnabledGroups sql.NullString sLazyConnectionEnabled sql.NullBool + sLocalMFAEnabled sql.NullBool sExtraPeerApprovalEnabled sql.NullBool sExtraUserApprovalRequired sql.NullBool sExtraIntegratedValidator sql.NullString @@ -1557,6 +1557,7 @@ func (s *SqlStore) getAccount(ctx context.Context, accountID string) (*types.Acc &sJWTGroupsEnabled, &sJWTGroupsClaimName, &sJWTAllowGroups, &sRoutingPeerDNSResolutionEnabled, &sDNSDomain, &sNetworkRange, &sNetworkRangeV6, &sIPv6EnabledGroups, &sLazyConnectionEnabled, + &sLocalMFAEnabled, &sExtraPeerApprovalEnabled, &sExtraUserApprovalRequired, &sExtraIntegratedValidator, &sExtraIntegratedValidatorGroups, ) @@ -1619,6 +1620,9 @@ func (s *SqlStore) getAccount(ctx context.Context, accountID string) (*types.Acc if sLazyConnectionEnabled.Valid { account.Settings.LazyConnectionEnabled = sLazyConnectionEnabled.Bool } + if sLocalMFAEnabled.Valid { + account.Settings.LocalMfaEnabled = sLocalMFAEnabled.Bool + } if sJWTAllowGroups.Valid { _ = json.Unmarshal([]byte(sJWTAllowGroups.String), &account.Settings.JWTAllowGroups) } @@ -3061,7 +3065,6 @@ func (s *SqlStore) AddPeerToAllGroup(ctx context.Context, accountID string, peer GroupID: groupID, PeerID: peerID, }).Error - if err != nil { return status.Errorf(status.Internal, "error adding peer to group 'All': %v", err) } @@ -3081,7 +3084,6 @@ func (s *SqlStore) AddPeerToGroup(ctx context.Context, accountID, peerID, groupI Columns: []clause.Column{{Name: "group_id"}, {Name: "peer_id"}}, DoNothing: true, }).Create(peer).Error - if err != nil { log.WithContext(ctx).Errorf("failed to add peer %s to group %s for account %s: %v", peerID, groupID, accountID, err) return status.Errorf(status.Internal, "failed to add peer to group") @@ -3094,7 +3096,6 @@ func (s *SqlStore) AddPeerToGroup(ctx context.Context, accountID, peerID, groupI func (s *SqlStore) RemovePeerFromGroup(ctx context.Context, peerID string, groupID string) error { err := s.db. Delete(&types.GroupPeer{}, "group_id = ? AND peer_id = ?", groupID, peerID).Error - if err != nil { log.WithContext(ctx).Errorf("failed to remove peer %s from group %s: %v", peerID, groupID, err) return status.Errorf(status.Internal, "failed to remove peer from group") @@ -3107,7 +3108,6 @@ func (s *SqlStore) RemovePeerFromGroup(ctx context.Context, peerID string, group func (s *SqlStore) RemovePeerFromAllGroups(ctx context.Context, peerID string) error { err := s.db. Delete(&types.GroupPeer{}, "peer_id = ?", peerID).Error - if err != nil { log.WithContext(ctx).Errorf("failed to remove peer %s from all groups: %v", peerID, err) return status.Errorf(status.Internal, "failed to remove peer from all groups") @@ -4964,7 +4964,6 @@ func (s *SqlStore) UpdateService(ctx context.Context, service *rpservice.Service return nil }) - if err != nil { log.WithContext(ctx).Errorf("failed to update service to store: %v", err) return status.Errorf(status.Internal, "failed to update service to store") @@ -5620,7 +5619,6 @@ func (s *SqlStore) getClusterUnanimousCapability(ctx context.Context, clusterAdd Where("cluster_address = ? AND status = ? AND last_seen > ?", clusterAddr, "connected", time.Now().Add(-proxyActiveThreshold)). Scan(&result).Error - if err != nil { log.WithContext(ctx).Errorf("query cluster capability %s for %s: %v", column, clusterAddr, err) return nil @@ -5662,7 +5660,6 @@ func (s *SqlStore) getClusterCapability(ctx context.Context, clusterAddr, column Where("cluster_address = ? AND status = ? AND last_seen > ?", clusterAddr, "connected", time.Now().Add(-proxyActiveThreshold)). Scan(&result).Error - if err != nil { log.WithContext(ctx).Errorf("query cluster capability %s for %s: %v", column, clusterAddr, err) return nil diff --git a/management/server/types/settings.go b/management/server/types/settings.go index 264a018d4..97ffa5e76 100644 --- a/management/server/types/settings.go +++ b/management/server/types/settings.go @@ -80,6 +80,10 @@ type Settings struct { // LocalAuthDisabled indicates if local (email/password) authentication is disabled. // This is a runtime-only field, not stored in the database. LocalAuthDisabled bool `gorm:"-"` + + // LocalMfaEnabled indicates if TOTP MFA is enabled for local users. + // Only applicable when the embedded IDP is enabled. + LocalMfaEnabled bool } // Copy copies the Settings struct @@ -108,6 +112,7 @@ func (s *Settings) Copy() *Settings { IPv6EnabledGroups: slices.Clone(s.IPv6EnabledGroups), EmbeddedIdpEnabled: s.EmbeddedIdpEnabled, LocalAuthDisabled: s.LocalAuthDisabled, + LocalMfaEnabled: s.LocalMfaEnabled, } if s.Extra != nil { settings.Extra = s.Extra.Copy() diff --git a/shared/management/http/api/openapi.yml b/shared/management/http/api/openapi.yml index 8e6ee54cc..82fca0782 100644 --- a/shared/management/http/api/openapi.yml +++ b/shared/management/http/api/openapi.yml @@ -381,6 +381,10 @@ components: type: boolean readOnly: true example: false + local_mfa_enabled: + description: Enables or disables TOTP multi-factor authentication for local users. Only applicable when the embedded identity provider is enabled. + type: boolean + example: false ipv6_enabled_groups: description: List of group IDs whose peers receive IPv6 overlay addresses. Peers not in any of these groups will not be allocated an IPv6 address. New accounts default to the All group. type: array diff --git a/shared/management/http/api/types.gen.go b/shared/management/http/api/types.gen.go index f8ea07be7..4b94ea01c 100644 --- a/shared/management/http/api/types.gen.go +++ b/shared/management/http/api/types.gen.go @@ -1486,6 +1486,9 @@ type AccountSettings struct { // LocalAuthDisabled Indicates whether local (email/password) authentication is disabled. When true, users can only authenticate via external identity providers. This is a read-only field. LocalAuthDisabled *bool `json:"local_auth_disabled,omitempty"` + // LocalMfaEnabled Enables or disables TOTP multi-factor authentication for local users. Only applicable when the embedded identity provider is enabled. + LocalMfaEnabled *bool `json:"local_mfa_enabled,omitempty"` + // NetworkRange Allows to define a custom network range for the account in CIDR format NetworkRange *string `json:"network_range,omitempty"` From afb83b3049c3c0b5b578f051e277562c6b8e513d Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Mon, 11 May 2026 16:58:49 +0900 Subject: [PATCH 02/29] [client] Use unique temp file and clean up on failure when writing ssh config (#6064) --- client/ssh/config/manager.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/client/ssh/config/manager.go b/client/ssh/config/manager.go index 01822ead6..b58bf2233 100644 --- a/client/ssh/config/manager.go +++ b/client/ssh/config/manager.go @@ -229,18 +229,31 @@ func (m *Manager) buildHostPatterns(peer PeerSSHInfo) []string { func (m *Manager) writeSSHConfig(sshConfig string) error { sshConfigPath := filepath.Join(m.sshConfigDir, m.sshConfigFile) - sshConfigPathTmp := sshConfigPath + ".tmp" if err := os.MkdirAll(m.sshConfigDir, 0755); err != nil { return fmt.Errorf("create SSH config directory %s: %w", m.sshConfigDir, err) } - if err := writeFileWithTimeout(sshConfigPathTmp, []byte(sshConfig), 0644); err != nil { - return fmt.Errorf("write SSH config file %s: %w", sshConfigPath, err) + tmp, err := os.CreateTemp(m.sshConfigDir, m.sshConfigFile+".*.tmp") + if err != nil { + return fmt.Errorf("create temp SSH config: %w", err) + } + tmpPath := tmp.Name() + defer func() { + if err := os.Remove(tmpPath); err != nil && !os.IsNotExist(err) { + log.Debugf("remove temp SSH config %s: %v", tmpPath, err) + } + }() + if err := tmp.Close(); err != nil { + return fmt.Errorf("close temp SSH config %s: %w", tmpPath, err) } - if err := os.Rename(sshConfigPathTmp, sshConfigPath); err != nil { - return fmt.Errorf("rename ssh config %s -> %s: %w", sshConfigPathTmp, sshConfigPath, err) + if err := writeFileWithTimeout(tmpPath, []byte(sshConfig), 0644); err != nil { + return fmt.Errorf("write SSH config file %s: %w", tmpPath, err) + } + + if err := os.Rename(tmpPath, sshConfigPath); err != nil { + return fmt.Errorf("rename SSH config %s -> %s: %w", tmpPath, sshConfigPath, err) } log.Infof("Created NetBird SSH client config: %s", sshConfigPath) From a852b3bd34f82c3d548874db4eb2b1b186320225 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Mon, 11 May 2026 16:59:13 +0900 Subject: [PATCH 03/29] [client, proxy] Harden uspfilter conntrack and share TCP relay (#5936) --- .../firewall/uspfilter/conntrack/cap_test.go | 125 ++++++ client/firewall/uspfilter/conntrack/common.go | 46 ++ .../uspfilter/conntrack/defaults_desktop.go | 11 + .../uspfilter/conntrack/defaults_mobile.go | 13 + client/firewall/uspfilter/conntrack/icmp.go | 50 ++- client/firewall/uspfilter/conntrack/tcp.go | 398 +++++++++++++----- .../uspfilter/conntrack/tcp_rst_bugs_test.go | 100 +++++ .../conntrack/tcp_state_bugs_test.go | 235 +++++++++++ client/firewall/uspfilter/conntrack/udp.go | 52 ++- client/firewall/uspfilter/filter.go | 42 +- client/firewall/uspfilter/forwarder/icmp.go | 31 +- client/firewall/uspfilter/forwarder/tcp.go | 75 +--- client/firewall/uspfilter/forwarder/udp.go | 16 +- client/firewall/uspfilter/log/log.go | 137 +++--- client/firewall/uspfilter/nat.go | 21 +- client/ssh/client/client.go | 26 +- client/ssh/common.go | 60 --- client/ssh/proxy/proxy.go | 5 +- client/ssh/server/port_forwarding.go | 4 +- client/ssh/server/server.go | 33 +- proxy/internal/tcp/peekedconn.go | 6 + proxy/internal/tcp/relay.go | 156 ------- proxy/internal/tcp/relay_test.go | 15 +- proxy/internal/tcp/router.go | 8 +- util/netrelay/relay.go | 238 +++++++++++ util/netrelay/relay_test.go | 221 ++++++++++ 26 files changed, 1629 insertions(+), 495 deletions(-) create mode 100644 client/firewall/uspfilter/conntrack/cap_test.go create mode 100644 client/firewall/uspfilter/conntrack/defaults_desktop.go create mode 100644 client/firewall/uspfilter/conntrack/defaults_mobile.go create mode 100644 client/firewall/uspfilter/conntrack/tcp_rst_bugs_test.go create mode 100644 client/firewall/uspfilter/conntrack/tcp_state_bugs_test.go delete mode 100644 proxy/internal/tcp/relay.go create mode 100644 util/netrelay/relay.go create mode 100644 util/netrelay/relay_test.go diff --git a/client/firewall/uspfilter/conntrack/cap_test.go b/client/firewall/uspfilter/conntrack/cap_test.go new file mode 100644 index 000000000..ee6b72e7f --- /dev/null +++ b/client/firewall/uspfilter/conntrack/cap_test.go @@ -0,0 +1,125 @@ +package conntrack + +import ( + "net/netip" + "testing" + + "github.com/google/gopacket/layers" + "github.com/stretchr/testify/require" +) + +func TestTCPCapEvicts(t *testing.T) { + t.Setenv(EnvTCPMaxEntries, "4") + + tracker := NewTCPTracker(DefaultTCPTimeout, logger, flowLogger) + defer tracker.Close() + require.Equal(t, 4, tracker.maxEntries) + + src := netip.MustParseAddr("100.64.0.1") + dst := netip.MustParseAddr("100.64.0.2") + + for i := 0; i < 10; i++ { + tracker.TrackOutbound(src, dst, uint16(10000+i), 80, TCPSyn, 0) + } + require.LessOrEqual(t, len(tracker.connections), 4, + "TCP table must not exceed the configured cap") + require.Greater(t, len(tracker.connections), 0, + "some entries must remain after eviction") + + // The most recently admitted flow must be present: eviction must make + // room for new entries, not silently drop them. + require.Contains(t, tracker.connections, + ConnKey{SrcIP: src, DstIP: dst, SrcPort: uint16(10009), DstPort: 80}, + "newest TCP flow must be admitted after eviction") + // A pre-cap flow must have been evicted to fit the last one. + require.NotContains(t, tracker.connections, + ConnKey{SrcIP: src, DstIP: dst, SrcPort: uint16(10000), DstPort: 80}, + "oldest TCP flow should have been evicted") +} + +func TestTCPCapPrefersTombstonedForEviction(t *testing.T) { + t.Setenv(EnvTCPMaxEntries, "3") + + tracker := NewTCPTracker(DefaultTCPTimeout, logger, flowLogger) + defer tracker.Close() + + src := netip.MustParseAddr("100.64.0.1") + dst := netip.MustParseAddr("100.64.0.2") + + // Fill to cap with 3 live connections. + for i := 0; i < 3; i++ { + tracker.TrackOutbound(src, dst, uint16(20000+i), 80, TCPSyn, 0) + } + require.Len(t, tracker.connections, 3) + + // Tombstone one by sending RST through IsValidInbound. + tombstonedKey := ConnKey{SrcIP: src, DstIP: dst, SrcPort: 20001, DstPort: 80} + require.True(t, tracker.IsValidInbound(dst, src, 80, 20001, TCPRst|TCPAck, 0)) + require.True(t, tracker.connections[tombstonedKey].IsTombstone()) + + // Another live connection forces eviction. The tombstone must go first. + tracker.TrackOutbound(src, dst, uint16(29999), 80, TCPSyn, 0) + + _, tombstonedStillPresent := tracker.connections[tombstonedKey] + require.False(t, tombstonedStillPresent, + "tombstoned entry should be evicted before live entries") + require.LessOrEqual(t, len(tracker.connections), 3) + + // Both live pre-cap entries must survive: eviction must prefer the + // tombstone, not just satisfy the size bound by dropping any entry. + require.Contains(t, tracker.connections, + ConnKey{SrcIP: src, DstIP: dst, SrcPort: uint16(20000), DstPort: 80}, + "live entries must not be evicted while a tombstone exists") + require.Contains(t, tracker.connections, + ConnKey{SrcIP: src, DstIP: dst, SrcPort: uint16(20002), DstPort: 80}, + "live entries must not be evicted while a tombstone exists") +} + +func TestUDPCapEvicts(t *testing.T) { + t.Setenv(EnvUDPMaxEntries, "5") + + tracker := NewUDPTracker(DefaultUDPTimeout, logger, flowLogger) + defer tracker.Close() + require.Equal(t, 5, tracker.maxEntries) + + src := netip.MustParseAddr("100.64.0.1") + dst := netip.MustParseAddr("100.64.0.2") + + for i := 0; i < 12; i++ { + tracker.TrackOutbound(src, dst, uint16(30000+i), 53, 0) + } + require.LessOrEqual(t, len(tracker.connections), 5) + require.Greater(t, len(tracker.connections), 0) + + require.Contains(t, tracker.connections, + ConnKey{SrcIP: src, DstIP: dst, SrcPort: uint16(30011), DstPort: 53}, + "newest UDP flow must be admitted after eviction") + require.NotContains(t, tracker.connections, + ConnKey{SrcIP: src, DstIP: dst, SrcPort: uint16(30000), DstPort: 53}, + "oldest UDP flow should have been evicted") +} + +func TestICMPCapEvicts(t *testing.T) { + t.Setenv(EnvICMPMaxEntries, "3") + + tracker := NewICMPTracker(DefaultICMPTimeout, logger, flowLogger) + defer tracker.Close() + require.Equal(t, 3, tracker.maxEntries) + + src := netip.MustParseAddr("100.64.0.1") + dst := netip.MustParseAddr("100.64.0.2") + + echoReq := layers.CreateICMPv4TypeCode(uint8(layers.ICMPv4TypeEchoRequest), 0) + for i := 0; i < 8; i++ { + tracker.TrackOutbound(src, dst, uint16(i), echoReq, nil, 64) + } + require.LessOrEqual(t, len(tracker.connections), 3) + require.Greater(t, len(tracker.connections), 0) + + require.Contains(t, tracker.connections, + ICMPConnKey{SrcIP: src, DstIP: dst, ID: uint16(7)}, + "newest ICMP flow must be admitted after eviction") + require.NotContains(t, tracker.connections, + ICMPConnKey{SrcIP: src, DstIP: dst, ID: uint16(0)}, + "oldest ICMP flow should have been evicted") +} diff --git a/client/firewall/uspfilter/conntrack/common.go b/client/firewall/uspfilter/conntrack/common.go index 88e90317c..e497a0bff 100644 --- a/client/firewall/uspfilter/conntrack/common.go +++ b/client/firewall/uspfilter/conntrack/common.go @@ -3,15 +3,61 @@ package conntrack import ( "net" "net/netip" + "os" "strconv" "sync/atomic" "time" "github.com/google/uuid" + nblog "github.com/netbirdio/netbird/client/firewall/uspfilter/log" nftypes "github.com/netbirdio/netbird/client/internal/netflow/types" ) +// evictSampleSize bounds how many map entries we scan per eviction call. +// Keeps eviction O(1) even at cap under sustained load; the sampled-LRU +// heuristic is good enough for a conntrack table that only overflows under +// abuse. +const evictSampleSize = 8 + +// envDuration parses an os.Getenv(name) as a time.Duration. Falls back to +// def on empty or invalid; logs a warning on invalid. +func envDuration(logger *nblog.Logger, name string, def time.Duration) time.Duration { + v := os.Getenv(name) + if v == "" { + return def + } + d, err := time.ParseDuration(v) + if err != nil { + logger.Warn3("invalid %s=%q: %v, using default", name, v, err) + return def + } + if d <= 0 { + logger.Warn2("invalid %s=%q: must be positive, using default", name, v) + return def + } + return d +} + +// envInt parses an os.Getenv(name) as an int. Falls back to def on empty, +// invalid, or non-positive. Logs a warning on invalid input. +func envInt(logger *nblog.Logger, name string, def int) int { + v := os.Getenv(name) + if v == "" { + return def + } + n, err := strconv.Atoi(v) + switch { + case err != nil: + logger.Warn3("invalid %s=%q: %v, using default", name, v, err) + return def + case n <= 0: + logger.Warn2("invalid %s=%q: must be positive, using default", name, v) + return def + } + return n +} + // BaseConnTrack provides common fields and locking for all connection types type BaseConnTrack struct { FlowId uuid.UUID diff --git a/client/firewall/uspfilter/conntrack/defaults_desktop.go b/client/firewall/uspfilter/conntrack/defaults_desktop.go new file mode 100644 index 000000000..2f07f5984 --- /dev/null +++ b/client/firewall/uspfilter/conntrack/defaults_desktop.go @@ -0,0 +1,11 @@ +//go:build !ios && !android + +package conntrack + +// Default per-tracker entry caps on desktop/server platforms. These mirror +// typical Linux netfilter nf_conntrack_max territory with ample headroom. +const ( + DefaultMaxTCPEntries = 65536 + DefaultMaxUDPEntries = 16384 + DefaultMaxICMPEntries = 2048 +) diff --git a/client/firewall/uspfilter/conntrack/defaults_mobile.go b/client/firewall/uspfilter/conntrack/defaults_mobile.go new file mode 100644 index 000000000..c9e05d229 --- /dev/null +++ b/client/firewall/uspfilter/conntrack/defaults_mobile.go @@ -0,0 +1,13 @@ +//go:build ios || android + +package conntrack + +// Default per-tracker entry caps on mobile platforms. iOS network extensions +// are capped at ~50 MB; Android runs under aggressive memory pressure. These +// values keep conntrack footprint well under 5 MB worst case (TCPConnTrack +// is ~200 B plus map overhead). +const ( + DefaultMaxTCPEntries = 4096 + DefaultMaxUDPEntries = 2048 + DefaultMaxICMPEntries = 512 +) diff --git a/client/firewall/uspfilter/conntrack/icmp.go b/client/firewall/uspfilter/conntrack/icmp.go index a48215ca9..3c96548b5 100644 --- a/client/firewall/uspfilter/conntrack/icmp.go +++ b/client/firewall/uspfilter/conntrack/icmp.go @@ -50,6 +50,9 @@ type ICMPConnTrack struct { ICMPCode uint8 } +// EnvICMPMaxEntries caps the ICMP conntrack table size. +const EnvICMPMaxEntries = "NB_CONNTRACK_ICMP_MAX" + // ICMPTracker manages ICMP connection states type ICMPTracker struct { logger *nblog.Logger @@ -58,6 +61,7 @@ type ICMPTracker struct { cleanupTicker *time.Ticker tickerCancel context.CancelFunc mutex sync.RWMutex + maxEntries int flowLogger nftypes.FlowLogger } @@ -171,6 +175,7 @@ func NewICMPTracker(timeout time.Duration, logger *nblog.Logger, flowLogger nfty timeout: timeout, cleanupTicker: time.NewTicker(ICMPCleanupInterval), tickerCancel: cancel, + maxEntries: envInt(logger, EnvICMPMaxEntries, DefaultMaxICMPEntries), flowLogger: flowLogger, } @@ -257,7 +262,9 @@ func (t *ICMPTracker) track( // non echo requests don't need tracking if typ != uint8(layers.ICMPv4TypeEchoRequest) { - t.logger.Trace3("New %s ICMP connection %s - %s", direction, key, icmpInfo) + if t.logger.Enabled(nblog.LevelTrace) { + t.logger.Trace3("New %s ICMP connection %s - %s", direction, key, icmpInfo) + } t.sendStartEvent(direction, srcIP, dstIP, typ, code, ruleId, size) return } @@ -276,10 +283,15 @@ func (t *ICMPTracker) track( conn.UpdateCounters(direction, size) t.mutex.Lock() + if t.maxEntries > 0 && len(t.connections) >= t.maxEntries { + t.evictOneLocked() + } t.connections[key] = conn t.mutex.Unlock() - t.logger.Trace3("New %s ICMP connection %s - %s", direction, key, icmpInfo) + if t.logger.Enabled(nblog.LevelTrace) { + t.logger.Trace3("New %s ICMP connection %s - %s", direction, key, icmpInfo) + } t.sendEvent(nftypes.TypeStart, conn, ruleId) } @@ -323,6 +335,34 @@ func (t *ICMPTracker) cleanupRoutine(ctx context.Context) { } } +// evictOneLocked removes one entry to make room. Caller must hold t.mutex. +// Bounded sample scan: picks the oldest among up to evictSampleSize entries. +func (t *ICMPTracker) evictOneLocked() { + var candKey ICMPConnKey + var candSeen int64 + haveCand := false + sampled := 0 + + for k, c := range t.connections { + seen := c.lastSeen.Load() + if !haveCand || seen < candSeen { + candKey = k + candSeen = seen + haveCand = true + } + sampled++ + if sampled >= evictSampleSize { + break + } + } + if haveCand { + if evicted := t.connections[candKey]; evicted != nil { + t.sendEvent(nftypes.TypeEnd, evicted, nil) + } + delete(t.connections, candKey) + } +} + func (t *ICMPTracker) cleanup() { t.mutex.Lock() defer t.mutex.Unlock() @@ -331,8 +371,10 @@ func (t *ICMPTracker) cleanup() { if conn.timeoutExceeded(t.timeout) { delete(t.connections, key) - t.logger.Trace5("Removed ICMP connection %s (timeout) [in: %d Pkts/%d B out: %d Pkts/%d B]", - key, conn.PacketsRx.Load(), conn.BytesRx.Load(), conn.PacketsTx.Load(), conn.BytesTx.Load()) + if t.logger.Enabled(nblog.LevelTrace) { + t.logger.Trace5("Removed ICMP connection %s (timeout) [in: %d Pkts/%d B out: %d Pkts/%d B]", + key, conn.PacketsRx.Load(), conn.BytesRx.Load(), conn.PacketsTx.Load(), conn.BytesTx.Load()) + } t.sendEvent(nftypes.TypeEnd, conn, nil) } } diff --git a/client/firewall/uspfilter/conntrack/tcp.go b/client/firewall/uspfilter/conntrack/tcp.go index 335a3abab..9edc9af22 100644 --- a/client/firewall/uspfilter/conntrack/tcp.go +++ b/client/firewall/uspfilter/conntrack/tcp.go @@ -38,6 +38,27 @@ const ( TCPHandshakeTimeout = 60 * time.Second // TCPCleanupInterval is how often we check for stale connections TCPCleanupInterval = 5 * time.Minute + // FinWaitTimeout bounds FIN_WAIT_1 / FIN_WAIT_2 / CLOSING states. + // Matches Linux netfilter nf_conntrack_tcp_timeout_fin_wait. + FinWaitTimeout = 60 * time.Second + // CloseWaitTimeout bounds CLOSE_WAIT. Matches Linux default; apps + // holding CloseWait longer than this should bump the env var. + CloseWaitTimeout = 60 * time.Second + // LastAckTimeout bounds LAST_ACK. Matches Linux default. + LastAckTimeout = 30 * time.Second +) + +// Env vars to override per-state teardown timeouts. Values parsed by +// time.ParseDuration (e.g. "120s", "2m"). Invalid values fall back to the +// defaults above with a warning. +const ( + EnvTCPFinWaitTimeout = "NB_CONNTRACK_TCP_FIN_WAIT_TIMEOUT" + EnvTCPCloseWaitTimeout = "NB_CONNTRACK_TCP_CLOSE_WAIT_TIMEOUT" + EnvTCPLastAckTimeout = "NB_CONNTRACK_TCP_LAST_ACK_TIMEOUT" + + // EnvTCPMaxEntries caps the TCP conntrack table size. Oldest entries + // (tombstones first) are evicted when the cap is reached. + EnvTCPMaxEntries = "NB_CONNTRACK_TCP_MAX" ) // TCPState represents the state of a TCP connection @@ -133,14 +154,18 @@ func (t *TCPConnTrack) SetTombstone() { // TCPTracker manages TCP connection states type TCPTracker struct { - logger *nblog.Logger - connections map[ConnKey]*TCPConnTrack - mutex sync.RWMutex - cleanupTicker *time.Ticker - tickerCancel context.CancelFunc - timeout time.Duration - waitTimeout time.Duration - flowLogger nftypes.FlowLogger + logger *nblog.Logger + connections map[ConnKey]*TCPConnTrack + mutex sync.RWMutex + cleanupTicker *time.Ticker + tickerCancel context.CancelFunc + timeout time.Duration + waitTimeout time.Duration + finWaitTimeout time.Duration + closeWaitTimeout time.Duration + lastAckTimeout time.Duration + maxEntries int + flowLogger nftypes.FlowLogger } // NewTCPTracker creates a new TCP connection tracker @@ -155,13 +180,17 @@ func NewTCPTracker(timeout time.Duration, logger *nblog.Logger, flowLogger nftyp ctx, cancel := context.WithCancel(context.Background()) tracker := &TCPTracker{ - logger: logger, - connections: make(map[ConnKey]*TCPConnTrack), - cleanupTicker: time.NewTicker(TCPCleanupInterval), - tickerCancel: cancel, - timeout: timeout, - waitTimeout: waitTimeout, - flowLogger: flowLogger, + logger: logger, + connections: make(map[ConnKey]*TCPConnTrack), + cleanupTicker: time.NewTicker(TCPCleanupInterval), + tickerCancel: cancel, + timeout: timeout, + waitTimeout: waitTimeout, + finWaitTimeout: envDuration(logger, EnvTCPFinWaitTimeout, FinWaitTimeout), + closeWaitTimeout: envDuration(logger, EnvTCPCloseWaitTimeout, CloseWaitTimeout), + lastAckTimeout: envDuration(logger, EnvTCPLastAckTimeout, LastAckTimeout), + maxEntries: envInt(logger, EnvTCPMaxEntries, DefaultMaxTCPEntries), + flowLogger: flowLogger, } go tracker.cleanupRoutine(ctx) @@ -209,6 +238,12 @@ func (t *TCPTracker) track(srcIP, dstIP netip.Addr, srcPort, dstPort uint16, fla if exists || flags&TCPSyn == 0 { return } + // Reject illegal SYN combinations (SYN+FIN, SYN+RST, …) so they don't + // create spurious conntrack entries. Not mandated by RFC 9293 but a + // common hardening (Linux netfilter/nftables rejects these too). + if !isValidFlagCombination(flags) { + return + } conn := &TCPConnTrack{ BaseConnTrack: BaseConnTrack{ @@ -225,20 +260,65 @@ func (t *TCPTracker) track(srcIP, dstIP netip.Addr, srcPort, dstPort uint16, fla conn.state.Store(int32(TCPStateNew)) conn.DNATOrigPort.Store(uint32(origPort)) - if origPort != 0 { - t.logger.Trace4("New %s TCP connection: %s (port DNAT %d -> %d)", direction, key, origPort, dstPort) - } else { - t.logger.Trace2("New %s TCP connection: %s", direction, key) + if t.logger.Enabled(nblog.LevelTrace) { + if origPort != 0 { + t.logger.Trace4("New %s TCP connection: %s (port DNAT %d -> %d)", direction, key, origPort, dstPort) + } else { + t.logger.Trace2("New %s TCP connection: %s", direction, key) + } } t.updateState(key, conn, flags, direction, size) t.mutex.Lock() + if t.maxEntries > 0 && len(t.connections) >= t.maxEntries { + t.evictOneLocked() + } t.connections[key] = conn t.mutex.Unlock() t.sendEvent(nftypes.TypeStart, conn, ruleID) } +// evictOneLocked removes one entry to make room. Caller must hold t.mutex. +// Bounded scan: samples up to evictSampleSize pseudo-random entries (Go map +// iteration order is randomized), preferring a tombstone. If no tombstone +// found in the sample, evicts the oldest among the sampled entries. O(1) +// worst case — cheap enough to run on every insert at cap during abuse. +func (t *TCPTracker) evictOneLocked() { + var candKey ConnKey + var candSeen int64 + haveCand := false + sampled := 0 + + for k, c := range t.connections { + if c.IsTombstone() { + delete(t.connections, k) + return + } + seen := c.lastSeen.Load() + if !haveCand || seen < candSeen { + candKey = k + candSeen = seen + haveCand = true + } + sampled++ + if sampled >= evictSampleSize { + break + } + } + if haveCand { + if evicted := t.connections[candKey]; evicted != nil { + // TypeEnd is already emitted at the state transition to + // TimeWait and when a connection is tombstoned. Only emit + // here when we're reaping a still-active flow. + if evicted.GetState() != TCPStateTimeWait && !evicted.IsTombstone() { + t.sendEvent(nftypes.TypeEnd, evicted, nil) + } + } + delete(t.connections, candKey) + } +} + // IsValidInbound checks if an inbound TCP packet matches a tracked connection func (t *TCPTracker) IsValidInbound(srcIP, dstIP netip.Addr, srcPort, dstPort uint16, flags uint8, size int) bool { key := ConnKey{ @@ -256,12 +336,19 @@ func (t *TCPTracker) IsValidInbound(srcIP, dstIP netip.Addr, srcPort, dstPort ui return false } + // Reject illegal flag combinations regardless of state. These never belong + // to a legitimate flow and must not advance or tear down state. + if !isValidFlagCombination(flags) { + if t.logger.Enabled(nblog.LevelWarn) { + t.logger.Warn3("TCP illegal flag combination %x for connection %s (state %s)", flags, key, conn.GetState()) + } + return false + } + currentState := conn.GetState() if !t.isValidStateForFlags(currentState, flags) { - t.logger.Warn3("TCP state %s is not valid with flags %x for connection %s", currentState, flags, key) - // allow all flags for established for now - if currentState == TCPStateEstablished { - return true + if t.logger.Enabled(nblog.LevelWarn) { + t.logger.Warn3("TCP state %s is not valid with flags %x for connection %s", currentState, flags, key) } return false } @@ -270,116 +357,208 @@ func (t *TCPTracker) IsValidInbound(srcIP, dstIP netip.Addr, srcPort, dstPort ui return true } -// updateState updates the TCP connection state based on flags +// updateState updates the TCP connection state based on flags. func (t *TCPTracker) updateState(key ConnKey, conn *TCPConnTrack, flags uint8, packetDir nftypes.Direction, size int) { - conn.UpdateLastSeen() conn.UpdateCounters(packetDir, size) + // Malformed flag combinations must not refresh lastSeen or drive state, + // otherwise spoofed packets keep a dead flow alive past its timeout. + if !isValidFlagCombination(flags) { + return + } + + conn.UpdateLastSeen() + currentState := conn.GetState() if flags&TCPRst != 0 { - if conn.CompareAndSwapState(currentState, TCPStateClosed) { - conn.SetTombstone() - t.logger.Trace6("TCP connection reset: %s (dir: %s) [in: %d Pkts/%d B, out: %d Pkts/%d B]", - key, packetDir, conn.PacketsRx.Load(), conn.BytesRx.Load(), conn.PacketsTx.Load(), conn.BytesTx.Load()) - t.sendEvent(nftypes.TypeEnd, conn, nil) - } + // Hardening beyond RFC 9293 §3.10.7.4: without sequence tracking we + // cannot apply the RFC 5961 in-window RST check, so we conservatively + // reject RSTs that the spec would accept (TIME-WAIT with in-window + // SEQ, SynSent from same direction as own SYN, etc.). + t.handleRst(key, conn, currentState, packetDir) return } - var newState TCPState - switch currentState { - case TCPStateNew: - if flags&TCPSyn != 0 && flags&TCPAck == 0 { - if conn.Direction == nftypes.Egress { - newState = TCPStateSynSent - } else { - newState = TCPStateSynReceived - } - } + newState := nextState(currentState, conn.Direction, packetDir, flags) + if newState == 0 || !conn.CompareAndSwapState(currentState, newState) { + return + } + t.onTransition(key, conn, currentState, newState, packetDir) +} - case TCPStateSynSent: - if flags&TCPSyn != 0 && flags&TCPAck != 0 { - if packetDir != conn.Direction { - newState = TCPStateEstablished - } else { - // Simultaneous open - newState = TCPStateSynReceived - } - } +// handleRst processes a RST segment. Late RSTs in TimeWait and spoofed RSTs +// from the SYN direction are ignored; otherwise the flow is tombstoned. +func (t *TCPTracker) handleRst(key ConnKey, conn *TCPConnTrack, currentState TCPState, packetDir nftypes.Direction) { + // TimeWait exists to absorb late segments; don't let a late RST + // tombstone the entry and break same-4-tuple reuse. + if currentState == TCPStateTimeWait { + return + } + // A RST from the same direction as the SYN cannot be a legitimate + // response and must not tear down a half-open connection. + if currentState == TCPStateSynSent && packetDir == conn.Direction { + return + } + if !conn.CompareAndSwapState(currentState, TCPStateClosed) { + return + } + conn.SetTombstone() + if t.logger.Enabled(nblog.LevelTrace) { + t.logger.Trace6("TCP connection reset: %s (dir: %s) [in: %d Pkts/%d B, out: %d Pkts/%d B]", + key, packetDir, conn.PacketsRx.Load(), conn.BytesRx.Load(), conn.PacketsTx.Load(), conn.BytesTx.Load()) + } + t.sendEvent(nftypes.TypeEnd, conn, nil) +} - case TCPStateSynReceived: - if flags&TCPAck != 0 && flags&TCPSyn == 0 { - if packetDir == conn.Direction { - newState = TCPStateEstablished - } - } +// stateTransition describes one state's transition logic. It receives the +// packet's flags plus whether the packet direction matches the connection's +// origin direction (same=true means same side as the SYN initiator). Return 0 +// for no transition. +type stateTransition func(flags uint8, connDir nftypes.Direction, same bool) TCPState - case TCPStateEstablished: - if flags&TCPFin != 0 { - if packetDir == conn.Direction { - newState = TCPStateFinWait1 - } else { - newState = TCPStateCloseWait - } - } +// stateTable maps each state to its transition function. Centralized here so +// nextState stays trivial and each rule is easy to read in isolation. +var stateTable = map[TCPState]stateTransition{ + TCPStateNew: transNew, + TCPStateSynSent: transSynSent, + TCPStateSynReceived: transSynReceived, + TCPStateEstablished: transEstablished, + TCPStateFinWait1: transFinWait1, + TCPStateFinWait2: transFinWait2, + TCPStateClosing: transClosing, + TCPStateCloseWait: transCloseWait, + TCPStateLastAck: transLastAck, +} - case TCPStateFinWait1: - if packetDir != conn.Direction { - switch { - case flags&TCPFin != 0 && flags&TCPAck != 0: - newState = TCPStateClosing - case flags&TCPFin != 0: - newState = TCPStateClosing - case flags&TCPAck != 0: - newState = TCPStateFinWait2 - } - } +// nextState returns the target TCP state for the given current state and +// packet, or 0 if the packet does not trigger a transition. +func nextState(currentState TCPState, connDir, packetDir nftypes.Direction, flags uint8) TCPState { + fn, ok := stateTable[currentState] + if !ok { + return 0 + } + return fn(flags, connDir, packetDir == connDir) +} - case TCPStateFinWait2: - if flags&TCPFin != 0 { - newState = TCPStateTimeWait +func transNew(flags uint8, connDir nftypes.Direction, _ bool) TCPState { + if flags&TCPSyn != 0 && flags&TCPAck == 0 { + if connDir == nftypes.Egress { + return TCPStateSynSent } + return TCPStateSynReceived + } + return 0 +} - case TCPStateClosing: - if flags&TCPAck != 0 { - newState = TCPStateTimeWait +func transSynSent(flags uint8, _ nftypes.Direction, same bool) TCPState { + if flags&TCPSyn != 0 && flags&TCPAck != 0 { + if same { + return TCPStateSynReceived // simultaneous open } + return TCPStateEstablished + } + return 0 +} - case TCPStateCloseWait: - if flags&TCPFin != 0 { - newState = TCPStateLastAck - } +func transSynReceived(flags uint8, _ nftypes.Direction, same bool) TCPState { + if flags&TCPAck != 0 && flags&TCPSyn == 0 && same { + return TCPStateEstablished + } + return 0 +} - case TCPStateLastAck: - if flags&TCPAck != 0 { - newState = TCPStateClosed - } +func transEstablished(flags uint8, _ nftypes.Direction, same bool) TCPState { + if flags&TCPFin == 0 { + return 0 + } + if same { + return TCPStateFinWait1 + } + return TCPStateCloseWait +} + +// transFinWait1 handles the active-close peer response. A FIN carrying our +// ACK piggybacked goes straight to TIME-WAIT (RFC 9293 §3.10.7.4, FIN-WAIT-1: +// "if our FIN has been ACKed... enter the TIME-WAIT state"); a lone FIN moves +// to CLOSING; a pure ACK of our FIN moves to FIN-WAIT-2. +func transFinWait1(flags uint8, _ nftypes.Direction, same bool) TCPState { + if same { + return 0 + } + if flags&TCPFin != 0 && flags&TCPAck != 0 { + return TCPStateTimeWait + } + switch { + case flags&TCPFin != 0: + return TCPStateClosing + case flags&TCPAck != 0: + return TCPStateFinWait2 + } + return 0 +} + +// transFinWait2 ignores own-side FIN retransmits; only the peer's FIN advances. +func transFinWait2(flags uint8, _ nftypes.Direction, same bool) TCPState { + if flags&TCPFin != 0 && !same { + return TCPStateTimeWait + } + return 0 +} + +// transClosing completes a simultaneous close on the peer's ACK. +func transClosing(flags uint8, _ nftypes.Direction, same bool) TCPState { + if flags&TCPAck != 0 && !same { + return TCPStateTimeWait + } + return 0 +} + +// transCloseWait only advances to LastAck when WE send FIN, ignoring peer retransmits. +func transCloseWait(flags uint8, _ nftypes.Direction, same bool) TCPState { + if flags&TCPFin != 0 && same { + return TCPStateLastAck + } + return 0 +} + +// transLastAck closes the flow only on the peer's ACK (not our own ACK retransmits). +func transLastAck(flags uint8, _ nftypes.Direction, same bool) TCPState { + if flags&TCPAck != 0 && !same { + return TCPStateClosed + } + return 0 +} + +// onTransition handles logging and flow-event emission after a successful +// state transition. TimeWait and Closed are terminal for flow accounting. +func (t *TCPTracker) onTransition(key ConnKey, conn *TCPConnTrack, from, to TCPState, packetDir nftypes.Direction) { + traceOn := t.logger.Enabled(nblog.LevelTrace) + if traceOn { + t.logger.Trace4("TCP connection %s transitioned from %s to %s (dir: %s)", key, from, to, packetDir) } - if newState != 0 && conn.CompareAndSwapState(currentState, newState) { - t.logger.Trace4("TCP connection %s transitioned from %s to %s (dir: %s)", key, currentState, newState, packetDir) - - switch newState { - case TCPStateTimeWait: + switch to { + case TCPStateTimeWait: + if traceOn { t.logger.Trace5("TCP connection %s completed [in: %d Pkts/%d B, out: %d Pkts/%d B]", key, conn.PacketsRx.Load(), conn.BytesRx.Load(), conn.PacketsTx.Load(), conn.BytesTx.Load()) - t.sendEvent(nftypes.TypeEnd, conn, nil) - - case TCPStateClosed: - conn.SetTombstone() + } + t.sendEvent(nftypes.TypeEnd, conn, nil) + case TCPStateClosed: + conn.SetTombstone() + if traceOn { t.logger.Trace5("TCP connection %s closed gracefully [in: %d Pkts/%d, B out: %d Pkts/%d B]", key, conn.PacketsRx.Load(), conn.BytesRx.Load(), conn.PacketsTx.Load(), conn.BytesTx.Load()) - t.sendEvent(nftypes.TypeEnd, conn, nil) } + t.sendEvent(nftypes.TypeEnd, conn, nil) } } -// isValidStateForFlags checks if the TCP flags are valid for the current connection state +// isValidStateForFlags checks if the TCP flags are valid for the current +// connection state. Caller must have already verified the flag combination is +// legal via isValidFlagCombination. func (t *TCPTracker) isValidStateForFlags(state TCPState, flags uint8) bool { - if !isValidFlagCombination(flags) { - return false - } if flags&TCPRst != 0 { if state == TCPStateSynSent { return flags&TCPAck != 0 @@ -449,15 +628,24 @@ func (t *TCPTracker) cleanup() { timeout = t.waitTimeout case TCPStateEstablished: timeout = t.timeout + case TCPStateFinWait1, TCPStateFinWait2, TCPStateClosing: + timeout = t.finWaitTimeout + case TCPStateCloseWait: + timeout = t.closeWaitTimeout + case TCPStateLastAck: + timeout = t.lastAckTimeout default: + // SynSent / SynReceived / New timeout = TCPHandshakeTimeout } if conn.timeoutExceeded(timeout) { delete(t.connections, key) - t.logger.Trace6("Cleaned up timed-out TCP connection %s (%s) [in: %d Pkts/%d, B out: %d Pkts/%d B]", - key, conn.GetState(), conn.PacketsRx.Load(), conn.BytesRx.Load(), conn.PacketsTx.Load(), conn.BytesTx.Load()) + if t.logger.Enabled(nblog.LevelTrace) { + t.logger.Trace6("Cleaned up timed-out TCP connection %s (%s) [in: %d Pkts/%d, B out: %d Pkts/%d B]", + key, conn.GetState(), conn.PacketsRx.Load(), conn.BytesRx.Load(), conn.PacketsTx.Load(), conn.BytesTx.Load()) + } // event already handled by state change if currentState != TCPStateTimeWait { diff --git a/client/firewall/uspfilter/conntrack/tcp_rst_bugs_test.go b/client/firewall/uspfilter/conntrack/tcp_rst_bugs_test.go new file mode 100644 index 000000000..81d4f5710 --- /dev/null +++ b/client/firewall/uspfilter/conntrack/tcp_rst_bugs_test.go @@ -0,0 +1,100 @@ +package conntrack + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/require" +) + +// RST hygiene tests: the tracker currently closes the flow on any RST that +// matches the 4-tuple, regardless of direction or state. These tests cover +// the minimum checks we want (no SEQ tracking). + +func TestTCPRstInSynSentWrongDirection(t *testing.T) { + tracker := NewTCPTracker(DefaultTCPTimeout, logger, flowLogger) + defer tracker.Close() + + srcIP := netip.MustParseAddr("100.64.0.1") + dstIP := netip.MustParseAddr("100.64.0.2") + srcPort := uint16(12345) + dstPort := uint16(80) + key := ConnKey{SrcIP: srcIP, DstIP: dstIP, SrcPort: srcPort, DstPort: dstPort} + + tracker.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPSyn, 0) + conn := tracker.connections[key] + require.Equal(t, TCPStateSynSent, conn.GetState()) + + // A RST arriving in the same direction as the SYN (i.e. TrackOutbound) + // cannot be a legitimate response. It must not close the connection. + tracker.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPRst|TCPAck, 0) + require.Equal(t, TCPStateSynSent, conn.GetState(), + "RST in same direction as SYN must not close connection") + require.False(t, conn.IsTombstone()) +} + +func TestTCPRstInTimeWaitIgnored(t *testing.T) { + tracker := NewTCPTracker(DefaultTCPTimeout, logger, flowLogger) + defer tracker.Close() + + srcIP := netip.MustParseAddr("100.64.0.1") + dstIP := netip.MustParseAddr("100.64.0.2") + srcPort := uint16(12345) + dstPort := uint16(80) + key := ConnKey{SrcIP: srcIP, DstIP: dstIP, SrcPort: srcPort, DstPort: dstPort} + + // Drive to TIME-WAIT via active close. + establishConnection(t, tracker, srcIP, dstIP, srcPort, dstPort) + tracker.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPFin|TCPAck, 0) + require.True(t, tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPAck, 0)) + require.True(t, tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPFin|TCPAck, 0)) + tracker.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPAck, 0) + + conn := tracker.connections[key] + require.Equal(t, TCPStateTimeWait, conn.GetState()) + require.False(t, conn.IsTombstone(), "TIME-WAIT must not be tombstoned") + + // Late RST during TIME-WAIT must not tombstone the entry (TIME-WAIT + // exists to absorb late segments). + tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPRst, 0) + require.Equal(t, TCPStateTimeWait, conn.GetState(), + "RST in TIME-WAIT must not transition state") + require.False(t, conn.IsTombstone(), + "RST in TIME-WAIT must not tombstone the entry") +} + +func TestTCPIllegalFlagCombos(t *testing.T) { + tracker := NewTCPTracker(DefaultTCPTimeout, logger, flowLogger) + defer tracker.Close() + + srcIP := netip.MustParseAddr("100.64.0.1") + dstIP := netip.MustParseAddr("100.64.0.2") + srcPort := uint16(12345) + dstPort := uint16(80) + key := ConnKey{SrcIP: srcIP, DstIP: dstIP, SrcPort: srcPort, DstPort: dstPort} + + establishConnection(t, tracker, srcIP, dstIP, srcPort, dstPort) + conn := tracker.connections[key] + + // Illegal combos must be rejected and must not change state. + combos := []struct { + name string + flags uint8 + }{ + {"SYN+RST", TCPSyn | TCPRst}, + {"FIN+RST", TCPFin | TCPRst}, + {"SYN+FIN", TCPSyn | TCPFin}, + {"SYN+FIN+RST", TCPSyn | TCPFin | TCPRst}, + } + + for _, c := range combos { + t.Run(c.name, func(t *testing.T) { + before := conn.GetState() + valid := tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, c.flags, 0) + require.False(t, valid, "illegal flag combo must be rejected: %s", c.name) + require.Equal(t, before, conn.GetState(), + "illegal flag combo must not change state") + require.False(t, conn.IsTombstone()) + }) + } +} diff --git a/client/firewall/uspfilter/conntrack/tcp_state_bugs_test.go b/client/firewall/uspfilter/conntrack/tcp_state_bugs_test.go new file mode 100644 index 000000000..32112cd58 --- /dev/null +++ b/client/firewall/uspfilter/conntrack/tcp_state_bugs_test.go @@ -0,0 +1,235 @@ +package conntrack + +import ( + "net/netip" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// These tests exercise cases where the TCP state machine currently advances +// on retransmitted or wrong-direction segments and tears the flow down +// prematurely. They are expected to fail until the direction checks are added. + +func TestTCPCloseWaitRetransmittedPeerFIN(t *testing.T) { + tracker := NewTCPTracker(DefaultTCPTimeout, logger, flowLogger) + defer tracker.Close() + + srcIP := netip.MustParseAddr("100.64.0.1") + dstIP := netip.MustParseAddr("100.64.0.2") + srcPort := uint16(12345) + dstPort := uint16(80) + key := ConnKey{SrcIP: srcIP, DstIP: dstIP, SrcPort: srcPort, DstPort: dstPort} + + establishConnection(t, tracker, srcIP, dstIP, srcPort, dstPort) + + // Peer sends FIN -> CloseWait (our app has not yet closed). + valid := tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPFin|TCPAck, 0) + require.True(t, valid) + conn := tracker.connections[key] + require.Equal(t, TCPStateCloseWait, conn.GetState()) + + // Peer retransmits their FIN (ACK may have been delayed). We have NOT + // sent our FIN yet, so state must remain CloseWait. + valid = tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPFin|TCPAck, 0) + require.True(t, valid, "retransmitted peer FIN must still be accepted") + require.Equal(t, TCPStateCloseWait, conn.GetState(), + "retransmitted peer FIN must not advance CloseWait to LastAck") + + // Our app finally closes -> LastAck. + tracker.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPFin|TCPAck, 0) + require.Equal(t, TCPStateLastAck, conn.GetState()) + + // Peer ACK closes. + valid = tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPAck, 0) + require.True(t, valid) + require.Equal(t, TCPStateClosed, conn.GetState()) +} + +func TestTCPFinWait2RetransmittedOwnFIN(t *testing.T) { + tracker := NewTCPTracker(DefaultTCPTimeout, logger, flowLogger) + defer tracker.Close() + + srcIP := netip.MustParseAddr("100.64.0.1") + dstIP := netip.MustParseAddr("100.64.0.2") + srcPort := uint16(12345) + dstPort := uint16(80) + key := ConnKey{SrcIP: srcIP, DstIP: dstIP, SrcPort: srcPort, DstPort: dstPort} + + establishConnection(t, tracker, srcIP, dstIP, srcPort, dstPort) + + // We initiate close. + tracker.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPFin|TCPAck, 0) + valid := tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPAck, 0) + require.True(t, valid) + conn := tracker.connections[key] + require.Equal(t, TCPStateFinWait2, conn.GetState()) + + // Stray retransmit of our own FIN (same direction as originator) must + // NOT advance FinWait2 to TimeWait; only the peer's FIN should. + tracker.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPFin|TCPAck, 0) + require.Equal(t, TCPStateFinWait2, conn.GetState(), + "own FIN retransmit must not advance FinWait2 to TimeWait") + + // Peer FIN -> TimeWait. + valid = tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPFin|TCPAck, 0) + require.True(t, valid) + require.Equal(t, TCPStateTimeWait, conn.GetState()) +} + +func TestTCPLastAckDirectionCheck(t *testing.T) { + tracker := NewTCPTracker(DefaultTCPTimeout, logger, flowLogger) + defer tracker.Close() + + srcIP := netip.MustParseAddr("100.64.0.1") + dstIP := netip.MustParseAddr("100.64.0.2") + srcPort := uint16(12345) + dstPort := uint16(80) + key := ConnKey{SrcIP: srcIP, DstIP: dstIP, SrcPort: srcPort, DstPort: dstPort} + + establishConnection(t, tracker, srcIP, dstIP, srcPort, dstPort) + + // Drive to LastAck: peer FIN -> CloseWait, our FIN -> LastAck. + require.True(t, tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPFin|TCPAck, 0)) + tracker.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPFin|TCPAck, 0) + conn := tracker.connections[key] + require.Equal(t, TCPStateLastAck, conn.GetState()) + + // Our own ACK retransmit (same direction as originator) must NOT close. + tracker.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPAck, 0) + require.Equal(t, TCPStateLastAck, conn.GetState(), + "own ACK retransmit in LastAck must not transition to Closed") + + // Peer's ACK -> Closed. + require.True(t, tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPAck, 0)) + require.Equal(t, TCPStateClosed, conn.GetState()) +} + +func TestTCPFinWait1OwnAckDoesNotAdvance(t *testing.T) { + tracker := NewTCPTracker(DefaultTCPTimeout, logger, flowLogger) + defer tracker.Close() + + srcIP := netip.MustParseAddr("100.64.0.1") + dstIP := netip.MustParseAddr("100.64.0.2") + srcPort := uint16(12345) + dstPort := uint16(80) + key := ConnKey{SrcIP: srcIP, DstIP: dstIP, SrcPort: srcPort, DstPort: dstPort} + + establishConnection(t, tracker, srcIP, dstIP, srcPort, dstPort) + + tracker.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPFin|TCPAck, 0) + conn := tracker.connections[key] + require.Equal(t, TCPStateFinWait1, conn.GetState()) + + // Our own ACK retransmit (same direction as originator) must not advance. + tracker.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPAck, 0) + require.Equal(t, TCPStateFinWait1, conn.GetState(), + "own ACK in FinWait1 must not advance to FinWait2") +} + +func TestTCPPerStateTeardownTimeouts(t *testing.T) { + // Verify cleanup reaps entries in each teardown state at the configured + // per-state timeout, not at the single handshake timeout. + t.Setenv(EnvTCPFinWaitTimeout, "50ms") + t.Setenv(EnvTCPCloseWaitTimeout, "80ms") + t.Setenv(EnvTCPLastAckTimeout, "30ms") + + dstIP := netip.MustParseAddr("100.64.0.2") + dstPort := uint16(80) + + // Drives a connection to the target state, forces its lastSeen well + // beyond the configured timeout, runs cleanup, and asserts reaping. + cases := []struct { + name string + // drive takes a fresh tracker and returns the conn key after + // transitioning the flow into the intended teardown state. + drive func(t *testing.T, tr *TCPTracker, srcIP netip.Addr, srcPort uint16) (ConnKey, TCPState) + }{ + { + name: "FinWait1", + drive: func(t *testing.T, tr *TCPTracker, srcIP netip.Addr, srcPort uint16) (ConnKey, TCPState) { + establishConnection(t, tr, srcIP, dstIP, srcPort, dstPort) + tr.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPFin|TCPAck, 0) // → FinWait1 + return ConnKey{SrcIP: srcIP, DstIP: dstIP, SrcPort: srcPort, DstPort: dstPort}, TCPStateFinWait1 + }, + }, + { + name: "FinWait2", + drive: func(t *testing.T, tr *TCPTracker, srcIP netip.Addr, srcPort uint16) (ConnKey, TCPState) { + establishConnection(t, tr, srcIP, dstIP, srcPort, dstPort) + tr.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPFin|TCPAck, 0) // FinWait1 + require.True(t, tr.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPAck, 0)) // → FinWait2 + return ConnKey{SrcIP: srcIP, DstIP: dstIP, SrcPort: srcPort, DstPort: dstPort}, TCPStateFinWait2 + }, + }, + { + name: "CloseWait", + drive: func(t *testing.T, tr *TCPTracker, srcIP netip.Addr, srcPort uint16) (ConnKey, TCPState) { + establishConnection(t, tr, srcIP, dstIP, srcPort, dstPort) + require.True(t, tr.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPFin|TCPAck, 0)) // → CloseWait + return ConnKey{SrcIP: srcIP, DstIP: dstIP, SrcPort: srcPort, DstPort: dstPort}, TCPStateCloseWait + }, + }, + { + name: "LastAck", + drive: func(t *testing.T, tr *TCPTracker, srcIP netip.Addr, srcPort uint16) (ConnKey, TCPState) { + establishConnection(t, tr, srcIP, dstIP, srcPort, dstPort) + require.True(t, tr.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPFin|TCPAck, 0)) // CloseWait + tr.TrackOutbound(srcIP, dstIP, srcPort, dstPort, TCPFin|TCPAck, 0) // → LastAck + return ConnKey{SrcIP: srcIP, DstIP: dstIP, SrcPort: srcPort, DstPort: dstPort}, TCPStateLastAck + }, + }, + } + + // Use a unique source port per subtest so nothing aliases. + port := uint16(12345) + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + tracker := NewTCPTracker(DefaultTCPTimeout, logger, flowLogger) + defer tracker.Close() + + require.Equal(t, 50*time.Millisecond, tracker.finWaitTimeout) + require.Equal(t, 80*time.Millisecond, tracker.closeWaitTimeout) + require.Equal(t, 30*time.Millisecond, tracker.lastAckTimeout) + + srcIP := netip.MustParseAddr("100.64.0.1") + port++ + key, wantState := c.drive(t, tracker, srcIP, port) + conn := tracker.connections[key] + require.NotNil(t, conn) + require.Equal(t, wantState, conn.GetState()) + + // Age the entry past the largest per-state timeout. + conn.lastSeen.Store(time.Now().Add(-500 * time.Millisecond).UnixNano()) + tracker.cleanup() + _, exists := tracker.connections[key] + require.False(t, exists, "%s entry should be reaped", c.name) + }) + } +} + +func TestTCPEstablishedPSHACKInFinStates(t *testing.T) { + // Verifies FIN|PSH|ACK and bare ACK keepalives are not dropped in FIN + // teardown states, which some stacks emit during close. + tracker := NewTCPTracker(DefaultTCPTimeout, logger, flowLogger) + defer tracker.Close() + + srcIP := netip.MustParseAddr("100.64.0.1") + dstIP := netip.MustParseAddr("100.64.0.2") + srcPort := uint16(12345) + dstPort := uint16(80) + + establishConnection(t, tracker, srcIP, dstIP, srcPort, dstPort) + + // Peer FIN -> CloseWait. + require.True(t, tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPFin|TCPAck, 0)) + + // Peer pushes trailing data + FIN|PSH|ACK (legal). + require.True(t, tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPFin|TCPPush|TCPAck, 100), + "FIN|PSH|ACK in CloseWait must be accepted") + + // Bare ACK keepalive from peer in CloseWait must be accepted. + require.True(t, tracker.IsValidInbound(dstIP, srcIP, dstPort, srcPort, TCPAck, 0), + "bare ACK in CloseWait must be accepted") +} diff --git a/client/firewall/uspfilter/conntrack/udp.go b/client/firewall/uspfilter/conntrack/udp.go index a3b6a418b..335c5832a 100644 --- a/client/firewall/uspfilter/conntrack/udp.go +++ b/client/firewall/uspfilter/conntrack/udp.go @@ -17,6 +17,9 @@ const ( DefaultUDPTimeout = 30 * time.Second // UDPCleanupInterval is how often we check for stale connections UDPCleanupInterval = 15 * time.Second + + // EnvUDPMaxEntries caps the UDP conntrack table size. + EnvUDPMaxEntries = "NB_CONNTRACK_UDP_MAX" ) // UDPConnTrack represents a UDP connection state @@ -34,6 +37,7 @@ type UDPTracker struct { cleanupTicker *time.Ticker tickerCancel context.CancelFunc mutex sync.RWMutex + maxEntries int flowLogger nftypes.FlowLogger } @@ -51,6 +55,7 @@ func NewUDPTracker(timeout time.Duration, logger *nblog.Logger, flowLogger nftyp timeout: timeout, cleanupTicker: time.NewTicker(UDPCleanupInterval), tickerCancel: cancel, + maxEntries: envInt(logger, EnvUDPMaxEntries, DefaultMaxUDPEntries), flowLogger: flowLogger, } @@ -117,13 +122,18 @@ func (t *UDPTracker) track(srcIP netip.Addr, dstIP netip.Addr, srcPort uint16, d conn.UpdateCounters(direction, size) t.mutex.Lock() + if t.maxEntries > 0 && len(t.connections) >= t.maxEntries { + t.evictOneLocked() + } t.connections[key] = conn t.mutex.Unlock() - if origPort != 0 { - t.logger.Trace4("New %s UDP connection: %s (port DNAT %d -> %d)", direction, key, origPort, dstPort) - } else { - t.logger.Trace2("New %s UDP connection: %s", direction, key) + if t.logger.Enabled(nblog.LevelTrace) { + if origPort != 0 { + t.logger.Trace4("New %s UDP connection: %s (port DNAT %d -> %d)", direction, key, origPort, dstPort) + } else { + t.logger.Trace2("New %s UDP connection: %s", direction, key) + } } t.sendEvent(nftypes.TypeStart, conn, ruleID) } @@ -151,6 +161,34 @@ func (t *UDPTracker) IsValidInbound(srcIP netip.Addr, dstIP netip.Addr, srcPort return true } +// evictOneLocked removes one entry to make room. Caller must hold t.mutex. +// Bounded sample: picks the oldest among up to evictSampleSize entries. +func (t *UDPTracker) evictOneLocked() { + var candKey ConnKey + var candSeen int64 + haveCand := false + sampled := 0 + + for k, c := range t.connections { + seen := c.lastSeen.Load() + if !haveCand || seen < candSeen { + candKey = k + candSeen = seen + haveCand = true + } + sampled++ + if sampled >= evictSampleSize { + break + } + } + if haveCand { + if evicted := t.connections[candKey]; evicted != nil { + t.sendEvent(nftypes.TypeEnd, evicted, nil) + } + delete(t.connections, candKey) + } +} + // cleanupRoutine periodically removes stale connections func (t *UDPTracker) cleanupRoutine(ctx context.Context) { defer t.cleanupTicker.Stop() @@ -173,8 +211,10 @@ func (t *UDPTracker) cleanup() { if conn.timeoutExceeded(t.timeout) { delete(t.connections, key) - t.logger.Trace5("Removed UDP connection %s (timeout) [in: %d Pkts/%d B, out: %d Pkts/%d B]", - key, conn.PacketsRx.Load(), conn.BytesRx.Load(), conn.PacketsTx.Load(), conn.BytesTx.Load()) + if t.logger.Enabled(nblog.LevelTrace) { + t.logger.Trace5("Removed UDP connection %s (timeout) [in: %d Pkts/%d B, out: %d Pkts/%d B]", + key, conn.PacketsRx.Load(), conn.BytesRx.Load(), conn.PacketsTx.Load(), conn.BytesTx.Load()) + } t.sendEvent(nftypes.TypeEnd, conn, nil) } } diff --git a/client/firewall/uspfilter/filter.go b/client/firewall/uspfilter/filter.go index 5ecd08abf..91866dcab 100644 --- a/client/firewall/uspfilter/filter.go +++ b/client/firewall/uspfilter/filter.go @@ -787,7 +787,9 @@ func (m *Manager) filterOutbound(packetData []byte, size int) bool { srcIP, dstIP := m.extractIPs(d) if !srcIP.IsValid() { - m.logger.Error1("Unknown network layer: %v", d.decoded[0]) + if m.logger.Enabled(nblog.LevelError) { + m.logger.Error1("Unknown network layer: %v", d.decoded[0]) + } return false } @@ -901,7 +903,9 @@ func (m *Manager) clampTCPMSS(packetData []byte, d *decoder) bool { return false } - m.logger.Trace2("Clamped TCP MSS from %d to %d", currentMSS, mssClampValue) + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace2("Clamped TCP MSS from %d to %d", currentMSS, mssClampValue) + } return true } @@ -1044,11 +1048,13 @@ func (m *Manager) filterInbound(packetData []byte, size int) bool { // TODO: pass fragments of routed packets to forwarder if fragment { - if d.decoded[0] == layers.LayerTypeIPv4 { - m.logger.Trace4("packet is a fragment: src=%v dst=%v id=%v flags=%v", - srcIP, dstIP, d.ip4.Id, d.ip4.Flags) - } else { - m.logger.Trace2("packet is an IPv6 fragment: src=%v dst=%v", srcIP, dstIP) + if m.logger.Enabled(nblog.LevelTrace) { + if d.decoded[0] == layers.LayerTypeIPv4 { + m.logger.Trace4("packet is a fragment: src=%v dst=%v id=%v flags=%v", + srcIP, dstIP, d.ip4.Id, d.ip4.Flags) + } else { + m.logger.Trace2("packet is an IPv6 fragment: src=%v dst=%v", srcIP, dstIP) + } } return false } @@ -1091,8 +1097,10 @@ func (m *Manager) handleLocalTraffic(d *decoder, srcIP, dstIP netip.Addr, packet pnum := getProtocolFromPacket(d) srcPort, dstPort := getPortsFromPacket(d) - m.logger.Trace6("Dropping local packet (ACL denied): rule_id=%s proto=%v src=%s:%d dst=%s:%d", - ruleID, pnum, srcIP, srcPort, dstIP, dstPort) + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace6("Dropping local packet (ACL denied): rule_id=%s proto=%v src=%s:%d dst=%s:%d", + ruleID, pnum, srcIP, srcPort, dstIP, dstPort) + } m.flowLogger.StoreEvent(nftypes.EventFields{ FlowID: uuid.New(), @@ -1142,8 +1150,10 @@ func (m *Manager) handleForwardedLocalTraffic(packetData []byte) bool { func (m *Manager) handleRoutedTraffic(d *decoder, srcIP, dstIP netip.Addr, packetData []byte, size int) bool { // Drop if routing is disabled if !m.routingEnabled.Load() { - m.logger.Trace2("Dropping routed packet (routing disabled): src=%s dst=%s", - srcIP, dstIP) + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace2("Dropping routed packet (routing disabled): src=%s dst=%s", + srcIP, dstIP) + } return true } @@ -1160,8 +1170,10 @@ func (m *Manager) handleRoutedTraffic(d *decoder, srcIP, dstIP netip.Addr, packe if !pass { proto := getProtocolFromPacket(d) - m.logger.Trace6("Dropping routed packet (ACL denied): rule_id=%s proto=%v src=%s:%d dst=%s:%d", - ruleID, proto, srcIP, srcPort, dstIP, dstPort) + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace6("Dropping routed packet (ACL denied): rule_id=%s proto=%v src=%s:%d dst=%s:%d", + ruleID, proto, srcIP, srcPort, dstIP, dstPort) + } m.flowLogger.StoreEvent(nftypes.EventFields{ FlowID: uuid.New(), @@ -1287,7 +1299,9 @@ func getPortsFromPacket(d *decoder) (srcPort, dstPort uint16) { // It returns true, true if the packet is a fragment and valid. func (m *Manager) isValidPacket(d *decoder, packetData []byte) (bool, bool) { if err := d.decodePacket(packetData); err != nil { - m.logger.Trace1("couldn't decode packet, err: %s", err) + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace1("couldn't decode packet, err: %s", err) + } return false, false } diff --git a/client/firewall/uspfilter/forwarder/icmp.go b/client/firewall/uspfilter/forwarder/icmp.go index 3922c2052..d6d4e705e 100644 --- a/client/firewall/uspfilter/forwarder/icmp.go +++ b/client/firewall/uspfilter/forwarder/icmp.go @@ -13,6 +13,7 @@ import ( "gvisor.dev/gvisor/pkg/tcpip/header" "gvisor.dev/gvisor/pkg/tcpip/stack" + nblog "github.com/netbirdio/netbird/client/firewall/uspfilter/log" nftypes "github.com/netbirdio/netbird/client/internal/netflow/types" ) @@ -97,8 +98,10 @@ func (f *Forwarder) forwardICMPPacket(id stack.TransportEndpointID, payload []by return nil, fmt.Errorf("write ICMP packet: %w", err) } - f.logger.Trace3("forwarder: Forwarded ICMP packet %v type %v code %v", - epID(id), icmpType, icmpCode) + if f.logger.Enabled(nblog.LevelTrace) { + f.logger.Trace3("forwarder: Forwarded ICMP packet %v type %v code %v", + epID(id), icmpType, icmpCode) + } return conn, nil } @@ -121,12 +124,14 @@ func (f *Forwarder) handleICMPViaSocket(flowID uuid.UUID, id stack.TransportEndp txBytes := f.handleEchoResponse(conn, id, v6) rtt := time.Since(sendTime).Round(10 * time.Microsecond) - proto := "ICMP" - if v6 { - proto = "ICMPv6" + if f.logger.Enabled(nblog.LevelTrace) { + proto := "ICMP" + if v6 { + proto = "ICMPv6" + } + f.logger.Trace5("forwarder: Forwarded %s echo reply %v type %v code %v (rtt=%v, raw socket)", + proto, epID(id), icmpType, icmpCode, rtt) } - f.logger.Trace5("forwarder: Forwarded %s echo reply %v type %v code %v (rtt=%v, raw socket)", - proto, epID(id), icmpType, icmpCode, rtt) f.sendICMPEvent(nftypes.TypeEnd, flowID, id, icmpType, icmpCode, uint64(rxBytes), uint64(txBytes)) } @@ -224,13 +229,17 @@ func (f *Forwarder) handleICMPViaPing(flowID uuid.UUID, id stack.TransportEndpoi } rtt := time.Since(pingStart).Round(10 * time.Microsecond) - f.logger.Trace3("forwarder: Forwarded ICMP echo request %v type %v code %v", - epID(id), icmpType, icmpCode) + if f.logger.Enabled(nblog.LevelTrace) { + f.logger.Trace3("forwarder: Forwarded ICMP echo request %v type %v code %v", + epID(id), icmpType, icmpCode) + } txBytes := f.synthesizeEchoReply(id, icmpData) - f.logger.Trace4("forwarder: Forwarded ICMP echo reply %v type %v code %v (rtt=%v, ping binary)", - epID(id), icmpType, icmpCode, rtt) + if f.logger.Enabled(nblog.LevelTrace) { + f.logger.Trace4("forwarder: Forwarded ICMP echo reply %v type %v code %v (rtt=%v, ping binary)", + epID(id), icmpType, icmpCode, rtt) + } f.sendICMPEvent(nftypes.TypeEnd, flowID, id, icmpType, icmpCode, uint64(rxBytes), uint64(txBytes)) } diff --git a/client/firewall/uspfilter/forwarder/tcp.go b/client/firewall/uspfilter/forwarder/tcp.go index 8844463f5..c65ebcde0 100644 --- a/client/firewall/uspfilter/forwarder/tcp.go +++ b/client/firewall/uspfilter/forwarder/tcp.go @@ -1,11 +1,8 @@ package forwarder import ( - "context" - "io" "net" "strconv" - "sync" "github.com/google/uuid" @@ -15,7 +12,9 @@ import ( "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" "gvisor.dev/gvisor/pkg/waiter" + nblog "github.com/netbirdio/netbird/client/firewall/uspfilter/log" nftypes "github.com/netbirdio/netbird/client/internal/netflow/types" + "github.com/netbirdio/netbird/util/netrelay" ) // handleTCP is called by the TCP forwarder for new connections. @@ -37,7 +36,9 @@ func (f *Forwarder) handleTCP(r *tcp.ForwarderRequest) { outConn, err := (&net.Dialer{}).DialContext(f.ctx, "tcp", dialAddr) if err != nil { r.Complete(true) - f.logger.Trace2("forwarder: dial error for %v: %v", epID(id), err) + if f.logger.Enabled(nblog.LevelTrace) { + f.logger.Trace2("forwarder: dial error for %v: %v", epID(id), err) + } return } @@ -60,64 +61,22 @@ func (f *Forwarder) handleTCP(r *tcp.ForwarderRequest) { inConn := gonet.NewTCPConn(&wq, ep) success = true - f.logger.Trace1("forwarder: established TCP connection %v", epID(id)) + if f.logger.Enabled(nblog.LevelTrace) { + f.logger.Trace1("forwarder: established TCP connection %v", epID(id)) + } go f.proxyTCP(id, inConn, outConn, ep, flowID) } func (f *Forwarder) proxyTCP(id stack.TransportEndpointID, inConn *gonet.TCPConn, outConn net.Conn, ep tcpip.Endpoint, flowID uuid.UUID) { + // netrelay.Relay copies bidirectionally with proper half-close propagation + // and fully closes both conns before returning. + bytesFromInToOut, bytesFromOutToIn := netrelay.Relay(f.ctx, inConn, outConn, netrelay.Options{ + Logger: f.logger, + }) - ctx, cancel := context.WithCancel(f.ctx) - defer cancel() - - go func() { - <-ctx.Done() - // Close connections and endpoint. - if err := inConn.Close(); err != nil && !isClosedError(err) { - f.logger.Debug1("forwarder: inConn close error: %v", err) - } - if err := outConn.Close(); err != nil && !isClosedError(err) { - f.logger.Debug1("forwarder: outConn close error: %v", err) - } - - ep.Close() - }() - - var wg sync.WaitGroup - wg.Add(2) - - var ( - bytesFromInToOut int64 // bytes from client to server (tx for client) - bytesFromOutToIn int64 // bytes from server to client (rx for client) - errInToOut error - errOutToIn error - ) - - go func() { - bytesFromInToOut, errInToOut = io.Copy(outConn, inConn) - cancel() - wg.Done() - }() - - go func() { - - bytesFromOutToIn, errOutToIn = io.Copy(inConn, outConn) - cancel() - wg.Done() - }() - - wg.Wait() - - if errInToOut != nil { - if !isClosedError(errInToOut) { - f.logger.Error2("proxyTCP: copy error (in → out) for %s: %v", epID(id), errInToOut) - } - } - if errOutToIn != nil { - if !isClosedError(errOutToIn) { - f.logger.Error2("proxyTCP: copy error (out → in) for %s: %v", epID(id), errOutToIn) - } - } + // Close the netstack endpoint after both conns are drained. + ep.Close() var rxPackets, txPackets uint64 if tcpStats, ok := ep.Stats().(*tcp.Stats); ok { @@ -126,7 +85,9 @@ func (f *Forwarder) proxyTCP(id stack.TransportEndpointID, inConn *gonet.TCPConn txPackets = tcpStats.SegmentsReceived.Value() } - f.logger.Trace5("forwarder: Removed TCP connection %s [in: %d Pkts/%d B, out: %d Pkts/%d B]", epID(id), rxPackets, bytesFromOutToIn, txPackets, bytesFromInToOut) + if f.logger.Enabled(nblog.LevelTrace) { + f.logger.Trace5("forwarder: Removed TCP connection %s [in: %d Pkts/%d B, out: %d Pkts/%d B]", epID(id), rxPackets, bytesFromOutToIn, txPackets, bytesFromInToOut) + } f.sendTCPEvent(nftypes.TypeEnd, flowID, id, uint64(bytesFromOutToIn), uint64(bytesFromInToOut), rxPackets, txPackets) } diff --git a/client/firewall/uspfilter/forwarder/udp.go b/client/firewall/uspfilter/forwarder/udp.go index c92fa1f32..d840ef06b 100644 --- a/client/firewall/uspfilter/forwarder/udp.go +++ b/client/firewall/uspfilter/forwarder/udp.go @@ -125,7 +125,9 @@ func (f *udpForwarder) cleanup() { delete(f.conns, idle.id) f.Unlock() - f.logger.Trace1("forwarder: cleaned up idle UDP connection %v", epID(idle.id)) + if f.logger.Enabled(nblog.LevelTrace) { + f.logger.Trace1("forwarder: cleaned up idle UDP connection %v", epID(idle.id)) + } } } } @@ -144,7 +146,9 @@ func (f *Forwarder) handleUDP(r *udp.ForwarderRequest) bool { _, exists := f.udpForwarder.conns[id] f.udpForwarder.RUnlock() if exists { - f.logger.Trace1("forwarder: existing UDP connection for %v", epID(id)) + if f.logger.Enabled(nblog.LevelTrace) { + f.logger.Trace1("forwarder: existing UDP connection for %v", epID(id)) + } return true } @@ -206,7 +210,9 @@ func (f *Forwarder) handleUDP(r *udp.ForwarderRequest) bool { f.udpForwarder.Unlock() success = true - f.logger.Trace1("forwarder: established UDP connection %v", epID(id)) + if f.logger.Enabled(nblog.LevelTrace) { + f.logger.Trace1("forwarder: established UDP connection %v", epID(id)) + } go f.proxyUDP(connCtx, pConn, id, ep) return true @@ -265,7 +271,9 @@ func (f *Forwarder) proxyUDP(ctx context.Context, pConn *udpPacketConn, id stack txPackets = udpStats.PacketsReceived.Value() } - f.logger.Trace5("forwarder: Removed UDP connection %s [in: %d Pkts/%d B, out: %d Pkts/%d B]", epID(id), rxPackets, rxBytes, txPackets, txBytes) + if f.logger.Enabled(nblog.LevelTrace) { + f.logger.Trace5("forwarder: Removed UDP connection %s [in: %d Pkts/%d B, out: %d Pkts/%d B]", epID(id), rxPackets, rxBytes, txPackets, txBytes) + } f.udpForwarder.Lock() delete(f.udpForwarder.conns, id) diff --git a/client/firewall/uspfilter/log/log.go b/client/firewall/uspfilter/log/log.go index c6ca55e70..03e7d4809 100644 --- a/client/firewall/uspfilter/log/log.go +++ b/client/firewall/uspfilter/log/log.go @@ -53,16 +53,17 @@ var levelStrings = map[Level]string{ } type logMessage struct { - level Level - format string - arg1 any - arg2 any - arg3 any - arg4 any - arg5 any - arg6 any - arg7 any - arg8 any + level Level + argCount uint8 + format string + arg1 any + arg2 any + arg3 any + arg4 any + arg5 any + arg6 any + arg7 any + arg8 any } // Logger is a high-performance, non-blocking logger @@ -107,6 +108,13 @@ func (l *Logger) SetLevel(level Level) { log.Debugf("Set uspfilter logger loglevel to %v", levelStrings[level]) } +// Enabled reports whether the given level is currently logged. Callers on the +// hot path should guard log sites with this to avoid boxing arguments into +// any when the level is off. +func (l *Logger) Enabled(level Level) bool { + return l.level.Load() >= uint32(level) +} + func (l *Logger) Error(format string) { if l.level.Load() >= uint32(LevelError) { select { @@ -155,7 +163,7 @@ func (l *Logger) Trace(format string) { func (l *Logger) Error1(format string, arg1 any) { if l.level.Load() >= uint32(LevelError) { select { - case l.msgChannel <- logMessage{level: LevelError, format: format, arg1: arg1}: + case l.msgChannel <- logMessage{level: LevelError, argCount: 1, format: format, arg1: arg1}: default: } } @@ -164,7 +172,16 @@ func (l *Logger) Error1(format string, arg1 any) { func (l *Logger) Error2(format string, arg1, arg2 any) { if l.level.Load() >= uint32(LevelError) { select { - case l.msgChannel <- logMessage{level: LevelError, format: format, arg1: arg1, arg2: arg2}: + case l.msgChannel <- logMessage{level: LevelError, argCount: 2, format: format, arg1: arg1, arg2: arg2}: + default: + } + } +} + +func (l *Logger) Warn2(format string, arg1, arg2 any) { + if l.level.Load() >= uint32(LevelWarn) { + select { + case l.msgChannel <- logMessage{level: LevelWarn, argCount: 2, format: format, arg1: arg1, arg2: arg2}: default: } } @@ -173,7 +190,7 @@ func (l *Logger) Error2(format string, arg1, arg2 any) { func (l *Logger) Warn3(format string, arg1, arg2, arg3 any) { if l.level.Load() >= uint32(LevelWarn) { select { - case l.msgChannel <- logMessage{level: LevelWarn, format: format, arg1: arg1, arg2: arg2, arg3: arg3}: + case l.msgChannel <- logMessage{level: LevelWarn, argCount: 3, format: format, arg1: arg1, arg2: arg2, arg3: arg3}: default: } } @@ -182,7 +199,7 @@ func (l *Logger) Warn3(format string, arg1, arg2, arg3 any) { func (l *Logger) Warn4(format string, arg1, arg2, arg3, arg4 any) { if l.level.Load() >= uint32(LevelWarn) { select { - case l.msgChannel <- logMessage{level: LevelWarn, format: format, arg1: arg1, arg2: arg2, arg3: arg3, arg4: arg4}: + case l.msgChannel <- logMessage{level: LevelWarn, argCount: 4, format: format, arg1: arg1, arg2: arg2, arg3: arg3, arg4: arg4}: default: } } @@ -191,7 +208,7 @@ func (l *Logger) Warn4(format string, arg1, arg2, arg3, arg4 any) { func (l *Logger) Debug1(format string, arg1 any) { if l.level.Load() >= uint32(LevelDebug) { select { - case l.msgChannel <- logMessage{level: LevelDebug, format: format, arg1: arg1}: + case l.msgChannel <- logMessage{level: LevelDebug, argCount: 1, format: format, arg1: arg1}: default: } } @@ -200,7 +217,7 @@ func (l *Logger) Debug1(format string, arg1 any) { func (l *Logger) Debug2(format string, arg1, arg2 any) { if l.level.Load() >= uint32(LevelDebug) { select { - case l.msgChannel <- logMessage{level: LevelDebug, format: format, arg1: arg1, arg2: arg2}: + case l.msgChannel <- logMessage{level: LevelDebug, argCount: 2, format: format, arg1: arg1, arg2: arg2}: default: } } @@ -209,16 +226,59 @@ func (l *Logger) Debug2(format string, arg1, arg2 any) { func (l *Logger) Debug3(format string, arg1, arg2, arg3 any) { if l.level.Load() >= uint32(LevelDebug) { select { - case l.msgChannel <- logMessage{level: LevelDebug, format: format, arg1: arg1, arg2: arg2, arg3: arg3}: + case l.msgChannel <- logMessage{level: LevelDebug, argCount: 3, format: format, arg1: arg1, arg2: arg2, arg3: arg3}: default: } } } +// Debugf is the variadic shape. Dispatches to Debug/Debug1/Debug2/Debug3 +// to avoid allocating an args slice on the fast path when the arg count is +// known (0-3). Args beyond 3 land on the general variadic path; callers on +// the hot path should prefer DebugN for known counts. +func (l *Logger) Debugf(format string, args ...any) { + if l.level.Load() < uint32(LevelDebug) { + return + } + switch len(args) { + case 0: + l.Debug(format) + case 1: + l.Debug1(format, args[0]) + case 2: + l.Debug2(format, args[0], args[1]) + case 3: + l.Debug3(format, args[0], args[1], args[2]) + default: + l.sendVariadic(LevelDebug, format, args) + } +} + +// sendVariadic packs a slice of arguments into a logMessage and non-blocking +// enqueues it. Used for arg counts beyond the fixed-arity fast paths. Args +// beyond the 8-arg slot limit are dropped so callers don't produce silently +// empty log lines via uint8 wraparound in argCount. +func (l *Logger) sendVariadic(level Level, format string, args []any) { + const maxArgs = 8 + n := len(args) + if n > maxArgs { + n = maxArgs + } + msg := logMessage{level: level, argCount: uint8(n), format: format} + slots := [maxArgs]*any{&msg.arg1, &msg.arg2, &msg.arg3, &msg.arg4, &msg.arg5, &msg.arg6, &msg.arg7, &msg.arg8} + for i := 0; i < n; i++ { + *slots[i] = args[i] + } + select { + case l.msgChannel <- msg: + default: + } +} + func (l *Logger) Trace1(format string, arg1 any) { if l.level.Load() >= uint32(LevelTrace) { select { - case l.msgChannel <- logMessage{level: LevelTrace, format: format, arg1: arg1}: + case l.msgChannel <- logMessage{level: LevelTrace, argCount: 1, format: format, arg1: arg1}: default: } } @@ -227,7 +287,7 @@ func (l *Logger) Trace1(format string, arg1 any) { func (l *Logger) Trace2(format string, arg1, arg2 any) { if l.level.Load() >= uint32(LevelTrace) { select { - case l.msgChannel <- logMessage{level: LevelTrace, format: format, arg1: arg1, arg2: arg2}: + case l.msgChannel <- logMessage{level: LevelTrace, argCount: 2, format: format, arg1: arg1, arg2: arg2}: default: } } @@ -236,7 +296,7 @@ func (l *Logger) Trace2(format string, arg1, arg2 any) { func (l *Logger) Trace3(format string, arg1, arg2, arg3 any) { if l.level.Load() >= uint32(LevelTrace) { select { - case l.msgChannel <- logMessage{level: LevelTrace, format: format, arg1: arg1, arg2: arg2, arg3: arg3}: + case l.msgChannel <- logMessage{level: LevelTrace, argCount: 3, format: format, arg1: arg1, arg2: arg2, arg3: arg3}: default: } } @@ -245,7 +305,7 @@ func (l *Logger) Trace3(format string, arg1, arg2, arg3 any) { func (l *Logger) Trace4(format string, arg1, arg2, arg3, arg4 any) { if l.level.Load() >= uint32(LevelTrace) { select { - case l.msgChannel <- logMessage{level: LevelTrace, format: format, arg1: arg1, arg2: arg2, arg3: arg3, arg4: arg4}: + case l.msgChannel <- logMessage{level: LevelTrace, argCount: 4, format: format, arg1: arg1, arg2: arg2, arg3: arg3, arg4: arg4}: default: } } @@ -254,7 +314,7 @@ func (l *Logger) Trace4(format string, arg1, arg2, arg3, arg4 any) { func (l *Logger) Trace5(format string, arg1, arg2, arg3, arg4, arg5 any) { if l.level.Load() >= uint32(LevelTrace) { select { - case l.msgChannel <- logMessage{level: LevelTrace, format: format, arg1: arg1, arg2: arg2, arg3: arg3, arg4: arg4, arg5: arg5}: + case l.msgChannel <- logMessage{level: LevelTrace, argCount: 5, format: format, arg1: arg1, arg2: arg2, arg3: arg3, arg4: arg4, arg5: arg5}: default: } } @@ -263,7 +323,7 @@ func (l *Logger) Trace5(format string, arg1, arg2, arg3, arg4, arg5 any) { func (l *Logger) Trace6(format string, arg1, arg2, arg3, arg4, arg5, arg6 any) { if l.level.Load() >= uint32(LevelTrace) { select { - case l.msgChannel <- logMessage{level: LevelTrace, format: format, arg1: arg1, arg2: arg2, arg3: arg3, arg4: arg4, arg5: arg5, arg6: arg6}: + case l.msgChannel <- logMessage{level: LevelTrace, argCount: 6, format: format, arg1: arg1, arg2: arg2, arg3: arg3, arg4: arg4, arg5: arg5, arg6: arg6}: default: } } @@ -273,7 +333,7 @@ func (l *Logger) Trace6(format string, arg1, arg2, arg3, arg4, arg5, arg6 any) { func (l *Logger) Trace8(format string, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8 any) { if l.level.Load() >= uint32(LevelTrace) { select { - case l.msgChannel <- logMessage{level: LevelTrace, format: format, arg1: arg1, arg2: arg2, arg3: arg3, arg4: arg4, arg5: arg5, arg6: arg6, arg7: arg7, arg8: arg8}: + case l.msgChannel <- logMessage{level: LevelTrace, argCount: 8, format: format, arg1: arg1, arg2: arg2, arg3: arg3, arg4: arg4, arg5: arg5, arg6: arg6, arg7: arg7, arg8: arg8}: default: } } @@ -286,35 +346,8 @@ func (l *Logger) formatMessage(buf *[]byte, msg logMessage) { *buf = append(*buf, levelStrings[msg.level]...) *buf = append(*buf, ' ') - // Count non-nil arguments for switch - argCount := 0 - if msg.arg1 != nil { - argCount++ - if msg.arg2 != nil { - argCount++ - if msg.arg3 != nil { - argCount++ - if msg.arg4 != nil { - argCount++ - if msg.arg5 != nil { - argCount++ - if msg.arg6 != nil { - argCount++ - if msg.arg7 != nil { - argCount++ - if msg.arg8 != nil { - argCount++ - } - } - } - } - } - } - } - } - var formatted string - switch argCount { + switch msg.argCount { case 0: formatted = msg.format case 1: diff --git a/client/firewall/uspfilter/nat.go b/client/firewall/uspfilter/nat.go index 0d411c21e..5d51c1538 100644 --- a/client/firewall/uspfilter/nat.go +++ b/client/firewall/uspfilter/nat.go @@ -11,6 +11,7 @@ import ( "github.com/google/gopacket/layers" firewall "github.com/netbirdio/netbird/client/firewall/manager" + nblog "github.com/netbirdio/netbird/client/firewall/uspfilter/log" ) var ( @@ -262,11 +263,15 @@ func (m *Manager) translateOutboundDNAT(packetData []byte, d *decoder) bool { } if err := m.rewritePacketIP(packetData, d, translatedIP, false); err != nil { - m.logger.Error1("failed to rewrite packet destination: %v", err) + if m.logger.Enabled(nblog.LevelError) { + m.logger.Error1("failed to rewrite packet destination: %v", err) + } return false } - m.logger.Trace2("DNAT: %s -> %s", dstIP, translatedIP) + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace2("DNAT: %s -> %s", dstIP, translatedIP) + } return true } @@ -283,11 +288,15 @@ func (m *Manager) translateInboundReverse(packetData []byte, d *decoder) bool { } if err := m.rewritePacketIP(packetData, d, originalIP, true); err != nil { - m.logger.Error1("failed to rewrite packet source: %v", err) + if m.logger.Enabled(nblog.LevelError) { + m.logger.Error1("failed to rewrite packet source: %v", err) + } return false } - m.logger.Trace2("Reverse DNAT: %s -> %s", srcIP, originalIP) + if m.logger.Enabled(nblog.LevelTrace) { + m.logger.Trace2("Reverse DNAT: %s -> %s", srcIP, originalIP) + } return true } @@ -612,7 +621,9 @@ func (m *Manager) applyPortRule(packetData []byte, d *decoder, srcIP, dstIP neti } if err := rewriteFn(packetData, d, rule.targetPort, destinationPortOffset); err != nil { - m.logger.Error1("failed to rewrite port: %v", err) + if m.logger.Enabled(nblog.LevelError) { + m.logger.Error1("failed to rewrite port: %v", err) + } return false } d.dnatOrigPort = rule.origPort diff --git a/client/ssh/client/client.go b/client/ssh/client/client.go index 7f72a72cf..ebf8eb794 100644 --- a/client/ssh/client/client.go +++ b/client/ssh/client/client.go @@ -25,6 +25,7 @@ import ( nbssh "github.com/netbirdio/netbird/client/ssh" "github.com/netbirdio/netbird/client/ssh/detection" "github.com/netbirdio/netbird/util" + "github.com/netbirdio/netbird/util/netrelay" ) const ( @@ -536,7 +537,7 @@ func (c *Client) LocalPortForward(ctx context.Context, localAddr, remoteAddr str continue } - go c.handleLocalForward(localConn, remoteAddr) + go c.handleLocalForward(ctx, localConn, remoteAddr) } }() @@ -548,7 +549,7 @@ func (c *Client) LocalPortForward(ctx context.Context, localAddr, remoteAddr str } // handleLocalForward handles a single local port forwarding connection -func (c *Client) handleLocalForward(localConn net.Conn, remoteAddr string) { +func (c *Client) handleLocalForward(ctx context.Context, localConn net.Conn, remoteAddr string) { defer func() { if err := localConn.Close(); err != nil { log.Debugf("local port forwarding: close local connection: %v", err) @@ -571,7 +572,7 @@ func (c *Client) handleLocalForward(localConn net.Conn, remoteAddr string) { } }() - nbssh.BidirectionalCopy(log.NewEntry(log.StandardLogger()), localConn, channel) + netrelay.Relay(ctx, localConn, channel, netrelay.Options{Logger: log.NewEntry(log.StandardLogger())}) } // RemotePortForward sets up remote port forwarding, binding on remote and forwarding to localAddr @@ -653,16 +654,19 @@ func (c *Client) handleRemoteForwardChannels(ctx context.Context, localAddr stri select { case <-ctx.Done(): return - case newChan := <-channelRequests: + case newChan, ok := <-channelRequests: + if !ok { + return + } if newChan != nil { - go c.handleRemoteForwardChannel(newChan, localAddr) + go c.handleRemoteForwardChannel(ctx, newChan, localAddr) } } } } // handleRemoteForwardChannel handles a single forwarded-tcpip channel -func (c *Client) handleRemoteForwardChannel(newChan ssh.NewChannel, localAddr string) { +func (c *Client) handleRemoteForwardChannel(ctx context.Context, newChan ssh.NewChannel, localAddr string) { channel, reqs, err := newChan.Accept() if err != nil { return @@ -675,8 +679,14 @@ func (c *Client) handleRemoteForwardChannel(newChan ssh.NewChannel, localAddr st go ssh.DiscardRequests(reqs) - localConn, err := net.Dial("tcp", localAddr) + // Bound the dial so a black-holed localAddr can't pin the accepted SSH + // channel open indefinitely; the relay itself runs under the outer ctx. + dialCtx, cancelDial := context.WithTimeout(ctx, 10*time.Second) + var dialer net.Dialer + localConn, err := dialer.DialContext(dialCtx, "tcp", localAddr) + cancelDial() if err != nil { + log.Debugf("remote port forwarding: dial %s: %v", localAddr, err) return } defer func() { @@ -685,7 +695,7 @@ func (c *Client) handleRemoteForwardChannel(newChan ssh.NewChannel, localAddr st } }() - nbssh.BidirectionalCopy(log.NewEntry(log.StandardLogger()), localConn, channel) + netrelay.Relay(ctx, localConn, channel, netrelay.Options{Logger: log.NewEntry(log.StandardLogger())}) } // tcpipForwardMsg represents the structure for tcpip-forward requests diff --git a/client/ssh/common.go b/client/ssh/common.go index f6aec5f9c..92e647b7d 100644 --- a/client/ssh/common.go +++ b/client/ssh/common.go @@ -194,63 +194,3 @@ func buildAddressList(hostname string, remote net.Addr) []string { return addresses } -// BidirectionalCopy copies data bidirectionally between two io.ReadWriter connections. -// It waits for both directions to complete before returning. -// The caller is responsible for closing the connections. -func BidirectionalCopy(logger *log.Entry, rw1, rw2 io.ReadWriter) { - done := make(chan struct{}, 2) - - go func() { - if _, err := io.Copy(rw2, rw1); err != nil && !isExpectedCopyError(err) { - logger.Debugf("copy error (1->2): %v", err) - } - done <- struct{}{} - }() - - go func() { - if _, err := io.Copy(rw1, rw2); err != nil && !isExpectedCopyError(err) { - logger.Debugf("copy error (2->1): %v", err) - } - done <- struct{}{} - }() - - <-done - <-done -} - -func isExpectedCopyError(err error) bool { - return errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) -} - -// BidirectionalCopyWithContext copies data bidirectionally between two io.ReadWriteCloser connections. -// It waits for both directions to complete or for context cancellation before returning. -// Both connections are closed when the function returns. -func BidirectionalCopyWithContext(logger *log.Entry, ctx context.Context, conn1, conn2 io.ReadWriteCloser) { - done := make(chan struct{}, 2) - - go func() { - if _, err := io.Copy(conn2, conn1); err != nil && !isExpectedCopyError(err) { - logger.Debugf("copy error (1->2): %v", err) - } - done <- struct{}{} - }() - - go func() { - if _, err := io.Copy(conn1, conn2); err != nil && !isExpectedCopyError(err) { - logger.Debugf("copy error (2->1): %v", err) - } - done <- struct{}{} - }() - - select { - case <-ctx.Done(): - case <-done: - select { - case <-ctx.Done(): - case <-done: - } - } - - _ = conn1.Close() - _ = conn2.Close() -} diff --git a/client/ssh/proxy/proxy.go b/client/ssh/proxy/proxy.go index eb659fe21..73b50122c 100644 --- a/client/ssh/proxy/proxy.go +++ b/client/ssh/proxy/proxy.go @@ -23,6 +23,7 @@ import ( "github.com/netbirdio/netbird/client/proto" nbssh "github.com/netbirdio/netbird/client/ssh" "github.com/netbirdio/netbird/client/ssh/detection" + "github.com/netbirdio/netbird/util/netrelay" "github.com/netbirdio/netbird/version" ) @@ -352,7 +353,7 @@ func (p *SSHProxy) directTCPIPHandler(_ *ssh.Server, _ *cryptossh.ServerConn, ne } go cryptossh.DiscardRequests(clientReqs) - nbssh.BidirectionalCopyWithContext(log.NewEntry(log.StandardLogger()), sshCtx, clientChan, backendChan) + netrelay.Relay(sshCtx, clientChan, backendChan, netrelay.Options{Logger: log.NewEntry(log.StandardLogger())}) } func (p *SSHProxy) sftpSubsystemHandler(s ssh.Session, jwtToken string) { @@ -591,7 +592,7 @@ func (p *SSHProxy) handleForwardedChannel(sshCtx ssh.Context, sshConn *cryptossh } go cryptossh.DiscardRequests(clientReqs) - nbssh.BidirectionalCopyWithContext(log.NewEntry(log.StandardLogger()), sshCtx, clientChan, backendChan) + netrelay.Relay(sshCtx, clientChan, backendChan, netrelay.Options{Logger: log.NewEntry(log.StandardLogger())}) } func (p *SSHProxy) dialBackend(ctx context.Context, addr, user, jwtToken string) (*cryptossh.Client, error) { diff --git a/client/ssh/server/port_forwarding.go b/client/ssh/server/port_forwarding.go index f5ac66fca..a47fdb48a 100644 --- a/client/ssh/server/port_forwarding.go +++ b/client/ssh/server/port_forwarding.go @@ -17,7 +17,7 @@ import ( log "github.com/sirupsen/logrus" cryptossh "golang.org/x/crypto/ssh" - nbssh "github.com/netbirdio/netbird/client/ssh" + "github.com/netbirdio/netbird/util/netrelay" ) const privilegedPortThreshold = 1024 @@ -357,7 +357,7 @@ func (s *Server) handleRemoteForwardConnection(ctx ssh.Context, conn net.Conn, h return } - nbssh.BidirectionalCopyWithContext(logger, ctx, conn, channel) + netrelay.Relay(ctx, conn, channel, netrelay.Options{Logger: logger}) } // openForwardChannel creates an SSH forwarded-tcpip channel diff --git a/client/ssh/server/server.go b/client/ssh/server/server.go index de40d3091..6735e0f3b 100644 --- a/client/ssh/server/server.go +++ b/client/ssh/server/server.go @@ -8,9 +8,9 @@ import ( "fmt" "io" "net" - "strconv" "net/netip" "slices" + "strconv" "strings" "sync" "time" @@ -27,6 +27,7 @@ import ( "github.com/netbirdio/netbird/client/ssh/detection" "github.com/netbirdio/netbird/shared/auth" "github.com/netbirdio/netbird/shared/auth/jwt" + "github.com/netbirdio/netbird/util/netrelay" "github.com/netbirdio/netbird/version" ) @@ -53,6 +54,10 @@ const ( DefaultJWTMaxTokenAge = 10 * 60 ) +// directTCPIPDialTimeout bounds how long relayDirectTCPIP waits on a dial to +// the forwarded destination before rejecting the SSH channel. +const directTCPIPDialTimeout = 30 * time.Second + var ( ErrPrivilegedUserDisabled = errors.New(msgPrivilegedUserDisabled) ErrUserNotFound = errors.New("user not found") @@ -933,5 +938,29 @@ func (s *Server) directTCPIPHandler(srv *ssh.Server, conn *cryptossh.ServerConn, s.addConnectionPortForward(ctx.User(), ctx.RemoteAddr(), forwardAddr) logger.Infof("local port forwarding: %s", hostPort) - ssh.DirectTCPIPHandler(srv, conn, newChan, ctx) + s.relayDirectTCPIP(ctx, newChan, payload.Host, int(payload.Port), logger) +} + +// relayDirectTCPIP is a netrelay-based replacement for gliderlabs' +// DirectTCPIPHandler. The upstream handler closes both sides on the first +// EOF; netrelay.Relay propagates CloseWrite so each direction drains on its +// own terms. +func (s *Server) relayDirectTCPIP(ctx ssh.Context, newChan cryptossh.NewChannel, host string, port int, logger *log.Entry) { + dest := net.JoinHostPort(host, strconv.Itoa(port)) + + dialer := net.Dialer{Timeout: directTCPIPDialTimeout} + dconn, err := dialer.DialContext(ctx, "tcp", dest) + if err != nil { + _ = newChan.Reject(cryptossh.ConnectionFailed, err.Error()) + return + } + + ch, reqs, err := newChan.Accept() + if err != nil { + _ = dconn.Close() + return + } + go cryptossh.DiscardRequests(reqs) + + netrelay.Relay(ctx, dconn, ch, netrelay.Options{Logger: logger}) } diff --git a/proxy/internal/tcp/peekedconn.go b/proxy/internal/tcp/peekedconn.go index 26f3e5c7c..23a348352 100644 --- a/proxy/internal/tcp/peekedconn.go +++ b/proxy/internal/tcp/peekedconn.go @@ -25,6 +25,12 @@ func (c *peekedConn) Read(b []byte) (int, error) { return c.reader.Read(b) } +// halfCloser matches connections that support shutting down the write +// side while keeping the read side open (e.g. *net.TCPConn). +type halfCloser interface { + CloseWrite() error +} + // CloseWrite delegates to the underlying connection if it supports // half-close (e.g. *net.TCPConn). Without this, embedding net.Conn // as an interface hides the concrete type's CloseWrite method, making diff --git a/proxy/internal/tcp/relay.go b/proxy/internal/tcp/relay.go deleted file mode 100644 index 39949818d..000000000 --- a/proxy/internal/tcp/relay.go +++ /dev/null @@ -1,156 +0,0 @@ -package tcp - -import ( - "context" - "errors" - "io" - "net" - "sync" - "time" - - log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/proxy/internal/netutil" -) - -// errIdleTimeout is returned when a relay connection is closed due to inactivity. -var errIdleTimeout = errors.New("idle timeout") - -// DefaultIdleTimeout is the default idle timeout for TCP relay connections. -// A zero value disables idle timeout checking. -const DefaultIdleTimeout = 5 * time.Minute - -// halfCloser is implemented by connections that support half-close -// (e.g. *net.TCPConn). When one copy direction finishes, we signal -// EOF to the remote by closing the write side while keeping the read -// side open so the other direction can drain. -type halfCloser interface { - CloseWrite() error -} - -// copyBufPool avoids allocating a new 32KB buffer per io.Copy call. -var copyBufPool = sync.Pool{ - New: func() any { - buf := make([]byte, 32*1024) - return &buf - }, -} - -// Relay copies data bidirectionally between src and dst until both -// sides are done or the context is canceled. When idleTimeout is -// non-zero, each direction's read is deadline-guarded; if no data -// flows within the timeout the connection is torn down. When one -// direction finishes, it half-closes the write side of the -// destination (if supported) to signal EOF, allowing the other -// direction to drain gracefully before the full connection teardown. -func Relay(ctx context.Context, logger *log.Entry, src, dst net.Conn, idleTimeout time.Duration) (srcToDst, dstToSrc int64) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - go func() { - <-ctx.Done() - _ = src.Close() - _ = dst.Close() - }() - - var wg sync.WaitGroup - wg.Add(2) - - var errSrcToDst, errDstToSrc error - - go func() { - defer wg.Done() - srcToDst, errSrcToDst = copyWithIdleTimeout(dst, src, idleTimeout) - halfClose(dst) - cancel() - }() - - go func() { - defer wg.Done() - dstToSrc, errDstToSrc = copyWithIdleTimeout(src, dst, idleTimeout) - halfClose(src) - cancel() - }() - - wg.Wait() - - if errors.Is(errSrcToDst, errIdleTimeout) || errors.Is(errDstToSrc, errIdleTimeout) { - logger.Debug("relay closed due to idle timeout") - } - if errSrcToDst != nil && !isExpectedCopyError(errSrcToDst) { - logger.Debugf("relay copy error (src→dst): %v", errSrcToDst) - } - if errDstToSrc != nil && !isExpectedCopyError(errDstToSrc) { - logger.Debugf("relay copy error (dst→src): %v", errDstToSrc) - } - - return srcToDst, dstToSrc -} - -// copyWithIdleTimeout copies from src to dst using a pooled buffer. -// When idleTimeout > 0 it sets a read deadline on src before each -// read and treats a timeout as an idle-triggered close. -func copyWithIdleTimeout(dst io.Writer, src io.Reader, idleTimeout time.Duration) (int64, error) { - bufp := copyBufPool.Get().(*[]byte) - defer copyBufPool.Put(bufp) - - if idleTimeout <= 0 { - return io.CopyBuffer(dst, src, *bufp) - } - - conn, ok := src.(net.Conn) - if !ok { - return io.CopyBuffer(dst, src, *bufp) - } - - buf := *bufp - var total int64 - for { - if err := conn.SetReadDeadline(time.Now().Add(idleTimeout)); err != nil { - return total, err - } - nr, readErr := src.Read(buf) - if nr > 0 { - n, err := checkedWrite(dst, buf[:nr]) - total += n - if err != nil { - return total, err - } - } - if readErr != nil { - if netutil.IsTimeout(readErr) { - return total, errIdleTimeout - } - return total, readErr - } - } -} - -// checkedWrite writes buf to dst and returns the number of bytes written. -// It guards against short writes and negative counts per io.Copy convention. -func checkedWrite(dst io.Writer, buf []byte) (int64, error) { - nw, err := dst.Write(buf) - if nw < 0 || nw > len(buf) { - nw = 0 - } - if err != nil { - return int64(nw), err - } - if nw != len(buf) { - return int64(nw), io.ErrShortWrite - } - return int64(nw), nil -} - -func isExpectedCopyError(err error) bool { - return errors.Is(err, errIdleTimeout) || netutil.IsExpectedError(err) -} - -// halfClose attempts to half-close the write side of the connection. -// If the connection does not support half-close, this is a no-op. -func halfClose(conn net.Conn) { - if hc, ok := conn.(halfCloser); ok { - // Best-effort; the full close will follow shortly. - _ = hc.CloseWrite() - } -} diff --git a/proxy/internal/tcp/relay_test.go b/proxy/internal/tcp/relay_test.go index e42d65b9d..f83a0d155 100644 --- a/proxy/internal/tcp/relay_test.go +++ b/proxy/internal/tcp/relay_test.go @@ -13,8 +13,13 @@ import ( "github.com/stretchr/testify/require" "github.com/netbirdio/netbird/proxy/internal/netutil" + "github.com/netbirdio/netbird/util/netrelay" ) +func testRelay(ctx context.Context, logger *log.Entry, src, dst net.Conn, idleTimeout time.Duration) (int64, int64) { + return netrelay.Relay(ctx, src, dst, netrelay.Options{IdleTimeout: idleTimeout, Logger: logger}) +} + func TestRelay_BidirectionalCopy(t *testing.T) { srcClient, srcServer := net.Pipe() dstClient, dstServer := net.Pipe() @@ -41,7 +46,7 @@ func TestRelay_BidirectionalCopy(t *testing.T) { srcClient.Close() }() - s2d, d2s := Relay(ctx, logger, srcServer, dstServer, 0) + s2d, d2s := testRelay(ctx, logger, srcServer, dstServer, 0) assert.Equal(t, int64(len(srcData)), s2d, "bytes src→dst") assert.Equal(t, int64(len(dstData)), d2s, "bytes dst→src") @@ -58,7 +63,7 @@ func TestRelay_ContextCancellation(t *testing.T) { done := make(chan struct{}) go func() { - Relay(ctx, logger, srcServer, dstServer, 0) + testRelay(ctx, logger, srcServer, dstServer, 0) close(done) }() @@ -85,7 +90,7 @@ func TestRelay_OneSideClosed(t *testing.T) { done := make(chan struct{}) go func() { - Relay(ctx, logger, srcServer, dstServer, 0) + testRelay(ctx, logger, srcServer, dstServer, 0) close(done) }() @@ -129,7 +134,7 @@ func TestRelay_LargeTransfer(t *testing.T) { dstClient.Close() }() - s2d, _ := Relay(ctx, logger, srcServer, dstServer, 0) + s2d, _ := testRelay(ctx, logger, srcServer, dstServer, 0) assert.Equal(t, int64(len(data)), s2d, "should transfer all bytes") require.NoError(t, <-errCh) } @@ -182,7 +187,7 @@ func TestRelay_IdleTimeout(t *testing.T) { done := make(chan struct{}) var s2d, d2s int64 go func() { - s2d, d2s = Relay(ctx, logger, srcServer, dstServer, 200*time.Millisecond) + s2d, d2s = testRelay(ctx, logger, srcServer, dstServer, 200*time.Millisecond) close(done) }() diff --git a/proxy/internal/tcp/router.go b/proxy/internal/tcp/router.go index 9f8660aeb..05beb658b 100644 --- a/proxy/internal/tcp/router.go +++ b/proxy/internal/tcp/router.go @@ -16,6 +16,7 @@ import ( "github.com/netbirdio/netbird/proxy/internal/accesslog" "github.com/netbirdio/netbird/proxy/internal/restrict" "github.com/netbirdio/netbird/proxy/internal/types" + "github.com/netbirdio/netbird/util/netrelay" ) // defaultDialTimeout is the fallback dial timeout when no per-route @@ -528,11 +529,14 @@ func (r *Router) relayTCP(ctx context.Context, conn net.Conn, sni SNIHost, route idleTimeout := route.SessionIdleTimeout if idleTimeout <= 0 { - idleTimeout = DefaultIdleTimeout + idleTimeout = netrelay.DefaultIdleTimeout } start := time.Now() - s2d, d2s := Relay(svcCtx, entry, conn, backend, idleTimeout) + s2d, d2s := netrelay.Relay(svcCtx, conn, backend, netrelay.Options{ + IdleTimeout: idleTimeout, + Logger: entry, + }) elapsed := time.Since(start) if obs != nil { diff --git a/util/netrelay/relay.go b/util/netrelay/relay.go new file mode 100644 index 000000000..de44d5bcd --- /dev/null +++ b/util/netrelay/relay.go @@ -0,0 +1,238 @@ +// Package netrelay provides a bidirectional byte-copy helper for TCP-like +// connections with correct half-close propagation. +// +// When one direction reads EOF, the write side of the opposite connection is +// half-closed (CloseWrite) so the peer sees FIN, then the second direction is +// allowed to drain to its own EOF before both connections are fully closed. +// This preserves TCP half-close semantics (e.g. shutdown(SHUT_WR)) that the +// naive "cancel-both-on-first-EOF" pattern breaks. +package netrelay + +import ( + "context" + "errors" + "io" + "net" + "sync" + "sync/atomic" + "syscall" + "time" +) + +// DebugLogger is the minimal interface netrelay uses to surface teardown +// errors. Both *logrus.Entry and *nblog.Logger (via its Debugf method) +// satisfy it, so callers can pass whichever they already use without an +// adapter. Debugf is the only required method; callers with richer +// loggers just expose this one shape here. +type DebugLogger interface { + Debugf(format string, args ...any) +} + +// DefaultIdleTimeout is a reasonable default for Options.IdleTimeout. Callers +// that want an idle timeout but have no specific preference can use this. +const DefaultIdleTimeout = 5 * time.Minute + +// halfCloser is implemented by connections that support half-close +// (e.g. *net.TCPConn, *gonet.TCPConn). +type halfCloser interface { + CloseWrite() error +} + +var copyBufPool = sync.Pool{ + New: func() any { + buf := make([]byte, 32*1024) + return &buf + }, +} + +// Options configures Relay behavior. The zero value is valid: no idle timeout, +// no logging. +type Options struct { + // IdleTimeout tears down the session if no bytes flow in either + // direction within this window. It is a connection-wide watchdog, so a + // long unidirectional transfer on one side keeps the other side alive. + // Zero disables idle tracking. + IdleTimeout time.Duration + // Logger receives debug-level copy/idle errors. Nil suppresses logging. + // Any logger with Debug/Debugf methods is accepted (logrus.Entry, + // uspfilter's nblog.Logger, etc.). + Logger DebugLogger +} + +// Relay copies bytes in both directions between a and b until both directions +// EOF or ctx is canceled. On each direction's EOF it half-closes the +// opposite conn's write side (best effort) so the peer sees FIN while the +// other direction drains. Both conns are fully closed when Relay returns. +// +// a and b only need to implement io.ReadWriteCloser; connections that also +// implement CloseWrite (e.g. *net.TCPConn, ssh.Channel) get proper half-close +// propagation. Options.IdleTimeout, when set, is enforced by a connection-wide +// watchdog that tracks reads in either direction. +// +// Return values are byte counts: aToB (a.Read → b.Write) and bToA (b.Read → +// a.Write). Errors are logged via Options.Logger when set; they are not +// returned because a relay always terminates on some kind of EOF/cancel. +func Relay(ctx context.Context, a, b io.ReadWriteCloser, opts Options) (aToB, bToA int64) { + ctx, cancel := context.WithCancel(ctx) + closeDone := make(chan struct{}) + defer func() { + cancel() + <-closeDone + }() + + go func() { + <-ctx.Done() + _ = a.Close() + _ = b.Close() + close(closeDone) + }() + + // Both sides must support CloseWrite to propagate half-close. If either + // doesn't, a direction's EOF can't be signaled to the peer and the other + // direction would block forever waiting for data; in that case we fall + // back to the cancel-both-on-first-EOF behavior. + _, aHC := a.(halfCloser) + _, bHC := b.(halfCloser) + halfCloseSupported := aHC && bHC + + var ( + lastActivity atomic.Int64 + idleHit atomic.Bool + ) + lastActivity.Store(time.Now().UnixNano()) + + if opts.IdleTimeout > 0 { + go watchdog(ctx, cancel, &lastActivity, &idleHit, opts.IdleTimeout) + } + + var wg sync.WaitGroup + wg.Add(2) + + var errAToB, errBToA error + + go func() { + defer wg.Done() + aToB, errAToB = copyTracked(b, a, &lastActivity) + if halfCloseSupported && isCleanEOF(errAToB) { + halfClose(b) + } else { + cancel() + } + }() + + go func() { + defer wg.Done() + bToA, errBToA = copyTracked(a, b, &lastActivity) + if halfCloseSupported && isCleanEOF(errBToA) { + halfClose(a) + } else { + cancel() + } + }() + + wg.Wait() + + if opts.Logger != nil { + if idleHit.Load() { + opts.Logger.Debugf("relay closed due to idle timeout") + } + if errAToB != nil && !isExpectedCopyError(errAToB) { + opts.Logger.Debugf("relay copy error (a→b): %v", errAToB) + } + if errBToA != nil && !isExpectedCopyError(errBToA) { + opts.Logger.Debugf("relay copy error (b→a): %v", errBToA) + } + } + + return aToB, bToA +} + +// watchdog enforces a connection-wide idle timeout. It cancels ctx when no +// activity has been seen on either direction for idle. It exits as soon as +// ctx is canceled so it doesn't outlive the relay. +func watchdog(ctx context.Context, cancel context.CancelFunc, lastActivity *atomic.Int64, idleHit *atomic.Bool, idle time.Duration) { + // Cap the tick at 50ms so detection latency stays bounded regardless of + // how large idle is, and fall back to idle/2 when that is smaller so + // very short timeouts (mainly in tests) are still caught promptly. + tick := min(idle/2, 50*time.Millisecond) + if tick <= 0 { + tick = time.Millisecond + } + t := time.NewTicker(tick) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + last := time.Unix(0, lastActivity.Load()) + if time.Since(last) >= idle { + idleHit.Store(true) + cancel() + return + } + } + } +} + +// copyTracked copies from src to dst using a pooled buffer, updating +// lastActivity on every successful read so a shared watchdog can enforce a +// connection-wide idle timeout. +func copyTracked(dst io.Writer, src io.Reader, lastActivity *atomic.Int64) (int64, error) { + bufp := copyBufPool.Get().(*[]byte) + defer copyBufPool.Put(bufp) + + buf := *bufp + var total int64 + for { + nr, readErr := src.Read(buf) + if nr > 0 { + lastActivity.Store(time.Now().UnixNano()) + n, werr := checkedWrite(dst, buf[:nr]) + total += n + if werr != nil { + return total, werr + } + } + if readErr != nil { + return total, readErr + } + } +} + +func checkedWrite(dst io.Writer, buf []byte) (int64, error) { + nw, err := dst.Write(buf) + if nw < 0 || nw > len(buf) { + nw = 0 + } + if err != nil { + return int64(nw), err + } + if nw != len(buf) { + return int64(nw), io.ErrShortWrite + } + return int64(nw), nil +} + +func halfClose(conn io.ReadWriteCloser) { + if hc, ok := conn.(halfCloser); ok { + _ = hc.CloseWrite() + } +} + +// isCleanEOF reports whether a copy terminated on a graceful end-of-stream. +// Only in that case is it correct to propagate the EOF via CloseWrite on the +// peer; any other error means the flow is broken and both directions should +// tear down. +func isCleanEOF(err error) bool { + return err == nil || errors.Is(err, io.EOF) +} + +func isExpectedCopyError(err error) bool { + return errors.Is(err, net.ErrClosed) || + errors.Is(err, context.Canceled) || + errors.Is(err, io.EOF) || + errors.Is(err, syscall.ECONNRESET) || + errors.Is(err, syscall.EPIPE) || + errors.Is(err, syscall.ECONNABORTED) +} diff --git a/util/netrelay/relay_test.go b/util/netrelay/relay_test.go new file mode 100644 index 000000000..0cb86eb0d --- /dev/null +++ b/util/netrelay/relay_test.go @@ -0,0 +1,221 @@ +package netrelay + +import ( + "io" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// tcpPair returns two connected loopback TCP conns. +func tcpPair(t *testing.T) (*net.TCPConn, *net.TCPConn) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer ln.Close() + + type result struct { + c *net.TCPConn + err error + } + ch := make(chan result, 1) + go func() { + c, err := ln.Accept() + if err != nil { + ch <- result{nil, err} + return + } + ch <- result{c.(*net.TCPConn), nil} + }() + + dial, err := net.Dial("tcp", ln.Addr().String()) + require.NoError(t, err) + + r := <-ch + require.NoError(t, r.err) + return dial.(*net.TCPConn), r.c +} + +// TestRelayHalfClose exercises the shutdown(SHUT_WR) scenario that the naive +// cancel-both-on-first-EOF pattern breaks. Client A shuts down its write +// side; B must still be able to write a full response and A must receive +// all of it before its read returns EOF. +func TestRelayHalfClose(t *testing.T) { + // Real peer pairs for each side of the relay. We relay between relayA + // and relayB. Peer A talks through relayA; peer B talks through relayB. + peerA, relayA := tcpPair(t) + relayB, peerB := tcpPair(t) + + defer peerA.Close() + defer peerB.Close() + + // Bound blocking reads/writes so a broken relay fails the test instead of + // hanging the test process. + deadline := time.Now().Add(5 * time.Second) + require.NoError(t, peerA.SetDeadline(deadline)) + require.NoError(t, peerB.SetDeadline(deadline)) + + ctx := t.Context() + + done := make(chan struct{}) + go func() { + Relay(ctx, relayA, relayB, Options{}) + close(done) + }() + + // Peer A sends a request, then half-closes its write side. + req := []byte("request-payload") + _, err := peerA.Write(req) + require.NoError(t, err) + require.NoError(t, peerA.CloseWrite()) + + // Peer B reads the request to EOF (FIN must have propagated). + got, err := io.ReadAll(peerB) + require.NoError(t, err) + require.Equal(t, req, got) + + // Peer B writes its response; peer A must receive all of it even though + // peer A's write side is already closed. + resp := make([]byte, 64*1024) + for i := range resp { + resp[i] = byte(i) + } + _, err = peerB.Write(resp) + require.NoError(t, err) + require.NoError(t, peerB.Close()) + + gotResp, err := io.ReadAll(peerA) + require.NoError(t, err) + require.Equal(t, resp, gotResp) + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("relay did not return") + } +} + +// TestRelayFullDuplex verifies bidirectional copy in the simple case. +func TestRelayFullDuplex(t *testing.T) { + peerA, relayA := tcpPair(t) + relayB, peerB := tcpPair(t) + defer peerA.Close() + defer peerB.Close() + + // Bound blocking reads/writes so a broken relay fails the test instead of + // hanging the test process. + deadline := time.Now().Add(5 * time.Second) + require.NoError(t, peerA.SetDeadline(deadline)) + require.NoError(t, peerB.SetDeadline(deadline)) + + ctx := t.Context() + + done := make(chan struct{}) + go func() { + Relay(ctx, relayA, relayB, Options{}) + close(done) + }() + + type result struct { + got []byte + err error + } + resA := make(chan result, 1) + resB := make(chan result, 1) + + msgAB := []byte("hello-from-a") + msgBA := []byte("hello-from-b") + + go func() { + if _, err := peerA.Write(msgAB); err != nil { + resA <- result{err: err} + return + } + buf := make([]byte, len(msgBA)) + _, err := io.ReadFull(peerA, buf) + resA <- result{got: buf, err: err} + _ = peerA.Close() + }() + + go func() { + if _, err := peerB.Write(msgBA); err != nil { + resB <- result{err: err} + return + } + buf := make([]byte, len(msgAB)) + _, err := io.ReadFull(peerB, buf) + resB <- result{got: buf, err: err} + _ = peerB.Close() + }() + + a, b := <-resA, <-resB + require.NoError(t, a.err) + require.Equal(t, msgBA, a.got) + require.NoError(t, b.err) + require.Equal(t, msgAB, b.got) + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("relay did not return") + } +} + +// TestRelayNoHalfCloseFallback ensures Relay terminates when the underlying +// conns don't support CloseWrite (e.g. net.Pipe). Without the fallback to +// cancel-both-on-first-EOF, the second direction would block forever. +func TestRelayNoHalfCloseFallback(t *testing.T) { + a1, a2 := net.Pipe() + b1, b2 := net.Pipe() + defer a1.Close() + defer b1.Close() + + ctx := t.Context() + done := make(chan struct{}) + go func() { + Relay(ctx, a2, b2, Options{}) + close(done) + }() + + // Close peer A's side; a2's Read will return EOF. + require.NoError(t, a1.Close()) + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("relay did not terminate when half-close is unsupported") + } +} + +// TestRelayIdleTimeout ensures the idle watchdog tears down a silent flow. +func TestRelayIdleTimeout(t *testing.T) { + peerA, relayA := tcpPair(t) + relayB, peerB := tcpPair(t) + defer peerA.Close() + defer peerB.Close() + + ctx := t.Context() + + const idle = 150 * time.Millisecond + + start := time.Now() + done := make(chan struct{}) + go func() { + Relay(ctx, relayA, relayB, Options{IdleTimeout: idle}) + close(done) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("relay did not close on idle") + } + + elapsed := time.Since(start) + require.GreaterOrEqual(t, elapsed, idle, + "relay must not close before the idle timeout elapses") + require.Less(t, elapsed, idle+500*time.Millisecond, + "relay should close shortly after the idle timeout") +} From 6b08e89c7bb318f51c24b467a34fe05e13e17fcf Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Mon, 11 May 2026 16:59:33 +0900 Subject: [PATCH 04/29] [relay] Preserve non-standard port in WS dialer URL prep (#6061) --- shared/relay/client/dialer/ws/ws.go | 32 ++++++---- shared/relay/client/dialer/ws/ws_test.go | 76 ++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 12 deletions(-) create mode 100644 shared/relay/client/dialer/ws/ws_test.go diff --git a/shared/relay/client/dialer/ws/ws.go b/shared/relay/client/dialer/ws/ws.go index 301486514..8a13ba126 100644 --- a/shared/relay/client/dialer/ws/ws.go +++ b/shared/relay/client/dialer/ws/ws.go @@ -9,7 +9,6 @@ import ( "net" "net/http" "net/url" - "strings" "github.com/coder/websocket" log "github.com/sirupsen/logrus" @@ -35,13 +34,7 @@ func (d Dialer) Dial(ctx context.Context, address, serverName string) (net.Conn, var underlying net.Conn opts := createDialOptions(serverName, &underlying) - parsedURL, err := url.Parse(wsURL) - if err != nil { - return nil, err - } - parsedURL.Path = relay.WebSocketURLPath - - wsConn, resp, err := websocket.Dial(ctx, parsedURL.String(), opts) + wsConn, resp, err := websocket.Dial(ctx, wsURL, opts) if err != nil { if errors.Is(err, context.Canceled) { return nil, err @@ -57,12 +50,27 @@ func (d Dialer) Dial(ctx context.Context, address, serverName string) (net.Conn, return conn, nil } +// prepareURL rewrites a rel://host[:port] or rels://host[:port] address into a +// ws://host[:port]/relay or wss://host[:port]/relay URL, preserving any +// non-standard port from the input. func prepareURL(address string) (string, error) { - if !strings.HasPrefix(address, "rel:") && !strings.HasPrefix(address, "rels:") { - return "", fmt.Errorf("unsupported scheme: %s", address) + parsed, err := url.Parse(address) + if err != nil { + return "", fmt.Errorf("parse relay address %q: %w", address, err) } - - return strings.Replace(address, "rel", "ws", 1), nil + switch parsed.Scheme { + case "rel": + parsed.Scheme = "ws" + case "rels": + parsed.Scheme = "wss" + default: + return "", fmt.Errorf("unsupported scheme: %s", parsed.Scheme) + } + if parsed.Host == "" { + return "", fmt.Errorf("missing host in relay address %q", address) + } + parsed.Path = relay.WebSocketURLPath + return parsed.String(), nil } // httpClientNbDialer builds the http client used by the websocket library. diff --git a/shared/relay/client/dialer/ws/ws_test.go b/shared/relay/client/dialer/ws/ws_test.go new file mode 100644 index 000000000..7357adbc0 --- /dev/null +++ b/shared/relay/client/dialer/ws/ws_test.go @@ -0,0 +1,76 @@ +package ws + +import ( + "testing" +) + +func TestPrepareURL(t *testing.T) { + tests := []struct { + name string + input string + want string + wantErr bool + }{ + { + name: "rel scheme with non-standard port", + input: "rel://test-domain-2:45678", + want: "ws://test-domain-2:45678/relay", + }, + { + name: "rels scheme with non-standard port", + input: "rels://test-domain-2:45678", + want: "wss://test-domain-2:45678/relay", + }, + { + name: "rel scheme without port", + input: "rel://test-domain-2", + want: "ws://test-domain-2/relay", + }, + { + name: "rels scheme without port", + input: "rels://test-domain-2", + want: "wss://test-domain-2/relay", + }, + { + name: "rel scheme with IP and port", + input: "rel://1.2.3.4:45678", + want: "ws://1.2.3.4:45678/relay", + }, + { + name: "rel scheme with hostname starting with rel", + input: "rel://relay.example.com:45678", + want: "ws://relay.example.com:45678/relay", + }, + { + name: "rel scheme with IPv6 and port", + input: "rel://[2001:db8::1]:45678", + want: "ws://[2001:db8::1]:45678/relay", + }, + { + name: "rels scheme with IPv6 loopback and port", + input: "rels://[::1]:45678", + want: "wss://[::1]:45678/relay", + }, + { + name: "unsupported scheme", + input: "http://test-domain-2:45678", + wantErr: true, + }, + { + name: "no scheme", + input: "test-domain-2:45678", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := prepareURL(tt.input) + if (err != nil) != tt.wantErr { + t.Fatalf("prepareURL(%q) err = %v, wantErr %v", tt.input, err, tt.wantErr) + } + if got != tt.want { + t.Errorf("prepareURL(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} From a4114a5e453bbbe287610ed0510de79ea057905d Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Mon, 11 May 2026 17:00:23 +0900 Subject: [PATCH 05/29] [client] Skip DNS upstream failover on definitive EDE (#6089) --- .github/workflows/golangci-lint.yml | 2 +- client/internal/dns/upstream.go | 99 +++++++++++++++++++- client/internal/dns/upstream_test.go | 129 +++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- util/capture/text.go | 33 ++++++- 6 files changed, 261 insertions(+), 8 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 62dfe9bce..7b7b32ec0 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -19,7 +19,7 @@ jobs: - name: codespell uses: codespell-project/actions-codespell@v2 with: - ignore_words_list: erro,clienta,hastable,iif,groupd,testin,groupe,cros,ans,deriver,te,userA + ignore_words_list: erro,clienta,hastable,iif,groupd,testin,groupe,cros,ans,deriver,te,userA,ede,additionals skip: go.mod,go.sum,**/proxy/web/** golangci: strategy: diff --git a/client/internal/dns/upstream.go b/client/internal/dns/upstream.go index a26536f6e..39064f26c 100644 --- a/client/internal/dns/upstream.go +++ b/client/internal/dns/upstream.go @@ -30,6 +30,27 @@ import ( var currentMTU uint16 = iface.DefaultMTU +// nonRetryableEDECodes lists EDE info codes (RFC 8914) for which a SERVFAIL +// from one upstream means another upstream would return the same answer: +// DNSSEC validation outcomes and policy-based blocks. Transient errors +// (network, cached, not ready) are not included. +var nonRetryableEDECodes = map[uint16]struct{}{ + dns.ExtendedErrorCodeUnsupportedDNSKEYAlgorithm: {}, + dns.ExtendedErrorCodeUnsupportedDSDigestType: {}, + dns.ExtendedErrorCodeDNSSECIndeterminate: {}, + dns.ExtendedErrorCodeDNSBogus: {}, + dns.ExtendedErrorCodeSignatureExpired: {}, + dns.ExtendedErrorCodeSignatureNotYetValid: {}, + dns.ExtendedErrorCodeDNSKEYMissing: {}, + dns.ExtendedErrorCodeRRSIGsMissing: {}, + dns.ExtendedErrorCodeNoZoneKeyBitSet: {}, + dns.ExtendedErrorCodeNSECMissing: {}, + dns.ExtendedErrorCodeBlocked: {}, + dns.ExtendedErrorCodeCensored: {}, + dns.ExtendedErrorCodeFiltered: {}, + dns.ExtendedErrorCodeProhibited: {}, +} + // privateClientIface is the subset of the WireGuard interface needed by GetClientPrivate. type privateClientIface interface { Name() string @@ -250,6 +271,18 @@ func (u *upstreamResolverBase) queryUpstream(parentCtx context.Context, w dns.Re var t time.Duration var err error + // Advertise EDNS0 so the upstream may include Extended DNS Errors + // (RFC 8914) in failure responses; we use those to short-circuit + // failover for definitive answers like DNSSEC validation failures. + // Operate on a copy so the inbound request is unchanged: a client that + // did not advertise EDNS0 must not see an OPT in the response. + hadEdns := r.IsEdns0() != nil + reqUp := r + if !hadEdns { + reqUp = r.Copy() + reqUp.SetEdns0(upstreamUDPSize(), false) + } + var startTime time.Time var upstreamProto *upstreamProtocolResult func() { @@ -257,7 +290,7 @@ func (u *upstreamResolverBase) queryUpstream(parentCtx context.Context, w dns.Re defer cancel() ctx, upstreamProto = contextWithupstreamProtocolResult(ctx) startTime = time.Now() - rm, t, err = u.upstreamClient.exchange(ctx, upstream.String(), r) + rm, t, err = u.upstreamClient.exchange(ctx, upstream.String(), reqUp) }() if err != nil { @@ -269,13 +302,49 @@ func (u *upstreamResolverBase) queryUpstream(parentCtx context.Context, w dns.Re } if rm.Rcode == dns.RcodeServerFailure || rm.Rcode == dns.RcodeRefused { + if code, ok := nonRetryableEDE(rm); ok { + resutil.SetMeta(w, "ede", edeName(code)) + if !hadEdns { + stripOPT(rm) + } + u.writeSuccessResponse(w, rm, upstream, r.Question[0].Name, t, upstreamProto, logger) + return nil + } return &upstreamFailure{upstream: upstream, reason: dns.RcodeToString[rm.Rcode]} } + if !hadEdns { + stripOPT(rm) + } u.writeSuccessResponse(w, rm, upstream, r.Question[0].Name, t, upstreamProto, logger) return nil } +// upstreamUDPSize returns the EDNS0 UDP buffer size we advertise to upstreams, +// derived from the tunnel MTU and bounded against underflow. +func upstreamUDPSize() uint16 { + if currentMTU > ipUDPHeaderSize { + return currentMTU - ipUDPHeaderSize + } + return dns.MinMsgSize +} + +// stripOPT removes any OPT pseudo-RRs from the response's Extra section so +// the response complies with RFC 6891 when the client did not advertise EDNS0. +func stripOPT(rm *dns.Msg) { + if len(rm.Extra) == 0 { + return + } + out := rm.Extra[:0] + for _, rr := range rm.Extra { + if _, ok := rr.(*dns.OPT); ok { + continue + } + out = append(out, rr) + } + rm.Extra = out +} + func (u *upstreamResolverBase) handleUpstreamError(err error, upstream netip.AddrPort, startTime time.Time) *upstreamFailure { if !errors.Is(err, context.DeadlineExceeded) && !isTimeout(err) { return &upstreamFailure{upstream: upstream, reason: err.Error()} @@ -337,6 +406,34 @@ func formatFailures(failures []upstreamFailure) string { return strings.Join(parts, ", ") } +// nonRetryableEDE returns the first non-retryable EDE code carried in the +// response, if any. +func nonRetryableEDE(rm *dns.Msg) (uint16, bool) { + opt := rm.IsEdns0() + if opt == nil { + return 0, false + } + for _, o := range opt.Option { + ede, ok := o.(*dns.EDNS0_EDE) + if !ok { + continue + } + if _, ok := nonRetryableEDECodes[ede.InfoCode]; ok { + return ede.InfoCode, true + } + } + return 0, false +} + +// edeName returns a human-readable name for an EDE code, falling back to +// the numeric code when unknown. +func edeName(code uint16) string { + if name, ok := dns.ExtendedErrorCodeToString[code]; ok { + return name + } + return fmt.Sprintf("EDE %d", code) +} + // ProbeAvailability tests all upstream servers simultaneously and // disables the resolver if none work func (u *upstreamResolverBase) ProbeAvailability(ctx context.Context) { diff --git a/client/internal/dns/upstream_test.go b/client/internal/dns/upstream_test.go index 1797fdad8..d6aec05ca 100644 --- a/client/internal/dns/upstream_test.go +++ b/client/internal/dns/upstream_test.go @@ -770,3 +770,132 @@ func TestExchangeWithFallback_TCPTruncatesToClientSize(t *testing.T) { assert.Less(t, len(rm2.Answer), 20, "small EDNS0 client should get fewer records") assert.True(t, rm2.Truncated, "response should be truncated for small buffer client") } + +func msgWithEDE(rcode int, codes ...uint16) *dns.Msg { + m := new(dns.Msg) + m.Response = true + m.Rcode = rcode + if len(codes) == 0 { + return m + } + opt := &dns.OPT{Hdr: dns.RR_Header{Name: ".", Rrtype: dns.TypeOPT}} + opt.SetUDPSize(dns.MinMsgSize) + for _, c := range codes { + opt.Option = append(opt.Option, &dns.EDNS0_EDE{InfoCode: c}) + } + m.Extra = append(m.Extra, opt) + return m +} + +func TestNonRetryableEDE(t *testing.T) { + tests := []struct { + name string + msg *dns.Msg + wantOK bool + wantCode uint16 + }{ + {name: "no edns0", msg: msgWithEDE(dns.RcodeServerFailure)}, + { + name: "opt without ede", + msg: func() *dns.Msg { + m := msgWithEDE(dns.RcodeServerFailure) + opt := &dns.OPT{Hdr: dns.RR_Header{Name: ".", Rrtype: dns.TypeOPT}} + opt.Option = append(opt.Option, &dns.EDNS0_NSID{Code: dns.EDNS0NSID}) + m.Extra = []dns.RR{opt} + return m + }(), + }, + {name: "ede dnsbogus", msg: msgWithEDE(dns.RcodeServerFailure, dns.ExtendedErrorCodeDNSBogus), wantOK: true, wantCode: dns.ExtendedErrorCodeDNSBogus}, + {name: "ede signature expired", msg: msgWithEDE(dns.RcodeServerFailure, dns.ExtendedErrorCodeSignatureExpired), wantOK: true, wantCode: dns.ExtendedErrorCodeSignatureExpired}, + {name: "ede blocked", msg: msgWithEDE(dns.RcodeServerFailure, dns.ExtendedErrorCodeBlocked), wantOK: true, wantCode: dns.ExtendedErrorCodeBlocked}, + {name: "ede prohibited", msg: msgWithEDE(dns.RcodeRefused, dns.ExtendedErrorCodeProhibited), wantOK: true, wantCode: dns.ExtendedErrorCodeProhibited}, + {name: "ede cached error retryable", msg: msgWithEDE(dns.RcodeServerFailure, dns.ExtendedErrorCodeCachedError)}, + {name: "ede network error retryable", msg: msgWithEDE(dns.RcodeServerFailure, dns.ExtendedErrorCodeNetworkError)}, + {name: "ede not ready retryable", msg: msgWithEDE(dns.RcodeServerFailure, dns.ExtendedErrorCodeNotReady)}, + { + name: "first non-retryable wins", + msg: msgWithEDE(dns.RcodeServerFailure, dns.ExtendedErrorCodeNetworkError, dns.ExtendedErrorCodeDNSBogus), + wantOK: true, + wantCode: dns.ExtendedErrorCodeDNSBogus, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + code, ok := nonRetryableEDE(tc.msg) + assert.Equal(t, tc.wantOK, ok, "ok should match") + if tc.wantOK { + assert.Equal(t, tc.wantCode, code, "code should match") + } + }) + } +} + +func TestEDEName(t *testing.T) { + assert.Equal(t, "DNSSEC Bogus", edeName(dns.ExtendedErrorCodeDNSBogus)) + assert.Equal(t, "Signature Expired", edeName(dns.ExtendedErrorCodeSignatureExpired)) + assert.Equal(t, "EDE 9999", edeName(9999), "unknown code falls back to numeric") +} + +func TestStripOPT(t *testing.T) { + rm := &dns.Msg{ + Extra: []dns.RR{ + &dns.OPT{Hdr: dns.RR_Header{Name: ".", Rrtype: dns.TypeOPT}}, + &dns.A{Hdr: dns.RR_Header{Name: "x.", Rrtype: dns.TypeA}, A: net.IPv4(1, 2, 3, 4)}, + }, + } + stripOPT(rm) + assert.Len(t, rm.Extra, 1, "OPT should be removed, A kept") + _, isOPT := rm.Extra[0].(*dns.OPT) + assert.False(t, isOPT, "remaining record must not be OPT") +} + +func TestUpstreamResolver_NonRetryableEDEShortCircuits(t *testing.T) { + upstream1 := netip.MustParseAddrPort("192.0.2.1:53") + upstream2 := netip.MustParseAddrPort("192.0.2.2:53") + + servfailWithEDE := msgWithEDE(dns.RcodeServerFailure, dns.ExtendedErrorCodeDNSBogus) + successResp := buildMockResponse(dns.RcodeSuccess, "192.0.2.100") + + var queried []string + tracking := &trackingMockClient{ + inner: &mockUpstreamResolverPerServer{ + responses: map[string]mockUpstreamResponse{ + upstream1.String(): {msg: servfailWithEDE}, + upstream2.String(): {msg: successResp}, + }, + rtt: time.Millisecond, + }, + queriedUpstreams: &queried, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + resolver := &upstreamResolverBase{ + ctx: ctx, + upstreamClient: tracking, + upstreamServers: []netip.AddrPort{upstream1, upstream2}, + upstreamTimeout: UpstreamTimeout, + } + + var written *dns.Msg + w := &test.MockResponseWriter{ + WriteMsgFunc: func(m *dns.Msg) error { + written = m + return nil + }, + } + + // Client query without EDNS0 must not see an OPT in the response. + q := new(dns.Msg).SetQuestion("example.com.", dns.TypeA) + resolver.ServeDNS(w, q) + + require.NotNil(t, written, "response must be written") + assert.Equal(t, dns.RcodeServerFailure, written.Rcode, "SERVFAIL must propagate") + assert.Len(t, queried, 1, "only first upstream should be queried") + assert.Equal(t, upstream1.String(), queried[0]) + for _, rr := range written.Extra { + _, isOPT := rr.(*dns.OPT) + assert.False(t, isOPT, "synthetic OPT must not leak to a non-EDNS0 client") + } +} diff --git a/go.mod b/go.mod index 84aeab941..5704887ce 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,7 @@ require ( github.com/lrh3321/ipset-go v0.0.0-20250619021614-54a0a98ace81 github.com/mdlayher/socket v0.5.1 github.com/mdp/qrterminal/v3 v3.2.1 - github.com/miekg/dns v1.1.59 + github.com/miekg/dns v1.1.72 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/netbirdio/management-integrations/integrations v0.0.0-20260416123949-2355d972be42 github.com/netbirdio/signal-dispatcher/dispatcher v0.0.0-20250805121659-6b4ac470ca45 diff --git a/go.sum b/go.sum index 851d1ce66..42652169c 100644 --- a/go.sum +++ b/go.sum @@ -455,8 +455,8 @@ github.com/mdp/qrterminal/v3 v3.2.1 h1:6+yQjiiOsSuXT5n9/m60E54vdgFsw0zhADHhHLrFe github.com/mdp/qrterminal/v3 v3.2.1/go.mod h1:jOTmXvnBsMy5xqLniO0R++Jmjs2sTm9dFSuQ5kpz/SU= github.com/mholt/acmez/v2 v2.0.1 h1:3/3N0u1pLjMK4sNEAFSI+bcvzbPhRpY383sy1kLHJ6k= github.com/mholt/acmez/v2 v2.0.1/go.mod h1:fX4c9r5jYwMyMsC+7tkYRxHibkOTgta5DIFGoe67e1U= -github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= -github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= +github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= +github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws= github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= diff --git a/util/capture/text.go b/util/capture/text.go index fbb26654e..a6a6dd28b 100644 --- a/util/capture/text.go +++ b/util/capture/text.go @@ -12,6 +12,7 @@ import ( "github.com/google/gopacket" "github.com/google/gopacket/layers" + "github.com/miekg/dns" ) // TextWriter writes human-readable one-line-per-packet summaries. @@ -594,19 +595,45 @@ func formatDNSResponse(d *layers.DNS, rd string, plen int) string { anCount := d.ANCount nsCount := d.NSCount arCount := d.ARCount + ede := formatEDE(d) if d.ResponseCode != layers.DNSResponseCodeNoErr { - return fmt.Sprintf("%04x %d/%d/%d %s (%d)", d.ID, anCount, nsCount, arCount, d.ResponseCode, plen) + return fmt.Sprintf("%04x %d/%d/%d %s%s (%d)", d.ID, anCount, nsCount, arCount, d.ResponseCode, ede, plen) } if anCount > 0 && len(d.Answers) > 0 { rr := d.Answers[0] if rdata := shortRData(&rr); rdata != "" { - return fmt.Sprintf("%04x %d/%d/%d %s %s (%d)", d.ID, anCount, nsCount, arCount, rr.Type, rdata, plen) + return fmt.Sprintf("%04x %d/%d/%d %s %s%s (%d)", d.ID, anCount, nsCount, arCount, rr.Type, rdata, ede, plen) } } - return fmt.Sprintf("%04x %d/%d/%d (%d)", d.ID, anCount, nsCount, arCount, plen) + return fmt.Sprintf("%04x %d/%d/%d%s (%d)", d.ID, anCount, nsCount, arCount, ede, plen) +} + +// dnsOPTCodeEDE is the EDNS0 option code for Extended DNS Errors (RFC 8914). +const dnsOPTCodeEDE layers.DNSOptionCode = layers.DNSOptionCode(dns.EDNS0EDE) + +// formatEDE returns " EDE=Name" for the first Extended DNS Error option +// found in the response, or empty string if none is present. +func formatEDE(d *layers.DNS) string { + for _, rr := range d.Additionals { + if rr.Type != layers.DNSTypeOPT { + continue + } + for _, opt := range rr.OPT { + if opt.Code != dnsOPTCodeEDE || len(opt.Data) < 2 { + continue + } + info := binary.BigEndian.Uint16(opt.Data[:2]) + name, ok := dns.ExtendedErrorCodeToString[info] + if !ok { + name = fmt.Sprintf("%d", info) + } + return " EDE=" + name + } + } + return "" } func shortRData(rr *layers.DNSResourceRecord) string { From 07cbfdbedec2804f2014c42049f23268b0dc2ec7 Mon Sep 17 00:00:00 2001 From: Vlad <4941176+crn4@users.noreply.github.com> Date: Mon, 11 May 2026 14:31:38 +0200 Subject: [PATCH 06/29] [proxy] feature: bring your own proxy (#5627) --- .../reverseproxy/domain/manager/manager.go | 22 +- .../domain/manager/manager_test.go | 110 +++++ .../modules/reverseproxy/proxy/manager.go | 8 +- .../reverseproxy/proxy/manager/manager.go | 64 ++- .../proxy/manager/manager_test.go | 337 +++++++++++++++ .../reverseproxy/proxy/manager_mock.go | 79 +++- .../modules/reverseproxy/proxy/proxy.go | 12 +- .../reverseproxy/proxytoken/handler.go | 195 +++++++++ .../reverseproxy/proxytoken/handler_test.go | 275 ++++++++++++ .../modules/reverseproxy/service/interface.go | 2 + .../reverseproxy/service/interface_mock.go | 29 ++ .../reverseproxy/service/manager/api.go | 24 + .../reverseproxy/service/manager/manager.go | 20 +- .../service/manager/manager_test.go | 6 +- management/internals/server/boot.go | 2 +- management/internals/shared/grpc/proxy.go | 192 ++++++-- .../shared/grpc/proxy_address_test.go | 29 ++ .../internals/shared/grpc/proxy_auth.go | 3 - .../shared/grpc/proxy_group_access_test.go | 18 + .../internals/shared/grpc/proxy_test.go | 55 +++ .../shared/grpc/validate_session_test.go | 28 +- management/server/account_test.go | 2 +- management/server/http/handler.go | 4 + .../proxy/auth_callback_integration_test.go | 9 + .../testing/testing_tools/channel/channel.go | 4 +- management/server/store/sql_store.go | 121 +++++- management/server/store/store.go | 13 +- management/server/store/store_mock.go | 157 ++++++- proxy/management_byop_integration_test.go | 409 ++++++++++++++++++ proxy/management_integration_test.go | 34 +- shared/management/http/api/openapi.yml | 165 +++++++ shared/management/http/api/types.gen.go | 41 ++ 32 files changed, 2352 insertions(+), 117 deletions(-) create mode 100644 management/internals/modules/reverseproxy/domain/manager/manager_test.go create mode 100644 management/internals/modules/reverseproxy/proxy/manager/manager_test.go create mode 100644 management/internals/modules/reverseproxy/proxytoken/handler.go create mode 100644 management/internals/modules/reverseproxy/proxytoken/handler_test.go create mode 100644 management/internals/shared/grpc/proxy_address_test.go create mode 100644 proxy/management_byop_integration_test.go diff --git a/management/internals/modules/reverseproxy/domain/manager/manager.go b/management/internals/modules/reverseproxy/domain/manager/manager.go index 2c4c1372e..ab899e0bf 100644 --- a/management/internals/modules/reverseproxy/domain/manager/manager.go +++ b/management/internals/modules/reverseproxy/domain/manager/manager.go @@ -31,6 +31,7 @@ type store interface { type proxyManager interface { GetActiveClusterAddresses(ctx context.Context) ([]string, error) + GetActiveClusterAddressesForAccount(ctx context.Context, accountID string) ([]string, error) ClusterSupportsCustomPorts(ctx context.Context, clusterAddr string) *bool ClusterRequireSubdomain(ctx context.Context, clusterAddr string) *bool ClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool @@ -71,8 +72,8 @@ func (m Manager) GetDomains(ctx context.Context, accountID, userID string) ([]*d var ret []*domain.Domain // Add connected proxy clusters as free domains. - // The cluster address itself is the free domain base (e.g., "eu.proxy.netbird.io"). - allowList, err := m.proxyManager.GetActiveClusterAddresses(ctx) + // For BYOP accounts, only their own cluster is returned; otherwise shared clusters. + allowList, err := m.getClusterAllowList(ctx, accountID) if err != nil { log.WithContext(ctx).Errorf("failed to get active proxy cluster addresses: %v", err) return nil, err @@ -126,8 +127,8 @@ func (m Manager) CreateDomain(ctx context.Context, accountID, userID, domainName return nil, status.NewPermissionDeniedError() } - // Verify the target cluster is in the available clusters - allowList, err := m.proxyManager.GetActiveClusterAddresses(ctx) + // Verify the target cluster is in the available clusters for this account + allowList, err := m.getClusterAllowList(ctx, accountID) if err != nil { return nil, fmt.Errorf("failed to get active proxy cluster addresses: %w", err) } @@ -273,7 +274,7 @@ func (m Manager) GetClusterDomains() []string { // For free domains (those ending with a known cluster suffix), the cluster is extracted from the domain. // For custom domains, the cluster is determined by checking the registered custom domain's target cluster. func (m Manager) DeriveClusterFromDomain(ctx context.Context, accountID, domain string) (string, error) { - allowList, err := m.proxyManager.GetActiveClusterAddresses(ctx) + allowList, err := m.getClusterAllowList(ctx, accountID) if err != nil { return "", fmt.Errorf("failed to get active proxy cluster addresses: %w", err) } @@ -298,6 +299,17 @@ func (m Manager) DeriveClusterFromDomain(ctx context.Context, accountID, domain return "", fmt.Errorf("domain %s does not match any available proxy cluster", domain) } +func (m Manager) getClusterAllowList(ctx context.Context, accountID string) ([]string, error) { + byopAddresses, err := m.proxyManager.GetActiveClusterAddressesForAccount(ctx, accountID) + if err != nil { + return nil, fmt.Errorf("get BYOP cluster addresses: %w", err) + } + if len(byopAddresses) > 0 { + return byopAddresses, nil + } + return m.proxyManager.GetActiveClusterAddresses(ctx) +} + func extractClusterFromCustomDomains(serviceDomain string, customDomains []*domain.Domain) (string, bool) { bestCluster := "" bestLen := -1 diff --git a/management/internals/modules/reverseproxy/domain/manager/manager_test.go b/management/internals/modules/reverseproxy/domain/manager/manager_test.go new file mode 100644 index 000000000..fdeb0765f --- /dev/null +++ b/management/internals/modules/reverseproxy/domain/manager/manager_test.go @@ -0,0 +1,110 @@ +package manager + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type mockProxyManager struct { + getActiveClusterAddressesFunc func(ctx context.Context) ([]string, error) + getActiveClusterAddressesForAccountFunc func(ctx context.Context, accountID string) ([]string, error) +} + +func (m *mockProxyManager) GetActiveClusterAddresses(ctx context.Context) ([]string, error) { + if m.getActiveClusterAddressesFunc != nil { + return m.getActiveClusterAddressesFunc(ctx) + } + return nil, nil +} + +func (m *mockProxyManager) GetActiveClusterAddressesForAccount(ctx context.Context, accountID string) ([]string, error) { + if m.getActiveClusterAddressesForAccountFunc != nil { + return m.getActiveClusterAddressesForAccountFunc(ctx, accountID) + } + return nil, nil +} + +func (m *mockProxyManager) ClusterSupportsCustomPorts(_ context.Context, _ string) *bool { + return nil +} + +func (m *mockProxyManager) ClusterRequireSubdomain(_ context.Context, _ string) *bool { + return nil +} + +func (m *mockProxyManager) ClusterSupportsCrowdSec(_ context.Context, _ string) *bool { + return nil +} + +func TestGetClusterAllowList_BYOPProxy(t *testing.T) { + pm := &mockProxyManager{ + getActiveClusterAddressesForAccountFunc: func(_ context.Context, accID string) ([]string, error) { + assert.Equal(t, "acc-123", accID) + return []string{"byop.example.com"}, nil + }, + getActiveClusterAddressesFunc: func(_ context.Context) ([]string, error) { + t.Fatal("should not call GetActiveClusterAddresses when BYOP addresses exist") + return nil, nil + }, + } + + mgr := Manager{proxyManager: pm} + result, err := mgr.getClusterAllowList(context.Background(), "acc-123") + require.NoError(t, err) + assert.Equal(t, []string{"byop.example.com"}, result) +} + +func TestGetClusterAllowList_NoBYOP_FallbackToShared(t *testing.T) { + pm := &mockProxyManager{ + getActiveClusterAddressesForAccountFunc: func(_ context.Context, _ string) ([]string, error) { + return nil, nil + }, + getActiveClusterAddressesFunc: func(_ context.Context) ([]string, error) { + return []string{"eu.proxy.netbird.io", "us.proxy.netbird.io"}, nil + }, + } + + mgr := Manager{proxyManager: pm} + result, err := mgr.getClusterAllowList(context.Background(), "acc-123") + require.NoError(t, err) + assert.Equal(t, []string{"eu.proxy.netbird.io", "us.proxy.netbird.io"}, result) +} + +func TestGetClusterAllowList_BYOPError_ReturnsError(t *testing.T) { + pm := &mockProxyManager{ + getActiveClusterAddressesForAccountFunc: func(_ context.Context, _ string) ([]string, error) { + return nil, errors.New("db error") + }, + getActiveClusterAddressesFunc: func(_ context.Context) ([]string, error) { + t.Fatal("should not call GetActiveClusterAddresses when BYOP lookup fails") + return nil, nil + }, + } + + mgr := Manager{proxyManager: pm} + result, err := mgr.getClusterAllowList(context.Background(), "acc-123") + require.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "BYOP cluster addresses") +} + +func TestGetClusterAllowList_BYOPEmptySlice_FallbackToShared(t *testing.T) { + pm := &mockProxyManager{ + getActiveClusterAddressesForAccountFunc: func(_ context.Context, _ string) ([]string, error) { + return []string{}, nil + }, + getActiveClusterAddressesFunc: func(_ context.Context) ([]string, error) { + return []string{"eu.proxy.netbird.io"}, nil + }, + } + + mgr := Manager{proxyManager: pm} + result, err := mgr.getClusterAllowList(context.Background(), "acc-123") + require.NoError(t, err) + assert.Equal(t, []string{"eu.proxy.netbird.io"}, result) +} + diff --git a/management/internals/modules/reverseproxy/proxy/manager.go b/management/internals/modules/reverseproxy/proxy/manager.go index 53c52b3aa..07ea6f0ab 100644 --- a/management/internals/modules/reverseproxy/proxy/manager.go +++ b/management/internals/modules/reverseproxy/proxy/manager.go @@ -11,15 +11,19 @@ import ( // Manager defines the interface for proxy operations type Manager interface { - Connect(ctx context.Context, proxyID, sessionID, clusterAddress, ipAddress string, capabilities *Capabilities) (*Proxy, error) + Connect(ctx context.Context, proxyID, sessionID, clusterAddress, ipAddress string, accountID *string, capabilities *Capabilities) (*Proxy, error) Disconnect(ctx context.Context, proxyID, sessionID string) error Heartbeat(ctx context.Context, p *Proxy) error GetActiveClusterAddresses(ctx context.Context) ([]string, error) - GetActiveClusters(ctx context.Context) ([]Cluster, error) + GetActiveClusterAddressesForAccount(ctx context.Context, accountID string) ([]string, error) ClusterSupportsCustomPorts(ctx context.Context, clusterAddr string) *bool ClusterRequireSubdomain(ctx context.Context, clusterAddr string) *bool ClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool CleanupStale(ctx context.Context, inactivityDuration time.Duration) error + GetAccountProxy(ctx context.Context, accountID string) (*Proxy, error) + CountAccountProxies(ctx context.Context, accountID string) (int64, error) + IsClusterAddressAvailable(ctx context.Context, clusterAddress, accountID string) (bool, error) + DeleteAccountCluster(ctx context.Context, clusterAddress, accountID string) error } // OIDCValidationConfig contains the OIDC configuration needed for token validation. diff --git a/management/internals/modules/reverseproxy/proxy/manager/manager.go b/management/internals/modules/reverseproxy/proxy/manager/manager.go index 341e8c943..b72a6ebe5 100644 --- a/management/internals/modules/reverseproxy/proxy/manager/manager.go +++ b/management/internals/modules/reverseproxy/proxy/manager/manager.go @@ -16,11 +16,16 @@ type store interface { DisconnectProxy(ctx context.Context, proxyID, sessionID string) error UpdateProxyHeartbeat(ctx context.Context, p *proxy.Proxy) error GetActiveProxyClusterAddresses(ctx context.Context) ([]string, error) - GetActiveProxyClusters(ctx context.Context) ([]proxy.Cluster, error) + GetActiveProxyClusterAddressesForAccount(ctx context.Context, accountID string) ([]string, error) + GetActiveProxyClusters(ctx context.Context, accountID string) ([]proxy.Cluster, error) GetClusterSupportsCustomPorts(ctx context.Context, clusterAddr string) *bool GetClusterRequireSubdomain(ctx context.Context, clusterAddr string) *bool GetClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool CleanupStaleProxies(ctx context.Context, inactivityDuration time.Duration) error + GetProxyByAccountID(ctx context.Context, accountID string) (*proxy.Proxy, error) + CountProxiesByAccountID(ctx context.Context, accountID string) (int64, error) + IsClusterAddressConflicting(ctx context.Context, clusterAddress, accountID string) (bool, error) + DeleteAccountCluster(ctx context.Context, clusterAddress, accountID string) error } // Manager handles all proxy operations @@ -44,7 +49,7 @@ func NewManager(store store, meter metric.Meter) (*Manager, error) { // Connect registers a new proxy connection in the database. // capabilities may be nil for old proxies that do not report them. -func (m Manager) Connect(ctx context.Context, proxyID, sessionID, clusterAddress, ipAddress string, capabilities *proxy.Capabilities) (*proxy.Proxy, error) { +func (m *Manager) Connect(ctx context.Context, proxyID, sessionID, clusterAddress, ipAddress string, accountID *string, capabilities *proxy.Capabilities) (*proxy.Proxy, error) { now := time.Now() var caps proxy.Capabilities if capabilities != nil { @@ -55,9 +60,10 @@ func (m Manager) Connect(ctx context.Context, proxyID, sessionID, clusterAddress SessionID: sessionID, ClusterAddress: clusterAddress, IPAddress: ipAddress, + AccountID: accountID, LastSeen: now, ConnectedAt: &now, - Status: "connected", + Status: proxy.StatusConnected, Capabilities: caps, } @@ -77,7 +83,7 @@ func (m Manager) Connect(ctx context.Context, proxyID, sessionID, clusterAddress } // Disconnect marks a proxy as disconnected in the database. -func (m Manager) Disconnect(ctx context.Context, proxyID, sessionID string) error { +func (m *Manager) Disconnect(ctx context.Context, proxyID, sessionID string) error { if err := m.store.DisconnectProxy(ctx, proxyID, sessionID); err != nil { log.WithContext(ctx).Errorf("failed to disconnect proxy %s session %s: %v", proxyID, sessionID, err) return err @@ -92,7 +98,7 @@ func (m Manager) Disconnect(ctx context.Context, proxyID, sessionID string) erro } // Heartbeat updates the proxy's last seen timestamp. -func (m Manager) Heartbeat(ctx context.Context, p *proxy.Proxy) error { +func (m *Manager) Heartbeat(ctx context.Context, p *proxy.Proxy) error { if err := m.store.UpdateProxyHeartbeat(ctx, p); err != nil { log.WithContext(ctx).Debugf("failed to update proxy %s heartbeat: %v", p.ID, err) return err @@ -104,7 +110,7 @@ func (m Manager) Heartbeat(ctx context.Context, p *proxy.Proxy) error { } // GetActiveClusterAddresses returns all unique cluster addresses for active proxies -func (m Manager) GetActiveClusterAddresses(ctx context.Context) ([]string, error) { +func (m *Manager) GetActiveClusterAddresses(ctx context.Context) ([]string, error) { addresses, err := m.store.GetActiveProxyClusterAddresses(ctx) if err != nil { log.WithContext(ctx).Errorf("failed to get active proxy cluster addresses: %v", err) @@ -113,16 +119,6 @@ func (m Manager) GetActiveClusterAddresses(ctx context.Context) ([]string, error return addresses, nil } -// GetActiveClusters returns all active proxy clusters with their connected proxy count. -func (m Manager) GetActiveClusters(ctx context.Context) ([]proxy.Cluster, error) { - clusters, err := m.store.GetActiveProxyClusters(ctx) - if err != nil { - log.WithContext(ctx).Errorf("failed to get active proxy clusters: %v", err) - return nil, err - } - return clusters, nil -} - // ClusterSupportsCustomPorts returns whether any active proxy in the cluster // supports custom ports. Returns nil when no proxy has reported capabilities. func (m Manager) ClusterSupportsCustomPorts(ctx context.Context, clusterAddr string) *bool { @@ -142,10 +138,44 @@ func (m Manager) ClusterSupportsCrowdSec(ctx context.Context, clusterAddr string } // CleanupStale removes proxies that haven't sent heartbeat in the specified duration -func (m Manager) CleanupStale(ctx context.Context, inactivityDuration time.Duration) error { +func (m *Manager) CleanupStale(ctx context.Context, inactivityDuration time.Duration) error { if err := m.store.CleanupStaleProxies(ctx, inactivityDuration); err != nil { log.WithContext(ctx).Errorf("failed to cleanup stale proxies: %v", err) return err } return nil } + +func (m *Manager) GetActiveClusterAddressesForAccount(ctx context.Context, accountID string) ([]string, error) { + addresses, err := m.store.GetActiveProxyClusterAddressesForAccount(ctx, accountID) + if err != nil { + log.WithContext(ctx).Errorf("failed to get active proxy cluster addresses for account %s: %v", accountID, err) + return nil, err + } + return addresses, nil +} + +func (m *Manager) GetAccountProxy(ctx context.Context, accountID string) (*proxy.Proxy, error) { + return m.store.GetProxyByAccountID(ctx, accountID) +} + +func (m *Manager) CountAccountProxies(ctx context.Context, accountID string) (int64, error) { + return m.store.CountProxiesByAccountID(ctx, accountID) +} + +func (m *Manager) IsClusterAddressAvailable(ctx context.Context, clusterAddress, accountID string) (bool, error) { + conflicting, err := m.store.IsClusterAddressConflicting(ctx, clusterAddress, accountID) + if err != nil { + return false, err + } + return !conflicting, nil +} + +func (m *Manager) DeleteAccountCluster(ctx context.Context, clusterAddress, accountID string) error { + if err := m.store.DeleteAccountCluster(ctx, clusterAddress, accountID); err != nil { + log.WithContext(ctx).Errorf("failed to delete cluster %s for account %s: %v", clusterAddress, accountID, err) + return err + } + return nil +} + diff --git a/management/internals/modules/reverseproxy/proxy/manager/manager_test.go b/management/internals/modules/reverseproxy/proxy/manager/manager_test.go new file mode 100644 index 000000000..3c53fe684 --- /dev/null +++ b/management/internals/modules/reverseproxy/proxy/manager/manager_test.go @@ -0,0 +1,337 @@ +package manager + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/metric/noop" + + "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy" +) + +type mockStore struct { + saveProxyFunc func(ctx context.Context, p *proxy.Proxy) error + disconnectProxyFunc func(ctx context.Context, proxyID, sessionID string) error + updateProxyHeartbeatFunc func(ctx context.Context, p *proxy.Proxy) error + getActiveProxyClusterAddressesFunc func(ctx context.Context) ([]string, error) + getActiveProxyClusterAddressesForAccFunc func(ctx context.Context, accountID string) ([]string, error) + cleanupStaleProxiesFunc func(ctx context.Context, d time.Duration) error + getProxyByAccountIDFunc func(ctx context.Context, accountID string) (*proxy.Proxy, error) + countProxiesByAccountIDFunc func(ctx context.Context, accountID string) (int64, error) + isClusterAddressConflictingFunc func(ctx context.Context, clusterAddress, accountID string) (bool, error) + deleteAccountClusterFunc func(ctx context.Context, clusterAddress, accountID string) error +} + +func (m *mockStore) SaveProxy(ctx context.Context, p *proxy.Proxy) error { + if m.saveProxyFunc != nil { + return m.saveProxyFunc(ctx, p) + } + return nil +} +func (m *mockStore) DisconnectProxy(ctx context.Context, proxyID, sessionID string) error { + if m.disconnectProxyFunc != nil { + return m.disconnectProxyFunc(ctx, proxyID, sessionID) + } + return nil +} +func (m *mockStore) UpdateProxyHeartbeat(ctx context.Context, p *proxy.Proxy) error { + if m.updateProxyHeartbeatFunc != nil { + return m.updateProxyHeartbeatFunc(ctx, p) + } + return nil +} +func (m *mockStore) GetActiveProxyClusterAddresses(ctx context.Context) ([]string, error) { + if m.getActiveProxyClusterAddressesFunc != nil { + return m.getActiveProxyClusterAddressesFunc(ctx) + } + return nil, nil +} +func (m *mockStore) GetActiveProxyClusterAddressesForAccount(ctx context.Context, accountID string) ([]string, error) { + if m.getActiveProxyClusterAddressesForAccFunc != nil { + return m.getActiveProxyClusterAddressesForAccFunc(ctx, accountID) + } + return nil, nil +} +func (m *mockStore) GetActiveProxyClusters(_ context.Context, _ string) ([]proxy.Cluster, error) { + return nil, nil +} +func (m *mockStore) CleanupStaleProxies(ctx context.Context, d time.Duration) error { + if m.cleanupStaleProxiesFunc != nil { + return m.cleanupStaleProxiesFunc(ctx, d) + } + return nil +} +func (m *mockStore) GetProxyByAccountID(ctx context.Context, accountID string) (*proxy.Proxy, error) { + if m.getProxyByAccountIDFunc != nil { + return m.getProxyByAccountIDFunc(ctx, accountID) + } + return nil, fmt.Errorf("proxy not found for account %s", accountID) +} +func (m *mockStore) CountProxiesByAccountID(ctx context.Context, accountID string) (int64, error) { + if m.countProxiesByAccountIDFunc != nil { + return m.countProxiesByAccountIDFunc(ctx, accountID) + } + return 0, nil +} +func (m *mockStore) IsClusterAddressConflicting(ctx context.Context, clusterAddress, accountID string) (bool, error) { + if m.isClusterAddressConflictingFunc != nil { + return m.isClusterAddressConflictingFunc(ctx, clusterAddress, accountID) + } + return false, nil +} +func (m *mockStore) DeleteAccountCluster(ctx context.Context, clusterAddress, accountID string) error { + if m.deleteAccountClusterFunc != nil { + return m.deleteAccountClusterFunc(ctx, clusterAddress, accountID) + } + return nil +} +func (m *mockStore) GetClusterSupportsCustomPorts(_ context.Context, _ string) *bool { + return nil +} +func (m *mockStore) GetClusterRequireSubdomain(_ context.Context, _ string) *bool { + return nil +} +func (m *mockStore) GetClusterSupportsCrowdSec(_ context.Context, _ string) *bool { + return nil +} + +func newTestManager(s store) *Manager { + meter := noop.NewMeterProvider().Meter("test") + m, err := NewManager(s, meter) + if err != nil { + panic(err) + } + return m +} + +func TestConnect_WithAccountID(t *testing.T) { + accountID := "acc-123" + + var savedProxy *proxy.Proxy + s := &mockStore{ + saveProxyFunc: func(_ context.Context, p *proxy.Proxy) error { + savedProxy = p + return nil + }, + } + + mgr := newTestManager(s) + _, err := mgr.Connect(context.Background(), "proxy-1", "session-1", "cluster.example.com", "10.0.0.1", &accountID, nil) + require.NoError(t, err) + + require.NotNil(t, savedProxy) + assert.Equal(t, "proxy-1", savedProxy.ID) + assert.Equal(t, "session-1", savedProxy.SessionID) + assert.Equal(t, "cluster.example.com", savedProxy.ClusterAddress) + assert.Equal(t, "10.0.0.1", savedProxy.IPAddress) + assert.Equal(t, &accountID, savedProxy.AccountID) + assert.Equal(t, proxy.StatusConnected, savedProxy.Status) + assert.NotNil(t, savedProxy.ConnectedAt) +} + +func TestConnect_WithoutAccountID(t *testing.T) { + var savedProxy *proxy.Proxy + s := &mockStore{ + saveProxyFunc: func(_ context.Context, p *proxy.Proxy) error { + savedProxy = p + return nil + }, + } + + mgr := newTestManager(s) + _, err := mgr.Connect(context.Background(), "proxy-1", "session-1", "eu.proxy.netbird.io", "10.0.0.1", nil, nil) + require.NoError(t, err) + + require.NotNil(t, savedProxy) + assert.Nil(t, savedProxy.AccountID) + assert.Equal(t, proxy.StatusConnected, savedProxy.Status) +} + +func TestConnect_StoreError(t *testing.T) { + s := &mockStore{ + saveProxyFunc: func(_ context.Context, _ *proxy.Proxy) error { + return errors.New("db error") + }, + } + + mgr := newTestManager(s) + _, err := mgr.Connect(context.Background(), "proxy-1", "session-1", "cluster.example.com", "10.0.0.1", nil, nil) + assert.Error(t, err) +} + +func TestIsClusterAddressAvailable(t *testing.T) { + tests := []struct { + name string + conflicting bool + storeErr error + wantResult bool + wantErr bool + }{ + { + name: "available - no conflict", + conflicting: false, + wantResult: true, + }, + { + name: "not available - conflict exists", + conflicting: true, + wantResult: false, + }, + { + name: "store error", + storeErr: errors.New("db error"), + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &mockStore{ + isClusterAddressConflictingFunc: func(_ context.Context, _, _ string) (bool, error) { + return tt.conflicting, tt.storeErr + }, + } + + mgr := newTestManager(s) + result, err := mgr.IsClusterAddressAvailable(context.Background(), "cluster.example.com", "acc-123") + if tt.wantErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantResult, result) + }) + } +} + +func TestCountAccountProxies(t *testing.T) { + tests := []struct { + name string + count int64 + storeErr error + wantCount int64 + wantErr bool + }{ + { + name: "no proxies", + count: 0, + wantCount: 0, + }, + { + name: "one proxy", + count: 1, + wantCount: 1, + }, + { + name: "store error", + storeErr: errors.New("db error"), + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &mockStore{ + countProxiesByAccountIDFunc: func(_ context.Context, _ string) (int64, error) { + return tt.count, tt.storeErr + }, + } + + mgr := newTestManager(s) + count, err := mgr.CountAccountProxies(context.Background(), "acc-123") + if tt.wantErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantCount, count) + }) + } +} + +func TestGetAccountProxy(t *testing.T) { + accountID := "acc-123" + + t.Run("found", func(t *testing.T) { + expected := &proxy.Proxy{ + ID: "proxy-1", + ClusterAddress: "byop.example.com", + AccountID: &accountID, + Status: proxy.StatusConnected, + } + s := &mockStore{ + getProxyByAccountIDFunc: func(_ context.Context, accID string) (*proxy.Proxy, error) { + assert.Equal(t, accountID, accID) + return expected, nil + }, + } + + mgr := newTestManager(s) + p, err := mgr.GetAccountProxy(context.Background(), accountID) + require.NoError(t, err) + assert.Equal(t, expected, p) + }) + + t.Run("not found", func(t *testing.T) { + s := &mockStore{ + getProxyByAccountIDFunc: func(_ context.Context, _ string) (*proxy.Proxy, error) { + return nil, errors.New("not found") + }, + } + + mgr := newTestManager(s) + _, err := mgr.GetAccountProxy(context.Background(), accountID) + assert.Error(t, err) + }) +} + +func TestDeleteAccountCluster(t *testing.T) { + t.Run("success", func(t *testing.T) { + var deletedCluster, deletedAccount string + s := &mockStore{ + deleteAccountClusterFunc: func(_ context.Context, clusterAddress, accountID string) error { + deletedCluster = clusterAddress + deletedAccount = accountID + return nil + }, + } + + mgr := newTestManager(s) + err := mgr.DeleteAccountCluster(context.Background(), "cluster.example.com", "acc-123") + require.NoError(t, err) + assert.Equal(t, "cluster.example.com", deletedCluster) + assert.Equal(t, "acc-123", deletedAccount) + }) + + t.Run("store error", func(t *testing.T) { + s := &mockStore{ + deleteAccountClusterFunc: func(_ context.Context, _, _ string) error { + return errors.New("db error") + }, + } + + mgr := newTestManager(s) + err := mgr.DeleteAccountCluster(context.Background(), "cluster.example.com", "acc-123") + assert.Error(t, err) + }) +} + +func TestGetActiveClusterAddressesForAccount(t *testing.T) { + expected := []string{"byop.example.com"} + s := &mockStore{ + getActiveProxyClusterAddressesForAccFunc: func(_ context.Context, accID string) ([]string, error) { + assert.Equal(t, "acc-123", accID) + return expected, nil + }, + } + + mgr := newTestManager(s) + result, err := mgr.GetActiveClusterAddressesForAccount(context.Background(), "acc-123") + require.NoError(t, err) + assert.Equal(t, expected, result) +} diff --git a/management/internals/modules/reverseproxy/proxy/manager_mock.go b/management/internals/modules/reverseproxy/proxy/manager_mock.go index 98d97b3c6..a0e360a1b 100644 --- a/management/internals/modules/reverseproxy/proxy/manager_mock.go +++ b/management/internals/modules/reverseproxy/proxy/manager_mock.go @@ -93,18 +93,18 @@ func (mr *MockManagerMockRecorder) ClusterSupportsCrowdSec(ctx, clusterAddr inte } // Connect mocks base method. -func (m *MockManager) Connect(ctx context.Context, proxyID, sessionID, clusterAddress, ipAddress string, capabilities *Capabilities) (*Proxy, error) { +func (m *MockManager) Connect(ctx context.Context, proxyID, sessionID, clusterAddress, ipAddress string, accountID *string, capabilities *Capabilities) (*Proxy, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Connect", ctx, proxyID, sessionID, clusterAddress, ipAddress, capabilities) + ret := m.ctrl.Call(m, "Connect", ctx, proxyID, sessionID, clusterAddress, ipAddress, accountID, capabilities) ret0, _ := ret[0].(*Proxy) ret1, _ := ret[1].(error) return ret0, ret1 } // Connect indicates an expected call of Connect. -func (mr *MockManagerMockRecorder) Connect(ctx, proxyID, sessionID, clusterAddress, ipAddress, capabilities interface{}) *gomock.Call { +func (mr *MockManagerMockRecorder) Connect(ctx, proxyID, sessionID, clusterAddress, ipAddress, accountID, capabilities interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Connect", reflect.TypeOf((*MockManager)(nil).Connect), ctx, proxyID, sessionID, clusterAddress, ipAddress, capabilities) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Connect", reflect.TypeOf((*MockManager)(nil).Connect), ctx, proxyID, sessionID, clusterAddress, ipAddress, accountID, capabilities) } // Disconnect mocks base method. @@ -136,19 +136,17 @@ func (mr *MockManagerMockRecorder) GetActiveClusterAddresses(ctx interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveClusterAddresses", reflect.TypeOf((*MockManager)(nil).GetActiveClusterAddresses), ctx) } -// GetActiveClusters mocks base method. -func (m *MockManager) GetActiveClusters(ctx context.Context) ([]Cluster, error) { +func (m *MockManager) GetActiveClusterAddressesForAccount(ctx context.Context, accountID string) ([]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetActiveClusters", ctx) - ret0, _ := ret[0].([]Cluster) + ret := m.ctrl.Call(m, "GetActiveClusterAddressesForAccount", ctx, accountID) + ret0, _ := ret[0].([]string) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetActiveClusters indicates an expected call of GetActiveClusters. -func (mr *MockManagerMockRecorder) GetActiveClusters(ctx interface{}) *gomock.Call { +func (mr *MockManagerMockRecorder) GetActiveClusterAddressesForAccount(ctx, accountID interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveClusters", reflect.TypeOf((*MockManager)(nil).GetActiveClusters), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveClusterAddressesForAccount", reflect.TypeOf((*MockManager)(nil).GetActiveClusterAddressesForAccount), ctx, accountID) } // Heartbeat mocks base method. @@ -165,6 +163,65 @@ func (mr *MockManagerMockRecorder) Heartbeat(ctx, p interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Heartbeat", reflect.TypeOf((*MockManager)(nil).Heartbeat), ctx, p) } +// GetAccountProxy mocks base method. +func (m *MockManager) GetAccountProxy(ctx context.Context, accountID string) (*Proxy, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAccountProxy", ctx, accountID) + ret0, _ := ret[0].(*Proxy) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAccountProxy indicates an expected call of GetAccountProxy. +func (mr *MockManagerMockRecorder) GetAccountProxy(ctx, accountID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAccountProxy", reflect.TypeOf((*MockManager)(nil).GetAccountProxy), ctx, accountID) +} + +// CountAccountProxies mocks base method. +func (m *MockManager) CountAccountProxies(ctx context.Context, accountID string) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountAccountProxies", ctx, accountID) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountAccountProxies indicates an expected call of CountAccountProxies. +func (mr *MockManagerMockRecorder) CountAccountProxies(ctx, accountID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountAccountProxies", reflect.TypeOf((*MockManager)(nil).CountAccountProxies), ctx, accountID) +} + +// IsClusterAddressAvailable mocks base method. +func (m *MockManager) IsClusterAddressAvailable(ctx context.Context, clusterAddress, accountID string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsClusterAddressAvailable", ctx, clusterAddress, accountID) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsClusterAddressAvailable indicates an expected call of IsClusterAddressAvailable. +func (mr *MockManagerMockRecorder) IsClusterAddressAvailable(ctx, clusterAddress, accountID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsClusterAddressAvailable", reflect.TypeOf((*MockManager)(nil).IsClusterAddressAvailable), ctx, clusterAddress, accountID) +} + +// DeleteAccountCluster mocks base method. +func (m *MockManager) DeleteAccountCluster(ctx context.Context, clusterAddress, accountID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAccountCluster", ctx, clusterAddress, accountID) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAccountCluster indicates an expected call of DeleteAccountCluster. +func (mr *MockManagerMockRecorder) DeleteAccountCluster(ctx, clusterAddress, accountID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAccountCluster", reflect.TypeOf((*MockManager)(nil).DeleteAccountCluster), ctx, clusterAddress, accountID) +} + // MockController is a mock of Controller interface. type MockController struct { ctrl *gomock.Controller diff --git a/management/internals/modules/reverseproxy/proxy/proxy.go b/management/internals/modules/reverseproxy/proxy/proxy.go index dcedb8811..64394799e 100644 --- a/management/internals/modules/reverseproxy/proxy/proxy.go +++ b/management/internals/modules/reverseproxy/proxy/proxy.go @@ -1,6 +1,13 @@ package proxy -import "time" +import ( + "time" +) + +const ( + StatusConnected = "connected" + StatusDisconnected = "disconnected" +) // Capabilities describes what a proxy can handle, as reported via gRPC. // Nil fields mean the proxy never reported this capability. @@ -21,6 +28,7 @@ type Proxy struct { SessionID string `gorm:"type:varchar(36)"` ClusterAddress string `gorm:"type:varchar(255);not null;index:idx_proxy_cluster_status"` IPAddress string `gorm:"type:varchar(45)"` + AccountID *string `gorm:"type:varchar(255);index:idx_proxy_account_id"` LastSeen time.Time `gorm:"not null;index:idx_proxy_last_seen"` ConnectedAt *time.Time DisconnectedAt *time.Time @@ -36,6 +44,8 @@ func (Proxy) TableName() string { // Cluster represents a group of proxy nodes serving the same address. type Cluster struct { + ID string Address string ConnectedProxies int + SelfHosted bool } diff --git a/management/internals/modules/reverseproxy/proxytoken/handler.go b/management/internals/modules/reverseproxy/proxytoken/handler.go new file mode 100644 index 000000000..728cdf723 --- /dev/null +++ b/management/internals/modules/reverseproxy/proxytoken/handler.go @@ -0,0 +1,195 @@ +package proxytoken + +import ( + "encoding/json" + "net/http" + "time" + + "github.com/gorilla/mux" + + nbcontext "github.com/netbirdio/netbird/management/server/context" + "github.com/netbirdio/netbird/management/server/permissions" + "github.com/netbirdio/netbird/management/server/permissions/modules" + "github.com/netbirdio/netbird/management/server/permissions/operations" + "github.com/netbirdio/netbird/management/server/store" + "github.com/netbirdio/netbird/management/server/types" + "github.com/netbirdio/netbird/shared/management/http/api" + "github.com/netbirdio/netbird/shared/management/http/util" + "github.com/netbirdio/netbird/shared/management/status" +) + +type handler struct { + store store.Store + permissionsManager permissions.Manager +} + +func RegisterEndpoints(s store.Store, permissionsManager permissions.Manager, router *mux.Router) { + h := &handler{store: s, permissionsManager: permissionsManager} + router.HandleFunc("/reverse-proxies/proxy-tokens", h.listTokens).Methods("GET", "OPTIONS") + router.HandleFunc("/reverse-proxies/proxy-tokens", h.createToken).Methods("POST", "OPTIONS") + router.HandleFunc("/reverse-proxies/proxy-tokens/{tokenId}", h.revokeToken).Methods("DELETE", "OPTIONS") +} + +func (h *handler) createToken(w http.ResponseWriter, r *http.Request) { + userAuth, err := nbcontext.GetUserAuthFromContext(r.Context()) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + + ok, err := h.permissionsManager.ValidateUserPermissions(r.Context(), userAuth.AccountId, userAuth.UserId, modules.Services, operations.Create) + if err != nil { + util.WriteErrorResponse("failed to validate permissions", http.StatusInternalServerError, w) + return + } + if !ok { + util.WriteErrorResponse("permission denied", http.StatusForbidden, w) + return + } + + var req api.ProxyTokenRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + util.WriteErrorResponse("couldn't parse JSON request", http.StatusBadRequest, w) + return + } + + if req.Name == "" || len(req.Name) > 255 { + util.WriteErrorResponse("name is required and must be at most 255 characters", http.StatusBadRequest, w) + return + } + + var expiresIn time.Duration + if req.ExpiresIn != nil { + if *req.ExpiresIn < 0 { + util.WriteErrorResponse("expires_in must be non-negative", http.StatusBadRequest, w) + return + } + if *req.ExpiresIn > 0 { + expiresIn = time.Duration(*req.ExpiresIn) * time.Second + } + } + + accountID := userAuth.AccountId + generated, err := types.CreateNewProxyAccessToken(req.Name, expiresIn, &accountID, userAuth.UserId) + if err != nil { + util.WriteErrorResponse("failed to generate token", http.StatusInternalServerError, w) + return + } + + if err := h.store.SaveProxyAccessToken(r.Context(), &generated.ProxyAccessToken); err != nil { + util.WriteErrorResponse("failed to save token", http.StatusInternalServerError, w) + return + } + + resp := toProxyTokenCreatedResponse(generated) + util.WriteJSONObject(r.Context(), w, resp) +} + +func (h *handler) listTokens(w http.ResponseWriter, r *http.Request) { + userAuth, err := nbcontext.GetUserAuthFromContext(r.Context()) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + + ok, err := h.permissionsManager.ValidateUserPermissions(r.Context(), userAuth.AccountId, userAuth.UserId, modules.Services, operations.Read) + if err != nil { + util.WriteErrorResponse("failed to validate permissions", http.StatusInternalServerError, w) + return + } + if !ok { + util.WriteErrorResponse("permission denied", http.StatusForbidden, w) + return + } + + tokens, err := h.store.GetProxyAccessTokensByAccountID(r.Context(), store.LockingStrengthNone, userAuth.AccountId) + if err != nil { + util.WriteErrorResponse("failed to list tokens", http.StatusInternalServerError, w) + return + } + + resp := make([]api.ProxyToken, 0, len(tokens)) + for _, token := range tokens { + resp = append(resp, toProxyTokenResponse(token)) + } + + util.WriteJSONObject(r.Context(), w, resp) +} + +func (h *handler) revokeToken(w http.ResponseWriter, r *http.Request) { + userAuth, err := nbcontext.GetUserAuthFromContext(r.Context()) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + + ok, err := h.permissionsManager.ValidateUserPermissions(r.Context(), userAuth.AccountId, userAuth.UserId, modules.Services, operations.Delete) + if err != nil { + util.WriteErrorResponse("failed to validate permissions", http.StatusInternalServerError, w) + return + } + if !ok { + util.WriteErrorResponse("permission denied", http.StatusForbidden, w) + return + } + + tokenID := mux.Vars(r)["tokenId"] + if tokenID == "" { + util.WriteErrorResponse("token ID is required", http.StatusBadRequest, w) + return + } + + token, err := h.store.GetProxyAccessTokenByID(r.Context(), store.LockingStrengthNone, tokenID) + if err != nil { + if s, ok := status.FromError(err); ok && s.ErrorType == status.NotFound { + util.WriteErrorResponse("token not found", http.StatusNotFound, w) + } else { + util.WriteErrorResponse("failed to retrieve token", http.StatusInternalServerError, w) + } + return + } + + if token.AccountID == nil || *token.AccountID != userAuth.AccountId { + util.WriteErrorResponse("token not found", http.StatusNotFound, w) + return + } + + if err := h.store.RevokeProxyAccessToken(r.Context(), tokenID); err != nil { + util.WriteErrorResponse("failed to revoke token", http.StatusInternalServerError, w) + return + } + + util.WriteJSONObject(r.Context(), w, util.EmptyObject{}) +} + +func toProxyTokenResponse(token *types.ProxyAccessToken) api.ProxyToken { + resp := api.ProxyToken{ + Id: token.ID, + Name: token.Name, + Revoked: token.Revoked, + } + if !token.CreatedAt.IsZero() { + resp.CreatedAt = token.CreatedAt + } + if token.ExpiresAt != nil { + resp.ExpiresAt = token.ExpiresAt + } + if token.LastUsed != nil { + resp.LastUsed = token.LastUsed + } + return resp +} + +func toProxyTokenCreatedResponse(generated *types.ProxyAccessTokenGenerated) api.ProxyTokenCreated { + base := toProxyTokenResponse(&generated.ProxyAccessToken) + plainToken := string(generated.PlainToken) + return api.ProxyTokenCreated{ + Id: base.Id, + Name: base.Name, + CreatedAt: base.CreatedAt, + ExpiresAt: base.ExpiresAt, + LastUsed: base.LastUsed, + Revoked: base.Revoked, + PlainToken: plainToken, + } +} diff --git a/management/internals/modules/reverseproxy/proxytoken/handler_test.go b/management/internals/modules/reverseproxy/proxytoken/handler_test.go new file mode 100644 index 000000000..a28752909 --- /dev/null +++ b/management/internals/modules/reverseproxy/proxytoken/handler_test.go @@ -0,0 +1,275 @@ +package proxytoken + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + nbcontext "github.com/netbirdio/netbird/management/server/context" + "github.com/netbirdio/netbird/management/server/permissions" + "github.com/netbirdio/netbird/management/server/permissions/modules" + "github.com/netbirdio/netbird/management/server/permissions/operations" + "github.com/netbirdio/netbird/management/server/store" + "github.com/netbirdio/netbird/management/server/types" + "github.com/netbirdio/netbird/shared/auth" + "github.com/netbirdio/netbird/shared/management/http/api" +) + +func authContext(accountID, userID string) context.Context { + return nbcontext.SetUserAuthInContext(context.Background(), auth.UserAuth{ + AccountId: accountID, + UserId: userID, + }) +} + +func TestCreateToken_AccountScoped(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + accountID := "acc-123" + var savedToken *types.ProxyAccessToken + + mockStore := store.NewMockStore(ctrl) + mockStore.EXPECT().SaveProxyAccessToken(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, token *types.ProxyAccessToken) error { + savedToken = token + return nil + }, + ) + + permsMgr := permissions.NewMockManager(ctrl) + permsMgr.EXPECT().ValidateUserPermissions(gomock.Any(), accountID, "user-1", modules.Services, operations.Create).Return(true, nil) + + h := &handler{ + store: mockStore, + permissionsManager: permsMgr, + } + + body := `{"name": "my-token"}` + req := httptest.NewRequest("POST", "/reverse-proxies/proxy-tokens", bytes.NewBufferString(body)) + req = req.WithContext(authContext(accountID, "user-1")) + w := httptest.NewRecorder() + + h.createToken(w, req) + assert.Equal(t, http.StatusOK, w.Code) + + var resp api.ProxyTokenCreated + require.NoError(t, json.NewDecoder(w.Body).Decode(&resp)) + + assert.NotEmpty(t, resp.PlainToken) + assert.Equal(t, "my-token", resp.Name) + assert.False(t, resp.Revoked) + + require.NotNil(t, savedToken) + require.NotNil(t, savedToken.AccountID) + assert.Equal(t, accountID, *savedToken.AccountID) + assert.Equal(t, "user-1", savedToken.CreatedBy) +} + +func TestCreateToken_WithExpiration(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + var savedToken *types.ProxyAccessToken + + mockStore := store.NewMockStore(ctrl) + mockStore.EXPECT().SaveProxyAccessToken(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, token *types.ProxyAccessToken) error { + savedToken = token + return nil + }, + ) + + permsMgr := permissions.NewMockManager(ctrl) + permsMgr.EXPECT().ValidateUserPermissions(gomock.Any(), "acc-123", "user-1", modules.Services, operations.Create).Return(true, nil) + + h := &handler{ + store: mockStore, + permissionsManager: permsMgr, + } + + body := `{"name": "expiring-token", "expires_in": 3600}` + req := httptest.NewRequest("POST", "/reverse-proxies/proxy-tokens", bytes.NewBufferString(body)) + req = req.WithContext(authContext("acc-123", "user-1")) + w := httptest.NewRecorder() + + h.createToken(w, req) + assert.Equal(t, http.StatusOK, w.Code) + + require.NotNil(t, savedToken) + require.NotNil(t, savedToken.ExpiresAt) + assert.True(t, savedToken.ExpiresAt.After(time.Now())) +} + +func TestCreateToken_EmptyName(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + permsMgr := permissions.NewMockManager(ctrl) + permsMgr.EXPECT().ValidateUserPermissions(gomock.Any(), "acc-123", "user-1", modules.Services, operations.Create).Return(true, nil) + + h := &handler{ + permissionsManager: permsMgr, + } + + body := `{"name": ""}` + req := httptest.NewRequest("POST", "/reverse-proxies/proxy-tokens", bytes.NewBufferString(body)) + req = req.WithContext(authContext("acc-123", "user-1")) + w := httptest.NewRecorder() + + h.createToken(w, req) + assert.Equal(t, http.StatusBadRequest, w.Code) +} + +func TestCreateToken_PermissionDenied(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + permsMgr := permissions.NewMockManager(ctrl) + permsMgr.EXPECT().ValidateUserPermissions(gomock.Any(), "acc-123", "user-1", modules.Services, operations.Create).Return(false, nil) + + h := &handler{ + permissionsManager: permsMgr, + } + + body := `{"name": "test"}` + req := httptest.NewRequest("POST", "/reverse-proxies/proxy-tokens", bytes.NewBufferString(body)) + req = req.WithContext(authContext("acc-123", "user-1")) + w := httptest.NewRecorder() + + h.createToken(w, req) + assert.Equal(t, http.StatusForbidden, w.Code) +} + +func TestListTokens(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + accountID := "acc-123" + now := time.Now() + + mockStore := store.NewMockStore(ctrl) + mockStore.EXPECT().GetProxyAccessTokensByAccountID(gomock.Any(), store.LockingStrengthNone, accountID).Return([]*types.ProxyAccessToken{ + {ID: "tok-1", Name: "token-1", AccountID: &accountID, CreatedAt: now, Revoked: false}, + {ID: "tok-2", Name: "token-2", AccountID: &accountID, CreatedAt: now, Revoked: true}, + }, nil) + + permsMgr := permissions.NewMockManager(ctrl) + permsMgr.EXPECT().ValidateUserPermissions(gomock.Any(), accountID, "user-1", modules.Services, operations.Read).Return(true, nil) + + h := &handler{ + store: mockStore, + permissionsManager: permsMgr, + } + + req := httptest.NewRequest("GET", "/reverse-proxies/proxy-tokens", nil) + req = req.WithContext(authContext(accountID, "user-1")) + w := httptest.NewRecorder() + + h.listTokens(w, req) + assert.Equal(t, http.StatusOK, w.Code) + + var resp []api.ProxyToken + require.NoError(t, json.NewDecoder(w.Body).Decode(&resp)) + require.Len(t, resp, 2) + assert.Equal(t, "tok-1", resp[0].Id) + assert.False(t, resp[0].Revoked) + assert.Equal(t, "tok-2", resp[1].Id) + assert.True(t, resp[1].Revoked) +} + +func TestRevokeToken_Success(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + accountID := "acc-123" + + mockStore := store.NewMockStore(ctrl) + mockStore.EXPECT().GetProxyAccessTokenByID(gomock.Any(), store.LockingStrengthNone, "tok-1").Return(&types.ProxyAccessToken{ + ID: "tok-1", + Name: "test-token", + AccountID: &accountID, + }, nil) + mockStore.EXPECT().RevokeProxyAccessToken(gomock.Any(), "tok-1").Return(nil) + + permsMgr := permissions.NewMockManager(ctrl) + permsMgr.EXPECT().ValidateUserPermissions(gomock.Any(), accountID, "user-1", modules.Services, operations.Delete).Return(true, nil) + + h := &handler{ + store: mockStore, + permissionsManager: permsMgr, + } + + req := httptest.NewRequest("DELETE", "/reverse-proxies/proxy-tokens/tok-1", nil) + req = req.WithContext(authContext(accountID, "user-1")) + req = mux.SetURLVars(req, map[string]string{"tokenId": "tok-1"}) + w := httptest.NewRecorder() + + h.revokeToken(w, req) + assert.Equal(t, http.StatusOK, w.Code) +} + +func TestRevokeToken_WrongAccount(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + otherAccount := "acc-other" + + mockStore := store.NewMockStore(ctrl) + mockStore.EXPECT().GetProxyAccessTokenByID(gomock.Any(), store.LockingStrengthNone, "tok-1").Return(&types.ProxyAccessToken{ + ID: "tok-1", + AccountID: &otherAccount, + }, nil) + + permsMgr := permissions.NewMockManager(ctrl) + permsMgr.EXPECT().ValidateUserPermissions(gomock.Any(), "acc-123", "user-1", modules.Services, operations.Delete).Return(true, nil) + + h := &handler{ + store: mockStore, + permissionsManager: permsMgr, + } + + req := httptest.NewRequest("DELETE", "/reverse-proxies/proxy-tokens/tok-1", nil) + req = req.WithContext(authContext("acc-123", "user-1")) + req = mux.SetURLVars(req, map[string]string{"tokenId": "tok-1"}) + w := httptest.NewRecorder() + + h.revokeToken(w, req) + assert.Equal(t, http.StatusNotFound, w.Code) +} + +func TestRevokeToken_ManagementWideToken(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStore := store.NewMockStore(ctrl) + mockStore.EXPECT().GetProxyAccessTokenByID(gomock.Any(), store.LockingStrengthNone, "tok-1").Return(&types.ProxyAccessToken{ + ID: "tok-1", + AccountID: nil, + }, nil) + + permsMgr := permissions.NewMockManager(ctrl) + permsMgr.EXPECT().ValidateUserPermissions(gomock.Any(), "acc-123", "user-1", modules.Services, operations.Delete).Return(true, nil) + + h := &handler{ + store: mockStore, + permissionsManager: permsMgr, + } + + req := httptest.NewRequest("DELETE", "/reverse-proxies/proxy-tokens/tok-1", nil) + req = req.WithContext(authContext("acc-123", "user-1")) + req = mux.SetURLVars(req, map[string]string{"tokenId": "tok-1"}) + w := httptest.NewRecorder() + + h.revokeToken(w, req) + assert.Equal(t, http.StatusNotFound, w.Code) +} diff --git a/management/internals/modules/reverseproxy/service/interface.go b/management/internals/modules/reverseproxy/service/interface.go index a49cbea35..6a94aa32b 100644 --- a/management/internals/modules/reverseproxy/service/interface.go +++ b/management/internals/modules/reverseproxy/service/interface.go @@ -10,6 +10,7 @@ import ( type Manager interface { GetActiveClusters(ctx context.Context, accountID, userID string) ([]proxy.Cluster, error) + DeleteAccountCluster(ctx context.Context, accountID, userID, clusterAddress string) error GetAllServices(ctx context.Context, accountID, userID string) ([]*Service, error) GetService(ctx context.Context, accountID, userID, serviceID string) (*Service, error) CreateService(ctx context.Context, accountID, userID string, service *Service) (*Service, error) @@ -28,4 +29,5 @@ type Manager interface { RenewServiceFromPeer(ctx context.Context, accountID, peerID, serviceID string) error StopServiceFromPeer(ctx context.Context, accountID, peerID, serviceID string) error StartExposeReaper(ctx context.Context) + GetServiceByDomain(ctx context.Context, domain string) (*Service, error) } diff --git a/management/internals/modules/reverseproxy/service/interface_mock.go b/management/internals/modules/reverseproxy/service/interface_mock.go index cc5ccbb8e..83b2162ed 100644 --- a/management/internals/modules/reverseproxy/service/interface_mock.go +++ b/management/internals/modules/reverseproxy/service/interface_mock.go @@ -79,6 +79,20 @@ func (mr *MockManagerMockRecorder) DeleteAllServices(ctx, accountID, userID inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllServices", reflect.TypeOf((*MockManager)(nil).DeleteAllServices), ctx, accountID, userID) } +// DeleteAccountCluster mocks base method. +func (m *MockManager) DeleteAccountCluster(ctx context.Context, accountID, userID, clusterAddress string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAccountCluster", ctx, accountID, userID, clusterAddress) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAccountCluster indicates an expected call of DeleteAccountCluster. +func (mr *MockManagerMockRecorder) DeleteAccountCluster(ctx, accountID, userID, clusterAddress interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAccountCluster", reflect.TypeOf((*MockManager)(nil).DeleteAccountCluster), ctx, accountID, userID, clusterAddress) +} + // DeleteService mocks base method. func (m *MockManager) DeleteService(ctx context.Context, accountID, userID, serviceID string) error { m.ctrl.T.Helper() @@ -138,6 +152,21 @@ func (mr *MockManagerMockRecorder) GetAllServices(ctx, accountID, userID interfa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllServices", reflect.TypeOf((*MockManager)(nil).GetAllServices), ctx, accountID, userID) } +// GetServiceByDomain mocks base method. +func (m *MockManager) GetServiceByDomain(ctx context.Context, domain string) (*Service, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetServiceByDomain", ctx, domain) + ret0, _ := ret[0].(*Service) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetServiceByDomain indicates an expected call of GetServiceByDomain. +func (mr *MockManagerMockRecorder) GetServiceByDomain(ctx, domain interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceByDomain", reflect.TypeOf((*MockManager)(nil).GetServiceByDomain), ctx, domain) +} + // GetGlobalServices mocks base method. func (m *MockManager) GetGlobalServices(ctx context.Context) ([]*Service, error) { m.ctrl.T.Helper() diff --git a/management/internals/modules/reverseproxy/service/manager/api.go b/management/internals/modules/reverseproxy/service/manager/api.go index cd81efa88..08272077c 100644 --- a/management/internals/modules/reverseproxy/service/manager/api.go +++ b/management/internals/modules/reverseproxy/service/manager/api.go @@ -35,6 +35,7 @@ func RegisterEndpoints(manager rpservice.Manager, domainManager domainmanager.Ma accesslogsmanager.RegisterEndpoints(router, accessLogsManager) router.HandleFunc("/reverse-proxies/clusters", h.getClusters).Methods("GET", "OPTIONS") + router.HandleFunc("/reverse-proxies/clusters/{clusterAddress}", h.deleteCluster).Methods("DELETE", "OPTIONS") router.HandleFunc("/reverse-proxies/services", h.getAllServices).Methods("GET", "OPTIONS") router.HandleFunc("/reverse-proxies/services", h.createService).Methods("POST", "OPTIONS") router.HandleFunc("/reverse-proxies/services/{serviceId}", h.getService).Methods("GET", "OPTIONS") @@ -195,10 +196,33 @@ func (h *handler) getClusters(w http.ResponseWriter, r *http.Request) { apiClusters := make([]api.ProxyCluster, 0, len(clusters)) for _, c := range clusters { apiClusters = append(apiClusters, api.ProxyCluster{ + Id: c.ID, Address: c.Address, ConnectedProxies: c.ConnectedProxies, + SelfHosted: c.SelfHosted, }) } util.WriteJSONObject(r.Context(), w, apiClusters) } + +func (h *handler) deleteCluster(w http.ResponseWriter, r *http.Request) { + userAuth, err := nbcontext.GetUserAuthFromContext(r.Context()) + if err != nil { + util.WriteError(r.Context(), err, w) + return + } + + clusterAddress := mux.Vars(r)["clusterAddress"] + if clusterAddress == "" { + util.WriteError(r.Context(), status.Errorf(status.InvalidArgument, "cluster address is required"), w) + return + } + + if err := h.manager.DeleteAccountCluster(r.Context(), userAuth.AccountId, userAuth.UserId, clusterAddress); err != nil { + util.WriteError(r.Context(), err, w) + return + } + + util.WriteJSONObject(r.Context(), w, util.EmptyObject{}) +} diff --git a/management/internals/modules/reverseproxy/service/manager/manager.go b/management/internals/modules/reverseproxy/service/manager/manager.go index d03a8dc82..c866d8f75 100644 --- a/management/internals/modules/reverseproxy/service/manager/manager.go +++ b/management/internals/modules/reverseproxy/service/manager/manager.go @@ -122,7 +122,21 @@ func (m *Manager) GetActiveClusters(ctx context.Context, accountID, userID strin return nil, status.NewPermissionDeniedError() } - return m.store.GetActiveProxyClusters(ctx) + return m.store.GetActiveProxyClusters(ctx, accountID) +} + +// DeleteAccountCluster removes all proxy registrations for the given cluster address +// owned by the account. +func (m *Manager) DeleteAccountCluster(ctx context.Context, accountID, userID, clusterAddress string) error { + ok, err := m.permissionsManager.ValidateUserPermissions(ctx, accountID, userID, modules.Services, operations.Delete) + if err != nil { + return status.NewPermissionValidationError(err) + } + if !ok { + return status.NewPermissionDeniedError() + } + + return m.store.DeleteAccountCluster(ctx, clusterAddress, accountID) } func (m *Manager) GetAllServices(ctx context.Context, accountID, userID string) ([]*service.Service, error) { @@ -986,6 +1000,10 @@ func (m *Manager) GetAccountServices(ctx context.Context, accountID string) ([]* return services, nil } +func (m *Manager) GetServiceByDomain(ctx context.Context, domain string) (*service.Service, error) { + return m.store.GetServiceByDomain(ctx, domain) +} + func (m *Manager) GetServiceIDByTargetID(ctx context.Context, accountID string, resourceID string) (string, error) { target, err := m.store.GetServiceTargetByTargetID(ctx, store.LockingStrengthNone, accountID, resourceID) if err != nil { diff --git a/management/internals/modules/reverseproxy/service/manager/manager_test.go b/management/internals/modules/reverseproxy/service/manager/manager_test.go index 46e79f1e5..47b8b3865 100644 --- a/management/internals/modules/reverseproxy/service/manager/manager_test.go +++ b/management/internals/modules/reverseproxy/service/manager/manager_test.go @@ -434,7 +434,7 @@ func TestDeletePeerService_SourcePeerValidation(t *testing.T) { t.Helper() tokenStore := nbgrpc.NewOneTimeTokenStore(context.Background(), testCacheStore(t)) pkceStore := nbgrpc.NewPKCEVerifierStore(context.Background(), testCacheStore(t)) - srv := nbgrpc.NewProxyServiceServer(nil, tokenStore, pkceStore, nbgrpc.ProxyOIDCConfig{}, nil, nil, nil) + srv := nbgrpc.NewProxyServiceServer(nil, tokenStore, pkceStore, nbgrpc.ProxyOIDCConfig{}, nil, nil, nil, nil) return srv } @@ -714,7 +714,7 @@ func setupIntegrationTest(t *testing.T) (*Manager, store.Store) { tokenStore := nbgrpc.NewOneTimeTokenStore(ctx, testCacheStore(t)) pkceStore := nbgrpc.NewPKCEVerifierStore(ctx, testCacheStore(t)) - proxySrv := nbgrpc.NewProxyServiceServer(nil, tokenStore, pkceStore, nbgrpc.ProxyOIDCConfig{}, nil, nil, nil) + proxySrv := nbgrpc.NewProxyServiceServer(nil, tokenStore, pkceStore, nbgrpc.ProxyOIDCConfig{}, nil, nil, nil, nil) proxyController, err := proxymanager.NewGRPCController(proxySrv, noop.NewMeterProvider().Meter("")) require.NoError(t, err) @@ -1138,7 +1138,7 @@ func TestDeleteService_DeletesTargets(t *testing.T) { tokenStore := nbgrpc.NewOneTimeTokenStore(ctx, testCacheStore(t)) pkceStore := nbgrpc.NewPKCEVerifierStore(ctx, testCacheStore(t)) - proxySrv := nbgrpc.NewProxyServiceServer(nil, tokenStore, pkceStore, nbgrpc.ProxyOIDCConfig{}, nil, nil, nil) + proxySrv := nbgrpc.NewProxyServiceServer(nil, tokenStore, pkceStore, nbgrpc.ProxyOIDCConfig{}, nil, nil, nil, nil) proxyController, err := proxymanager.NewGRPCController(proxySrv, noop.NewMeterProvider().Meter("")) require.NoError(t, err) diff --git a/management/internals/server/boot.go b/management/internals/server/boot.go index f2ab0a2c4..7c655f020 100644 --- a/management/internals/server/boot.go +++ b/management/internals/server/boot.go @@ -193,7 +193,7 @@ func (s *BaseServer) GRPCServer() *grpc.Server { func (s *BaseServer) ReverseProxyGRPCServer() *nbgrpc.ProxyServiceServer { return Create(s, func() *nbgrpc.ProxyServiceServer { - proxyService := nbgrpc.NewProxyServiceServer(s.AccessLogsManager(), s.ProxyTokenStore(), s.PKCEVerifierStore(), s.proxyOIDCConfig(), s.PeersManager(), s.UsersManager(), s.ProxyManager()) + proxyService := nbgrpc.NewProxyServiceServer(s.AccessLogsManager(), s.ProxyTokenStore(), s.PKCEVerifierStore(), s.proxyOIDCConfig(), s.PeersManager(), s.UsersManager(), s.ProxyManager(), s.Store()) s.AfterInit(func(s *BaseServer) { proxyService.SetServiceManager(s.ServiceManager()) proxyService.SetProxyController(s.ServiceProxyController()) diff --git a/management/internals/shared/grpc/proxy.go b/management/internals/shared/grpc/proxy.go index 6763a3ba3..9e5027547 100644 --- a/management/internals/shared/grpc/proxy.go +++ b/management/internals/shared/grpc/proxy.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "errors" "fmt" + "net" "net/http" "net/url" "os" @@ -50,6 +51,11 @@ type ProxyOIDCConfig struct { KeysLocation string } +// ProxyTokenChecker checks whether a proxy access token is still valid. +type ProxyTokenChecker interface { + IsProxyAccessTokenValid(ctx context.Context, tokenID string) (bool, error) +} + // ProxyServiceServer implements the ProxyService gRPC server type ProxyServiceServer struct { proto.UnimplementedProxyServiceServer @@ -78,6 +84,9 @@ type ProxyServiceServer struct { // Store for one-time authentication tokens tokenStore *OneTimeTokenStore + // Checker for proxy access token validity + tokenChecker ProxyTokenChecker + // OIDC configuration for proxy authentication oidcConfig ProxyOIDCConfig @@ -123,6 +132,8 @@ type proxyConnection struct { proxyID string sessionID string address string + accountID *string + tokenID string capabilities *proto.ProxyCapabilities stream proto.ProxyService_GetMappingUpdateServer sendChan chan *proto.GetMappingUpdateResponse @@ -130,8 +141,19 @@ type proxyConnection struct { cancel context.CancelFunc } +func enforceAccountScope(ctx context.Context, requestAccountID string) error { + token := GetProxyTokenFromContext(ctx) + if token == nil || token.AccountID == nil { + return nil + } + if requestAccountID == "" || *token.AccountID != requestAccountID { + return status.Errorf(codes.PermissionDenied, "account-scoped token cannot access account %s", requestAccountID) + } + return nil +} + // NewProxyServiceServer creates a new proxy service server. -func NewProxyServiceServer(accessLogMgr accesslogs.Manager, tokenStore *OneTimeTokenStore, pkceStore *PKCEVerifierStore, oidcConfig ProxyOIDCConfig, peersManager peers.Manager, usersManager users.Manager, proxyMgr proxy.Manager) *ProxyServiceServer { +func NewProxyServiceServer(accessLogMgr accesslogs.Manager, tokenStore *OneTimeTokenStore, pkceStore *PKCEVerifierStore, oidcConfig ProxyOIDCConfig, peersManager peers.Manager, usersManager users.Manager, proxyMgr proxy.Manager, tokenChecker ProxyTokenChecker) *ProxyServiceServer { ctx, cancel := context.WithCancel(context.Background()) s := &ProxyServiceServer{ accessLogManager: accessLogMgr, @@ -141,6 +163,7 @@ func NewProxyServiceServer(accessLogMgr accesslogs.Manager, tokenStore *OneTimeT peersManager: peersManager, usersManager: usersManager, proxyManager: proxyMgr, + tokenChecker: tokenChecker, snapshotBatchSize: snapshotBatchSizeFromEnv(), cancel: cancel, } @@ -200,6 +223,25 @@ func (s *ProxyServiceServer) GetMappingUpdate(req *proto.GetMappingUpdateRequest return status.Errorf(codes.InvalidArgument, "proxy address is invalid") } + var accountID *string + token := GetProxyTokenFromContext(ctx) + if token != nil && token.AccountID != nil { + accountID = token.AccountID + + available, err := s.proxyManager.IsClusterAddressAvailable(ctx, proxyAddress, *accountID) + if err != nil { + return status.Errorf(codes.Internal, "check cluster address: %v", err) + } + if !available { + return status.Errorf(codes.AlreadyExists, "cluster address %s is already in use", proxyAddress) + } + } + + var tokenID string + if token != nil { + tokenID = token.ID + } + sessionID := uuid.NewString() if old, loaded := s.connectedProxies.Load(proxyID); loaded { @@ -217,6 +259,8 @@ func (s *ProxyServiceServer) GetMappingUpdate(req *proto.GetMappingUpdateRequest proxyID: proxyID, sessionID: sessionID, address: proxyAddress, + accountID: accountID, + tokenID: tokenID, capabilities: req.GetCapabilities(), stream: stream, sendChan: make(chan *proto.GetMappingUpdateResponse, 100), @@ -224,7 +268,6 @@ func (s *ProxyServiceServer) GetMappingUpdate(req *proto.GetMappingUpdateRequest cancel: cancel, } - // Register proxy in database with capabilities var caps *proxy.Capabilities if c := req.GetCapabilities(); c != nil { caps = &proxy.Capabilities{ @@ -233,10 +276,13 @@ func (s *ProxyServiceServer) GetMappingUpdate(req *proto.GetMappingUpdateRequest SupportsCrowdsec: c.SupportsCrowdsec, } } - proxyRecord, err := s.proxyManager.Connect(ctx, proxyID, sessionID, proxyAddress, peerInfo, caps) + proxyRecord, err := s.proxyManager.Connect(ctx, proxyID, sessionID, proxyAddress, peerInfo, accountID, caps) if err != nil { - log.WithContext(ctx).Warnf("failed to register proxy %s in database: %v", proxyID, err) cancel() + if accountID != nil { + return status.Errorf(codes.Internal, "failed to register BYOP proxy: %v", err) + } + log.WithContext(ctx).Warnf("failed to register proxy %s in database: %v", proxyID, err) return status.Errorf(codes.Internal, "register proxy in database: %v", err) } @@ -266,6 +312,7 @@ func (s *ProxyServiceServer) GetMappingUpdate(req *proto.GetMappingUpdateRequest "session_id": sessionID, "address": proxyAddress, "cluster_addr": proxyAddress, + "account_id": accountID, "total_proxies": len(s.GetConnectedProxies()), }).Info("Proxy registered in cluster") defer func() { @@ -286,7 +333,7 @@ func (s *ProxyServiceServer) GetMappingUpdate(req *proto.GetMappingUpdateRequest log.Infof("Proxy %s session %s disconnected", proxyID, sessionID) }() - go s.heartbeat(connCtx, proxyRecord) + go s.heartbeat(connCtx, conn, proxyRecord) select { case err := <-errChan: @@ -298,8 +345,9 @@ func (s *ProxyServiceServer) GetMappingUpdate(req *proto.GetMappingUpdateRequest } } -// heartbeat updates the proxy's last_seen timestamp every minute -func (s *ProxyServiceServer) heartbeat(ctx context.Context, p *proxy.Proxy) { +// heartbeat updates the proxy's last_seen timestamp every minute and +// disconnects the proxy if its access token has been revoked. +func (s *ProxyServiceServer) heartbeat(ctx context.Context, conn *proxyConnection, p *proxy.Proxy) { ticker := time.NewTicker(1 * time.Minute) defer ticker.Stop() @@ -309,6 +357,19 @@ func (s *ProxyServiceServer) heartbeat(ctx context.Context, p *proxy.Proxy) { if err := s.proxyManager.Heartbeat(ctx, p); err != nil { log.WithContext(ctx).Debugf("Failed to update proxy %s heartbeat: %v", p.ID, err) } + + if conn.tokenID != "" && s.tokenChecker != nil { + valid, err := s.tokenChecker.IsProxyAccessTokenValid(ctx, conn.tokenID) + if err != nil { + log.WithContext(ctx).Warnf("failed to check token validity for proxy %s: %v", conn.proxyID, err) + continue + } + if !valid { + log.WithContext(ctx).Warnf("proxy %s token revoked or expired, disconnecting", conn.proxyID) + conn.cancel() + return + } + } case <-ctx.Done(): log.WithContext(ctx).Infof("proxy %s heartbeat stopped: context canceled", p.ID) return @@ -316,8 +377,6 @@ func (s *ProxyServiceServer) heartbeat(ctx context.Context, p *proxy.Proxy) { } } -// sendSnapshot sends the initial snapshot of services to the connecting proxy. -// Only entries matching the proxy's cluster address are sent. func (s *ProxyServiceServer) sendSnapshot(ctx context.Context, conn *proxyConnection) error { if !isProxyAddressValid(conn.address) { return fmt.Errorf("proxy address is invalid") @@ -355,7 +414,13 @@ func (s *ProxyServiceServer) sendSnapshot(ctx context.Context, conn *proxyConnec } func (s *ProxyServiceServer) snapshotServiceMappings(ctx context.Context, conn *proxyConnection) ([]*proto.ProxyMapping, error) { - services, err := s.serviceManager.GetGlobalServices(ctx) + var services []*rpservice.Service + var err error + if conn.accountID != nil { + services, err = s.serviceManager.GetAccountServices(ctx, *conn.accountID) + } else { + services, err = s.serviceManager.GetGlobalServices(ctx) + } if err != nil { return nil, fmt.Errorf("get services from store: %w", err) } @@ -380,8 +445,14 @@ func (s *ProxyServiceServer) snapshotServiceMappings(ctx context.Context, conn * return mappings, nil } -// isProxyAddressValid validates a proxy address +// isProxyAddressValid validates a proxy address (domain name or IP address) func isProxyAddressValid(addr string) bool { + if addr == "" { + return false + } + if net.ParseIP(addr) != nil { + return true + } _, err := domain.ValidateDomains([]string{addr}) return err == nil } @@ -405,6 +476,10 @@ func (s *ProxyServiceServer) sender(conn *proxyConnection, errChan chan<- error) func (s *ProxyServiceServer) SendAccessLog(ctx context.Context, req *proto.SendAccessLogRequest) (*proto.SendAccessLogResponse, error) { accessLog := req.GetLog() + if err := enforceAccountScope(ctx, accessLog.GetAccountId()); err != nil { + return nil, err + } + fields := log.Fields{ "service_id": accessLog.GetServiceId(), "account_id": accessLog.GetAccountId(), @@ -442,11 +517,32 @@ func (s *ProxyServiceServer) SendAccessLog(ctx context.Context, req *proto.SendA // Management should call this when services are created/updated/removed. // For create/update operations a unique one-time auth token is generated per // proxy so that every replica can independently authenticate with management. +// BYOP proxies only receive updates for their own account's services. func (s *ProxyServiceServer) SendServiceUpdate(update *proto.GetMappingUpdateResponse) { log.Debugf("Broadcasting service update to all connected proxy servers") + updateAccountIDs := make(map[string]struct{}) + for _, m := range update.Mapping { + if m.AccountId != "" { + updateAccountIDs[m.AccountId] = struct{}{} + } + } s.connectedProxies.Range(func(key, value interface{}) bool { conn := value.(*proxyConnection) - resp := s.perProxyMessage(update, conn.proxyID) + connUpdate := update + if conn.accountID != nil && len(updateAccountIDs) > 0 { + if _, ok := updateAccountIDs[*conn.accountID]; !ok { + return true + } + filtered := filterMappingsForAccount(update.Mapping, *conn.accountID) + if len(filtered) == 0 { + return true + } + connUpdate = &proto.GetMappingUpdateResponse{ + Mapping: filtered, + InitialSyncComplete: update.InitialSyncComplete, + } + } + resp := s.perProxyMessage(connUpdate, conn.proxyID) if resp == nil { log.Warnf("Token generation failed for proxy %s, disconnecting to force resync", conn.proxyID) conn.cancel() @@ -463,6 +559,26 @@ func (s *ProxyServiceServer) SendServiceUpdate(update *proto.GetMappingUpdateRes }) } +// ForceDisconnect cancels the gRPC stream for a connected proxy, causing it to disconnect. +func (s *ProxyServiceServer) ForceDisconnect(proxyID string) { + if connVal, ok := s.connectedProxies.Load(proxyID); ok { + conn := connVal.(*proxyConnection) + conn.cancel() + s.connectedProxies.Delete(proxyID) + log.WithFields(log.Fields{"proxyID": proxyID}).Info("force disconnected proxy") + } +} + +func filterMappingsForAccount(mappings []*proto.ProxyMapping, accountID string) []*proto.ProxyMapping { + var filtered []*proto.ProxyMapping + for _, m := range mappings { + if m.AccountId == accountID { + filtered = append(filtered, m) + } + } + return filtered +} + // GetConnectedProxies returns a list of connected proxy IDs func (s *ProxyServiceServer) GetConnectedProxies() []string { var proxies []string @@ -531,6 +647,9 @@ func (s *ProxyServiceServer) SendServiceUpdateToCluster(ctx context.Context, upd continue } conn := connVal.(*proxyConnection) + if conn.accountID != nil && update.AccountId != "" && *conn.accountID != update.AccountId { + continue + } if !proxyAcceptsMapping(conn, update) { log.WithContext(ctx).Debugf("Skipping proxy %s: does not support custom ports for mapping %s", proxyID, update.Id) continue @@ -618,6 +737,10 @@ func shallowCloneMapping(m *proto.ProxyMapping) *proto.ProxyMapping { } func (s *ProxyServiceServer) Authenticate(ctx context.Context, req *proto.AuthenticateRequest) (*proto.AuthenticateResponse, error) { + if err := enforceAccountScope(ctx, req.GetAccountId()); err != nil { + return nil, err + } + service, err := s.serviceManager.GetServiceByID(ctx, req.GetAccountId(), req.GetId()) if err != nil { log.WithContext(ctx).Debugf("failed to get service from store: %v", err) @@ -737,6 +860,10 @@ func (s *ProxyServiceServer) generateSessionToken(ctx context.Context, authentic // SendStatusUpdate handles status updates from proxy clients. func (s *ProxyServiceServer) SendStatusUpdate(ctx context.Context, req *proto.SendStatusUpdateRequest) (*proto.SendStatusUpdateResponse, error) { + if err := enforceAccountScope(ctx, req.GetAccountId()); err != nil { + return nil, err + } + accountID := req.GetAccountId() serviceID := req.GetServiceId() protoStatus := req.GetStatus() @@ -807,6 +934,10 @@ func protoStatusToInternal(protoStatus proto.ProxyStatus) rpservice.Status { // CreateProxyPeer handles proxy peer creation with one-time token authentication func (s *ProxyServiceServer) CreateProxyPeer(ctx context.Context, req *proto.CreateProxyPeerRequest) (*proto.CreateProxyPeerResponse, error) { + if err := enforceAccountScope(ctx, req.GetAccountId()); err != nil { + return nil, err + } + serviceID := req.GetServiceId() accountID := req.GetAccountId() token := req.GetToken() @@ -861,6 +992,10 @@ func strPtr(s string) *string { } func (s *ProxyServiceServer) GetOIDCURL(ctx context.Context, req *proto.GetOIDCURLRequest) (*proto.GetOIDCURLResponse, error) { + if err := enforceAccountScope(ctx, req.GetAccountId()); err != nil { + return nil, err + } + redirectURL, err := url.Parse(req.GetRedirectUrl()) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "parse redirect url: %v", err) @@ -989,21 +1124,9 @@ func (s *ProxyServiceServer) ValidateState(state string) (verifier, redirectURL // GenerateSessionToken creates a signed session JWT for the given domain and user. func (s *ProxyServiceServer) GenerateSessionToken(ctx context.Context, domain, userID string, method proxyauth.Method) (string, error) { - // Find the service by domain to get its signing key - services, err := s.serviceManager.GetGlobalServices(ctx) + service, err := s.getServiceByDomain(ctx, domain) if err != nil { - return "", fmt.Errorf("get services: %w", err) - } - - var service *rpservice.Service - for _, svc := range services { - if svc.Domain == domain { - service = svc - break - } - } - if service == nil { - return "", fmt.Errorf("service not found for domain: %s", domain) + return "", fmt.Errorf("service not found for domain %s: %w", domain, err) } if service.SessionPrivateKey == "" { @@ -1101,6 +1224,10 @@ func (s *ProxyServiceServer) ValidateSession(ctx context.Context, req *proto.Val }, nil } + if err := enforceAccountScope(ctx, service.AccountID); err != nil { + return nil, err + } + pubKeyBytes, err := base64.StdEncoding.DecodeString(service.SessionPublicKey) if err != nil { log.WithFields(log.Fields{ @@ -1184,18 +1311,7 @@ func (s *ProxyServiceServer) ValidateSession(ctx context.Context, req *proto.Val } func (s *ProxyServiceServer) getServiceByDomain(ctx context.Context, domain string) (*rpservice.Service, error) { - services, err := s.serviceManager.GetGlobalServices(ctx) - if err != nil { - return nil, fmt.Errorf("get services: %w", err) - } - - for _, service := range services { - if service.Domain == domain { - return service, nil - } - } - - return nil, fmt.Errorf("service not found for domain: %s", domain) + return s.serviceManager.GetServiceByDomain(ctx, domain) } func (s *ProxyServiceServer) checkGroupAccess(service *rpservice.Service, user *types.User) error { diff --git a/management/internals/shared/grpc/proxy_address_test.go b/management/internals/shared/grpc/proxy_address_test.go new file mode 100644 index 000000000..824a57226 --- /dev/null +++ b/management/internals/shared/grpc/proxy_address_test.go @@ -0,0 +1,29 @@ +package grpc + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsProxyAddressValid(t *testing.T) { + tests := []struct { + name string + addr string + valid bool + }{ + {name: "valid domain", addr: "eu.proxy.netbird.io", valid: true}, + {name: "valid subdomain", addr: "byop.proxy.example.com", valid: true}, + {name: "valid IPv4", addr: "10.0.0.1", valid: true}, + {name: "valid IPv4 public", addr: "203.0.113.10", valid: true}, + {name: "valid IPv6", addr: "::1", valid: true}, + {name: "valid IPv6 full", addr: "2001:db8::1", valid: true}, + {name: "empty string", addr: "", valid: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.valid, isProxyAddressValid(tt.addr)) + }) + } +} diff --git a/management/internals/shared/grpc/proxy_auth.go b/management/internals/shared/grpc/proxy_auth.go index dd593dfa0..9888e8eee 100644 --- a/management/internals/shared/grpc/proxy_auth.go +++ b/management/internals/shared/grpc/proxy_auth.go @@ -153,9 +153,6 @@ func (i *proxyAuthInterceptor) doValidateProxyToken(ctx context.Context) (*types return nil, status.Errorf(codes.Unauthenticated, "invalid token") } - // TODO: Enforce AccountID scope for "bring your own proxy" feature. - // Currently tokens are management-wide; AccountID field is reserved for future use. - if !token.IsValid() { return nil, status.Errorf(codes.Unauthenticated, "token expired or revoked") } diff --git a/management/internals/shared/grpc/proxy_group_access_test.go b/management/internals/shared/grpc/proxy_group_access_test.go index 0fa9a0dc1..46dad5b56 100644 --- a/management/internals/shared/grpc/proxy_group_access_test.go +++ b/management/internals/shared/grpc/proxy_group_access_test.go @@ -53,6 +53,10 @@ func (m *mockReverseProxyManager) DeleteService(ctx context.Context, accountID, return nil } +func (m *mockReverseProxyManager) DeleteAccountCluster(_ context.Context, _, _, _ string) error { + return nil +} + func (m *mockReverseProxyManager) SetCertificateIssuedAt(ctx context.Context, accountID, reverseProxyID string) error { return nil } @@ -91,6 +95,20 @@ func (m *mockReverseProxyManager) StopServiceFromPeer(_ context.Context, _, _, _ func (m *mockReverseProxyManager) StartExposeReaper(_ context.Context) {} +func (m *mockReverseProxyManager) GetServiceByDomain(_ context.Context, domain string) (*service.Service, error) { + if m.err != nil { + return nil, m.err + } + for _, services := range m.proxiesByAccount { + for _, svc := range services { + if svc.Domain == domain { + return svc, nil + } + } + } + return nil, errors.New("service not found for domain: " + domain) +} + func (m *mockReverseProxyManager) GetActiveClusters(_ context.Context, _, _ string) ([]proxy.Cluster, error) { return nil, nil } diff --git a/management/internals/shared/grpc/proxy_test.go b/management/internals/shared/grpc/proxy_test.go index 5a7a457df..0379edc6d 100644 --- a/management/internals/shared/grpc/proxy_test.go +++ b/management/internals/shared/grpc/proxy_test.go @@ -12,9 +12,12 @@ import ( cachestore "github.com/eko/gocache/lib/v4/store" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + grpcstatus "google.golang.org/grpc/status" "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy" nbcache "github.com/netbirdio/netbird/management/server/cache" + "github.com/netbirdio/netbird/management/server/types" "github.com/netbirdio/netbird/shared/management/proto" ) @@ -316,6 +319,58 @@ func TestValidateState_RejectsOldTwoPartFormat(t *testing.T) { assert.Contains(t, err.Error(), "invalid state format") } +func scopedCtx(accountID string) context.Context { + token := &types.ProxyAccessToken{ + ID: "token-1", + AccountID: &accountID, + } + return context.WithValue(context.Background(), ProxyTokenContextKey, token) +} + +func globalCtx() context.Context { + token := &types.ProxyAccessToken{ + ID: "token-global", + } + return context.WithValue(context.Background(), ProxyTokenContextKey, token) +} + +func TestEnforceAccountScope_AllowsMatchingAccount(t *testing.T) { + err := enforceAccountScope(scopedCtx("acc-1"), "acc-1") + assert.NoError(t, err) +} + +func TestEnforceAccountScope_BlocksMismatchedAccount(t *testing.T) { + err := enforceAccountScope(scopedCtx("acc-1"), "acc-2") + require.Error(t, err) + st, ok := grpcstatus.FromError(err) + require.True(t, ok) + assert.Equal(t, codes.PermissionDenied, st.Code()) +} + +func TestEnforceAccountScope_BlocksEmptyRequestAccountID(t *testing.T) { + err := enforceAccountScope(scopedCtx("acc-1"), "") + require.Error(t, err) + st, ok := grpcstatus.FromError(err) + require.True(t, ok) + assert.Equal(t, codes.PermissionDenied, st.Code()) +} + +func TestEnforceAccountScope_AllowsGlobalToken(t *testing.T) { + err := enforceAccountScope(globalCtx(), "acc-1") + assert.NoError(t, err) + + err = enforceAccountScope(globalCtx(), "acc-2") + assert.NoError(t, err) + + err = enforceAccountScope(globalCtx(), "") + assert.NoError(t, err) +} + +func TestEnforceAccountScope_AllowsNoTokenInContext(t *testing.T) { + err := enforceAccountScope(context.Background(), "acc-1") + assert.NoError(t, err) +} + func TestValidateState_RejectsInvalidHMAC(t *testing.T) { ctx := context.Background() pkceStore := NewPKCEVerifierStore(ctx, testCacheStore(t)) diff --git a/management/internals/shared/grpc/validate_session_test.go b/management/internals/shared/grpc/validate_session_test.go index d1d7fc8b7..6cd95f988 100644 --- a/management/internals/shared/grpc/validate_session_test.go +++ b/management/internals/shared/grpc/validate_session_test.go @@ -42,7 +42,7 @@ func setupValidateSessionTest(t *testing.T) *validateSessionTestSetup { tokenStore := NewOneTimeTokenStore(ctx, testCacheStore(t)) pkceStore := NewPKCEVerifierStore(ctx, testCacheStore(t)) - proxyService := NewProxyServiceServer(nil, tokenStore, pkceStore, ProxyOIDCConfig{}, nil, usersManager, proxyManager) + proxyService := NewProxyServiceServer(nil, tokenStore, pkceStore, ProxyOIDCConfig{}, nil, usersManager, proxyManager, nil) proxyService.SetServiceManager(serviceManager) createTestProxies(t, ctx, testStore) @@ -318,13 +318,17 @@ func (m *testValidateSessionServiceManager) StopServiceFromPeer(_ context.Contex func (m *testValidateSessionServiceManager) StartExposeReaper(_ context.Context) {} +func (m *testValidateSessionServiceManager) GetServiceByDomain(ctx context.Context, domain string) (*service.Service, error) { + return m.store.GetServiceByDomain(ctx, domain) +} + func (m *testValidateSessionServiceManager) GetActiveClusters(_ context.Context, _, _ string) ([]proxy.Cluster, error) { return nil, nil } type testValidateSessionProxyManager struct{} -func (m *testValidateSessionProxyManager) Connect(_ context.Context, _, _, _ string, _ *proxy.Capabilities) error { +func (m *testValidateSessionProxyManager) Connect(_ context.Context, _, _, _ string, _ *string, _ *proxy.Capabilities) error { return nil } @@ -340,6 +344,10 @@ func (m *testValidateSessionProxyManager) GetActiveClusterAddresses(_ context.Co return nil, nil } +func (m *testValidateSessionProxyManager) GetActiveClusterAddressesForAccount(_ context.Context, _ string) ([]string, error) { + return nil, nil +} + func (m *testValidateSessionProxyManager) GetActiveClusters(_ context.Context) ([]proxy.Cluster, error) { return nil, nil } @@ -348,6 +356,22 @@ func (m *testValidateSessionProxyManager) CleanupStale(_ context.Context, _ time return nil } +func (m *testValidateSessionProxyManager) GetAccountProxy(_ context.Context, _ string) (*proxy.Proxy, error) { + return nil, nil +} + +func (m *testValidateSessionProxyManager) CountAccountProxies(_ context.Context, _ string) (int64, error) { + return 0, nil +} + +func (m *testValidateSessionProxyManager) IsClusterAddressAvailable(_ context.Context, _, _ string) (bool, error) { + return true, nil +} + +func (m *testValidateSessionProxyManager) DeleteProxy(_ context.Context, _ string) error { + return nil +} + func (m *testValidateSessionProxyManager) ClusterSupportsCustomPorts(_ context.Context, _ string) *bool { return nil } diff --git a/management/server/account_test.go b/management/server/account_test.go index 6bb875f99..65b27df49 100644 --- a/management/server/account_test.go +++ b/management/server/account_test.go @@ -3113,7 +3113,7 @@ func createManager(t testing.TB) (*DefaultAccountManager, *update_channel.PeersU return nil, nil, err } - proxyGrpcServer := nbgrpc.NewProxyServiceServer(nil, nil, nil, nbgrpc.ProxyOIDCConfig{}, peersManager, nil, proxyManager) + proxyGrpcServer := nbgrpc.NewProxyServiceServer(nil, nil, nil, nbgrpc.ProxyOIDCConfig{}, peersManager, nil, proxyManager, nil) proxyController, err := proxymanager.NewGRPCController(proxyGrpcServer, noop.Meter{}) if err != nil { return nil, nil, err diff --git a/management/server/http/handler.go b/management/server/http/handler.go index b9ea605d3..1e2c710db 100644 --- a/management/server/http/handler.go +++ b/management/server/http/handler.go @@ -16,6 +16,7 @@ import ( "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs" "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service" + "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxytoken" reverseproxymanager "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service/manager" nbgrpc "github.com/netbirdio/netbird/management/internals/shared/grpc" @@ -144,6 +145,9 @@ func NewAPIHandler(ctx context.Context, accountManager account.Manager, networks if serviceManager != nil && reverseProxyDomainManager != nil { reverseproxymanager.RegisterEndpoints(serviceManager, *reverseProxyDomainManager, reverseProxyAccessLogsManager, permissionsManager, router) } + + proxytoken.RegisterEndpoints(accountManager.GetStore(), permissionsManager, router) + // Register OAuth callback handler for proxy authentication if proxyGRPCServer != nil { oauthHandler := proxy.NewAuthCallbackHandler(proxyGRPCServer, trustedHTTPProxies) diff --git a/management/server/http/handlers/proxy/auth_callback_integration_test.go b/management/server/http/handlers/proxy/auth_callback_integration_test.go index c99acab63..30d8aa0e7 100644 --- a/management/server/http/handlers/proxy/auth_callback_integration_test.go +++ b/management/server/http/handlers/proxy/auth_callback_integration_test.go @@ -216,6 +216,7 @@ func setupAuthCallbackTest(t *testing.T) *testSetup { nil, usersManager, nil, + nil, ) proxyService.SetServiceManager(&testServiceManager{store: testStore}) @@ -389,6 +390,10 @@ func (m *testServiceManager) DeleteService(_ context.Context, _, _, _ string) er return nil } +func (m *testServiceManager) DeleteAccountCluster(_ context.Context, _, _, _ string) error { + return nil +} + func (m *testServiceManager) SetCertificateIssuedAt(_ context.Context, _, _ string) error { return nil } @@ -435,6 +440,10 @@ func (m *testServiceManager) StopServiceFromPeer(_ context.Context, _, _, _ stri func (m *testServiceManager) StartExposeReaper(_ context.Context) {} +func (m *testServiceManager) GetServiceByDomain(ctx context.Context, domain string) (*service.Service, error) { + return m.store.GetServiceByDomain(ctx, domain) +} + func (m *testServiceManager) GetActiveClusters(_ context.Context, _, _ string) ([]nbproxy.Cluster, error) { return nil, nil } diff --git a/management/server/http/testing/testing_tools/channel/channel.go b/management/server/http/testing/testing_tools/channel/channel.go index 1a8b83c7e..3c4ea98d0 100644 --- a/management/server/http/testing/testing_tools/channel/channel.go +++ b/management/server/http/testing/testing_tools/channel/channel.go @@ -109,7 +109,7 @@ func BuildApiBlackBoxWithDBState(t testing_tools.TB, sqlFile string, expectedPee if err != nil { t.Fatalf("Failed to create proxy manager: %v", err) } - proxyServiceServer := nbgrpc.NewProxyServiceServer(accessLogsManager, proxyTokenStore, pkceverifierStore, nbgrpc.ProxyOIDCConfig{}, peersManager, userManager, proxyMgr) + proxyServiceServer := nbgrpc.NewProxyServiceServer(accessLogsManager, proxyTokenStore, pkceverifierStore, nbgrpc.ProxyOIDCConfig{}, peersManager, userManager, proxyMgr, nil) domainManager := manager.NewManager(store, proxyMgr, permissionsManager, am) serviceProxyController, err := proxymanager.NewGRPCController(proxyServiceServer, noopMeter) if err != nil { @@ -238,7 +238,7 @@ func BuildApiBlackBoxWithDBStateAndPeerChannel(t testing_tools.TB, sqlFile strin if err != nil { t.Fatalf("Failed to create proxy manager: %v", err) } - proxyServiceServer := nbgrpc.NewProxyServiceServer(accessLogsManager, proxyTokenStore, pkceverifierStore, nbgrpc.ProxyOIDCConfig{}, peersManager, userManager, proxyMgr) + proxyServiceServer := nbgrpc.NewProxyServiceServer(accessLogsManager, proxyTokenStore, pkceverifierStore, nbgrpc.ProxyOIDCConfig{}, peersManager, userManager, proxyMgr, nil) domainManager := manager.NewManager(store, proxyMgr, permissionsManager, am) serviceProxyController, err := proxymanager.NewGRPCController(proxyServiceServer, noopMeter) if err != nil { diff --git a/management/server/store/sql_store.go b/management/server/store/sql_store.go index 065a0d306..4c2f0be52 100644 --- a/management/server/store/sql_store.go +++ b/management/server/store/sql_store.go @@ -4513,6 +4513,47 @@ func (s *SqlStore) RevokeProxyAccessToken(ctx context.Context, tokenID string) e return nil } +func (s *SqlStore) GetProxyAccessTokensByAccountID(ctx context.Context, lockStrength LockingStrength, accountID string) ([]*types.ProxyAccessToken, error) { + tx := s.db + if lockStrength != LockingStrengthNone { + tx = tx.Clauses(clause.Locking{Strength: string(lockStrength)}) + } + + var tokens []*types.ProxyAccessToken + result := tx.Where("account_id = ?", accountID).Find(&tokens) + if result.Error != nil { + return nil, status.Errorf(status.Internal, "get proxy access tokens by account: %v", result.Error) + } + + return tokens, nil +} + +func (s *SqlStore) IsProxyAccessTokenValid(ctx context.Context, tokenID string) (bool, error) { + token, err := s.GetProxyAccessTokenByID(ctx, LockingStrengthNone, tokenID) + if err != nil { + return false, err + } + return token.IsValid(), nil +} + +func (s *SqlStore) GetProxyAccessTokenByID(ctx context.Context, lockStrength LockingStrength, tokenID string) (*types.ProxyAccessToken, error) { + tx := s.db + if lockStrength != LockingStrengthNone { + tx = tx.Clauses(clause.Locking{Strength: string(lockStrength)}) + } + + var token types.ProxyAccessToken + result := tx.Take(&token, idQueryCondition, tokenID) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return nil, status.Errorf(status.NotFound, "proxy access token not found") + } + return nil, status.Errorf(status.Internal, "get proxy access token by ID: %v", result.Error) + } + + return &token, nil +} + // MarkProxyAccessTokenUsed updates the last used timestamp for a proxy access token. func (s *SqlStore) MarkProxyAccessTokenUsed(ctx context.Context, tokenID string) error { result := s.db.Model(&types.ProxyAccessToken{}). @@ -5487,7 +5528,7 @@ func (s *SqlStore) DisconnectProxy(ctx context.Context, proxyID, sessionID strin Model(&proxy.Proxy{}). Where("id = ? AND session_id = ?", proxyID, sessionID). Updates(map[string]any{ - "status": "disconnected", + "status": proxy.StatusDisconnected, "disconnected_at": now, "last_seen": now, }) @@ -5518,7 +5559,7 @@ func (s *SqlStore) UpdateProxyHeartbeat(ctx context.Context, p *proxy.Proxy) err if result.RowsAffected == 0 { p.LastSeen = now p.ConnectedAt = &now - p.Status = "connected" + p.Status = proxy.StatusConnected if err := s.db.Create(p).Error; err != nil { log.WithContext(ctx).Debugf("proxy %s session %s: heartbeat fallback insert skipped: %v", p.ID, p.SessionID, err) } @@ -5527,13 +5568,15 @@ func (s *SqlStore) UpdateProxyHeartbeat(ctx context.Context, p *proxy.Proxy) err return nil } -// GetActiveProxyClusterAddresses returns all unique cluster addresses for active proxies +// GetActiveProxyClusterAddresses returns the unique cluster addresses of active +// shared proxies (those without an account scope). BYOP cluster addresses are +// excluded; use GetActiveProxyClusterAddressesForAccount to retrieve them. func (s *SqlStore) GetActiveProxyClusterAddresses(ctx context.Context) ([]string, error) { var addresses []string result := s.db. Model(&proxy.Proxy{}). - Where("status = ? AND last_seen > ?", "connected", time.Now().Add(-proxyActiveThreshold)). + Where("account_id IS NULL AND status = ? AND last_seen > ?", proxy.StatusConnected, time.Now().Add(-proxyActiveThreshold)). Distinct("cluster_address"). Pluck("cluster_address", &addresses) @@ -5545,13 +5588,75 @@ func (s *SqlStore) GetActiveProxyClusterAddresses(ctx context.Context) ([]string return addresses, nil } -// GetActiveProxyClusters returns all active proxy clusters with their connected proxy count. -func (s *SqlStore) GetActiveProxyClusters(ctx context.Context) ([]proxy.Cluster, error) { +func (s *SqlStore) GetActiveProxyClusterAddressesForAccount(ctx context.Context, accountID string) ([]string, error) { + var addresses []string + + result := s.db. + Model(&proxy.Proxy{}). + Where("account_id = ? AND status = ? AND last_seen > ?", accountID, proxy.StatusConnected, time.Now().Add(-proxyActiveThreshold)). + Distinct("cluster_address"). + Pluck("cluster_address", &addresses) + + if result.Error != nil { + return nil, status.Errorf(status.Internal, "failed to get active proxy cluster addresses for account") + } + + return addresses, nil +} + +func (s *SqlStore) GetProxyByAccountID(ctx context.Context, accountID string) (*proxy.Proxy, error) { + var p proxy.Proxy + result := s.db.Where("account_id = ?", accountID).Take(&p) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return nil, status.Errorf(status.NotFound, "proxy not found for account") + } + return nil, status.Errorf(status.Internal, "get proxy by account ID: %v", result.Error) + } + return &p, nil +} + +func (s *SqlStore) CountProxiesByAccountID(ctx context.Context, accountID string) (int64, error) { + var count int64 + result := s.db.Model(&proxy.Proxy{}).Where("account_id = ?", accountID).Count(&count) + if result.Error != nil { + return 0, status.Errorf(status.Internal, "count proxies by account ID: %v", result.Error) + } + return count, nil +} + +func (s *SqlStore) IsClusterAddressConflicting(ctx context.Context, clusterAddress, accountID string) (bool, error) { + var count int64 + result := s.db. + Model(&proxy.Proxy{}). + Where("cluster_address = ? AND (account_id IS NULL OR account_id != ?)", clusterAddress, accountID). + Count(&count) + if result.Error != nil { + return false, status.Errorf(status.Internal, "check cluster address conflict: %v", result.Error) + } + return count > 0, nil +} + +func (s *SqlStore) DeleteAccountCluster(ctx context.Context, clusterAddress, accountID string) error { + result := s.db. + Where("cluster_address = ? AND account_id = ?", clusterAddress, accountID). + Delete(&proxy.Proxy{}) + if result.Error != nil { + return status.Errorf(status.Internal, "delete account cluster: %v", result.Error) + } + if result.RowsAffected == 0 { + return status.Errorf(status.NotFound, "cluster not found") + } + return nil +} + +func (s *SqlStore) GetActiveProxyClusters(ctx context.Context, accountID string) ([]proxy.Cluster, error) { var clusters []proxy.Cluster result := s.db.Model(&proxy.Proxy{}). - Select("cluster_address as address, COUNT(*) as connected_proxies"). - Where("status = ? AND last_seen > ?", "connected", time.Now().Add(-proxyActiveThreshold)). + Select("MIN(id) as id, cluster_address as address, COUNT(*) as connected_proxies, COUNT(account_id) > 0 as self_hosted"). + Where("status = ? AND last_seen > ? AND (account_id IS NULL OR account_id = ?)", + proxy.StatusConnected, time.Now().Add(-proxyActiveThreshold), accountID). Group("cluster_address"). Scan(&clusters) diff --git a/management/server/store/store.go b/management/server/store/store.go index db98bc644..aa601c33f 100644 --- a/management/server/store/store.go +++ b/management/server/store/store.go @@ -114,6 +114,9 @@ type Store interface { GetProxyAccessTokenByHashedToken(ctx context.Context, lockStrength LockingStrength, hashedToken types.HashedProxyToken) (*types.ProxyAccessToken, error) GetAllProxyAccessTokens(ctx context.Context, lockStrength LockingStrength) ([]*types.ProxyAccessToken, error) + GetProxyAccessTokensByAccountID(ctx context.Context, lockStrength LockingStrength, accountID string) ([]*types.ProxyAccessToken, error) + GetProxyAccessTokenByID(ctx context.Context, lockStrength LockingStrength, tokenID string) (*types.ProxyAccessToken, error) + IsProxyAccessTokenValid(ctx context.Context, tokenID string) (bool, error) SaveProxyAccessToken(ctx context.Context, token *types.ProxyAccessToken) error RevokeProxyAccessToken(ctx context.Context, tokenID string) error MarkProxyAccessTokenUsed(ctx context.Context, tokenID string) error @@ -288,11 +291,16 @@ type Store interface { DisconnectProxy(ctx context.Context, proxyID, sessionID string) error UpdateProxyHeartbeat(ctx context.Context, p *proxy.Proxy) error GetActiveProxyClusterAddresses(ctx context.Context) ([]string, error) - GetActiveProxyClusters(ctx context.Context) ([]proxy.Cluster, error) + GetActiveProxyClusterAddressesForAccount(ctx context.Context, accountID string) ([]string, error) + GetActiveProxyClusters(ctx context.Context, accountID string) ([]proxy.Cluster, error) GetClusterSupportsCustomPorts(ctx context.Context, clusterAddr string) *bool GetClusterRequireSubdomain(ctx context.Context, clusterAddr string) *bool GetClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool CleanupStaleProxies(ctx context.Context, inactivityDuration time.Duration) error + GetProxyByAccountID(ctx context.Context, accountID string) (*proxy.Proxy, error) + CountProxiesByAccountID(ctx context.Context, accountID string) (int64, error) + IsClusterAddressConflicting(ctx context.Context, clusterAddress, accountID string) (bool, error) + DeleteAccountCluster(ctx context.Context, clusterAddress, accountID string) error GetCustomDomainsCounts(ctx context.Context) (total int64, validated int64, err error) @@ -496,6 +504,9 @@ func getMigrationsPostAuto(ctx context.Context) []migrationFunc { func(db *gorm.DB) error { return migration.CreateIndexIfNotExists[nbpeer.Peer](ctx, db, "idx_peers_key_unique", "key") }, + func(db *gorm.DB) error { + return migration.DropIndex[proxy.Proxy](ctx, db, "idx_proxy_account_id_unique") + }, } } diff --git a/management/server/store/store_mock.go b/management/server/store/store_mock.go index 6c2c9bbc3..9780c521e 100644 --- a/management/server/store/store_mock.go +++ b/management/server/store/store_mock.go @@ -166,20 +166,6 @@ func (mr *MockStoreMockRecorder) CleanupStaleProxies(ctx, inactivityDuration int return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanupStaleProxies", reflect.TypeOf((*MockStore)(nil).CleanupStaleProxies), ctx, inactivityDuration) } -// GetClusterSupportsCrowdSec mocks base method. -func (m *MockStore) GetClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetClusterSupportsCrowdSec", ctx, clusterAddr) - ret0, _ := ret[0].(*bool) - return ret0 -} - -// GetClusterSupportsCrowdSec indicates an expected call of GetClusterSupportsCrowdSec. -func (mr *MockStoreMockRecorder) GetClusterSupportsCrowdSec(ctx, clusterAddr interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterSupportsCrowdSec", reflect.TypeOf((*MockStore)(nil).GetClusterSupportsCrowdSec), ctx, clusterAddr) -} - // Close mocks base method. func (m *MockStore) Close(ctx context.Context) error { m.ctrl.T.Helper() @@ -238,6 +224,21 @@ func (mr *MockStoreMockRecorder) CountEphemeralServicesByPeer(ctx, lockStrength, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountEphemeralServicesByPeer", reflect.TypeOf((*MockStore)(nil).CountEphemeralServicesByPeer), ctx, lockStrength, accountID, peerID) } +// CountProxiesByAccountID mocks base method. +func (m *MockStore) CountProxiesByAccountID(ctx context.Context, accountID string) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountProxiesByAccountID", ctx, accountID) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountProxiesByAccountID indicates an expected call of CountProxiesByAccountID. +func (mr *MockStoreMockRecorder) CountProxiesByAccountID(ctx, accountID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountProxiesByAccountID", reflect.TypeOf((*MockStore)(nil).CountProxiesByAccountID), ctx, accountID) +} + // CreateAccessLog mocks base method. func (m *MockStore) CreateAccessLog(ctx context.Context, log *accesslogs.AccessLogEntry) error { m.ctrl.T.Helper() @@ -576,6 +577,20 @@ func (mr *MockStoreMockRecorder) DeletePostureChecks(ctx, accountID, postureChec return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeletePostureChecks", reflect.TypeOf((*MockStore)(nil).DeletePostureChecks), ctx, accountID, postureChecksID) } +// DeleteAccountCluster mocks base method. +func (m *MockStore) DeleteAccountCluster(ctx context.Context, clusterAddress, accountID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAccountCluster", ctx, clusterAddress, accountID) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAccountCluster indicates an expected call of DeleteAccountCluster. +func (mr *MockStoreMockRecorder) DeleteAccountCluster(ctx, clusterAddress, accountID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAccountCluster", reflect.TypeOf((*MockStore)(nil).DeleteAccountCluster), ctx, clusterAddress, accountID) +} + // DeleteRoute mocks base method. func (m *MockStore) DeleteRoute(ctx context.Context, accountID, routeID string) error { m.ctrl.T.Helper() @@ -1302,19 +1317,34 @@ func (mr *MockStoreMockRecorder) GetActiveProxyClusterAddresses(ctx interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveProxyClusterAddresses", reflect.TypeOf((*MockStore)(nil).GetActiveProxyClusterAddresses), ctx) } -// GetActiveProxyClusters mocks base method. -func (m *MockStore) GetActiveProxyClusters(ctx context.Context) ([]proxy.Cluster, error) { +// GetActiveProxyClusterAddressesForAccount mocks base method. +func (m *MockStore) GetActiveProxyClusterAddressesForAccount(ctx context.Context, accountID string) ([]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetActiveProxyClusters", ctx) + ret := m.ctrl.Call(m, "GetActiveProxyClusterAddressesForAccount", ctx, accountID) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetActiveProxyClusterAddressesForAccount indicates an expected call of GetActiveProxyClusterAddressesForAccount. +func (mr *MockStoreMockRecorder) GetActiveProxyClusterAddressesForAccount(ctx, accountID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveProxyClusterAddressesForAccount", reflect.TypeOf((*MockStore)(nil).GetActiveProxyClusterAddressesForAccount), ctx, accountID) +} + +// GetActiveProxyClusters mocks base method. +func (m *MockStore) GetActiveProxyClusters(ctx context.Context, accountID string) ([]proxy.Cluster, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetActiveProxyClusters", ctx, accountID) ret0, _ := ret[0].([]proxy.Cluster) ret1, _ := ret[1].(error) return ret0, ret1 } // GetActiveProxyClusters indicates an expected call of GetActiveProxyClusters. -func (mr *MockStoreMockRecorder) GetActiveProxyClusters(ctx interface{}) *gomock.Call { +func (mr *MockStoreMockRecorder) GetActiveProxyClusters(ctx, accountID interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveProxyClusters", reflect.TypeOf((*MockStore)(nil).GetActiveProxyClusters), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveProxyClusters", reflect.TypeOf((*MockStore)(nil).GetActiveProxyClusters), ctx, accountID) } // GetAllAccounts mocks base method. @@ -1390,6 +1420,20 @@ func (mr *MockStoreMockRecorder) GetClusterRequireSubdomain(ctx, clusterAddr int return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterRequireSubdomain", reflect.TypeOf((*MockStore)(nil).GetClusterRequireSubdomain), ctx, clusterAddr) } +// GetClusterSupportsCrowdSec mocks base method. +func (m *MockStore) GetClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetClusterSupportsCrowdSec", ctx, clusterAddr) + ret0, _ := ret[0].(*bool) + return ret0 +} + +// GetClusterSupportsCrowdSec indicates an expected call of GetClusterSupportsCrowdSec. +func (mr *MockStoreMockRecorder) GetClusterSupportsCrowdSec(ctx, clusterAddr interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterSupportsCrowdSec", reflect.TypeOf((*MockStore)(nil).GetClusterSupportsCrowdSec), ctx, clusterAddr) +} + // GetClusterSupportsCustomPorts mocks base method. func (m *MockStore) GetClusterSupportsCustomPorts(ctx context.Context, clusterAddr string) *bool { m.ctrl.T.Helper() @@ -1959,6 +2003,51 @@ func (mr *MockStoreMockRecorder) GetProxyAccessTokenByHashedToken(ctx, lockStren return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProxyAccessTokenByHashedToken", reflect.TypeOf((*MockStore)(nil).GetProxyAccessTokenByHashedToken), ctx, lockStrength, hashedToken) } +// GetProxyAccessTokenByID mocks base method. +func (m *MockStore) GetProxyAccessTokenByID(ctx context.Context, lockStrength LockingStrength, tokenID string) (*types2.ProxyAccessToken, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetProxyAccessTokenByID", ctx, lockStrength, tokenID) + ret0, _ := ret[0].(*types2.ProxyAccessToken) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetProxyAccessTokenByID indicates an expected call of GetProxyAccessTokenByID. +func (mr *MockStoreMockRecorder) GetProxyAccessTokenByID(ctx, lockStrength, tokenID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProxyAccessTokenByID", reflect.TypeOf((*MockStore)(nil).GetProxyAccessTokenByID), ctx, lockStrength, tokenID) +} + +// GetProxyAccessTokensByAccountID mocks base method. +func (m *MockStore) GetProxyAccessTokensByAccountID(ctx context.Context, lockStrength LockingStrength, accountID string) ([]*types2.ProxyAccessToken, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetProxyAccessTokensByAccountID", ctx, lockStrength, accountID) + ret0, _ := ret[0].([]*types2.ProxyAccessToken) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetProxyAccessTokensByAccountID indicates an expected call of GetProxyAccessTokensByAccountID. +func (mr *MockStoreMockRecorder) GetProxyAccessTokensByAccountID(ctx, lockStrength, accountID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProxyAccessTokensByAccountID", reflect.TypeOf((*MockStore)(nil).GetProxyAccessTokensByAccountID), ctx, lockStrength, accountID) +} + +// GetProxyByAccountID mocks base method. +func (m *MockStore) GetProxyByAccountID(ctx context.Context, accountID string) (*proxy.Proxy, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetProxyByAccountID", ctx, accountID) + ret0, _ := ret[0].(*proxy.Proxy) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetProxyByAccountID indicates an expected call of GetProxyByAccountID. +func (mr *MockStoreMockRecorder) GetProxyByAccountID(ctx, accountID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProxyByAccountID", reflect.TypeOf((*MockStore)(nil).GetProxyByAccountID), ctx, accountID) +} + // GetResourceGroups mocks base method. func (m *MockStore) GetResourceGroups(ctx context.Context, lockStrength LockingStrength, accountID, resourceID string) ([]*types2.Group, error) { m.ctrl.T.Helper() @@ -2391,6 +2480,21 @@ func (mr *MockStoreMockRecorder) IncrementSetupKeyUsage(ctx, setupKeyID interfac return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IncrementSetupKeyUsage", reflect.TypeOf((*MockStore)(nil).IncrementSetupKeyUsage), ctx, setupKeyID) } +// IsClusterAddressConflicting mocks base method. +func (m *MockStore) IsClusterAddressConflicting(ctx context.Context, clusterAddress, accountID string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsClusterAddressConflicting", ctx, clusterAddress, accountID) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsClusterAddressConflicting indicates an expected call of IsClusterAddressConflicting. +func (mr *MockStoreMockRecorder) IsClusterAddressConflicting(ctx, clusterAddress, accountID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsClusterAddressConflicting", reflect.TypeOf((*MockStore)(nil).IsClusterAddressConflicting), ctx, clusterAddress, accountID) +} + // IsPrimaryAccount mocks base method. func (m *MockStore) IsPrimaryAccount(ctx context.Context, accountID string) (bool, string, error) { m.ctrl.T.Helper() @@ -2407,6 +2511,21 @@ func (mr *MockStoreMockRecorder) IsPrimaryAccount(ctx, accountID interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsPrimaryAccount", reflect.TypeOf((*MockStore)(nil).IsPrimaryAccount), ctx, accountID) } +// IsProxyAccessTokenValid mocks base method. +func (m *MockStore) IsProxyAccessTokenValid(ctx context.Context, tokenID string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsProxyAccessTokenValid", ctx, tokenID) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsProxyAccessTokenValid indicates an expected call of IsProxyAccessTokenValid. +func (mr *MockStoreMockRecorder) IsProxyAccessTokenValid(ctx, tokenID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsProxyAccessTokenValid", reflect.TypeOf((*MockStore)(nil).IsProxyAccessTokenValid), ctx, tokenID) +} + // ListCustomDomains mocks base method. func (m *MockStore) ListCustomDomains(ctx context.Context, accountID string) ([]*domain.Domain, error) { m.ctrl.T.Helper() diff --git a/proxy/management_byop_integration_test.go b/proxy/management_byop_integration_test.go new file mode 100644 index 000000000..c0fbe682a --- /dev/null +++ b/proxy/management_byop_integration_test.go @@ -0,0 +1,409 @@ +package proxy + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/metric/noop" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + grpcstatus "google.golang.org/grpc/status" + + proxymanager "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy/manager" + "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service" + nbgrpc "github.com/netbirdio/netbird/management/internals/shared/grpc" + nbcache "github.com/netbirdio/netbird/management/server/cache" + "github.com/netbirdio/netbird/management/server/store" + "github.com/netbirdio/netbird/management/server/types" + "github.com/netbirdio/netbird/management/server/users" + "github.com/netbirdio/netbird/shared/management/proto" +) + +type byopTestSetup struct { + store store.Store + proxyService *nbgrpc.ProxyServiceServer + grpcServer *grpc.Server + grpcAddr string + cleanup func() + + accountA string + accountB string + accountAToken types.PlainProxyToken + accountBToken types.PlainProxyToken + accountACluster string + accountBCluster string +} + +func setupBYOPIntegrationTest(t *testing.T) *byopTestSetup { + t.Helper() + ctx := context.Background() + + testStore, storeCleanup, err := store.NewTestStoreFromSQL(ctx, "", t.TempDir()) + require.NoError(t, err) + + accountAID := "byop-account-a" + accountBID := "byop-account-b" + + for _, acc := range []*types.Account{ + {Id: accountAID, Domain: "a.test.com", DomainCategory: "private", IsDomainPrimaryAccount: true, CreatedAt: time.Now()}, + {Id: accountBID, Domain: "b.test.com", DomainCategory: "private", IsDomainPrimaryAccount: true, CreatedAt: time.Now()}, + } { + require.NoError(t, testStore.SaveAccount(ctx, acc)) + } + + pub, priv, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + pubKey := base64.StdEncoding.EncodeToString(pub) + privKey := base64.StdEncoding.EncodeToString(priv) + + clusterA := "byop-a.proxy.test" + clusterB := "byop-b.proxy.test" + + services := []*service.Service{ + { + ID: "svc-a1", AccountID: accountAID, Name: "App A1", + Domain: "app1." + clusterA, ProxyCluster: clusterA, Enabled: true, + SessionPrivateKey: privKey, SessionPublicKey: pubKey, + Targets: []*service.Target{{Path: strPtr("/"), Host: "10.0.0.1", Port: 8080, Protocol: "http", TargetId: "peer-a1", TargetType: "peer", Enabled: true}}, + }, + { + ID: "svc-a2", AccountID: accountAID, Name: "App A2", + Domain: "app2." + clusterA, ProxyCluster: clusterA, Enabled: true, + SessionPrivateKey: privKey, SessionPublicKey: pubKey, + Targets: []*service.Target{{Path: strPtr("/"), Host: "10.0.0.2", Port: 8080, Protocol: "http", TargetId: "peer-a2", TargetType: "peer", Enabled: true}}, + }, + { + ID: "svc-b1", AccountID: accountBID, Name: "App B1", + Domain: "app1." + clusterB, ProxyCluster: clusterB, Enabled: true, + SessionPrivateKey: privKey, SessionPublicKey: pubKey, + Targets: []*service.Target{{Path: strPtr("/"), Host: "10.0.0.3", Port: 8080, Protocol: "http", TargetId: "peer-b1", TargetType: "peer", Enabled: true}}, + }, + } + for _, svc := range services { + require.NoError(t, testStore.CreateService(ctx, svc)) + } + + tokenA, err := types.CreateNewProxyAccessToken("byop-token-a", 0, &accountAID, "admin-a") + require.NoError(t, err) + require.NoError(t, testStore.SaveProxyAccessToken(ctx, &tokenA.ProxyAccessToken)) + + tokenB, err := types.CreateNewProxyAccessToken("byop-token-b", 0, &accountBID, "admin-b") + require.NoError(t, err) + require.NoError(t, testStore.SaveProxyAccessToken(ctx, &tokenB.ProxyAccessToken)) + + cacheStore, err := nbcache.NewStore(ctx, 30*time.Minute, 10*time.Minute, 100) + require.NoError(t, err) + + tokenStore := nbgrpc.NewOneTimeTokenStore(ctx, cacheStore) + pkceStore := nbgrpc.NewPKCEVerifierStore(ctx, cacheStore) + + meter := noop.NewMeterProvider().Meter("test") + realProxyManager, err := proxymanager.NewManager(testStore, meter) + require.NoError(t, err) + + oidcConfig := nbgrpc.ProxyOIDCConfig{ + Issuer: "https://fake-issuer.example.com", + ClientID: "test-client", + HMACKey: []byte("test-hmac-key"), + } + + usersManager := users.NewManager(testStore) + + proxyService := nbgrpc.NewProxyServiceServer( + &testAccessLogManager{}, + tokenStore, + pkceStore, + oidcConfig, + nil, + usersManager, + realProxyManager, + nil, + ) + + svcMgr := &storeBackedServiceManager{store: testStore, tokenStore: tokenStore} + proxyService.SetServiceManager(svcMgr) + + proxyController := &testProxyController{} + proxyService.SetProxyController(proxyController) + + _, streamInterceptor, authClose := nbgrpc.NewProxyAuthInterceptors(testStore) + + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + grpcServer := grpc.NewServer(grpc.StreamInterceptor(streamInterceptor)) + proto.RegisterProxyServiceServer(grpcServer, proxyService) + + go func() { + if err := grpcServer.Serve(lis); err != nil { + t.Logf("gRPC server error: %v", err) + } + }() + + return &byopTestSetup{ + store: testStore, + proxyService: proxyService, + grpcServer: grpcServer, + grpcAddr: lis.Addr().String(), + cleanup: func() { + grpcServer.GracefulStop() + authClose() + storeCleanup() + }, + accountA: accountAID, + accountB: accountBID, + accountAToken: tokenA.PlainToken, + accountBToken: tokenB.PlainToken, + accountACluster: clusterA, + accountBCluster: clusterB, + } +} + +func byopContext(ctx context.Context, token types.PlainProxyToken) context.Context { + md := metadata.Pairs("authorization", "Bearer "+string(token)) + return metadata.NewOutgoingContext(ctx, md) +} + +func receiveBYOPMappings(t *testing.T, stream proto.ProxyService_GetMappingUpdateClient) []*proto.ProxyMapping { + t.Helper() + var mappings []*proto.ProxyMapping + for { + msg, err := stream.Recv() + require.NoError(t, err) + mappings = append(mappings, msg.GetMapping()...) + if msg.GetInitialSyncComplete() { + break + } + } + return mappings +} + +func TestIntegration_BYOPProxy_ReceivesOnlyAccountServices(t *testing.T) { + setup := setupBYOPIntegrationTest(t) + defer setup.cleanup() + + conn, err := grpc.NewClient(setup.grpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer conn.Close() + + client := proto.NewProxyServiceClient(conn) + + ctx, cancel := context.WithTimeout(byopContext(context.Background(), setup.accountAToken), 5*time.Second) + defer cancel() + + stream, err := client.GetMappingUpdate(ctx, &proto.GetMappingUpdateRequest{ + ProxyId: "byop-proxy-a", + Version: "test-v1", + Address: setup.accountACluster, + }) + require.NoError(t, err) + + mappings := receiveBYOPMappings(t, stream) + + assert.Len(t, mappings, 2, "BYOP proxy should receive only account A's 2 services") + for _, m := range mappings { + assert.Equal(t, setup.accountA, m.GetAccountId(), "all mappings should belong to account A") + t.Logf("received mapping: id=%s domain=%s account=%s", m.GetId(), m.GetDomain(), m.GetAccountId()) + } + + ids := map[string]bool{} + for _, m := range mappings { + ids[m.GetId()] = true + } + assert.True(t, ids["svc-a1"], "should contain svc-a1") + assert.True(t, ids["svc-a2"], "should contain svc-a2") + assert.False(t, ids["svc-b1"], "should NOT contain account B's svc-b1") +} + +func TestIntegration_BYOPProxy_AccountBReceivesOnlyItsServices(t *testing.T) { + setup := setupBYOPIntegrationTest(t) + defer setup.cleanup() + + conn, err := grpc.NewClient(setup.grpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer conn.Close() + + client := proto.NewProxyServiceClient(conn) + + ctx, cancel := context.WithTimeout(byopContext(context.Background(), setup.accountBToken), 5*time.Second) + defer cancel() + + stream, err := client.GetMappingUpdate(ctx, &proto.GetMappingUpdateRequest{ + ProxyId: "byop-proxy-b", + Version: "test-v1", + Address: setup.accountBCluster, + }) + require.NoError(t, err) + + mappings := receiveBYOPMappings(t, stream) + + assert.Len(t, mappings, 1, "BYOP proxy B should receive only 1 service") + assert.Equal(t, "svc-b1", mappings[0].GetId()) + assert.Equal(t, setup.accountB, mappings[0].GetAccountId()) +} + +func TestIntegration_BYOPProxy_MultiplePerAccount(t *testing.T) { + setup := setupBYOPIntegrationTest(t) + defer setup.cleanup() + + conn, err := grpc.NewClient(setup.grpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer conn.Close() + + client := proto.NewProxyServiceClient(conn) + + ctx1, cancel1 := context.WithTimeout(byopContext(context.Background(), setup.accountAToken), 5*time.Second) + defer cancel1() + + stream1, err := client.GetMappingUpdate(ctx1, &proto.GetMappingUpdateRequest{ + ProxyId: "byop-proxy-a-first", + Version: "test-v1", + Address: setup.accountACluster, + }) + require.NoError(t, err) + + mappings1 := receiveBYOPMappings(t, stream1) + assert.Len(t, mappings1, 2, "first BYOP proxy should receive account A's 2 services") + + ctx2, cancel2 := context.WithTimeout(byopContext(context.Background(), setup.accountAToken), 5*time.Second) + defer cancel2() + + stream2, err := client.GetMappingUpdate(ctx2, &proto.GetMappingUpdateRequest{ + ProxyId: "byop-proxy-a-second", + Version: "test-v1", + Address: setup.accountACluster, + }) + require.NoError(t, err) + + mappings2 := receiveBYOPMappings(t, stream2) + assert.Len(t, mappings2, 2, "second BYOP proxy from same account should also receive the 2 services") + for _, m := range mappings2 { + assert.Equal(t, setup.accountA, m.GetAccountId()) + } +} + +func TestIntegration_BYOPProxy_ClusterAddressConflict(t *testing.T) { + setup := setupBYOPIntegrationTest(t) + defer setup.cleanup() + + conn, err := grpc.NewClient(setup.grpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer conn.Close() + + client := proto.NewProxyServiceClient(conn) + + ctx1, cancel1 := context.WithTimeout(byopContext(context.Background(), setup.accountAToken), 5*time.Second) + defer cancel1() + + stream1, err := client.GetMappingUpdate(ctx1, &proto.GetMappingUpdateRequest{ + ProxyId: "byop-proxy-a-cluster", + Version: "test-v1", + Address: setup.accountACluster, + }) + require.NoError(t, err) + + _ = receiveBYOPMappings(t, stream1) + + ctx2, cancel2 := context.WithTimeout(byopContext(context.Background(), setup.accountBToken), 5*time.Second) + defer cancel2() + + stream2, err := client.GetMappingUpdate(ctx2, &proto.GetMappingUpdateRequest{ + ProxyId: "byop-proxy-b-conflict", + Version: "test-v1", + Address: setup.accountACluster, + }) + require.NoError(t, err) + + _, err = stream2.Recv() + require.Error(t, err) + + st, ok := grpcstatus.FromError(err) + require.True(t, ok) + assert.Equal(t, codes.AlreadyExists, st.Code(), "cluster address conflict should return AlreadyExists") + t.Logf("expected rejection: %s", st.Message()) +} + +func TestIntegration_BYOPProxy_SameProxyReconnects(t *testing.T) { + setup := setupBYOPIntegrationTest(t) + defer setup.cleanup() + + conn, err := grpc.NewClient(setup.grpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer conn.Close() + + client := proto.NewProxyServiceClient(conn) + + proxyID := "byop-proxy-reconnect" + + ctx1, cancel1 := context.WithTimeout(byopContext(context.Background(), setup.accountAToken), 5*time.Second) + stream1, err := client.GetMappingUpdate(ctx1, &proto.GetMappingUpdateRequest{ + ProxyId: proxyID, + Version: "test-v1", + Address: setup.accountACluster, + }) + require.NoError(t, err) + + firstMappings := receiveBYOPMappings(t, stream1) + cancel1() + + time.Sleep(200 * time.Millisecond) + + ctx2, cancel2 := context.WithTimeout(byopContext(context.Background(), setup.accountAToken), 5*time.Second) + defer cancel2() + + stream2, err := client.GetMappingUpdate(ctx2, &proto.GetMappingUpdateRequest{ + ProxyId: proxyID, + Version: "test-v1", + Address: setup.accountACluster, + }) + require.NoError(t, err) + + secondMappings := receiveBYOPMappings(t, stream2) + + assert.Equal(t, len(firstMappings), len(secondMappings), "reconnect should receive same mappings") + + firstIDs := map[string]bool{} + for _, m := range firstMappings { + firstIDs[m.GetId()] = true + } + for _, m := range secondMappings { + assert.True(t, firstIDs[m.GetId()], "mapping %s should be present on reconnect", m.GetId()) + } +} + +func TestIntegration_BYOPProxy_UnauthenticatedRejected(t *testing.T) { + setup := setupBYOPIntegrationTest(t) + defer setup.cleanup() + + conn, err := grpc.NewClient(setup.grpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer conn.Close() + + client := proto.NewProxyServiceClient(conn) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + stream, err := client.GetMappingUpdate(ctx, &proto.GetMappingUpdateRequest{ + ProxyId: "no-auth-proxy", + Version: "test-v1", + Address: "some.cluster.io", + }) + require.NoError(t, err) + + _, err = stream.Recv() + require.Error(t, err) + + st, ok := grpcstatus.FromError(err) + require.True(t, ok) + assert.Equal(t, codes.Unauthenticated, st.Code()) +} diff --git a/proxy/management_integration_test.go b/proxy/management_integration_test.go index 99bbdad0c..9fd3d2ce9 100644 --- a/proxy/management_integration_test.go +++ b/proxy/management_integration_test.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "encoding/base64" "errors" + "fmt" "net" "sync" "sync/atomic" @@ -140,6 +141,7 @@ func setupIntegrationTest(t *testing.T) *integrationTestSetup { nil, usersManager, proxyManager, + nil, ) // Use store-backed service manager @@ -201,8 +203,8 @@ func (m *testAccessLogManager) GetAllAccessLogs(_ context.Context, _, _ string, // testProxyManager is a mock implementation of proxy.Manager for testing. type testProxyManager struct{} -func (m *testProxyManager) Connect(_ context.Context, proxyID, sessionID, _, _ string, _ *nbproxy.Capabilities) (*nbproxy.Proxy, error) { - return &nbproxy.Proxy{ID: proxyID, SessionID: sessionID, Status: "connected"}, nil +func (m *testProxyManager) Connect(_ context.Context, proxyID, sessionID, _, _ string, _ *string, _ *nbproxy.Capabilities) (*nbproxy.Proxy, error) { + return &nbproxy.Proxy{ID: proxyID, SessionID: sessionID, Status: nbproxy.StatusConnected}, nil } func (m *testProxyManager) Disconnect(_ context.Context, _, _ string) error { @@ -217,6 +219,10 @@ func (m *testProxyManager) GetActiveClusterAddresses(_ context.Context) ([]strin return nil, nil } +func (m *testProxyManager) GetActiveClusterAddressesForAccount(_ context.Context, _ string) ([]string, error) { + return nil, nil +} + func (m *testProxyManager) GetActiveClusters(_ context.Context) ([]nbproxy.Cluster, error) { return nil, nil } @@ -237,6 +243,22 @@ func (m *testProxyManager) CleanupStale(_ context.Context, _ time.Duration) erro return nil } +func (m *testProxyManager) GetAccountProxy(_ context.Context, accountID string) (*nbproxy.Proxy, error) { + return nil, fmt.Errorf("proxy not found for account %s", accountID) +} + +func (m *testProxyManager) CountAccountProxies(_ context.Context, _ string) (int64, error) { + return 0, nil +} + +func (m *testProxyManager) IsClusterAddressAvailable(_ context.Context, _, _ string) (bool, error) { + return true, nil +} + +func (m *testProxyManager) DeleteAccountCluster(_ context.Context, _, _ string) error { + return nil +} + // testProxyController is a mock implementation of rpservice.ProxyController for testing. type testProxyController struct{} @@ -290,6 +312,10 @@ func (m *storeBackedServiceManager) DeleteService(ctx context.Context, accountID return nil } +func (m *storeBackedServiceManager) DeleteAccountCluster(_ context.Context, _, _, _ string) error { + return nil +} + func (m *storeBackedServiceManager) SetCertificateIssuedAt(ctx context.Context, accountID, serviceID string) error { return nil } @@ -336,6 +362,10 @@ func (m *storeBackedServiceManager) StopServiceFromPeer(_ context.Context, _, _, func (m *storeBackedServiceManager) StartExposeReaper(_ context.Context) {} +func (m *storeBackedServiceManager) GetServiceByDomain(ctx context.Context, domain string) (*service.Service, error) { + return m.store.GetServiceByDomain(ctx, domain) +} + func (m *storeBackedServiceManager) GetActiveClusters(_ context.Context, _, _ string) ([]nbproxy.Cluster, error) { return nil, nil } diff --git a/shared/management/http/api/openapi.yml b/shared/management/http/api/openapi.yml index 82fca0782..942f3aa45 100644 --- a/shared/management/http/api/openapi.yml +++ b/shared/management/http/api/openapi.yml @@ -3355,10 +3355,64 @@ components: example: false required: - enabled + ProxyTokenRequest: + type: object + properties: + name: + type: string + description: Human-readable token name + example: "my-proxy-token" + expires_in: + type: integer + minimum: 0 + description: Token expiration in seconds (0 = never expires) + example: 0 + required: + - name + ProxyToken: + type: object + properties: + id: + type: string + name: + type: string + expires_at: + type: string + format: date-time + created_at: + type: string + format: date-time + last_used: + type: string + format: date-time + revoked: + type: boolean + required: + - id + - name + - created_at + - revoked + ProxyTokenCreated: + type: object + description: Returned on creation — plain_token is shown only once + allOf: + - $ref: '#/components/schemas/ProxyToken' + - type: object + properties: + plain_token: + type: string + description: The plain text token (shown only once) + example: "nbx_abc123..." + required: + - plain_token ProxyCluster: type: object description: A proxy cluster represents a group of proxy nodes serving the same address properties: + id: + type: string + description: Unique identifier of a proxy in this cluster + example: "chlfq4q5r8kc73b0qjpg" address: type: string description: Cluster address used for CNAME targets @@ -3367,9 +3421,15 @@ components: type: integer description: Number of proxy nodes connected in this cluster example: 3 + self_hosted: + type: boolean + description: Whether this cluster is a self-hosted (BYOP) proxy managed by the account owner + example: false required: + - id - address - connected_proxies + - self_hosted ReverseProxyDomainType: type: string description: Type of Reverse Proxy Domain @@ -11375,6 +11435,111 @@ paths: "$ref": "#/components/responses/forbidden" '500': "$ref": "#/components/responses/internal_error" + /api/reverse-proxies/clusters/{clusterAddress}: + delete: + summary: Delete a self-hosted proxy cluster + description: Removes all self-hosted (BYOP) proxy registrations for the given cluster address owned by the account. + tags: [ Services ] + security: + - BearerAuth: [ ] + - TokenAuth: [ ] + parameters: + - in: path + name: clusterAddress + required: true + schema: + type: string + description: The address of the proxy cluster + responses: + '200': + description: Proxy cluster deleted successfully + content: { } + '400': + "$ref": "#/components/responses/bad_request" + '401': + "$ref": "#/components/responses/requires_authentication" + '403': + "$ref": "#/components/responses/forbidden" + '404': + "$ref": "#/components/responses/not_found" + '500': + "$ref": "#/components/responses/internal_error" + /api/reverse-proxies/proxy-tokens: + get: + summary: List Proxy Tokens + description: Returns all proxy access tokens for the account + tags: [ Self-Hosted Proxies ] + security: + - BearerAuth: [ ] + - TokenAuth: [ ] + responses: + '200': + description: A JSON Array of proxy tokens + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/ProxyToken' + '401': + "$ref": "#/components/responses/requires_authentication" + '403': + "$ref": "#/components/responses/forbidden" + '500': + "$ref": "#/components/responses/internal_error" + post: + summary: Create a Proxy Token + description: Generate an account-scoped proxy access token for self-hosted proxy registration + tags: [ Self-Hosted Proxies ] + security: + - BearerAuth: [ ] + - TokenAuth: [ ] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ProxyTokenRequest' + responses: + '200': + description: Proxy token created (plain token shown once) + content: + application/json: + schema: + $ref: '#/components/schemas/ProxyTokenCreated' + '400': + "$ref": "#/components/responses/bad_request" + '401': + "$ref": "#/components/responses/requires_authentication" + '403': + "$ref": "#/components/responses/forbidden" + '500': + "$ref": "#/components/responses/internal_error" + /api/reverse-proxies/proxy-tokens/{tokenId}: + delete: + summary: Revoke a Proxy Token + description: Revoke an account-scoped proxy access token + tags: [ Self-Hosted Proxies ] + security: + - BearerAuth: [ ] + - TokenAuth: [ ] + parameters: + - in: path + name: tokenId + required: true + schema: + type: string + description: The unique identifier of the proxy token + responses: + '200': + description: Token revoked + '401': + "$ref": "#/components/responses/requires_authentication" + '403': + "$ref": "#/components/responses/forbidden" + '404': + "$ref": "#/components/responses/not_found" + '500': + "$ref": "#/components/responses/internal_error" /api/reverse-proxies/services: get: summary: List all Services diff --git a/shared/management/http/api/types.gen.go b/shared/management/http/api/types.gen.go index 4b94ea01c..b3bb475a9 100644 --- a/shared/management/http/api/types.gen.go +++ b/shared/management/http/api/types.gen.go @@ -3785,11 +3785,49 @@ type ProxyAccessLogsResponse struct { // ProxyCluster A proxy cluster represents a group of proxy nodes serving the same address type ProxyCluster struct { + // Id Unique identifier of a proxy in this cluster + Id string `json:"id"` + // Address Cluster address used for CNAME targets Address string `json:"address"` // ConnectedProxies Number of proxy nodes connected in this cluster ConnectedProxies int `json:"connected_proxies"` + + // SelfHosted Whether this cluster is a self-hosted (BYOP) proxy managed by the account owner + SelfHosted bool `json:"self_hosted"` +} + +// ProxyToken defines model for ProxyToken. +type ProxyToken struct { + CreatedAt time.Time `json:"created_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + Id string `json:"id"` + LastUsed *time.Time `json:"last_used,omitempty"` + Name string `json:"name"` + Revoked bool `json:"revoked"` +} + +// ProxyTokenCreated defines model for ProxyTokenCreated. +type ProxyTokenCreated struct { + CreatedAt time.Time `json:"created_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + Id string `json:"id"` + LastUsed *time.Time `json:"last_used,omitempty"` + Name string `json:"name"` + + // PlainToken The plain text token (shown only once) + PlainToken string `json:"plain_token"` + Revoked bool `json:"revoked"` +} + +// ProxyTokenRequest defines model for ProxyTokenRequest. +type ProxyTokenRequest struct { + // ExpiresIn Token expiration in seconds (0 = never expires) + ExpiresIn *int `json:"expires_in,omitempty"` + + // Name Human-readable token name + Name string `json:"name"` } // Resource defines model for Resource. @@ -5160,6 +5198,9 @@ type PutApiPostureChecksPostureCheckIdJSONRequestBody = PostureCheckUpdate // PostApiReverseProxiesDomainsJSONRequestBody defines body for PostApiReverseProxiesDomains for application/json ContentType. type PostApiReverseProxiesDomainsJSONRequestBody = ReverseProxyDomainRequest +// PostApiReverseProxiesProxyTokensJSONRequestBody defines body for PostApiReverseProxiesProxyTokens for application/json ContentType. +type PostApiReverseProxiesProxyTokensJSONRequestBody = ProxyTokenRequest + // PostApiReverseProxiesServicesJSONRequestBody defines body for PostApiReverseProxiesServices for application/json ContentType. type PostApiReverseProxiesServicesJSONRequestBody = ServiceRequest From 946ce4c3da24126ae34cc3b482277923863cbddb Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Tue, 12 May 2026 00:48:21 +0900 Subject: [PATCH 07/29] [client] Fix --config flag default to point at profile path (#6122) --- client/cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/cmd/root.go b/client/cmd/root.go index 29d4328a1..0a0aa4197 100644 --- a/client/cmd/root.go +++ b/client/cmd/root.go @@ -143,7 +143,7 @@ func init() { rootCmd.PersistentFlags().StringVar(&preSharedKey, preSharedKeyFlag, "", "Sets WireGuard PreSharedKey property. If set, then only peers that have the same key can communicate.") rootCmd.PersistentFlags().StringVarP(&hostName, "hostname", "n", "", "Sets a custom hostname for the device") rootCmd.PersistentFlags().BoolVarP(&anonymizeFlag, "anonymize", "A", false, "anonymize IP addresses and non-netbird.io domains in logs and status output") - rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", defaultConfigPath, "Overrides the default profile file location") + rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", profilemanager.DefaultConfigPath, "Overrides the default profile file location") rootCmd.AddCommand(upCmd) rootCmd.AddCommand(downCmd) From 96672dd1f8b116d2c40339bb94c61ed812c508e0 Mon Sep 17 00:00:00 2001 From: Nicolas Frati Date: Tue, 12 May 2026 13:50:35 +0200 Subject: [PATCH 08/29] [management] chores: update dex version (#6124) * chores: update dex version * chore: update dex fork --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5704887ce..7c1a95e79 100644 --- a/go.mod +++ b/go.mod @@ -341,8 +341,8 @@ replace github.com/cloudflare/circl => codeberg.org/cunicu/circl v0.0.0-20230801 replace github.com/pion/ice/v4 => github.com/netbirdio/ice/v4 v4.0.0-20250908184934-6202be846b51 -replace github.com/dexidp/dex => github.com/netbirdio/dex v0.244.1-0.20260415145816-a0c6b40ff9f2 +replace github.com/dexidp/dex => github.com/netbirdio/dex v0.244.1-0.20260512110716-8d70ad8647c1 -replace github.com/dexidp/dex/api/v2 => github.com/netbirdio/dex/api/v2 v2.0.0-20260415145816-a0c6b40ff9f2 +replace github.com/dexidp/dex/api/v2 => github.com/netbirdio/dex/api/v2 v2.0.0-20260512110716-8d70ad8647c1 replace github.com/mailru/easyjson => github.com/netbirdio/easyjson v0.9.0 diff --git a/go.sum b/go.sum index 42652169c..53789f49d 100644 --- a/go.sum +++ b/go.sum @@ -485,10 +485,10 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/netbirdio/dex v0.244.1-0.20260415145816-a0c6b40ff9f2 h1:AP7OM/JnTogod3rVcLsMuilSG94kWQCr3z6R4rfVXnc= -github.com/netbirdio/dex v0.244.1-0.20260415145816-a0c6b40ff9f2/go.mod h1:+trSlzHNmdJGvz0oLEyyiuaPstUeD7YO6B3Fx9nyziY= -github.com/netbirdio/dex/api/v2 v2.0.0-20260415145816-a0c6b40ff9f2 h1:HEEGJPsVw7/p7SEL3HWP4vaInxHo8OJSEaOkHpUAk+M= -github.com/netbirdio/dex/api/v2 v2.0.0-20260415145816-a0c6b40ff9f2/go.mod h1:awuTyT29CYALpEyET0S307EgNlPWrc7fFKRAyhsO45M= +github.com/netbirdio/dex v0.244.1-0.20260512110716-8d70ad8647c1 h1:4TaYr9O4xX0D2kszeOLclTiCbA3eHq3xWV+9ILJbIYs= +github.com/netbirdio/dex v0.244.1-0.20260512110716-8d70ad8647c1/go.mod h1:IHH+H8vK2GfqtIt5u/5OdPh18yk0oDHuj2vz5+Goetg= +github.com/netbirdio/dex/api/v2 v2.0.0-20260512110716-8d70ad8647c1 h1:neE7z+FPUkldl3faK/Jt+hJK2L+1XfQ1W33TQhU9m88= +github.com/netbirdio/dex/api/v2 v2.0.0-20260512110716-8d70ad8647c1/go.mod h1:awuTyT29CYALpEyET0S307EgNlPWrc7fFKRAyhsO45M= github.com/netbirdio/easyjson v0.9.0 h1:6Nw2lghSVuy8RSkAYDhDv1thBVEmfVbKZnV7T7Z6Aus= github.com/netbirdio/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/netbirdio/ice/v4 v4.0.0-20250908184934-6202be846b51 h1:Ov4qdafATOgGMB1wbSuh+0aAHcwz9hdvB6VZjh1mVMI= From 1224d6e1eeb04d6d43ca05f3bdbdc697b0b7a182 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Tue, 12 May 2026 21:52:56 +0900 Subject: [PATCH 09/29] [client] Persist management URL and pre-shared key overrides on login (#6065) --- client/server/login_overrides_test.go | 93 +++++++++++++++++++++++++++ client/server/server.go | 33 +++++++++- 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 client/server/login_overrides_test.go diff --git a/client/server/login_overrides_test.go b/client/server/login_overrides_test.go new file mode 100644 index 000000000..c45557c59 --- /dev/null +++ b/client/server/login_overrides_test.go @@ -0,0 +1,93 @@ +package server + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/internal/profilemanager" +) + +func TestPersistLoginOverrides(t *testing.T) { + strPtr := func(s string) *string { return &s } + + tests := []struct { + name string + initialMgmtURL string + initialPSK string + newMgmtURL string + newPSK *string + wantMgmtURL string + wantPSK string + }{ + { + name: "persist new management URL", + initialMgmtURL: "https://old.example.com:33073", + newMgmtURL: "https://new.example.com:33073", + wantMgmtURL: "https://new.example.com:33073", + }, + { + name: "persist new pre-shared key", + initialMgmtURL: "https://existing.example.com:33073", + initialPSK: "old-key", + newPSK: strPtr("new-key"), + wantMgmtURL: "https://existing.example.com:33073", + wantPSK: "new-key", + }, + { + name: "persist both", + initialMgmtURL: "https://old.example.com:33073", + initialPSK: "old-key", + newMgmtURL: "https://new.example.com:33073", + newPSK: strPtr("new-key"), + wantMgmtURL: "https://new.example.com:33073", + wantPSK: "new-key", + }, + { + name: "no inputs preserves existing", + initialMgmtURL: "https://existing.example.com:33073", + initialPSK: "existing-key", + wantMgmtURL: "https://existing.example.com:33073", + wantPSK: "existing-key", + }, + { + name: "empty PSK pointer is ignored", + initialMgmtURL: "https://existing.example.com:33073", + initialPSK: "existing-key", + newPSK: strPtr(""), + wantMgmtURL: "https://existing.example.com:33073", + wantPSK: "existing-key", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + origDefault := profilemanager.DefaultConfigPath + t.Cleanup(func() { profilemanager.DefaultConfigPath = origDefault }) + + dir := t.TempDir() + profilemanager.DefaultConfigPath = filepath.Join(dir, "default.json") + + seed := profilemanager.ConfigInput{ + ConfigPath: profilemanager.DefaultConfigPath, + ManagementURL: tt.initialMgmtURL, + } + if tt.initialPSK != "" { + seed.PreSharedKey = strPtr(tt.initialPSK) + } + _, err := profilemanager.UpdateOrCreateConfig(seed) + require.NoError(t, err, "seed config") + + activeProf := &profilemanager.ActiveProfileState{Name: "default"} + err = persistLoginOverrides(activeProf, tt.newMgmtURL, tt.newPSK) + require.NoError(t, err, "persistLoginOverrides") + + cfg, err := profilemanager.ReadConfig(profilemanager.DefaultConfigPath) + require.NoError(t, err, "read back config") + + require.Equal(t, tt.wantMgmtURL, cfg.ManagementURL.String(), "management URL") + require.Equal(t, tt.wantPSK, cfg.PreSharedKey, "pre-shared key") + }) + } +} diff --git a/client/server/server.go b/client/server/server.go index bc8de8f9f..397fb37e4 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -490,6 +490,11 @@ func (s *Server) Login(callerCtx context.Context, msg *proto.LoginRequest) (*pro s.mutex.Unlock() + if err := persistLoginOverrides(activeProf, msg.ManagementUrl, msg.OptionalPreSharedKey); err != nil { + log.Errorf("failed to persist login overrides: %v", err) + return nil, fmt.Errorf("persist login overrides: %w", err) + } + config, _, err := s.getConfig(activeProf) if err != nil { log.Errorf("failed to get active profile config: %v", err) @@ -964,7 +969,7 @@ func (s *Server) handleActiveProfileLogout(ctx context.Context) (*proto.LogoutRe return &proto.LogoutResponse{}, nil } -// GetConfig reads config file and returns Config and whether the config file already existed. Errors out if it does not exist +// getConfig reads config file and returns Config and whether the config file already existed. Errors out if it does not exist func (s *Server) getConfig(activeProf *profilemanager.ActiveProfileState) (*profilemanager.Config, bool, error) { cfgPath, err := activeProf.FilePath() if err != nil { @@ -1766,3 +1771,29 @@ func sendTerminalNotification() error { return wallCmd.Wait() } + +// persistLoginOverrides writes management URL and pre-shared key from a LoginRequest to the +// active profile config so that subsequent reads pick them up. Empty/nil values are ignored. +func persistLoginOverrides(activeProf *profilemanager.ActiveProfileState, managementURL string, preSharedKey *string) error { + if preSharedKey != nil && *preSharedKey == "" { + preSharedKey = nil + } + if managementURL == "" && preSharedKey == nil { + return nil + } + + cfgPath, err := activeProf.FilePath() + if err != nil { + return fmt.Errorf("active profile file path: %w", err) + } + + input := profilemanager.ConfigInput{ + ConfigPath: cfgPath, + ManagementURL: managementURL, + PreSharedKey: preSharedKey, + } + if _, err := profilemanager.UpdateOrCreateConfig(input); err != nil { + return fmt.Errorf("update config: %w", err) + } + return nil +} From 9126a192ca3f6845b798c948e1b1bc05bb7db965 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Tue, 12 May 2026 22:05:53 +0900 Subject: [PATCH 10/29] [client] Set 0644 perms on SSH client config after os.CreateTemp (#6126) --- client/ssh/config/manager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/ssh/config/manager.go b/client/ssh/config/manager.go index b58bf2233..20695cb4d 100644 --- a/client/ssh/config/manager.go +++ b/client/ssh/config/manager.go @@ -252,6 +252,10 @@ func (m *Manager) writeSSHConfig(sshConfig string) error { return fmt.Errorf("write SSH config file %s: %w", tmpPath, err) } + if err := os.Chmod(tmpPath, 0644); err != nil { + return fmt.Errorf("chmod SSH config file %s: %w", tmpPath, err) + } + if err := os.Rename(tmpPath, sshConfigPath); err != nil { return fmt.Errorf("rename SSH config %s -> %s: %w", tmpPath, sshConfigPath, err) } From ab2a8794e7a41693fff303725c96399ca190e8ff Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Thu, 14 May 2026 12:30:42 +0200 Subject: [PATCH 11/29] [client] Add short flags for status command options (#6137) * [client] Add short flags for status command options * uppercase filters --- client/cmd/status.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/client/cmd/status.go b/client/cmd/status.go index dae30e854..103b3044a 100644 --- a/client/cmd/status.go +++ b/client/cmd/status.go @@ -43,16 +43,16 @@ func init() { ipsFilterMap = make(map[string]struct{}) prefixNamesFilterMap = make(map[string]struct{}) statusCmd.PersistentFlags().BoolVarP(&detailFlag, "detail", "d", false, "display detailed status information in human-readable format") - statusCmd.PersistentFlags().BoolVar(&jsonFlag, "json", false, "display detailed status information in json format") - statusCmd.PersistentFlags().BoolVar(&yamlFlag, "yaml", false, "display detailed status information in yaml format") - statusCmd.PersistentFlags().BoolVar(&ipv4Flag, "ipv4", false, "display only NetBird IPv4 of this peer, e.g., --ipv4 will output 100.64.0.33") - statusCmd.PersistentFlags().BoolVar(&ipv6Flag, "ipv6", false, "display only NetBird IPv6 of this peer") + statusCmd.PersistentFlags().BoolVarP(&jsonFlag, "json", "j", false, "display detailed status information in json format") + statusCmd.PersistentFlags().BoolVarP(&yamlFlag, "yaml", "y", false, "display detailed status information in yaml format") + statusCmd.PersistentFlags().BoolVarP(&ipv4Flag, "ipv4", "4", false, "display only NetBird IPv4 of this peer, e.g., --ipv4 will output 100.64.0.33") + statusCmd.PersistentFlags().BoolVarP(&ipv6Flag, "ipv6", "6", false, "display only NetBird IPv6 of this peer") statusCmd.MarkFlagsMutuallyExclusive("detail", "json", "yaml", "ipv4", "ipv6") - statusCmd.PersistentFlags().StringSliceVar(&ipsFilter, "filter-by-ips", []string{}, "filters the detailed output by a list of one or more IPs (v4 or v6), e.g., --filter-by-ips 100.64.0.100,fd00::1") - statusCmd.PersistentFlags().StringSliceVar(&prefixNamesFilter, "filter-by-names", []string{}, "filters the detailed output by a list of one or more peer FQDN or hostnames, e.g., --filter-by-names peer-a,peer-b.netbird.cloud") - statusCmd.PersistentFlags().StringVar(&statusFilter, "filter-by-status", "", "filters the detailed output by connection status(idle|connecting|connected), e.g., --filter-by-status connected") - statusCmd.PersistentFlags().StringVar(&connectionTypeFilter, "filter-by-connection-type", "", "filters the detailed output by connection type (P2P|Relayed), e.g., --filter-by-connection-type P2P") - statusCmd.PersistentFlags().StringVar(&checkFlag, "check", "", "run a health check and exit with code 0 on success, 1 on failure (live|ready|startup)") + statusCmd.PersistentFlags().StringSliceVarP(&ipsFilter, "filter-by-ips", "I", []string{}, "filters the detailed output by a list of one or more IPs (v4 or v6), e.g., --filter-by-ips 100.64.0.100,fd00::1") + statusCmd.PersistentFlags().StringSliceVarP(&prefixNamesFilter, "filter-by-names", "N", []string{}, "filters the detailed output by a list of one or more peer FQDN or hostnames, e.g., --filter-by-names peer-a,peer-b.netbird.cloud") + statusCmd.PersistentFlags().StringVarP(&statusFilter, "filter-by-status", "S", "", "filters the detailed output by connection status(idle|connecting|connected), e.g., --filter-by-status connected") + statusCmd.PersistentFlags().StringVarP(&connectionTypeFilter, "filter-by-connection-type", "T", "", "filters the detailed output by connection type (P2P|Relayed), e.g., --filter-by-connection-type P2P") + statusCmd.PersistentFlags().StringVarP(&checkFlag, "check", "C", "", "run a health check and exit with code 0 on success, 1 on failure (live|ready|startup)") } func statusFunc(cmd *cobra.Command, args []string) error { From 77b479286e399660ef2bdcbe7983363946660574 Mon Sep 17 00:00:00 2001 From: Vlad <4941176+crn4@users.noreply.github.com> Date: Thu, 14 May 2026 13:27:50 +0200 Subject: [PATCH 12/29] [management] fix offline statuses for public proxy clusters (#6133) --- .../reverseproxy/domain/manager/manager.go | 23 ++++++- .../domain/manager/manager_test.go | 60 ++++++++++++++++--- .../reverseproxy/service/manager/manager.go | 59 +++++++++--------- management/server/store/sql_store.go | 28 +++++++-- management/server/store/sql_store_test.go | 52 ++++++++++++++++ 5 files changed, 177 insertions(+), 45 deletions(-) diff --git a/management/internals/modules/reverseproxy/domain/manager/manager.go b/management/internals/modules/reverseproxy/domain/manager/manager.go index ab899e0bf..2790b5f20 100644 --- a/management/internals/modules/reverseproxy/domain/manager/manager.go +++ b/management/internals/modules/reverseproxy/domain/manager/manager.go @@ -304,10 +304,27 @@ func (m Manager) getClusterAllowList(ctx context.Context, accountID string) ([]s if err != nil { return nil, fmt.Errorf("get BYOP cluster addresses: %w", err) } - if len(byopAddresses) > 0 { - return byopAddresses, nil + publicAddresses, err := m.proxyManager.GetActiveClusterAddresses(ctx) + if err != nil { + return nil, fmt.Errorf("get public cluster addresses: %w", err) } - return m.proxyManager.GetActiveClusterAddresses(ctx) + seen := make(map[string]struct{}, len(byopAddresses)+len(publicAddresses)) + merged := make([]string, 0, len(byopAddresses)+len(publicAddresses)) + for _, addr := range byopAddresses { + if _, ok := seen[addr]; ok { + continue + } + seen[addr] = struct{}{} + merged = append(merged, addr) + } + for _, addr := range publicAddresses { + if _, ok := seen[addr]; ok { + continue + } + seen[addr] = struct{}{} + merged = append(merged, addr) + } + return merged, nil } func extractClusterFromCustomDomains(serviceDomain string, customDomains []*domain.Domain) (string, bool) { diff --git a/management/internals/modules/reverseproxy/domain/manager/manager_test.go b/management/internals/modules/reverseproxy/domain/manager/manager_test.go index fdeb0765f..5e7bbfc36 100644 --- a/management/internals/modules/reverseproxy/domain/manager/manager_test.go +++ b/management/internals/modules/reverseproxy/domain/manager/manager_test.go @@ -40,22 +40,37 @@ func (m *mockProxyManager) ClusterSupportsCrowdSec(_ context.Context, _ string) return nil } -func TestGetClusterAllowList_BYOPProxy(t *testing.T) { +func TestGetClusterAllowList_BYOPMergedWithPublic(t *testing.T) { pm := &mockProxyManager{ getActiveClusterAddressesForAccountFunc: func(_ context.Context, accID string) ([]string, error) { assert.Equal(t, "acc-123", accID) return []string{"byop.example.com"}, nil }, getActiveClusterAddressesFunc: func(_ context.Context) ([]string, error) { - t.Fatal("should not call GetActiveClusterAddresses when BYOP addresses exist") - return nil, nil + return []string{"eu.proxy.netbird.io"}, nil }, } mgr := Manager{proxyManager: pm} result, err := mgr.getClusterAllowList(context.Background(), "acc-123") require.NoError(t, err) - assert.Equal(t, []string{"byop.example.com"}, result) + assert.Equal(t, []string{"byop.example.com", "eu.proxy.netbird.io"}, result) +} + +func TestGetClusterAllowList_DeduplicatesBYOPAndPublic(t *testing.T) { + pm := &mockProxyManager{ + getActiveClusterAddressesForAccountFunc: func(_ context.Context, _ string) ([]string, error) { + return []string{"shared.example.com", "byop.example.com"}, nil + }, + getActiveClusterAddressesFunc: func(_ context.Context) ([]string, error) { + return []string{"shared.example.com", "eu.proxy.netbird.io"}, nil + }, + } + + mgr := Manager{proxyManager: pm} + result, err := mgr.getClusterAllowList(context.Background(), "acc-123") + require.NoError(t, err) + assert.Equal(t, []string{"shared.example.com", "byop.example.com", "eu.proxy.netbird.io"}, result) } func TestGetClusterAllowList_NoBYOP_FallbackToShared(t *testing.T) { @@ -79,10 +94,6 @@ func TestGetClusterAllowList_BYOPError_ReturnsError(t *testing.T) { getActiveClusterAddressesForAccountFunc: func(_ context.Context, _ string) ([]string, error) { return nil, errors.New("db error") }, - getActiveClusterAddressesFunc: func(_ context.Context) ([]string, error) { - t.Fatal("should not call GetActiveClusterAddresses when BYOP lookup fails") - return nil, nil - }, } mgr := Manager{proxyManager: pm} @@ -92,6 +103,23 @@ func TestGetClusterAllowList_BYOPError_ReturnsError(t *testing.T) { assert.Contains(t, err.Error(), "BYOP cluster addresses") } +func TestGetClusterAllowList_PublicError_ReturnsError(t *testing.T) { + pm := &mockProxyManager{ + getActiveClusterAddressesForAccountFunc: func(_ context.Context, _ string) ([]string, error) { + return []string{"byop.example.com"}, nil + }, + getActiveClusterAddressesFunc: func(_ context.Context) ([]string, error) { + return nil, errors.New("db error") + }, + } + + mgr := Manager{proxyManager: pm} + result, err := mgr.getClusterAllowList(context.Background(), "acc-123") + require.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "public cluster addresses") +} + func TestGetClusterAllowList_BYOPEmptySlice_FallbackToShared(t *testing.T) { pm := &mockProxyManager{ getActiveClusterAddressesForAccountFunc: func(_ context.Context, _ string) ([]string, error) { @@ -108,3 +136,19 @@ func TestGetClusterAllowList_BYOPEmptySlice_FallbackToShared(t *testing.T) { assert.Equal(t, []string{"eu.proxy.netbird.io"}, result) } +func TestGetClusterAllowList_PublicEmpty_BYOPOnly(t *testing.T) { + pm := &mockProxyManager{ + getActiveClusterAddressesForAccountFunc: func(_ context.Context, _ string) ([]string, error) { + return []string{"byop.example.com"}, nil + }, + getActiveClusterAddressesFunc: func(_ context.Context) ([]string, error) { + return nil, nil + }, + } + + mgr := Manager{proxyManager: pm} + result, err := mgr.getClusterAllowList(context.Background(), "acc-123") + require.NoError(t, err) + assert.Equal(t, []string{"byop.example.com"}, result) +} + diff --git a/management/internals/modules/reverseproxy/service/manager/manager.go b/management/internals/modules/reverseproxy/service/manager/manager.go index c866d8f75..4a8598afb 100644 --- a/management/internals/modules/reverseproxy/service/manager/manager.go +++ b/management/internals/modules/reverseproxy/service/manager/manager.go @@ -306,6 +306,10 @@ func (m *Manager) validateSubdomainRequirement(ctx context.Context, domain, clus func (m *Manager) persistNewService(ctx context.Context, accountID string, svc *service.Service) error { customPorts := m.clusterCustomPorts(ctx, svc) + if err := validateTargetReferences(ctx, m.store, accountID, svc.Targets); err != nil { + return err + } + return m.store.ExecuteInTransaction(ctx, func(transaction store.Store) error { if svc.Domain != "" { if err := m.checkDomainAvailable(ctx, transaction, svc.Domain, ""); err != nil { @@ -321,10 +325,6 @@ func (m *Manager) persistNewService(ctx context.Context, accountID string, svc * return err } - if err := validateTargetReferences(ctx, transaction, accountID, svc.Targets); err != nil { - return err - } - if err := transaction.CreateService(ctx, svc); err != nil { return fmt.Errorf("create service: %w", err) } @@ -435,6 +435,10 @@ func (m *Manager) assignPort(ctx context.Context, tx store.Store, cluster string func (m *Manager) persistNewEphemeralService(ctx context.Context, accountID, peerID string, svc *service.Service) error { customPorts := m.clusterCustomPorts(ctx, svc) + if err := validateTargetReferences(ctx, m.store, accountID, svc.Targets); err != nil { + return err + } + return m.store.ExecuteInTransaction(ctx, func(transaction store.Store) error { if err := m.validateEphemeralPreconditions(ctx, transaction, accountID, peerID, svc); err != nil { return err @@ -448,10 +452,6 @@ func (m *Manager) persistNewEphemeralService(ctx context.Context, accountID, pee return err } - if err := validateTargetReferences(ctx, transaction, accountID, svc.Targets); err != nil { - return err - } - if err := transaction.CreateService(ctx, svc); err != nil { return fmt.Errorf("create service: %w", err) } @@ -552,10 +552,22 @@ func (m *Manager) persistServiceUpdate(ctx context.Context, accountID string, se svcForCaps.ProxyCluster = effectiveCluster customPorts := m.clusterCustomPorts(ctx, &svcForCaps) + if err := validateTargetReferences(ctx, m.store, accountID, service.Targets); err != nil { + return nil, err + } + + // Validate subdomain requirement *before* the transaction: the underlying + // capability lookup talks to the main DB pool, and SQLite's single-connection + // pool would self-deadlock if this ran while the tx already held the only + // connection. + if err := m.validateSubdomainRequirement(ctx, service.Domain, effectiveCluster); err != nil { + return nil, err + } + var updateInfo serviceUpdateInfo err = m.store.ExecuteInTransaction(ctx, func(transaction store.Store) error { - return m.executeServiceUpdate(ctx, transaction, accountID, service, &updateInfo, customPorts) + return m.executeServiceUpdate(ctx, transaction, accountID, service, &updateInfo, customPorts, effectiveCluster) }) return &updateInfo, err @@ -585,7 +597,7 @@ func (m *Manager) resolveEffectiveCluster(ctx context.Context, accountID string, return existing.ProxyCluster, nil } -func (m *Manager) executeServiceUpdate(ctx context.Context, transaction store.Store, accountID string, service *service.Service, updateInfo *serviceUpdateInfo, customPorts *bool) error { +func (m *Manager) executeServiceUpdate(ctx context.Context, transaction store.Store, accountID string, service *service.Service, updateInfo *serviceUpdateInfo, customPorts *bool, effectiveCluster string) error { existingService, err := transaction.GetServiceByID(ctx, store.LockingStrengthUpdate, accountID, service.ID) if err != nil { return err @@ -603,17 +615,13 @@ func (m *Manager) executeServiceUpdate(ctx context.Context, transaction store.St updateInfo.domainChanged = existingService.Domain != service.Domain if updateInfo.domainChanged { - if err := m.handleDomainChange(ctx, transaction, accountID, service); err != nil { + if err := m.handleDomainChange(ctx, transaction, service, effectiveCluster); err != nil { return err } } else { service.ProxyCluster = existingService.ProxyCluster } - if err := m.validateSubdomainRequirement(ctx, service.Domain, service.ProxyCluster); err != nil { - return err - } - m.preserveExistingAuthSecrets(service, existingService) if err := validateHeaderAuthValues(service.Auth.HeaderAuths); err != nil { return err @@ -628,9 +636,6 @@ func (m *Manager) executeServiceUpdate(ctx context.Context, transaction store.St if err := m.checkPortConflict(ctx, transaction, service); err != nil { return err } - if err := validateTargetReferences(ctx, transaction, accountID, service.Targets); err != nil { - return err - } if err := transaction.UpdateService(ctx, service); err != nil { return fmt.Errorf("update service: %w", err) } @@ -638,20 +643,18 @@ func (m *Manager) executeServiceUpdate(ctx context.Context, transaction store.St return nil } -func (m *Manager) handleDomainChange(ctx context.Context, transaction store.Store, accountID string, svc *service.Service) error { +// handleDomainChange validates the new domain is free inside the transaction +// and applies the pre-resolved cluster (computed outside the tx by +// resolveEffectiveCluster). It must NOT call clusterDeriver here: that talks +// to the main DB pool and would self-deadlock under SQLite (max_open_conns=1) +// because the transaction already holds the only connection. +func (m *Manager) handleDomainChange(ctx context.Context, transaction store.Store, svc *service.Service, effectiveCluster string) error { if err := m.checkDomainAvailable(ctx, transaction, svc.Domain, svc.ID); err != nil { return err } - - if m.clusterDeriver != nil { - newCluster, err := m.clusterDeriver.DeriveClusterFromDomain(ctx, accountID, svc.Domain) - if err != nil { - log.WithError(err).Warnf("could not derive cluster from domain %s", svc.Domain) - } else { - svc.ProxyCluster = newCluster - } + if effectiveCluster != "" { + svc.ProxyCluster = effectiveCluster } - return nil } diff --git a/management/server/store/sql_store.go b/management/server/store/sql_store.go index 4c2f0be52..893ee2168 100644 --- a/management/server/store/sql_store.go +++ b/management/server/store/sql_store.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "net/netip" + "net/url" "os" "path/filepath" "runtime" @@ -2794,12 +2795,27 @@ func NewSqliteStore(ctx context.Context, dataDir string, metrics telemetry.AppMe connStr = filepath.Join(dataDir, filePath) } - // Append query parameters: user-provided take precedence, otherwise default to cache=shared on non-Windows - if hasQuery { - connStr += "?" + query - } else if runtime.GOOS != "windows" { + // Compose query parameters. User-provided ?_busy_timeout (or its mattn alias + // ?_timeout) overrides our default; otherwise inject 30s so SQLite waits at + // most that long on a lock instead of blocking the only Go-side connection. + // mattn/go-sqlite3 applies PRAGMA from the DSN on every fresh connection, so + // the value survives ConnMaxIdleTime/ConnMaxLifetime recycling. cache=shared + // stays the default on non-Windows for the same reason as before. + parsed, _ := url.ParseQuery(query) + var defaults []string + if parsed.Get("_busy_timeout") == "" && parsed.Get("_timeout") == "" { + defaults = append(defaults, "_busy_timeout=30000") + } + if !hasQuery && runtime.GOOS != "windows" { // To avoid `The process cannot access the file because it is being used by another process` on Windows - connStr += "?cache=shared" + defaults = append(defaults, "cache=shared") + } + parts := defaults + if hasQuery { + parts = append(parts, query) + } + if len(parts) > 0 { + connStr += "?" + strings.Join(parts, "&") } db, err := gorm.Open(sqlite.Open(connStr), getGormConfig()) @@ -3402,7 +3418,7 @@ func (s *SqlStore) IncrementNetworkSerial(ctx context.Context, accountId string) } func (s *SqlStore) ExecuteInTransaction(ctx context.Context, operation func(store Store) error) error { - timeoutCtx, cancel := context.WithTimeout(context.Background(), s.transactionTimeout) + timeoutCtx, cancel := context.WithTimeout(ctx, s.transactionTimeout) defer cancel() startTime := time.Now() diff --git a/management/server/store/sql_store_test.go b/management/server/store/sql_store_test.go index 2819265c3..7515add62 100644 --- a/management/server/store/sql_store_test.go +++ b/management/server/store/sql_store_test.go @@ -4592,3 +4592,55 @@ func TestSqlStore_DeleteZoneDNSRecords(t *testing.T) { require.NoError(t, err) assert.Equal(t, 0, len(remainingRecords)) } + +// TestNewSqliteStore_BusyTimeoutApplied opens a fresh SQLite store and verifies +// that the _busy_timeout DSN parameter took effect at the driver level. Without +// this, lock contention on the single SQLite connection waits indefinitely on +// the Go side and can be hidden behind the 5-minute transactionTimeout. +func TestNewSqliteStore_BusyTimeoutApplied(t *testing.T) { + dir := t.TempDir() + store, err := NewSqliteStore(context.Background(), dir, nil, true) + require.NoError(t, err) + t.Cleanup(func() { + _ = store.Close(context.Background()) + }) + + sqlDB, err := store.db.DB() + require.NoError(t, err) + row := sqlDB.QueryRow("PRAGMA busy_timeout") + var busyTimeout int + require.NoError(t, row.Scan(&busyTimeout)) + assert.Equal(t, 30000, busyTimeout, "SQLite busy_timeout must be set via DSN so it survives connection recycling") +} + +// TestNewSqliteStore_BusyTimeoutRespectsUserOverride confirms that an operator +// passing _busy_timeout or its mattn alias _timeout via NB_STORE_ENGINE_SQLITE_FILE +// wins over our 30s default. This guards the DSN merge logic in NewSqliteStore. +func TestNewSqliteStore_BusyTimeoutRespectsUserOverride(t *testing.T) { + cases := []struct { + name string + envFile string + expected int + }{ + {name: "explicit _busy_timeout wins", envFile: "store.db?_busy_timeout=5000", expected: 5000}, + {name: "alias _timeout wins", envFile: "store.db?_timeout=7000", expected: 7000}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("NB_STORE_ENGINE_SQLITE_FILE", tc.envFile) + dir := t.TempDir() + store, err := NewSqliteStore(context.Background(), dir, nil, true) + require.NoError(t, err) + t.Cleanup(func() { + _ = store.Close(context.Background()) + }) + + sqlDB, err := store.db.DB() + require.NoError(t, err) + row := sqlDB.QueryRow("PRAGMA busy_timeout") + var busyTimeout int + require.NoError(t, row.Scan(&busyTimeout)) + assert.Equal(t, tc.expected, busyTimeout) + }) + } +} From ea9fab4396fc5513f7c62e3465dd361dc8bb9e91 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Thu, 14 May 2026 23:05:33 +0900 Subject: [PATCH 13/29] [management] Allocate and preserve IPv6 overlay addresses for embedded proxy peers (#6132) --- management/server/account.go | 12 +++++++++ management/server/peer.go | 17 +++++++----- management/server/types/account.go | 30 ++++++++++----------- management/server/types/group.go | 5 +++- management/server/types/ipv6_groups_test.go | 30 +++++++++++++++++++++ 5 files changed, 70 insertions(+), 24 deletions(-) diff --git a/management/server/account.go b/management/server/account.go index 364c0c37b..77a46a069 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -2487,6 +2487,18 @@ func (am *DefaultAccountManager) buildIPv6AllowedPeers(ctx context.Context, tran allowedPeers[peerID] = struct{}{} } } + + // Embedded proxy peers sit outside regular group membership but must + // participate in any v6-enabled overlay to reach v6-only peers. + peers, err := transaction.GetAccountPeers(ctx, store.LockingStrengthNone, accountID, "", "") + if err != nil { + return nil, fmt.Errorf("get peers: %w", err) + } + for _, p := range peers { + if p.ProxyMeta.Embedded { + allowedPeers[p.ID] = struct{}{} + } + } return allowedPeers, nil } diff --git a/management/server/peer.go b/management/server/peer.go index 8a39fbbb8..c3b130ba2 100644 --- a/management/server/peer.go +++ b/management/server/peer.go @@ -762,16 +762,19 @@ func (am *DefaultAccountManager) AddPeer(ctx context.Context, accountID, setupKe newPeer.IP = freeIP if len(settings.IPv6EnabledGroups) > 0 && network.NetV6.IP != nil { - var allGroupID string - if !peer.ProxyMeta.Embedded { - allGroup, err := am.Store.GetGroupByName(ctx, store.LockingStrengthNone, accountID, "All") - if err != nil { - log.WithContext(ctx).Debugf("get All group for IPv6 allocation: %v", err) - } else { + // Embedded proxy peers are not group members but participate in any + // IPv6-enabled overlay so reverse-proxy traffic reaches v6-only peers. + allocate := peer.ProxyMeta.Embedded + if !allocate { + var allGroupID string + if allGroup, err := am.Store.GetGroupByName(ctx, store.LockingStrengthNone, accountID, types.GroupAllName); err == nil { allGroupID = allGroup.ID + } else { + log.WithContext(ctx).Debugf("get All group for IPv6 allocation: %v", err) } + allocate = peerWillHaveIPv6(settings, peerAddConfig.GroupsToAdd, allGroupID) } - if peerWillHaveIPv6(settings, peerAddConfig.GroupsToAdd, allGroupID) { + if allocate { v6Prefix, err := netip.ParsePrefix(network.NetV6.String()) if err != nil { return nil, nil, nil, fmt.Errorf("parse IPv6 prefix: %w", err) diff --git a/management/server/types/account.go b/management/server/types/account.go index 49600163a..870333a60 100644 --- a/management/server/types/account.go +++ b/management/server/types/account.go @@ -598,28 +598,21 @@ func (a *Account) GetPeerGroups(peerID string) LookupMap { return groupList } -// PeerIPv6Allowed reports whether the given peer is in any of the account's IPv6 enabled groups. +// PeerIPv6Allowed reports whether the given peer participates in the IPv6 overlay. // Returns false if IPv6 is disabled or no groups are configured. func (a *Account) PeerIPv6Allowed(peerID string) bool { - if len(a.Settings.IPv6EnabledGroups) == 0 { - return false - } - - for _, groupID := range a.Settings.IPv6EnabledGroups { - group, ok := a.Groups[groupID] - if !ok { - continue - } - if slices.Contains(group.Peers, peerID) { - return true - } - } - return false + _, ok := a.peerIPv6AllowedSet()[peerID] + return ok } -// peerIPv6AllowedSet returns a set of peer IDs that belong to any IPv6-enabled group. +// peerIPv6AllowedSet returns the set of peer IDs that participate in the IPv6 overlay: +// members of any IPv6-enabled group, plus every embedded proxy peer (which sit outside +// regular group membership but must reach v6-enabled peers). func (a *Account) peerIPv6AllowedSet() map[string]struct{} { result := make(map[string]struct{}) + if len(a.Settings.IPv6EnabledGroups) == 0 { + return result + } for _, groupID := range a.Settings.IPv6EnabledGroups { group, ok := a.Groups[groupID] if !ok { @@ -629,6 +622,11 @@ func (a *Account) peerIPv6AllowedSet() map[string]struct{} { result[peerID] = struct{}{} } } + for id, p := range a.Peers { + if p != nil && p.ProxyMeta.Embedded { + result[id] = struct{}{} + } + } return result } diff --git a/management/server/types/group.go b/management/server/types/group.go index 00fdf7a69..b4f50080a 100644 --- a/management/server/types/group.go +++ b/management/server/types/group.go @@ -92,9 +92,12 @@ func (g *Group) HasPeers() bool { return len(g.Peers) > 0 } +// GroupAllName is the reserved name of the default group that contains every peer in an account. +const GroupAllName = "All" + // IsGroupAll checks if the group is a default "All" group. func (g *Group) IsGroupAll() bool { - return g.Name == "All" + return g.Name == GroupAllName } // AddPeer adds peerID to Peers if not present, returning true if added. diff --git a/management/server/types/ipv6_groups_test.go b/management/server/types/ipv6_groups_test.go index 5151e1b1f..766a9c92c 100644 --- a/management/server/types/ipv6_groups_test.go +++ b/management/server/types/ipv6_groups_test.go @@ -232,3 +232,33 @@ func TestIPv6RecalculationOnGroupChange(t *testing.T) { assert.True(t, account.PeerIPv6Allowed("peer3"), "peer3 now in infra") }) } + +func TestPeerIPv6AllowedEmbeddedProxy(t *testing.T) { + account := &Account{ + Peers: map[string]*nbpeer.Peer{ + "peer1": {ID: "peer1"}, + "proxy": {ID: "proxy", ProxyMeta: nbpeer.ProxyMeta{Embedded: true, Cluster: "netbird.test"}}, + }, + Groups: map[string]*Group{ + "group-devs": {ID: "group-devs", Peers: []string{"peer1"}}, + }, + Settings: &Settings{}, + } + + t.Run("embedded proxy allowed when any v6 group exists, without group membership", func(t *testing.T) { + account.Settings.IPv6EnabledGroups = []string{"group-devs"} + assert.True(t, account.PeerIPv6Allowed("proxy"), "embedded proxy participates in v6 overlay") + assert.True(t, account.PeerIPv6Allowed("peer1"), "regular peer in enabled group still allowed") + }) + + t.Run("embedded proxy denied when no v6 group enabled", func(t *testing.T) { + account.Settings.IPv6EnabledGroups = nil + assert.False(t, account.PeerIPv6Allowed("proxy"), "v6 disabled account-wide denies embedded proxies too") + }) + + t.Run("non-embedded peer outside any enabled group is not pulled in", func(t *testing.T) { + account.Settings.IPv6EnabledGroups = []string{"group-devs"} + account.Peers["lonely"] = &nbpeer.Peer{ID: "lonely"} + assert.False(t, account.PeerIPv6Allowed("lonely"), "embedded-proxy bypass must not leak to regular peers") + }) +} From 3f914090cbb345707a88b5edb20d9c1351873b4c Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Thu, 14 May 2026 23:22:53 +0900 Subject: [PATCH 14/29] [client] Bracket IPv6 in embed listeners, expand debug bundle (#6134) --- client/embed/embed.go | 4 +- client/internal/debug/debug.go | 36 +++-- client/internal/debug/debug_linux.go | 195 +++++++++++++++++------- client/internal/debug/debug_nonlinux.go | 5 + 4 files changed, 178 insertions(+), 62 deletions(-) diff --git a/client/embed/embed.go b/client/embed/embed.go index 4b9445b97..8b669e547 100644 --- a/client/embed/embed.go +++ b/client/embed/embed.go @@ -336,7 +336,7 @@ func (c *Client) ListenTCP(address string) (net.Listener, error) { if err != nil { return nil, fmt.Errorf("split host port: %w", err) } - listenAddr := fmt.Sprintf("%s:%s", addr, port) + listenAddr := net.JoinHostPort(addr.String(), port) tcpAddr, err := net.ResolveTCPAddr("tcp", listenAddr) if err != nil { @@ -357,7 +357,7 @@ func (c *Client) ListenUDP(address string) (net.PacketConn, error) { if err != nil { return nil, fmt.Errorf("split host port: %w", err) } - listenAddr := fmt.Sprintf("%s:%s", addr, port) + listenAddr := net.JoinHostPort(addr.String(), port) udpAddr, err := net.ResolveUDPAddr("udp", listenAddr) if err != nil { diff --git a/client/internal/debug/debug.go b/client/internal/debug/debug.go index 9c50f02b3..ebaf71b21 100644 --- a/client/internal/debug/debug.go +++ b/client/internal/debug/debug.go @@ -45,8 +45,11 @@ netbird.out: Most recent, anonymized stdout log file of the NetBird client. routes.txt: Detailed system routing table in tabular format including destination, gateway, interface, metrics, and protocol information, if --system-info flag was provided. interfaces.txt: Anonymized network interface information, if --system-info flag was provided. ip_rules.txt: Detailed IP routing rules in tabular format including priority, source, destination, interfaces, table, and action information (Linux only), if --system-info flag was provided. -iptables.txt: Anonymized iptables rules with packet counters, if --system-info flag was provided. -nftables.txt: Anonymized nftables rules with packet counters, if --system-info flag was provided. +iptables.txt: Anonymized iptables (IPv4) rules with packet counters, if --system-info flag was provided. +ip6tables.txt: Anonymized ip6tables (IPv6) rules with packet counters, if --system-info flag was provided. +ipset.txt: Anonymized ipset list output, if --system-info flag was provided. +nftables.txt: Anonymized nftables rules with packet counters across all families (ip, ip6, inet, etc.), if --system-info flag was provided. +sysctls.txt: Forwarding, reverse-path filter, source-validation, and conntrack accounting sysctl values that the NetBird client may read or modify, if --system-info flag was provided (Linux only). resolv.conf: DNS resolver configuration from /etc/resolv.conf (Unix systems only), if --system-info flag was provided. scutil_dns.txt: DNS configuration from scutil --dns (macOS only), if --system-info flag was provided. resolved_domains.txt: Anonymized resolved domain IP addresses from the status recorder. @@ -165,22 +168,33 @@ The config.txt file contains anonymized configuration information of the NetBird Other non-sensitive configuration options are included without anonymization. Firewall Rules (Linux only) -The bundle includes two separate firewall rule files: +The bundle includes the following firewall-related files: iptables.txt: -- Complete iptables ruleset with packet counters using 'iptables -v -n -L' +- IPv4 iptables ruleset with packet counters using 'iptables-save' and 'iptables -v -n -L' - Includes all tables (filter, nat, mangle, raw, security) - Shows packet and byte counters for each rule - All IP addresses are anonymized - Chain names, table names, and other non-sensitive information remain unchanged +ip6tables.txt: +- IPv6 ip6tables ruleset with packet counters using 'ip6tables-save' and 'ip6tables -v -n -L' +- Same table coverage and anonymization as iptables.txt +- Omitted when ip6tables is not installed or no IPv6 rules are present + +ipset.txt: +- Output of 'ipset list' (family-agnostic) +- IP addresses are anonymized; set names and types remain unchanged + nftables.txt: -- Complete nftables ruleset obtained via 'nft -a list ruleset' +- Complete nftables ruleset across all families (ip, ip6, inet, arp, bridge, netdev) via 'nft -a list ruleset' - Includes rule handle numbers and packet counters -- All tables, chains, and rules are included -- Shows packet and byte counters for each rule -- All IP addresses are anonymized -- Chain names, table names, and other non-sensitive information remain unchanged +- All IP addresses are anonymized; chain/table names remain unchanged + +sysctls.txt: +- Forwarding (IPv4 + IPv6, global and per-interface), reverse-path filter, source-validation, conntrack accounting, and TCP-related sysctls that netbird may read or modify +- Per-interface keys are enumerated from /proc/sys/net/ipv{4,6}/conf +- Interface names anonymized when --anonymize is set IP Rules (Linux only) The ip_rules.txt file contains detailed IP routing rule information: @@ -412,6 +426,10 @@ func (g *BundleGenerator) addSystemInfo() { log.Errorf("failed to add firewall rules to debug bundle: %v", err) } + if err := g.addSysctls(); err != nil { + log.Errorf("failed to add sysctls to debug bundle: %v", err) + } + if err := g.addDNSInfo(); err != nil { log.Errorf("failed to add DNS info to debug bundle: %v", err) } diff --git a/client/internal/debug/debug_linux.go b/client/internal/debug/debug_linux.go index aedf88b79..40d864eda 100644 --- a/client/internal/debug/debug_linux.go +++ b/client/internal/debug/debug_linux.go @@ -124,15 +124,18 @@ func getSystemdLogs(serviceName string) (string, error) { // addFirewallRules collects and adds firewall rules to the archive func (g *BundleGenerator) addFirewallRules() error { log.Info("Collecting firewall rules") - iptablesRules, err := collectIPTablesRules() + g.addIPTablesRulesToBundle("iptables-save", "iptables", "iptables.txt") + g.addIPTablesRulesToBundle("ip6tables-save", "ip6tables", "ip6tables.txt") + + ipsetOutput, err := collectIPSets() if err != nil { - log.Warnf("Failed to collect iptables rules: %v", err) + log.Warnf("Failed to collect ipset information: %v", err) } else { if g.anonymize { - iptablesRules = g.anonymizer.AnonymizeString(iptablesRules) + ipsetOutput = g.anonymizer.AnonymizeString(ipsetOutput) } - if err := g.addFileToZip(strings.NewReader(iptablesRules), "iptables.txt"); err != nil { - log.Warnf("Failed to add iptables rules to bundle: %v", err) + if err := g.addFileToZip(strings.NewReader(ipsetOutput), "ipset.txt"); err != nil { + log.Warnf("Failed to add ipset output to bundle: %v", err) } } @@ -151,44 +154,65 @@ func (g *BundleGenerator) addFirewallRules() error { return nil } -// collectIPTablesRules collects rules using both iptables-save and verbose listing -func collectIPTablesRules() (string, error) { - var builder strings.Builder - - saveOutput, err := collectIPTablesSave() +// addIPTablesRulesToBundle collects iptables/ip6tables rules and writes them to the bundle. +func (g *BundleGenerator) addIPTablesRulesToBundle(saveBin, listBin, filename string) { + rules, err := collectIPTablesRules(saveBin, listBin) if err != nil { - log.Warnf("Failed to collect iptables rules using iptables-save: %v", err) - } else { - builder.WriteString("=== iptables-save output ===\n") + log.Warnf("Failed to collect %s rules: %v", listBin, err) + return + } + if g.anonymize { + rules = g.anonymizer.AnonymizeString(rules) + } + if err := g.addFileToZip(strings.NewReader(rules), filename); err != nil { + log.Warnf("Failed to add %s rules to bundle: %v", listBin, err) + } +} + +// collectIPTablesRules collects rules using both and verbose listing via . +// Returns an error when neither command produced any output (e.g. the binary is missing), +// so the caller can skip writing an empty file. +func collectIPTablesRules(saveBin, listBin string) (string, error) { + var builder strings.Builder + var collected bool + var firstErr error + + saveOutput, err := runCommand(saveBin) + switch { + case err != nil: + firstErr = err + log.Warnf("Failed to collect %s output: %v", saveBin, err) + case strings.TrimSpace(saveOutput) == "": + log.Debugf("%s produced no output, skipping", saveBin) + default: + builder.WriteString(fmt.Sprintf("=== %s output ===\n", saveBin)) builder.WriteString(saveOutput) builder.WriteString("\n") + collected = true } - ipsetOutput, err := collectIPSets() - if err != nil { - log.Warnf("Failed to collect ipset information: %v", err) - } else { - builder.WriteString("=== ipset list output ===\n") - builder.WriteString(ipsetOutput) - builder.WriteString("\n") - } - - builder.WriteString("=== iptables -v -n -L output ===\n") + listHeader := fmt.Sprintf("=== %s -v -n -L output ===\n", listBin) + builder.WriteString(listHeader) tables := []string{"filter", "nat", "mangle", "raw", "security"} - for _, table := range tables { - builder.WriteString(fmt.Sprintf("*%s\n", table)) - - stats, err := getTableStatistics(table) + stats, err := runCommand(listBin, "-v", "-n", "-L", "-t", table) if err != nil { - log.Warnf("Failed to get statistics for table %s: %v", table, err) + if firstErr == nil { + firstErr = err + } + log.Warnf("Failed to get %s statistics for table %s: %v", listBin, table, err) continue } + builder.WriteString(fmt.Sprintf("*%s\n", table)) builder.WriteString(stats) builder.WriteString("\n") + collected = true } + if !collected { + return "", fmt.Errorf("collect %s rules: %w", listBin, firstErr) + } return builder.String(), nil } @@ -214,34 +238,15 @@ func collectIPSets() (string, error) { return ipsets, nil } -// collectIPTablesSave uses iptables-save to get rule definitions -func collectIPTablesSave() (string, error) { - cmd := exec.Command("iptables-save") +// runCommand executes a command and returns its stdout, wrapping stderr in the error on failure. +func runCommand(name string, args ...string) (string, error) { + cmd := exec.Command(name, args...) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return "", fmt.Errorf("execute iptables-save: %w (stderr: %s)", err, stderr.String()) - } - - rules := stdout.String() - if strings.TrimSpace(rules) == "" { - return "", fmt.Errorf("no iptables rules found") - } - - return rules, nil -} - -// getTableStatistics gets verbose statistics for an entire table using iptables command -func getTableStatistics(table string) (string, error) { - cmd := exec.Command("iptables", "-v", "-n", "-L", "-t", table) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("execute iptables -v -n -L: %w (stderr: %s)", err, stderr.String()) + return "", fmt.Errorf("execute %s: %w (stderr: %s)", name, err, stderr.String()) } return stdout.String(), nil @@ -804,3 +809,91 @@ func formatSetKeyType(keyType nftables.SetDatatype) string { return fmt.Sprintf("type-%v", keyType) } } + +// addSysctls collects forwarding and netbird-managed sysctl values and writes them to the bundle. +func (g *BundleGenerator) addSysctls() error { + log.Info("Collecting sysctls") + content := collectSysctls() + if g.anonymize { + content = g.anonymizer.AnonymizeString(content) + } + if err := g.addFileToZip(strings.NewReader(content), "sysctls.txt"); err != nil { + return fmt.Errorf("add sysctls to bundle: %w", err) + } + return nil +} + +// collectSysctls reads every sysctl that the netbird client may modify, plus +// global IPv4/IPv6 forwarding, and returns a formatted dump grouped by topic. +// Per-interface values are enumerated by listing /proc/sys/net/ipv{4,6}/conf. +func collectSysctls() string { + var builder strings.Builder + + writeSysctlGroup(&builder, "forwarding", []string{ + "net.ipv4.ip_forward", + "net.ipv6.conf.all.forwarding", + "net.ipv6.conf.default.forwarding", + }) + writeSysctlGroup(&builder, "ipv4 per-interface forwarding", listInterfaceSysctls("ipv4", "forwarding")) + writeSysctlGroup(&builder, "ipv6 per-interface forwarding", listInterfaceSysctls("ipv6", "forwarding")) + writeSysctlGroup(&builder, "rp_filter", append( + []string{"net.ipv4.conf.all.rp_filter", "net.ipv4.conf.default.rp_filter"}, + listInterfaceSysctls("ipv4", "rp_filter")..., + )) + writeSysctlGroup(&builder, "src_valid_mark", append( + []string{"net.ipv4.conf.all.src_valid_mark", "net.ipv4.conf.default.src_valid_mark"}, + listInterfaceSysctls("ipv4", "src_valid_mark")..., + )) + writeSysctlGroup(&builder, "conntrack", []string{ + "net.netfilter.nf_conntrack_acct", + "net.netfilter.nf_conntrack_tcp_loose", + }) + writeSysctlGroup(&builder, "tcp", []string{ + "net.ipv4.tcp_tw_reuse", + }) + + return builder.String() +} + +func writeSysctlGroup(builder *strings.Builder, title string, keys []string) { + builder.WriteString(fmt.Sprintf("=== %s ===\n", title)) + for _, key := range keys { + value, err := readSysctl(key) + if err != nil { + builder.WriteString(fmt.Sprintf("%s = \n", key, err)) + continue + } + builder.WriteString(fmt.Sprintf("%s = %s\n", key, value)) + } + builder.WriteString("\n") +} + +// listInterfaceSysctls returns net.ipvX.conf.. keys for every +// interface present in /proc/sys/net/ipvX/conf, skipping "all" and "default" +// (callers add those explicitly so they appear first). +func listInterfaceSysctls(family, leaf string) []string { + dir := fmt.Sprintf("/proc/sys/net/%s/conf", family) + entries, err := os.ReadDir(dir) + if err != nil { + return nil + } + var keys []string + for _, e := range entries { + name := e.Name() + if name == "all" || name == "default" { + continue + } + keys = append(keys, fmt.Sprintf("net.%s.conf.%s.%s", family, name, leaf)) + } + sort.Strings(keys) + return keys +} + +func readSysctl(key string) (string, error) { + path := fmt.Sprintf("/proc/sys/%s", strings.ReplaceAll(key, ".", "/")) + value, err := os.ReadFile(path) + if err != nil { + return "", err + } + return strings.TrimSpace(string(value)), nil +} diff --git a/client/internal/debug/debug_nonlinux.go b/client/internal/debug/debug_nonlinux.go index ace53bd94..878fee40f 100644 --- a/client/internal/debug/debug_nonlinux.go +++ b/client/internal/debug/debug_nonlinux.go @@ -17,3 +17,8 @@ func (g *BundleGenerator) addIPRules() error { // IP rules are only supported on Linux return nil } + +func (g *BundleGenerator) addSysctls() error { + // Sysctl collection is only supported on Linux + return nil +} From 07e5450117dd0451aaeefc18729a822115587e69 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Thu, 14 May 2026 23:42:40 +0900 Subject: [PATCH 15/29] [management] Bracket IPv6 reverse-proxy target hosts when building URL Host field (#6141) --- .../modules/reverseproxy/service/service.go | 18 ++++- .../reverseproxy/service/service_test.go | 77 +++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/management/internals/modules/reverseproxy/service/service.go b/management/internals/modules/reverseproxy/service/service.go index 769e037bc..166a66a5f 100644 --- a/management/internals/modules/reverseproxy/service/service.go +++ b/management/internals/modules/reverseproxy/service/service.go @@ -381,13 +381,14 @@ func (s *Service) buildPathMappings() []*proto.PathMapping { } // HTTP/HTTPS: build full URL + hostNoBrackets := strings.TrimSuffix(strings.TrimPrefix(target.Host, "["), "]") targetURL := url.URL{ Scheme: target.Protocol, - Host: target.Host, + Host: bracketIPv6Host(hostNoBrackets), Path: "/", } if target.Port > 0 && !isDefaultPort(target.Protocol, target.Port) { - targetURL.Host = net.JoinHostPort(targetURL.Host, strconv.FormatUint(uint64(target.Port), 10)) + targetURL.Host = net.JoinHostPort(hostNoBrackets, strconv.FormatUint(uint64(target.Port), 10)) } path := "/" @@ -405,6 +406,19 @@ func (s *Service) buildPathMappings() []*proto.PathMapping { return pathMappings } +// bracketIPv6Host wraps host in square brackets when it is an IPv6 literal, as +// required for the Host field of net/url.URL (RFC 3986 §3.2.2). v4-mapped IPv6 +// addresses are bracketed too since their textual form contains colons. +func bracketIPv6Host(host string) string { + if strings.HasPrefix(host, "[") { + return host + } + if addr, err := netip.ParseAddr(host); err == nil && addr.Is6() { + return "[" + host + "]" + } + return host +} + func operationToProtoType(op Operation) proto.ProxyMappingUpdateType { switch op { case Create: diff --git a/management/internals/modules/reverseproxy/service/service_test.go b/management/internals/modules/reverseproxy/service/service_test.go index ff54cb79f..f1349ff65 100644 --- a/management/internals/modules/reverseproxy/service/service_test.go +++ b/management/internals/modules/reverseproxy/service/service_test.go @@ -351,6 +351,83 @@ func TestToProtoMapping_PortInTargetURL(t *testing.T) { port: 80, wantTarget: "https://10.0.0.1:80/", }, + { + name: "domain host without port is unchanged", + protocol: "http", + host: "example.com", + port: 0, + wantTarget: "http://example.com/", + }, + { + name: "domain host with non-default port is unchanged", + protocol: "http", + host: "example.com", + port: 8080, + wantTarget: "http://example.com:8080/", + }, + { + name: "ipv6 host without port is bracketed", + protocol: "http", + host: "fb00:cafe:1::3", + port: 0, + wantTarget: "http://[fb00:cafe:1::3]/", + }, + { + name: "ipv6 host with default port omits port and brackets host", + protocol: "http", + host: "fb00:cafe:1::3", + port: 80, + wantTarget: "http://[fb00:cafe:1::3]/", + }, + { + name: "ipv6 host with non-default port is bracketed", + protocol: "http", + host: "fb00:cafe:1::3", + port: 8080, + wantTarget: "http://[fb00:cafe:1::3]:8080/", + }, + { + name: "ipv6 loopback without port is bracketed", + protocol: "http", + host: "::1", + port: 0, + wantTarget: "http://[::1]/", + }, + { + name: "ipv6 host with 5-digit port is bracketed", + protocol: "http", + host: "fb00:cafe::1", + port: 18080, + wantTarget: "http://[fb00:cafe::1]:18080/", + }, + { + name: "pre-bracketed ipv6 without port stays single-bracketed", + protocol: "http", + host: "[fb00:cafe::1]", + port: 0, + wantTarget: "http://[fb00:cafe::1]/", + }, + { + name: "pre-bracketed ipv6 with port is not double-bracketed", + protocol: "http", + host: "[fb00:cafe::1]", + port: 8080, + wantTarget: "http://[fb00:cafe::1]:8080/", + }, + { + name: "v4-mapped ipv6 host without port is bracketed", + protocol: "http", + host: "::ffff:10.0.0.1", + port: 0, + wantTarget: "http://[::ffff:10.0.0.1]/", + }, + { + name: "full-form 8-group ipv6 without port is bracketed", + protocol: "http", + host: "fb00:cafe:1:0:0:0:0:3", + port: 0, + wantTarget: "http://[fb00:cafe:1:0:0:0:0:3]/", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 2ccae7ec479c106efb6d7a7edff4bb55affb2aa4 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Fri, 15 May 2026 23:58:47 +0900 Subject: [PATCH 16/29] [client] Mirror v4 exit selection onto v6 pair and honour SkipAutoApply per route (#6150) --- client/internal/routemanager/manager.go | 5 +- .../internal/routeselector/routeselector.go | 84 ++++++----- .../routeselector/routeselector_test.go | 131 ++++++++++++++++++ client/ui/network.go | 10 +- 4 files changed, 197 insertions(+), 33 deletions(-) diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index e5d9363ca..907f1f592 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -704,7 +704,10 @@ func (m *DefaultManager) collectExitNodeInfo(clientRoutes route.HAMap) exitNodeI } func (m *DefaultManager) isExitNodeRoute(routes []*route.Route) bool { - return len(routes) > 0 && routes[0].Network.String() == vars.ExitNodeCIDR + if len(routes) == 0 { + return false + } + return route.IsV4DefaultRoute(routes[0].Network) || route.IsV6DefaultRoute(routes[0].Network) } func (m *DefaultManager) categorizeUserSelection(netID route.NetID, info *exitNodeInfo) { diff --git a/client/internal/routeselector/routeselector.go b/client/internal/routeselector/routeselector.go index 30afc013b..2ddc24bf2 100644 --- a/client/internal/routeselector/routeselector.go +++ b/client/internal/routeselector/routeselector.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "slices" + "strings" "sync" "github.com/hashicorp/go-multierror" @@ -12,10 +13,6 @@ import ( "github.com/netbirdio/netbird/route" ) -const ( - exitNodeCIDR = "0.0.0.0/0" -) - type RouteSelector struct { mu sync.RWMutex deselectedRoutes map[route.NetID]struct{} @@ -124,13 +121,7 @@ func (rs *RouteSelector) IsSelected(routeID route.NetID) bool { rs.mu.RLock() defer rs.mu.RUnlock() - if rs.deselectAll { - return false - } - - _, deselected := rs.deselectedRoutes[routeID] - isSelected := !deselected - return isSelected + return rs.isSelectedLocked(routeID) } // FilterSelected removes unselected routes from the provided map. @@ -144,23 +135,22 @@ func (rs *RouteSelector) FilterSelected(routes route.HAMap) route.HAMap { filtered := route.HAMap{} for id, rt := range routes { - netID := id.NetID() - _, deselected := rs.deselectedRoutes[netID] - if !deselected { + if !rs.isDeselectedLocked(id.NetID()) { filtered[id] = rt } } return filtered } -// HasUserSelectionForRoute returns true if the user has explicitly selected or deselected this specific route +// HasUserSelectionForRoute returns true if the user has explicitly selected or deselected this route. +// Intended for exit-node code paths: a v6 exit-node pair (e.g. "MyExit-v6") with no explicit state of +// its own inherits its v4 base's state, so legacy persisted selections that predate v6 pairing +// transparently apply to the synthesized v6 entry. func (rs *RouteSelector) HasUserSelectionForRoute(routeID route.NetID) bool { rs.mu.RLock() defer rs.mu.RUnlock() - _, selected := rs.selectedRoutes[routeID] - _, deselected := rs.deselectedRoutes[routeID] - return selected || deselected + return rs.hasUserSelectionForRouteLocked(rs.effectiveNetID(routeID)) } func (rs *RouteSelector) FilterSelectedExitNodes(routes route.HAMap) route.HAMap { @@ -174,7 +164,7 @@ func (rs *RouteSelector) FilterSelectedExitNodes(routes route.HAMap) route.HAMap filtered := make(route.HAMap, len(routes)) for id, rt := range routes { netID := id.NetID() - if rs.isDeselected(netID) { + if rs.isDeselectedLocked(netID) { continue } @@ -189,13 +179,48 @@ func (rs *RouteSelector) FilterSelectedExitNodes(routes route.HAMap) route.HAMap return filtered } -func (rs *RouteSelector) isDeselected(netID route.NetID) bool { +// effectiveNetID returns the v4 base for a "-v6" exit pair entry that has no explicit +// state of its own, so selections made on the v4 entry govern the v6 entry automatically. +// Only call this from exit-node-specific code paths: applying it to a non-exit "-v6" route +// would make it inherit unrelated v4 state. Must be called with rs.mu held. +func (rs *RouteSelector) effectiveNetID(id route.NetID) route.NetID { + name := string(id) + if !strings.HasSuffix(name, route.V6ExitSuffix) { + return id + } + if _, ok := rs.selectedRoutes[id]; ok { + return id + } + if _, ok := rs.deselectedRoutes[id]; ok { + return id + } + return route.NetID(strings.TrimSuffix(name, route.V6ExitSuffix)) +} + +func (rs *RouteSelector) isSelectedLocked(routeID route.NetID) bool { + if rs.deselectAll { + return false + } + _, deselected := rs.deselectedRoutes[routeID] + return !deselected +} + +func (rs *RouteSelector) isDeselectedLocked(netID route.NetID) bool { + if rs.deselectAll { + return true + } _, deselected := rs.deselectedRoutes[netID] - return deselected || rs.deselectAll + return deselected +} + +func (rs *RouteSelector) hasUserSelectionForRouteLocked(routeID route.NetID) bool { + _, selected := rs.selectedRoutes[routeID] + _, deselected := rs.deselectedRoutes[routeID] + return selected || deselected } func isExitNode(rt []*route.Route) bool { - return len(rt) > 0 && rt[0].Network.String() == exitNodeCIDR + return len(rt) > 0 && (route.IsV4DefaultRoute(rt[0].Network) || route.IsV6DefaultRoute(rt[0].Network)) } func (rs *RouteSelector) applyExitNodeFilter( @@ -204,26 +229,23 @@ func (rs *RouteSelector) applyExitNodeFilter( rt []*route.Route, out route.HAMap, ) { - - if rs.hasUserSelections() { - // user made explicit selects/deselects - if rs.IsSelected(netID) { + // Exit-node path: apply the v4/v6 pair mirror so a deselect on the v4 base also + // drops the synthesized v6 entry that lacks its own explicit state. + effective := rs.effectiveNetID(netID) + if rs.hasUserSelectionForRouteLocked(effective) { + if rs.isSelectedLocked(effective) { out[id] = rt } return } - // no explicit selections: only include routes marked !SkipAutoApply (=AutoApply) + // no explicit selection for this route: defer to management's SkipAutoApply flag sel := collectSelected(rt) if len(sel) > 0 { out[id] = sel } } -func (rs *RouteSelector) hasUserSelections() bool { - return len(rs.selectedRoutes) > 0 || len(rs.deselectedRoutes) > 0 -} - func collectSelected(rt []*route.Route) []*route.Route { var sel []*route.Route for _, r := range rt { diff --git a/client/internal/routeselector/routeselector_test.go b/client/internal/routeselector/routeselector_test.go index 5faea2456..3f0d9f120 100644 --- a/client/internal/routeselector/routeselector_test.go +++ b/client/internal/routeselector/routeselector_test.go @@ -330,6 +330,137 @@ func TestRouteSelector_FilterSelectedExitNodes(t *testing.T) { assert.Len(t, filtered, 0) // No routes should be selected } +// TestRouteSelector_V6ExitPairInherits covers the v4/v6 exit-node pair selection +// mirror. The mirror is scoped to exit-node code paths: HasUserSelectionForRoute +// and FilterSelectedExitNodes resolve a "-v6" entry without explicit state to its +// v4 base, so legacy persisted selections that predate v6 pairing transparently +// apply to the synthesized v6 entry. General lookups (IsSelected, FilterSelected) +// stay literal so unrelated routes named "*-v6" don't inherit unrelated state. +func TestRouteSelector_V6ExitPairInherits(t *testing.T) { + all := []route.NetID{"exit1", "exit1-v6", "exit2", "exit2-v6", "corp", "corp-v6"} + + t.Run("HasUserSelectionForRoute mirrors deselected v4 base", func(t *testing.T) { + rs := routeselector.NewRouteSelector() + require.NoError(t, rs.DeselectRoutes([]route.NetID{"exit1"}, all)) + + assert.True(t, rs.HasUserSelectionForRoute("exit1-v6"), "v6 pair sees v4 base's user selection") + + // unrelated v6 with no v4 base touched is unaffected + assert.False(t, rs.HasUserSelectionForRoute("exit2-v6")) + }) + + t.Run("IsSelected stays literal for non-exit lookups", func(t *testing.T) { + rs := routeselector.NewRouteSelector() + require.NoError(t, rs.DeselectRoutes([]route.NetID{"corp"}, all)) + + // A non-exit route literally named "corp-v6" must not inherit "corp"'s state + // via the mirror; the mirror only applies in exit-node code paths. + assert.False(t, rs.IsSelected("corp")) + assert.True(t, rs.IsSelected("corp-v6"), "non-exit *-v6 routes must not inherit unrelated v4 state") + }) + + t.Run("explicit v6 state overrides v4 base in filter", func(t *testing.T) { + rs := routeselector.NewRouteSelector() + require.NoError(t, rs.DeselectRoutes([]route.NetID{"exit1"}, all)) + require.NoError(t, rs.SelectRoutes([]route.NetID{"exit1-v6"}, true, all)) + + v4Route := &route.Route{NetID: "exit1", Network: netip.MustParsePrefix("0.0.0.0/0")} + v6Route := &route.Route{NetID: "exit1-v6", Network: netip.MustParsePrefix("::/0")} + routes := route.HAMap{ + "exit1|0.0.0.0/0": {v4Route}, + "exit1-v6|::/0": {v6Route}, + } + + filtered := rs.FilterSelectedExitNodes(routes) + assert.NotContains(t, filtered, route.HAUniqueID("exit1|0.0.0.0/0")) + assert.Contains(t, filtered, route.HAUniqueID("exit1-v6|::/0"), "explicit v6 select wins over v4 base") + }) + + t.Run("non-v6-suffix routes unaffected", func(t *testing.T) { + rs := routeselector.NewRouteSelector() + require.NoError(t, rs.DeselectRoutes([]route.NetID{"exit1"}, all)) + + // A route literally named "exit1-something" must not pair-resolve. + assert.False(t, rs.HasUserSelectionForRoute("exit1-something")) + }) + + t.Run("filter v6 paired with deselected v4 base", func(t *testing.T) { + rs := routeselector.NewRouteSelector() + require.NoError(t, rs.DeselectRoutes([]route.NetID{"exit1"}, all)) + + v4Route := &route.Route{NetID: "exit1", Network: netip.MustParsePrefix("0.0.0.0/0")} + v6Route := &route.Route{NetID: "exit1-v6", Network: netip.MustParsePrefix("::/0")} + routes := route.HAMap{ + "exit1|0.0.0.0/0": {v4Route}, + "exit1-v6|::/0": {v6Route}, + } + + filtered := rs.FilterSelectedExitNodes(routes) + assert.Empty(t, filtered, "deselecting v4 base must also drop the v6 pair") + }) + + t.Run("non-exit *-v6 routes pass through FilterSelectedExitNodes", func(t *testing.T) { + rs := routeselector.NewRouteSelector() + require.NoError(t, rs.DeselectRoutes([]route.NetID{"corp"}, all)) + + // A non-default-route entry named "corp-v6" is not an exit node and + // must not be skipped because its v4 base "corp" is deselected. + corpV6 := &route.Route{NetID: "corp-v6", Network: netip.MustParsePrefix("10.0.0.0/8")} + routes := route.HAMap{ + "corp-v6|10.0.0.0/8": {corpV6}, + } + + filtered := rs.FilterSelectedExitNodes(routes) + assert.Contains(t, filtered, route.HAUniqueID("corp-v6|10.0.0.0/8"), + "non-exit *-v6 routes must not inherit unrelated v4 state in FilterSelectedExitNodes") + }) +} + +// TestRouteSelector_SkipAutoApplyPerRoute verifies that management's +// SkipAutoApply flag governs each untouched route independently, even when +// the user has explicit selections on other routes. +func TestRouteSelector_SkipAutoApplyPerRoute(t *testing.T) { + autoApplied := &route.Route{ + NetID: "Auto", + Network: netip.MustParsePrefix("0.0.0.0/0"), + SkipAutoApply: false, + } + skipApply := &route.Route{ + NetID: "Skip", + Network: netip.MustParsePrefix("0.0.0.0/0"), + SkipAutoApply: true, + } + routes := route.HAMap{ + "Auto|0.0.0.0/0": {autoApplied}, + "Skip|0.0.0.0/0": {skipApply}, + } + + rs := routeselector.NewRouteSelector() + // User makes an unrelated explicit selection elsewhere. + require.NoError(t, rs.DeselectRoutes([]route.NetID{"Unrelated"}, []route.NetID{"Auto", "Skip", "Unrelated"})) + + filtered := rs.FilterSelectedExitNodes(routes) + assert.Contains(t, filtered, route.HAUniqueID("Auto|0.0.0.0/0"), "AutoApply route should be included") + assert.NotContains(t, filtered, route.HAUniqueID("Skip|0.0.0.0/0"), "SkipAutoApply route should be excluded without explicit user selection") +} + +// TestRouteSelector_V6ExitIsExitNode verifies that ::/0 routes are recognized +// as exit nodes by the selector's filter path. +func TestRouteSelector_V6ExitIsExitNode(t *testing.T) { + v6Exit := &route.Route{ + NetID: "V6Only", + Network: netip.MustParsePrefix("::/0"), + SkipAutoApply: true, + } + routes := route.HAMap{ + "V6Only|::/0": {v6Exit}, + } + + rs := routeselector.NewRouteSelector() + filtered := rs.FilterSelectedExitNodes(routes) + assert.Empty(t, filtered, "::/0 should be treated as an exit node and respect SkipAutoApply") +} + func TestRouteSelector_NewRoutesBehavior(t *testing.T) { initialRoutes := []route.NetID{"route1", "route2", "route3"} newRoutes := []route.NetID{"route1", "route2", "route3", "route4", "route5"} diff --git a/client/ui/network.go b/client/ui/network.go index 1619f78a2..cd5d23558 100644 --- a/client/ui/network.go +++ b/client/ui/network.go @@ -193,7 +193,15 @@ func getOverlappingNetworks(routes []*proto.Network) []*proto.Network { } func isDefaultRoute(routeRange string) bool { - return routeRange == "0.0.0.0/0" || routeRange == "::/0" + // routeRange is the merged display string from the daemon, e.g. "0.0.0.0/0", + // "::/0", or "0.0.0.0/0, ::/0" when a v4 exit node has a paired v6 entry. + for _, part := range strings.Split(routeRange, ",") { + switch strings.TrimSpace(part) { + case "0.0.0.0/0", "::/0": + return true + } + } + return false } func getExitNodeNetworks(routes []*proto.Network) []*proto.Network { From 9ed2e2a5b463077f8abe3e3926695f5dc9411e29 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Sat, 16 May 2026 00:07:38 +0900 Subject: [PATCH 17/29] [client] Drop DNS probes for passive health projection (#5971) --- client/internal/connect.go | 2 - client/internal/dns/host.go | 12 + client/internal/dns/host_android.go | 19 +- client/internal/dns/host_ios.go | 9 + client/internal/dns/host_windows.go | 121 ++- client/internal/dns/hosts_dns_holder.go | 1 + client/internal/dns/local/local.go | 2 - client/internal/dns/mock_server.go | 9 +- client/internal/dns/network_manager_unix.go | 211 ++++- client/internal/dns/server.go | 928 ++++++++++++-------- client/internal/dns/server_android.go | 2 +- client/internal/dns/server_test.go | 698 +++++++++++++-- client/internal/dns/systemd_linux.go | 151 +++- client/internal/dns/upstream.go | 683 +++++++------- client/internal/dns/upstream_android.go | 5 +- client/internal/dns/upstream_general.go | 5 +- client/internal/dns/upstream_ios.go | 17 +- client/internal/dns/upstream_test.go | 227 +++-- client/internal/engine.go | 16 +- client/internal/routemanager/manager.go | 34 + client/internal/routemanager/mock.go | 9 + client/ios/NetBirdSDK/client.go | 6 +- 22 files changed, 2294 insertions(+), 873 deletions(-) diff --git a/client/internal/connect.go b/client/internal/connect.go index 8c0e9b1ba..ea884818f 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -116,7 +116,6 @@ func (c *ConnectClient) RunOniOS( fileDescriptor int32, networkChangeListener listener.NetworkChangeListener, dnsManager dns.IosDnsManager, - dnsAddresses []netip.AddrPort, stateFilePath string, ) error { // Set GC percent to 5% to reduce memory usage as iOS only allows 50MB of memory for the extension. @@ -126,7 +125,6 @@ func (c *ConnectClient) RunOniOS( FileDescriptor: fileDescriptor, NetworkChangeListener: networkChangeListener, DnsManager: dnsManager, - HostDNSAddresses: dnsAddresses, StateFilePath: stateFilePath, } return c.run(mobileDependency, nil, "") diff --git a/client/internal/dns/host.go b/client/internal/dns/host.go index f7dc46a6b..48eacef29 100644 --- a/client/internal/dns/host.go +++ b/client/internal/dns/host.go @@ -16,6 +16,10 @@ type hostManager interface { restoreHostDNS() error supportCustomPort() bool string() string + // getOriginalNameservers returns the OS-side resolvers used as PriorityFallback + // upstreams: pre-takeover snapshots on desktop, the OS-pushed list on Android, + // hardcoded Quad9 on iOS, nil for noop / mock. + getOriginalNameservers() []netip.Addr } type SystemDNSSettings struct { @@ -131,3 +135,11 @@ func (n noopHostConfigurator) supportCustomPort() bool { func (n noopHostConfigurator) string() string { return "noop" } + +func (n noopHostConfigurator) getOriginalNameservers() []netip.Addr { + return nil +} + +func (m *mockHostConfigurator) getOriginalNameservers() []netip.Addr { + return nil +} diff --git a/client/internal/dns/host_android.go b/client/internal/dns/host_android.go index dfa3e5712..48b3e0301 100644 --- a/client/internal/dns/host_android.go +++ b/client/internal/dns/host_android.go @@ -1,14 +1,20 @@ package dns import ( + "net/netip" + "github.com/netbirdio/netbird/client/internal/statemanager" ) +// androidHostManager is a noop on the OS side (Android's VPN service handles +// DNS for us) but tracks the OS-reported resolver list pushed via +// OnUpdatedHostDNSServer so it can serve as the fallback nameserver source. type androidHostManager struct { + holder *hostsDNSHolder } -func newHostManager() (*androidHostManager, error) { - return &androidHostManager{}, nil +func newHostManager(holder *hostsDNSHolder) (*androidHostManager, error) { + return &androidHostManager{holder: holder}, nil } func (a androidHostManager) applyDNSConfig(HostDNSConfig, *statemanager.Manager) error { @@ -26,3 +32,12 @@ func (a androidHostManager) supportCustomPort() bool { func (a androidHostManager) string() string { return "none" } + +func (a androidHostManager) getOriginalNameservers() []netip.Addr { + hosts := a.holder.get() + out := make([]netip.Addr, 0, len(hosts)) + for ap := range hosts { + out = append(out, ap.Addr()) + } + return out +} diff --git a/client/internal/dns/host_ios.go b/client/internal/dns/host_ios.go index 1c0ac63e9..860bb8b50 100644 --- a/client/internal/dns/host_ios.go +++ b/client/internal/dns/host_ios.go @@ -3,6 +3,7 @@ package dns import ( "encoding/json" "fmt" + "net/netip" log "github.com/sirupsen/logrus" @@ -20,6 +21,14 @@ func newHostManager(dnsManager IosDnsManager) (*iosHostManager, error) { }, nil } +func (a iosHostManager) getOriginalNameservers() []netip.Addr { + // Quad9 v4+v6: 9.9.9.9, 2620:fe::fe. + return []netip.Addr{ + netip.AddrFrom4([4]byte{9, 9, 9, 9}), + netip.AddrFrom16([16]byte{0x26, 0x20, 0x00, 0xfe, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xfe}), + } +} + func (a iosHostManager) applyDNSConfig(config HostDNSConfig, _ *statemanager.Manager) error { jsonData, err := json.Marshal(config) if err != nil { diff --git a/client/internal/dns/host_windows.go b/client/internal/dns/host_windows.go index 4a8cf8cec..4f6ece532 100644 --- a/client/internal/dns/host_windows.go +++ b/client/internal/dns/host_windows.go @@ -7,6 +7,7 @@ import ( "io" "net/netip" "os/exec" + "slices" "strings" "syscall" "time" @@ -44,9 +45,11 @@ const ( nrptMaxDomainsPerRule = 50 - interfaceConfigPath = `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces` - interfaceConfigNameServerKey = "NameServer" - interfaceConfigSearchListKey = "SearchList" + interfaceConfigPath = `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces` + interfaceConfigPathV6 = `SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces` + interfaceConfigNameServerKey = "NameServer" + interfaceConfigDhcpNameSrvKey = "DhcpNameServer" + interfaceConfigSearchListKey = "SearchList" // Network interface DNS registration settings disableDynamicUpdateKey = "DisableDynamicUpdate" @@ -67,10 +70,11 @@ const ( ) type registryConfigurator struct { - guid string - routingAll bool - gpo bool - nrptEntryCount int + guid string + routingAll bool + gpo bool + nrptEntryCount int + origNameservers []netip.Addr } func newHostManager(wgInterface WGIface) (*registryConfigurator, error) { @@ -94,6 +98,17 @@ func newHostManager(wgInterface WGIface) (*registryConfigurator, error) { gpo: useGPO, } + origNameservers, err := configurator.captureOriginalNameservers() + switch { + case err != nil: + log.Warnf("capture original nameservers from non-WG adapters: %v", err) + case len(origNameservers) == 0: + log.Warnf("no original nameservers captured from non-WG adapters; DNS fallback will be empty") + default: + log.Debugf("captured %d original nameservers from non-WG adapters: %v", len(origNameservers), origNameservers) + } + configurator.origNameservers = origNameservers + if err := configurator.configureInterface(); err != nil { log.Errorf("failed to configure interface settings: %v", err) } @@ -101,6 +116,98 @@ func newHostManager(wgInterface WGIface) (*registryConfigurator, error) { return configurator, nil } +// captureOriginalNameservers reads DNS addresses from every Tcpip(6) interface +// registry key except the WG adapter. v4 and v6 servers live in separate +// hives (Tcpip vs Tcpip6) keyed by the same interface GUID. +func (r *registryConfigurator) captureOriginalNameservers() ([]netip.Addr, error) { + seen := make(map[netip.Addr]struct{}) + var out []netip.Addr + var merr *multierror.Error + for _, root := range []string{interfaceConfigPath, interfaceConfigPathV6} { + addrs, err := r.captureFromTcpipRoot(root) + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("%s: %w", root, err)) + continue + } + for _, addr := range addrs { + if _, dup := seen[addr]; dup { + continue + } + seen[addr] = struct{}{} + out = append(out, addr) + } + } + return out, nberrors.FormatErrorOrNil(merr) +} + +func (r *registryConfigurator) captureFromTcpipRoot(rootPath string) ([]netip.Addr, error) { + root, err := registry.OpenKey(registry.LOCAL_MACHINE, rootPath, registry.READ) + if err != nil { + return nil, fmt.Errorf("open key: %w", err) + } + defer closer(root) + + guids, err := root.ReadSubKeyNames(-1) + if err != nil { + return nil, fmt.Errorf("read subkeys: %w", err) + } + + var out []netip.Addr + for _, guid := range guids { + if strings.EqualFold(guid, r.guid) { + continue + } + out = append(out, readInterfaceNameservers(rootPath, guid)...) + } + return out, nil +} + +func readInterfaceNameservers(rootPath, guid string) []netip.Addr { + keyPath := rootPath + "\\" + guid + k, err := registry.OpenKey(registry.LOCAL_MACHINE, keyPath, registry.QUERY_VALUE) + if err != nil { + return nil + } + defer closer(k) + + // Static NameServer wins over DhcpNameServer for actual resolution. + for _, name := range []string{interfaceConfigNameServerKey, interfaceConfigDhcpNameSrvKey} { + raw, _, err := k.GetStringValue(name) + if err != nil || raw == "" { + continue + } + if out := parseRegistryNameservers(raw); len(out) > 0 { + return out + } + } + return nil +} + +func parseRegistryNameservers(raw string) []netip.Addr { + var out []netip.Addr + for _, field := range strings.FieldsFunc(raw, func(r rune) bool { return r == ',' || r == ' ' || r == '\t' }) { + addr, err := netip.ParseAddr(strings.TrimSpace(field)) + if err != nil { + continue + } + addr = addr.Unmap() + if !addr.IsValid() || addr.IsUnspecified() { + continue + } + // Drop unzoned link-local: not routable without a scope id. If + // the user wrote "fe80::1%eth0" ParseAddr preserves the zone. + if addr.IsLinkLocalUnicast() && addr.Zone() == "" { + continue + } + out = append(out, addr) + } + return out +} + +func (r *registryConfigurator) getOriginalNameservers() []netip.Addr { + return slices.Clone(r.origNameservers) +} + func (r *registryConfigurator) supportCustomPort() bool { return false } diff --git a/client/internal/dns/hosts_dns_holder.go b/client/internal/dns/hosts_dns_holder.go index 980d917a7..9ecc397be 100644 --- a/client/internal/dns/hosts_dns_holder.go +++ b/client/internal/dns/hosts_dns_holder.go @@ -25,6 +25,7 @@ func (h *hostsDNSHolder) set(list []netip.AddrPort) { h.mutex.Unlock() } +//nolint:unused func (h *hostsDNSHolder) get() map[netip.AddrPort]struct{} { h.mutex.RLock() l := h.unprotectedDNSList diff --git a/client/internal/dns/local/local.go b/client/internal/dns/local/local.go index e9d310f00..4a75a76b6 100644 --- a/client/internal/dns/local/local.go +++ b/client/internal/dns/local/local.go @@ -76,8 +76,6 @@ func (d *Resolver) ID() types.HandlerID { return "local-resolver" } -func (d *Resolver) ProbeAvailability(context.Context) {} - // ServeDNS handles a DNS request func (d *Resolver) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { logger := log.WithFields(log.Fields{ diff --git a/client/internal/dns/mock_server.go b/client/internal/dns/mock_server.go index 548b1f54f..31fedd9e5 100644 --- a/client/internal/dns/mock_server.go +++ b/client/internal/dns/mock_server.go @@ -9,6 +9,7 @@ import ( dnsconfig "github.com/netbirdio/netbird/client/internal/dns/config" nbdns "github.com/netbirdio/netbird/dns" + "github.com/netbirdio/netbird/route" "github.com/netbirdio/netbird/shared/management/domain" ) @@ -70,10 +71,6 @@ func (m *MockServer) SearchDomains() []string { return make([]string, 0) } -// ProbeAvailability mocks implementation of ProbeAvailability from the Server interface -func (m *MockServer) ProbeAvailability() { -} - func (m *MockServer) UpdateServerConfig(domains dnsconfig.ServerDomains) error { if m.UpdateServerConfigFunc != nil { return m.UpdateServerConfigFunc(domains) @@ -85,8 +82,8 @@ func (m *MockServer) PopulateManagementDomain(mgmtURL *url.URL) error { return nil } -// SetRouteChecker mock implementation of SetRouteChecker from Server interface -func (m *MockServer) SetRouteChecker(func(netip.Addr) bool) { +// SetRouteSources mock implementation of SetRouteSources from Server interface +func (m *MockServer) SetRouteSources(selected, active func() route.HAMap) { // Mock implementation - no-op } diff --git a/client/internal/dns/network_manager_unix.go b/client/internal/dns/network_manager_unix.go index 66d82dcd7..3932e78b7 100644 --- a/client/internal/dns/network_manager_unix.go +++ b/client/internal/dns/network_manager_unix.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "net/netip" + "slices" "strings" "time" @@ -32,6 +33,15 @@ const ( networkManagerDbusDeviceGetAppliedConnectionMethod = networkManagerDbusDeviceInterface + ".GetAppliedConnection" networkManagerDbusDeviceReapplyMethod = networkManagerDbusDeviceInterface + ".Reapply" networkManagerDbusDeviceDeleteMethod = networkManagerDbusDeviceInterface + ".Delete" + networkManagerDbusDeviceIp4ConfigProperty = networkManagerDbusDeviceInterface + ".Ip4Config" + networkManagerDbusDeviceIp6ConfigProperty = networkManagerDbusDeviceInterface + ".Ip6Config" + networkManagerDbusDeviceIfaceProperty = networkManagerDbusDeviceInterface + ".Interface" + networkManagerDbusGetDevicesMethod = networkManagerDest + ".GetDevices" + networkManagerDbusIp4ConfigInterface = "org.freedesktop.NetworkManager.IP4Config" + networkManagerDbusIp6ConfigInterface = "org.freedesktop.NetworkManager.IP6Config" + networkManagerDbusIp4ConfigNameserverDataProperty = networkManagerDbusIp4ConfigInterface + ".NameserverData" + networkManagerDbusIp4ConfigNameserversProperty = networkManagerDbusIp4ConfigInterface + ".Nameservers" + networkManagerDbusIp6ConfigNameserversProperty = networkManagerDbusIp6ConfigInterface + ".Nameservers" networkManagerDbusDefaultBehaviorFlag networkManagerConfigBehavior = 0 networkManagerDbusIPv4Key = "ipv4" networkManagerDbusIPv6Key = "ipv6" @@ -51,9 +61,10 @@ var supportedNetworkManagerVersionConstraints = []string{ } type networkManagerDbusConfigurator struct { - dbusLinkObject dbus.ObjectPath - routingAll bool - ifaceName string + dbusLinkObject dbus.ObjectPath + routingAll bool + ifaceName string + origNameservers []netip.Addr } // the types below are based on dbus specification, each field is mapped to a dbus type @@ -92,10 +103,200 @@ func newNetworkManagerDbusConfigurator(wgInterface string) (*networkManagerDbusC log.Debugf("got network manager dbus Link Object: %s from net interface %s", s, wgInterface) - return &networkManagerDbusConfigurator{ + c := &networkManagerDbusConfigurator{ dbusLinkObject: dbus.ObjectPath(s), ifaceName: wgInterface, - }, nil + } + + origNameservers, err := c.captureOriginalNameservers() + switch { + case err != nil: + log.Warnf("capture original nameservers from NetworkManager: %v", err) + case len(origNameservers) == 0: + log.Warnf("no original nameservers captured from non-WG NetworkManager devices; DNS fallback will be empty") + default: + log.Debugf("captured %d original nameservers from non-WG NetworkManager devices: %v", len(origNameservers), origNameservers) + } + c.origNameservers = origNameservers + return c, nil +} + +// captureOriginalNameservers reads DNS servers from every NM device's +// IP4Config / IP6Config except our WG device. +func (n *networkManagerDbusConfigurator) captureOriginalNameservers() ([]netip.Addr, error) { + devices, err := networkManagerListDevices() + if err != nil { + return nil, fmt.Errorf("list devices: %w", err) + } + + seen := make(map[netip.Addr]struct{}) + var out []netip.Addr + for _, dev := range devices { + if dev == n.dbusLinkObject { + continue + } + ifaceName := readNetworkManagerDeviceInterface(dev) + for _, addr := range readNetworkManagerDeviceDNS(dev) { + addr = addr.Unmap() + if !addr.IsValid() || addr.IsUnspecified() { + continue + } + // IP6Config.Nameservers is a byte slice without zone info; + // reattach the device's interface name so a captured fe80::… + // stays routable. + if addr.IsLinkLocalUnicast() && ifaceName != "" { + addr = addr.WithZone(ifaceName) + } + if _, dup := seen[addr]; dup { + continue + } + seen[addr] = struct{}{} + out = append(out, addr) + } + } + return out, nil +} + +func readNetworkManagerDeviceInterface(devicePath dbus.ObjectPath) string { + obj, closeConn, err := getDbusObject(networkManagerDest, devicePath) + if err != nil { + return "" + } + defer closeConn() + v, err := obj.GetProperty(networkManagerDbusDeviceIfaceProperty) + if err != nil { + return "" + } + s, _ := v.Value().(string) + return s +} + +func networkManagerListDevices() ([]dbus.ObjectPath, error) { + obj, closeConn, err := getDbusObject(networkManagerDest, networkManagerDbusObjectNode) + if err != nil { + return nil, fmt.Errorf("dbus NetworkManager: %w", err) + } + defer closeConn() + var devs []dbus.ObjectPath + if err := obj.Call(networkManagerDbusGetDevicesMethod, dbusDefaultFlag).Store(&devs); err != nil { + return nil, err + } + return devs, nil +} + +func readNetworkManagerDeviceDNS(devicePath dbus.ObjectPath) []netip.Addr { + obj, closeConn, err := getDbusObject(networkManagerDest, devicePath) + if err != nil { + return nil + } + defer closeConn() + + var out []netip.Addr + if path := readNetworkManagerConfigPath(obj, networkManagerDbusDeviceIp4ConfigProperty); path != "" { + out = append(out, readIPv4ConfigDNS(path)...) + } + if path := readNetworkManagerConfigPath(obj, networkManagerDbusDeviceIp6ConfigProperty); path != "" { + out = append(out, readIPv6ConfigDNS(path)...) + } + return out +} + +func readNetworkManagerConfigPath(obj dbus.BusObject, property string) dbus.ObjectPath { + v, err := obj.GetProperty(property) + if err != nil { + return "" + } + path, ok := v.Value().(dbus.ObjectPath) + if !ok || path == "/" { + return "" + } + return path +} + +func readIPv4ConfigDNS(path dbus.ObjectPath) []netip.Addr { + obj, closeConn, err := getDbusObject(networkManagerDest, path) + if err != nil { + return nil + } + defer closeConn() + + // NameserverData (NM 1.13+) carries strings; older NMs only expose the + // legacy uint32 Nameservers property. + if out := readIPv4NameserverData(obj); len(out) > 0 { + return out + } + return readIPv4LegacyNameservers(obj) +} + +func readIPv4NameserverData(obj dbus.BusObject) []netip.Addr { + v, err := obj.GetProperty(networkManagerDbusIp4ConfigNameserverDataProperty) + if err != nil { + return nil + } + entries, ok := v.Value().([]map[string]dbus.Variant) + if !ok { + return nil + } + var out []netip.Addr + for _, entry := range entries { + addrVar, ok := entry["address"] + if !ok { + continue + } + s, ok := addrVar.Value().(string) + if !ok { + continue + } + if a, err := netip.ParseAddr(s); err == nil { + out = append(out, a) + } + } + return out +} + +func readIPv4LegacyNameservers(obj dbus.BusObject) []netip.Addr { + v, err := obj.GetProperty(networkManagerDbusIp4ConfigNameserversProperty) + if err != nil { + return nil + } + raw, ok := v.Value().([]uint32) + if !ok { + return nil + } + out := make([]netip.Addr, 0, len(raw)) + for _, n := range raw { + var b [4]byte + binary.LittleEndian.PutUint32(b[:], n) + out = append(out, netip.AddrFrom4(b)) + } + return out +} + +func readIPv6ConfigDNS(path dbus.ObjectPath) []netip.Addr { + obj, closeConn, err := getDbusObject(networkManagerDest, path) + if err != nil { + return nil + } + defer closeConn() + v, err := obj.GetProperty(networkManagerDbusIp6ConfigNameserversProperty) + if err != nil { + return nil + } + raw, ok := v.Value().([][]byte) + if !ok { + return nil + } + out := make([]netip.Addr, 0, len(raw)) + for _, b := range raw { + if a, ok := netip.AddrFromSlice(b); ok { + out = append(out, a) + } + } + return out +} + +func (n *networkManagerDbusConfigurator) getOriginalNameservers() []netip.Addr { + return slices.Clone(n.origNameservers) } func (n *networkManagerDbusConfigurator) supportCustomPort() bool { diff --git a/client/internal/dns/server.go b/client/internal/dns/server.go index 6fe2e21b6..e689f3586 100644 --- a/client/internal/dns/server.go +++ b/client/internal/dns/server.go @@ -6,11 +6,10 @@ import ( "fmt" "net/netip" "net/url" - "os" - "runtime" - "strconv" + "slices" "strings" "sync" + "time" "github.com/miekg/dns" "github.com/mitchellh/hashstructure/v2" @@ -25,11 +24,31 @@ import ( "github.com/netbirdio/netbird/client/internal/listener" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/statemanager" + "github.com/netbirdio/netbird/client/proto" nbdns "github.com/netbirdio/netbird/dns" + "github.com/netbirdio/netbird/route" "github.com/netbirdio/netbird/shared/management/domain" ) -const envSkipDNSProbe = "NB_SKIP_DNS_PROBE" +const ( + // healthLookback must exceed the upstream query timeout so one + // query per refresh cycle is enough to keep a group marked healthy. + healthLookback = 60 * time.Second + nsGroupHealthRefreshInterval = 10 * time.Second + // defaultWarningDelayBase is the starting grace window before a + // "Nameserver group unreachable" event fires for a group that's + // never been healthy and only has overlay upstreams with no + // Connected peer. Per-server and overridable; see warningDelayFor. + defaultWarningDelayBase = 30 * time.Second + // warningDelayBonusCap caps the route-count bonus added to the + // base grace window. See warningDelayFor. + warningDelayBonusCap = 30 * time.Second +) + +// errNoUsableNameservers signals that a merged-domain group has no usable +// upstream servers. Callers should skip the group without treating it as a +// build failure. +var errNoUsableNameservers = errors.New("no usable nameservers") // ReadyListener is a notification mechanism what indicate the server is ready to handle host dns address changes type ReadyListener interface { @@ -54,10 +73,9 @@ type Server interface { UpdateDNSServer(serial uint64, update nbdns.Config) error OnUpdatedHostDNSServer(addrs []netip.AddrPort) SearchDomains() []string - ProbeAvailability() UpdateServerConfig(domains dnsconfig.ServerDomains) error PopulateManagementDomain(mgmtURL *url.URL) error - SetRouteChecker(func(netip.Addr) bool) + SetRouteSources(selected, active func() route.HAMap) SetFirewall(Firewall) } @@ -66,12 +84,47 @@ type nsGroupsByDomain struct { groups []*nbdns.NameServerGroup } -// hostManagerWithOriginalNS extends the basic hostManager interface -type hostManagerWithOriginalNS interface { - hostManager - getOriginalNameservers() []netip.Addr +// nsGroupID identifies a nameserver group by the tuple (server list, domain +// list) so config updates produce stable IDs across recomputations. +type nsGroupID string + +// nsHealthSnapshot is the input to projectNSGroupHealth, captured under +// s.mux so projection runs lock-free. +type nsHealthSnapshot struct { + groups []*nbdns.NameServerGroup + merged map[netip.AddrPort]UpstreamHealth + selected route.HAMap + active route.HAMap } +// nsGroupProj holds per-group state for the emission rules. +type nsGroupProj struct { + // unhealthySince is the start of the current Unhealthy streak, + // zero when the group is not currently Unhealthy. + unhealthySince time.Time + // everHealthy is sticky: once the group has been Healthy at least + // once this session, subsequent failures skip warningDelay. + everHealthy bool + // warningActive tracks whether we've already published a warning + // for the current streak, so recovery emits iff a warning did. + warningActive bool +} + +// nsGroupVerdict is the outcome of evaluateNSGroupHealth. +type nsGroupVerdict int + +const ( + // nsVerdictUndecided means no upstream has a fresh observation + // (startup before first query, or records aged past healthLookback). + nsVerdictUndecided nsGroupVerdict = iota + // nsVerdictHealthy means at least one upstream's most-recent + // in-lookback observation is a success. + nsVerdictHealthy + // nsVerdictUnhealthy means at least one upstream has a recent + // failure and none has a fresher success. + nsVerdictUnhealthy +) + // DefaultServer dns server object type DefaultServer struct { ctx context.Context @@ -100,26 +153,46 @@ type DefaultServer struct { permanent bool hostsDNSHolder *hostsDNSHolder + // fallbackHandler is the upstream resolver currently registered at + // PriorityFallback. Tracked so registerFallback can Stop() the previous + // instance instead of leaking its context. + fallbackHandler handlerWithStop + // make sense on mobile only searchDomainNotifier *notifier iosDnsManager IosDnsManager statusRecorder *peer.Status stateManager *statemanager.Manager - routeMatch func(netip.Addr) bool + // selectedRoutes returns admin-enabled client routes. + selectedRoutes func() route.HAMap + // activeRoutes returns the subset whose peer is in StatusConnected. + activeRoutes func() route.HAMap - probeMu sync.Mutex - probeCancel context.CancelFunc - probeWg sync.WaitGroup + nsGroups []*nbdns.NameServerGroup + healthProjectMu sync.Mutex + // nsGroupProj is the per-group state used by the emission rules. + // Accessed only under healthProjectMu. + nsGroupProj map[nsGroupID]*nsGroupProj + // warningDelayBase is the base grace window for health projection. + // Set at construction, mutated only by tests. Read by the + // refresher goroutine so never change it while one is running. + warningDelayBase time.Duration + // healthRefresh is buffered=1; writers coalesce, senders never block. + // See refreshHealth for the lock-order rationale. + healthRefresh chan struct{} } type handlerWithStop interface { dns.Handler Stop() - ProbeAvailability(context.Context) ID() types.HandlerID } +type upstreamHealthReporter interface { + UpstreamHealth() map[netip.AddrPort]UpstreamHealth +} + type handlerWrapper struct { domain string handler handlerWithStop @@ -174,7 +247,6 @@ func NewDefaultServerPermanentUpstream( ds.hostsDNSHolder.set(hostsDnsList) ds.permanent = true - ds.addHostRootZone() ds.currentConfig = dnsConfigToHostDNSConfig(config, ds.service.RuntimeIP(), ds.service.RuntimePort()) ds.searchDomainNotifier = newNotifier(ds.SearchDomains()) ds.searchDomainNotifier.setListener(listener) @@ -182,21 +254,17 @@ func NewDefaultServerPermanentUpstream( return ds } -// NewDefaultServerIos returns a new dns server. It optimized for ios +// NewDefaultServerIos returns a new dns server. It optimized for ios. func NewDefaultServerIos( ctx context.Context, wgInterface WGIface, iosDnsManager IosDnsManager, - hostsDnsList []netip.AddrPort, statusRecorder *peer.Status, disableSys bool, ) *DefaultServer { - log.Debugf("iOS host dns address list is: %v", hostsDnsList) ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder, nil, disableSys) ds.iosDnsManager = iosDnsManager - ds.hostsDNSHolder.set(hostsDnsList) ds.permanent = true - ds.addHostRootZone() return ds } @@ -230,6 +298,8 @@ func newDefaultServer( hostManager: &noopHostConfigurator{}, mgmtCacheResolver: mgmtCacheResolver, currentConfigHash: ^uint64(0), // Initialize to max uint64 to ensure first config is always applied + warningDelayBase: defaultWarningDelayBase, + healthRefresh: make(chan struct{}, 1), } // register with root zone, handler chain takes care of the routing @@ -238,12 +308,26 @@ func newDefaultServer( return defaultServer } -// SetRouteChecker sets the function used by upstream resolvers to determine -// whether an IP is routed through the tunnel. -func (s *DefaultServer) SetRouteChecker(f func(netip.Addr) bool) { +// SetRouteSources wires the route-manager accessors used by health +// projection to classify each upstream for emission timing. +func (s *DefaultServer) SetRouteSources(selected, active func() route.HAMap) { s.mux.Lock() defer s.mux.Unlock() - s.routeMatch = f + s.selectedRoutes = selected + s.activeRoutes = active + + // Permanent / iOS constructors build the root handler before the + // engine wires route sources, so its selectedRoutes callback would + // otherwise remain nil and overlay upstreams would be classified + // as public. Propagate the new accessors to existing handlers. + type routeSettable interface { + setSelectedRoutes(func() route.HAMap) + } + for _, entry := range s.dnsMuxMap { + if h, ok := entry.handler.(routeSettable); ok { + h.setSelectedRoutes(selected) + } + } } // RegisterHandler registers a handler for the given domains with the given priority. @@ -256,7 +340,6 @@ func (s *DefaultServer) RegisterHandler(domains domain.List, handler dns.Handler // TODO: This will take over zones for non-wildcard domains, for which we might not have a handler in the chain for _, domain := range domains { - // convert to zone with simple ref counter s.extraDomains[toZone(domain)]++ } if !s.batchMode { @@ -357,6 +440,8 @@ func (s *DefaultServer) Initialize() (err error) { s.stateManager.RegisterState(&ShutdownState{}) + s.startHealthRefresher() + // Keep using noop host manager if dns off requested or running in netstack mode. // Netstack mode currently doesn't have a way to receive DNS requests. // TODO: Use listener on localhost in netstack mode when running as root. @@ -370,6 +455,13 @@ func (s *DefaultServer) Initialize() (err error) { return fmt.Errorf("initialize: %w", err) } s.hostManager = hostManager + // On mobile-permanent setups the seeded host DNS list is the only + // source until the first network-map arrives; register it now so DNS + // works in that window. Desktop host managers register fallback when + // applyConfiguration runs. + if s.permanent { + s.registerFallback() + } return nil } @@ -394,13 +486,7 @@ func (s *DefaultServer) SetFirewall(fw Firewall) { // Stop stops the server func (s *DefaultServer) Stop() { - s.probeMu.Lock() - if s.probeCancel != nil { - s.probeCancel() - } s.ctxCancel() - s.probeMu.Unlock() - s.probeWg.Wait() s.shutdownWg.Wait() s.mux.Lock() @@ -411,6 +497,13 @@ func (s *DefaultServer) Stop() { } clear(s.extraDomains) + + // Clear health projection state so a subsequent Start doesn't + // inherit sticky flags (notably everHealthy) that would bypass + // the grace window during the next peer handshake. + s.healthProjectMu.Lock() + s.nsGroupProj = nil + s.healthProjectMu.Unlock() } func (s *DefaultServer) disableDNS() (retErr error) { @@ -424,10 +517,9 @@ func (s *DefaultServer) disableDNS() (retErr error) { return nil } - // Deregister original nameservers if they were registered as fallback - if srvs, ok := s.hostManager.(hostManagerWithOriginalNS); ok && len(srvs.getOriginalNameservers()) > 0 { - log.Debugf("deregistering original nameservers as fallback handlers") - s.deregisterHandler([]string{nbdns.RootZone}, PriorityFallback) + if s.fallbackHandler != nil { + log.Debugf("deregistering fallback handlers") + s.clearFallback() } if err := s.hostManager.restoreHostDNS(); err != nil { @@ -441,27 +533,16 @@ func (s *DefaultServer) disableDNS() (retErr error) { return nil } -// OnUpdatedHostDNSServer update the DNS servers addresses for root zones -// It will be applied if the mgm server do not enforce DNS settings for root zone +// OnUpdatedHostDNSServer updates the fallback DNS upstreams. Called by Android +// outside the engine's sync mux when the OS reports a network change, so it +// takes s.mux to serialize against host manager swaps in Initialize/enableDNS. func (s *DefaultServer) OnUpdatedHostDNSServer(hostsDnsList []netip.AddrPort) { s.hostsDNSHolder.set(hostsDnsList) - - // Check if there's any root handler - var hasRootHandler bool - for _, handler := range s.dnsMuxMap { - if handler.domain == nbdns.RootZone { - hasRootHandler = true - break - } - } - - if hasRootHandler { - log.Debugf("on new host DNS config but skip to apply it") - return - } - log.Debugf("update host DNS settings: %+v", hostsDnsList) - s.addHostRootZone() + + s.mux.Lock() + defer s.mux.Unlock() + s.registerFallback() } // UpdateDNSServer processes an update received from the management service @@ -520,69 +601,6 @@ func (s *DefaultServer) SearchDomains() []string { return searchDomains } -// ProbeAvailability tests each upstream group's servers for availability -// and deactivates the group if no server responds. -// If a previous probe is still running, it will be cancelled before starting a new one. -func (s *DefaultServer) ProbeAvailability() { - if val := os.Getenv(envSkipDNSProbe); val != "" { - skipProbe, err := strconv.ParseBool(val) - if err != nil { - log.Warnf("failed to parse %s: %v", envSkipDNSProbe, err) - } - if skipProbe { - log.Infof("skipping DNS probe due to %s", envSkipDNSProbe) - return - } - } - - s.probeMu.Lock() - - // don't start probes on a stopped server - if s.ctx.Err() != nil { - s.probeMu.Unlock() - return - } - - // cancel any running probe - if s.probeCancel != nil { - s.probeCancel() - s.probeCancel = nil - } - - // wait for the previous probe goroutines to finish while holding - // the mutex so no other caller can start a new probe concurrently - s.probeWg.Wait() - - // start a new probe - probeCtx, probeCancel := context.WithCancel(s.ctx) - s.probeCancel = probeCancel - - s.probeWg.Add(1) - defer s.probeWg.Done() - - // Snapshot handlers under s.mux to avoid racing with updateMux/dnsMuxMap writers. - s.mux.Lock() - handlers := make([]handlerWithStop, 0, len(s.dnsMuxMap)) - for _, mux := range s.dnsMuxMap { - handlers = append(handlers, mux.handler) - } - s.mux.Unlock() - - var wg sync.WaitGroup - for _, handler := range handlers { - wg.Add(1) - go func(h handlerWithStop) { - defer wg.Done() - h.ProbeAvailability(probeCtx) - }(handler) - } - - s.probeMu.Unlock() - - wg.Wait() - probeCancel() -} - func (s *DefaultServer) UpdateServerConfig(domains dnsconfig.ServerDomains) error { s.mux.Lock() defer s.mux.Unlock() @@ -746,19 +764,17 @@ func (s *DefaultServer) applyHostConfig() { s.currentConfigHash = hash } - s.registerFallback(config) + s.registerFallback() } // registerFallback registers original nameservers as low-priority fallback handlers. -func (s *DefaultServer) registerFallback(config HostDNSConfig) { - hostMgrWithNS, ok := s.hostManager.(hostManagerWithOriginalNS) - if !ok { - return - } - - originalNameservers := hostMgrWithNS.getOriginalNameservers() +// Replaces and Stop()s the previously-registered fallback handler so its +// context is released rather than leaked until GC. +func (s *DefaultServer) registerFallback() { + originalNameservers := s.hostManager.getOriginalNameservers() if len(originalNameservers) == 0 { - s.deregisterHandler([]string{nbdns.RootZone}, PriorityFallback) + log.Debugf("no fallback upstreams to register; clearing PriorityFallback handler") + s.clearFallback() return } @@ -775,21 +791,28 @@ func (s *DefaultServer) registerFallback(config HostDNSConfig) { log.Errorf("failed to create upstream resolver for original nameservers: %v", err) return } - handler.routeMatch = s.routeMatch + handler.selectedRoutes = s.selectedRoutes + var servers []netip.AddrPort for _, ns := range originalNameservers { - if ns == config.ServerIP { - log.Debugf("skipping original nameserver %s as it is the same as the server IP %s", ns, config.ServerIP) - continue - } - - addrPort := netip.AddrPortFrom(ns, DefaultPort) - handler.upstreamServers = append(handler.upstreamServers, addrPort) + servers = append(servers, netip.AddrPortFrom(ns, DefaultPort)) } - handler.deactivate = func(error) { /* always active */ } - handler.reactivate = func() { /* always active */ } + handler.addRace(servers) + prev := s.fallbackHandler + s.fallbackHandler = handler s.registerHandler([]string{nbdns.RootZone}, handler, PriorityFallback) + if prev != nil { + prev.Stop() + } +} + +func (s *DefaultServer) clearFallback() { + s.deregisterHandler([]string{nbdns.RootZone}, PriorityFallback) + if s.fallbackHandler != nil { + s.fallbackHandler.Stop() + s.fallbackHandler = nil + } } func (s *DefaultServer) buildLocalHandlerUpdate(customZones []nbdns.CustomZone) ([]handlerWrapper, []nbdns.CustomZone, error) { @@ -847,100 +870,99 @@ func (s *DefaultServer) buildUpstreamHandlerUpdate(nameServerGroups []*nbdns.Nam groupedNS := groupNSGroupsByDomain(nameServerGroups) for _, domainGroup := range groupedNS { - basePriority := PriorityUpstream + priority := PriorityUpstream if domainGroup.domain == nbdns.RootZone { - basePriority = PriorityDefault + priority = PriorityDefault } - updates, err := s.createHandlersForDomainGroup(domainGroup, basePriority) + update, err := s.buildMergedDomainHandler(domainGroup, priority) if err != nil { + if errors.Is(err, errNoUsableNameservers) { + log.Errorf("no usable nameservers for domain=%s", domainGroup.domain) + continue + } return nil, err } - muxUpdates = append(muxUpdates, updates...) + muxUpdates = append(muxUpdates, *update) } return muxUpdates, nil } -func (s *DefaultServer) createHandlersForDomainGroup(domainGroup nsGroupsByDomain, basePriority int) ([]handlerWrapper, error) { - var muxUpdates []handlerWrapper +// buildMergedDomainHandler merges every nameserver group that targets the +// same domain into one handler whose inner groups are raced in parallel. +func (s *DefaultServer) buildMergedDomainHandler(domainGroup nsGroupsByDomain, priority int) (*handlerWrapper, error) { + handler, err := newUpstreamResolver( + s.ctx, + s.wgInterface, + s.statusRecorder, + s.hostsDNSHolder, + domain.Domain(domainGroup.domain), + ) + if err != nil { + return nil, fmt.Errorf("create upstream resolver: %v", err) + } + handler.selectedRoutes = s.selectedRoutes - for i, nsGroup := range domainGroup.groups { - // Decrement priority by handler index (0, 1, 2, ...) to avoid conflicts - priority := basePriority - i - - // Check if we're about to overlap with the next priority tier - if s.leaksPriority(domainGroup, basePriority, priority) { - break - } - - log.Debugf("creating handler for domain=%s with priority=%d", domainGroup.domain, priority) - handler, err := newUpstreamResolver( - s.ctx, - s.wgInterface, - s.statusRecorder, - s.hostsDNSHolder, - domainGroup.domain, - ) - if err != nil { - return nil, fmt.Errorf("create upstream resolver: %v", err) - } - handler.routeMatch = s.routeMatch - - for _, ns := range nsGroup.NameServers { - if ns.NSType != nbdns.UDPNameServerType { - log.Warnf("skipping nameserver %s with type %s, this peer supports only %s", - ns.IP.String(), ns.NSType.String(), nbdns.UDPNameServerType.String()) - continue - } - - if ns.IP == s.service.RuntimeIP() { - log.Warnf("skipping nameserver %s as it matches our DNS server IP, preventing potential loop", ns.IP) - continue - } - - handler.upstreamServers = append(handler.upstreamServers, ns.AddrPort()) - } - - if len(handler.upstreamServers) == 0 { - handler.Stop() - log.Errorf("received a nameserver group with an invalid nameserver list") + for _, nsGroup := range domainGroup.groups { + servers := s.filterNameServers(nsGroup.NameServers) + if len(servers) == 0 { + log.Warnf("nameserver group for domain=%s yielded no usable servers, skipping", domainGroup.domain) continue } - - // when upstream fails to resolve domain several times over all it servers - // it will calls this hook to exclude self from the configuration and - // reapply DNS settings, but it not touch the original configuration and serial number - // because it is temporal deactivation until next try - // - // after some period defined by upstream it tries to reactivate self by calling this hook - // everything we need here is just to re-apply current configuration because it already - // contains this upstream settings (temporal deactivation not removed it) - handler.deactivate, handler.reactivate = s.upstreamCallbacks(nsGroup, handler, priority) - - muxUpdates = append(muxUpdates, handlerWrapper{ - domain: domainGroup.domain, - handler: handler, - priority: priority, - }) + handler.addRace(servers) } - return muxUpdates, nil + if len(handler.upstreamServers) == 0 { + handler.Stop() + return nil, errNoUsableNameservers + } + + log.Debugf("creating merged handler for domain=%s with %d group(s) priority=%d", domainGroup.domain, len(handler.upstreamServers), priority) + + return &handlerWrapper{ + domain: domainGroup.domain, + handler: handler, + priority: priority, + }, nil } -func (s *DefaultServer) leaksPriority(domainGroup nsGroupsByDomain, basePriority int, priority int) bool { - if basePriority == PriorityUpstream && priority <= PriorityDefault { - log.Warnf("too many handlers for domain=%s, would overlap with default priority tier (diff=%d). Skipping remaining handlers", - domainGroup.domain, PriorityUpstream-PriorityDefault) - return true - } - if basePriority == PriorityDefault && priority <= PriorityFallback { - log.Warnf("too many handlers for domain=%s, would overlap with fallback priority tier (diff=%d). Skipping remaining handlers", - domainGroup.domain, PriorityDefault-PriorityFallback) - return true +func (s *DefaultServer) filterNameServers(nameServers []nbdns.NameServer) []netip.AddrPort { + var out []netip.AddrPort + for _, ns := range nameServers { + if ns.NSType != nbdns.UDPNameServerType { + log.Warnf("skipping nameserver %s with type %s, this peer supports only %s", + ns.IP.String(), ns.NSType.String(), nbdns.UDPNameServerType.String()) + continue + } + if ns.IP == s.service.RuntimeIP() { + log.Warnf("skipping nameserver %s as it matches our DNS server IP, preventing potential loop", ns.IP) + continue + } + out = append(out, ns.AddrPort()) } + return out +} - return false +// usableNameServers returns the subset of nameServers the handler would +// actually query. Matches filterNameServers without the warning logs, so +// it's safe to call on every health-projection tick. +func (s *DefaultServer) usableNameServers(nameServers []nbdns.NameServer) []netip.AddrPort { + var runtimeIP netip.Addr + if s.service != nil { + runtimeIP = s.service.RuntimeIP() + } + var out []netip.AddrPort + for _, ns := range nameServers { + if ns.NSType != nbdns.UDPNameServerType { + continue + } + if runtimeIP.IsValid() && ns.IP == runtimeIP { + continue + } + out = append(out, ns.AddrPort()) + } + return out } func (s *DefaultServer) updateMux(muxUpdates []handlerWrapper) { @@ -951,175 +973,356 @@ func (s *DefaultServer) updateMux(muxUpdates []handlerWrapper) { } muxUpdateMap := make(registeredHandlerMap) - var containsRootUpdate bool for _, update := range muxUpdates { - if update.domain == nbdns.RootZone { - containsRootUpdate = true - } s.registerHandler([]string{update.domain}, update.handler, update.priority) muxUpdateMap[update.handler.ID()] = update } - // If there's no root update and we had a root handler, restore it - if !containsRootUpdate { - for _, existing := range s.dnsMuxMap { - if existing.domain == nbdns.RootZone { - s.addHostRootZone() - break - } - } - } - s.dnsMuxMap = muxUpdateMap } -// upstreamCallbacks returns two functions, the first one is used to deactivate -// the upstream resolver from the configuration, the second one is used to -// reactivate it. Not allowed to call reactivate before deactivate. -func (s *DefaultServer) upstreamCallbacks( - nsGroup *nbdns.NameServerGroup, - handler dns.Handler, - priority int, -) (deactivate func(error), reactivate func()) { - var removeIndex map[string]int - deactivate = func(err error) { - s.mux.Lock() - defer s.mux.Unlock() - - l := log.WithField("nameservers", nsGroup.NameServers) - l.Info("Temporarily deactivating nameservers group due to timeout") - - removeIndex = make(map[string]int) - for _, domain := range nsGroup.Domains { - removeIndex[domain] = -1 - } - if nsGroup.Primary { - removeIndex[nbdns.RootZone] = -1 - s.currentConfig.RouteAll = false - s.deregisterHandler([]string{nbdns.RootZone}, priority) - } - - for i, item := range s.currentConfig.Domains { - if _, found := removeIndex[item.Domain]; found { - s.currentConfig.Domains[i].Disabled = true - s.deregisterHandler([]string{item.Domain}, priority) - removeIndex[item.Domain] = i - } - } - - // Always apply host config when nameserver goes down, regardless of batch mode - s.applyHostConfig() - - go func() { - if err := s.stateManager.PersistState(s.ctx); err != nil { - l.Errorf("Failed to persist dns state: %v", err) - } - }() - - if runtime.GOOS == "android" && nsGroup.Primary && len(s.hostsDNSHolder.get()) > 0 { - s.addHostRootZone() - } - - s.updateNSState(nsGroup, err, false) - } - - reactivate = func() { - s.mux.Lock() - defer s.mux.Unlock() - - for domain, i := range removeIndex { - if i == -1 || i >= len(s.currentConfig.Domains) || s.currentConfig.Domains[i].Domain != domain { - continue - } - s.currentConfig.Domains[i].Disabled = false - s.registerHandler([]string{domain}, handler, priority) - } - - l := log.WithField("nameservers", nsGroup.NameServers) - l.Debug("reactivate temporary disabled nameserver group") - - if nsGroup.Primary { - s.currentConfig.RouteAll = true - s.registerHandler([]string{nbdns.RootZone}, handler, priority) - } - - // Always apply host config when nameserver reactivates, regardless of batch mode - s.applyHostConfig() - - s.updateNSState(nsGroup, nil, true) - } - return -} - -func (s *DefaultServer) addHostRootZone() { - hostDNSServers := s.hostsDNSHolder.get() - if len(hostDNSServers) == 0 { - log.Debug("no host DNS servers available, skipping root zone handler creation") - return - } - - handler, err := newUpstreamResolver( - s.ctx, - s.wgInterface, - s.statusRecorder, - s.hostsDNSHolder, - nbdns.RootZone, - ) - if err != nil { - log.Errorf("unable to create a new upstream resolver, error: %v", err) - return - } - handler.routeMatch = s.routeMatch - - handler.upstreamServers = maps.Keys(hostDNSServers) - handler.deactivate = func(error) {} - handler.reactivate = func() {} - - s.registerHandler([]string{nbdns.RootZone}, handler, PriorityDefault) -} - +// updateNSGroupStates records the new group set and pokes the refresher. +// Must hold s.mux; projection runs async (see refreshHealth for why). func (s *DefaultServer) updateNSGroupStates(groups []*nbdns.NameServerGroup) { - var states []peer.NSGroupState + s.nsGroups = groups + select { + case s.healthRefresh <- struct{}{}: + default: + } +} - for _, group := range groups { - var servers []netip.AddrPort - for _, ns := range group.NameServers { - servers = append(servers, ns.AddrPort()) +// refreshHealth runs one projection cycle. Must not be called while +// holding s.mux: the route callbacks re-enter routemanager's lock. +func (s *DefaultServer) refreshHealth() { + s.mux.Lock() + groups := s.nsGroups + merged := s.collectUpstreamHealth() + selFn := s.selectedRoutes + actFn := s.activeRoutes + s.mux.Unlock() + + var selected, active route.HAMap + if selFn != nil { + selected = selFn() + } + if actFn != nil { + active = actFn() + } + + s.projectNSGroupHealth(nsHealthSnapshot{ + groups: groups, + merged: merged, + selected: selected, + active: active, + }) +} + +// projectNSGroupHealth applies the emission rules to the snapshot and +// publishes the resulting NSGroupStates. Serialized by healthProjectMu, +// lock-free wrt s.mux. +// +// Rules: +// - Healthy: emit recovery iff warningActive; set everHealthy. +// - Unhealthy: stamp unhealthySince on streak start; emit warning +// iff any of immediate / everHealthy / elapsed >= effective delay. +// - Undecided: no-op. +// +// "Immediate" means the group has at least one upstream that's public +// or overlay+Connected: no peer-startup race to wait out. +func (s *DefaultServer) projectNSGroupHealth(snap nsHealthSnapshot) { + if s.statusRecorder == nil { + return + } + + s.healthProjectMu.Lock() + defer s.healthProjectMu.Unlock() + + if s.nsGroupProj == nil { + s.nsGroupProj = make(map[nsGroupID]*nsGroupProj) + } + + now := time.Now() + delay := s.warningDelay(haMapRouteCount(snap.selected)) + states := make([]peer.NSGroupState, 0, len(snap.groups)) + seen := make(map[nsGroupID]struct{}, len(snap.groups)) + for _, group := range snap.groups { + servers := s.usableNameServers(group.NameServers) + if len(servers) == 0 { + continue + } + verdict, groupErr := evaluateNSGroupHealth(snap.merged, servers, now) + id := generateGroupKey(group) + seen[id] = struct{}{} + + immediate := s.groupHasImmediateUpstream(servers, snap) + + p, known := s.nsGroupProj[id] + if !known { + p = &nsGroupProj{} + s.nsGroupProj[id] = p } - state := peer.NSGroupState{ - ID: generateGroupKey(group), + enabled := true + switch verdict { + case nsVerdictHealthy: + enabled = s.projectHealthy(p, servers) + case nsVerdictUnhealthy: + enabled = s.projectUnhealthy(p, servers, immediate, now, delay) + case nsVerdictUndecided: + // Stay Available until evidence says otherwise, unless a + // warning is already active for this group. Also clear any + // prior Unhealthy streak so a later Unhealthy verdict starts + // a fresh grace window rather than inheriting a stale one. + p.unhealthySince = time.Time{} + enabled = !p.warningActive + groupErr = nil + } + + states = append(states, peer.NSGroupState{ + ID: string(id), Servers: servers, Domains: group.Domains, - // The probe will determine the state, default enabled - Enabled: true, - Error: nil, - } - states = append(states, state) + Enabled: enabled, + Error: groupErr, + }) } - s.statusRecorder.UpdateDNSStates(states) -} - -func (s *DefaultServer) updateNSState(nsGroup *nbdns.NameServerGroup, err error, enabled bool) { - states := s.statusRecorder.GetDNSStates() - id := generateGroupKey(nsGroup) - for i, state := range states { - if state.ID == id { - states[i].Enabled = enabled - states[i].Error = err - break + for id := range s.nsGroupProj { + if _, ok := seen[id]; !ok { + delete(s.nsGroupProj, id) } } s.statusRecorder.UpdateDNSStates(states) } -func generateGroupKey(nsGroup *nbdns.NameServerGroup) string { - var servers []string +// projectHealthy records a healthy tick on p and publishes a recovery +// event iff a warning was active for the current streak. Returns the +// Enabled flag to record in NSGroupState. +func (s *DefaultServer) projectHealthy(p *nsGroupProj, servers []netip.AddrPort) bool { + p.everHealthy = true + p.unhealthySince = time.Time{} + if !p.warningActive { + return true + } + log.Debugf("DNS health: group [%s] recovered, emitting event", joinAddrPorts(servers)) + s.statusRecorder.PublishEvent( + proto.SystemEvent_INFO, + proto.SystemEvent_DNS, + "Nameserver group recovered", + "DNS servers are reachable again.", + map[string]string{"upstreams": joinAddrPorts(servers)}, + ) + p.warningActive = false + return true +} + +// projectUnhealthy records an unhealthy tick on p, publishes the +// warning when the emission rules fire, and returns the Enabled flag +// to record in NSGroupState. +func (s *DefaultServer) projectUnhealthy(p *nsGroupProj, servers []netip.AddrPort, immediate bool, now time.Time, delay time.Duration) bool { + streakStart := p.unhealthySince.IsZero() + if streakStart { + p.unhealthySince = now + } + reason := unhealthyEmitReason(immediate, p.everHealthy, now.Sub(p.unhealthySince), delay) + switch { + case reason != "" && !p.warningActive: + log.Debugf("DNS health: group [%s] unreachable, emitting event (reason=%s)", joinAddrPorts(servers), reason) + s.statusRecorder.PublishEvent( + proto.SystemEvent_WARNING, + proto.SystemEvent_DNS, + "Nameserver group unreachable", + "Unable to reach one or more DNS servers. This might affect your ability to connect to some services.", + map[string]string{"upstreams": joinAddrPorts(servers)}, + ) + p.warningActive = true + case streakStart && reason == "": + // One line per streak, not per tick. + log.Debugf("DNS health: group [%s] unreachable but holding warning for up to %v (overlay-routed, no connected peer)", joinAddrPorts(servers), delay) + } + return false +} + +// warningDelay returns the grace window for the given selected-route +// count. Scales gently: +1s per 100 routes, capped by +// warningDelayBonusCap. Parallel handshakes mean handshake time grows +// much slower than route count, so linear scaling would overcorrect. +// +// TODO: revisit the scaling curve with real-world data — the current +// values are a reasonable starting point, not a measured fit. +func (s *DefaultServer) warningDelay(routeCount int) time.Duration { + bonus := time.Duration(routeCount/100) * time.Second + if bonus > warningDelayBonusCap { + bonus = warningDelayBonusCap + } + return s.warningDelayBase + bonus +} + +// groupHasImmediateUpstream reports whether the group has at least one +// upstream in a classification that bypasses the grace window: public +// (outside the overlay range and not routed), or overlay/routed with a +// Connected peer. +// +// TODO(ipv6): include the v6 overlay prefix once it's plumbed in. +func (s *DefaultServer) groupHasImmediateUpstream(servers []netip.AddrPort, snap nsHealthSnapshot) bool { + var overlayV4 netip.Prefix + if s.wgInterface != nil { + overlayV4 = s.wgInterface.Address().Network + } + for _, srv := range servers { + addr := srv.Addr().Unmap() + overlay := overlayV4.IsValid() && overlayV4.Contains(addr) + selMatched, selDynamic := haMapContains(snap.selected, addr) + // Treat an unknown (dynamic selected route) as possibly routed: + // the upstream might reach through a dynamic route whose Network + // hasn't resolved yet, and classifying as public would bypass + // the startup grace window. + routed := selMatched || selDynamic + if !overlay && !routed { + return true + } + if actMatched, _ := haMapContains(snap.active, addr); actMatched { + return true + } + } + return false +} + +// collectUpstreamHealth merges health snapshots across handlers, keeping +// the most recent success and failure per upstream when an address appears +// in more than one handler. +func (s *DefaultServer) collectUpstreamHealth() map[netip.AddrPort]UpstreamHealth { + merged := make(map[netip.AddrPort]UpstreamHealth) + for _, entry := range s.dnsMuxMap { + reporter, ok := entry.handler.(upstreamHealthReporter) + if !ok { + continue + } + for addr, h := range reporter.UpstreamHealth() { + existing, have := merged[addr] + if !have { + merged[addr] = h + continue + } + if h.LastOk.After(existing.LastOk) { + existing.LastOk = h.LastOk + } + if h.LastFail.After(existing.LastFail) { + existing.LastFail = h.LastFail + existing.LastErr = h.LastErr + } + merged[addr] = existing + } + } + return merged +} + +func (s *DefaultServer) startHealthRefresher() { + s.shutdownWg.Add(1) + go func() { + defer s.shutdownWg.Done() + ticker := time.NewTicker(nsGroupHealthRefreshInterval) + defer ticker.Stop() + for { + select { + case <-s.ctx.Done(): + return + case <-ticker.C: + case <-s.healthRefresh: + } + s.refreshHealth() + } + }() +} + +// evaluateNSGroupHealth decides a group's verdict from query records +// alone. Per upstream, the most-recent-in-lookback observation wins. +// Group is Healthy if any upstream is fresh-working, Unhealthy if any +// is fresh-broken with no fresh-working sibling, Undecided otherwise. +func evaluateNSGroupHealth(merged map[netip.AddrPort]UpstreamHealth, servers []netip.AddrPort, now time.Time) (nsGroupVerdict, error) { + anyWorking := false + anyBroken := false + var mostRecentFail time.Time + var mostRecentErr string + + for _, srv := range servers { + h, ok := merged[srv] + if !ok { + continue + } + switch classifyUpstreamHealth(h, now) { + case upstreamFresh: + anyWorking = true + case upstreamBroken: + anyBroken = true + if h.LastFail.After(mostRecentFail) { + mostRecentFail = h.LastFail + mostRecentErr = h.LastErr + } + } + } + + if anyWorking { + return nsVerdictHealthy, nil + } + if anyBroken { + if mostRecentErr == "" { + return nsVerdictUnhealthy, nil + } + return nsVerdictUnhealthy, errors.New(mostRecentErr) + } + return nsVerdictUndecided, nil +} + +// upstreamClassification is the per-upstream verdict within healthLookback. +type upstreamClassification int + +const ( + upstreamStale upstreamClassification = iota + upstreamFresh + upstreamBroken +) + +// classifyUpstreamHealth compares the last ok and last fail timestamps +// against healthLookback and returns which one (if any) counts. Fresh +// wins when both are in-window and ok is newer; broken otherwise. +func classifyUpstreamHealth(h UpstreamHealth, now time.Time) upstreamClassification { + okRecent := !h.LastOk.IsZero() && now.Sub(h.LastOk) <= healthLookback + failRecent := !h.LastFail.IsZero() && now.Sub(h.LastFail) <= healthLookback + switch { + case okRecent && failRecent: + if h.LastOk.After(h.LastFail) { + return upstreamFresh + } + return upstreamBroken + case okRecent: + return upstreamFresh + case failRecent: + return upstreamBroken + } + return upstreamStale +} + +func joinAddrPorts(servers []netip.AddrPort) string { + parts := make([]string, 0, len(servers)) + for _, s := range servers { + parts = append(parts, s.String()) + } + return strings.Join(parts, ", ") +} + +// generateGroupKey returns a stable identity for an NS group so health +// state (everHealthy / warningActive) survives reorderings in the +// configured nameserver or domain lists. +func generateGroupKey(nsGroup *nbdns.NameServerGroup) nsGroupID { + servers := make([]string, 0, len(nsGroup.NameServers)) for _, ns := range nsGroup.NameServers { servers = append(servers, ns.AddrPort().String()) } - return fmt.Sprintf("%v_%v", servers, nsGroup.Domains) + slices.Sort(servers) + domains := slices.Clone(nsGroup.Domains) + slices.Sort(domains) + return nsGroupID(fmt.Sprintf("%v_%v", servers, domains)) } // groupNSGroupsByDomain groups nameserver groups by their match domains @@ -1161,6 +1364,21 @@ func toZone(d domain.Domain) domain.Domain { ) } +// unhealthyEmitReason returns the tag of the rule that fires the +// warning now, or "" if the group is still inside its grace window. +func unhealthyEmitReason(immediate, everHealthy bool, elapsed, delay time.Duration) string { + switch { + case immediate: + return "immediate" + case everHealthy: + return "ever-healthy" + case elapsed >= delay: + return "grace-elapsed" + default: + return "" + } +} + // PopulateManagementDomain populates the DNS cache with management domain func (s *DefaultServer) PopulateManagementDomain(mgmtURL *url.URL) error { if s.mgmtCacheResolver != nil { diff --git a/client/internal/dns/server_android.go b/client/internal/dns/server_android.go index 7ca12d69d..b2cb26f65 100644 --- a/client/internal/dns/server_android.go +++ b/client/internal/dns/server_android.go @@ -1,5 +1,5 @@ package dns func (s *DefaultServer) initialize() (manager hostManager, err error) { - return newHostManager() + return newHostManager(s.hostsDNSHolder) } diff --git a/client/internal/dns/server_test.go b/client/internal/dns/server_test.go index 1026a29fc..722c2abd7 100644 --- a/client/internal/dns/server_test.go +++ b/client/internal/dns/server_test.go @@ -6,7 +6,7 @@ import ( "net" "net/netip" "os" - "strings" + "runtime" "testing" "time" @@ -15,6 +15,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "golang.zx2c4.com/wireguard/tun/netstack" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" @@ -31,8 +32,10 @@ import ( "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/statemanager" "github.com/netbirdio/netbird/client/internal/stdnet" + "github.com/netbirdio/netbird/client/proto" nbdns "github.com/netbirdio/netbird/dns" "github.com/netbirdio/netbird/formatter" + "github.com/netbirdio/netbird/route" "github.com/netbirdio/netbird/shared/management/domain" ) @@ -101,16 +104,17 @@ func init() { formatter.SetTextFormatter(log.StandardLogger()) } -func generateDummyHandler(domain string, servers []nbdns.NameServer) *upstreamResolverBase { +func generateDummyHandler(d string, servers []nbdns.NameServer) *upstreamResolverBase { var srvs []netip.AddrPort for _, srv := range servers { srvs = append(srvs, srv.AddrPort()) } - return &upstreamResolverBase{ - domain: domain, - upstreamServers: srvs, - cancel: func() {}, + u := &upstreamResolverBase{ + domain: domain.Domain(d), + cancel: func() {}, } + u.addRace(srvs) + return u } func TestUpdateDNSServer(t *testing.T) { @@ -653,74 +657,8 @@ func TestDNSServerStartStop(t *testing.T) { } } -func TestDNSServerUpstreamDeactivateCallback(t *testing.T) { - hostManager := &mockHostConfigurator{} - server := DefaultServer{ - ctx: context.Background(), - service: NewServiceViaMemory(&mocWGIface{}), - localResolver: local.NewResolver(), - handlerChain: NewHandlerChain(), - hostManager: hostManager, - currentConfig: HostDNSConfig{ - Domains: []DomainConfig{ - {false, "domain0", false}, - {false, "domain1", false}, - {false, "domain2", false}, - }, - }, - statusRecorder: peer.NewRecorder("mgm"), - } - - var domainsUpdate string - hostManager.applyDNSConfigFunc = func(config HostDNSConfig, statemanager *statemanager.Manager) error { - domains := []string{} - for _, item := range config.Domains { - if item.Disabled { - continue - } - domains = append(domains, item.Domain) - } - domainsUpdate = strings.Join(domains, ",") - return nil - } - - deactivate, reactivate := server.upstreamCallbacks(&nbdns.NameServerGroup{ - Domains: []string{"domain1"}, - NameServers: []nbdns.NameServer{ - {IP: netip.MustParseAddr("8.8.0.0"), NSType: nbdns.UDPNameServerType, Port: 53}, - }, - }, nil, 0) - - deactivate(nil) - expected := "domain0,domain2" - domains := []string{} - for _, item := range server.currentConfig.Domains { - if item.Disabled { - continue - } - domains = append(domains, item.Domain) - } - got := strings.Join(domains, ",") - if expected != got { - t.Errorf("expected domains list: %q, got %q", expected, got) - } - - reactivate() - expected = "domain0,domain1,domain2" - domains = []string{} - for _, item := range server.currentConfig.Domains { - if item.Disabled { - continue - } - domains = append(domains, item.Domain) - } - got = strings.Join(domains, ",") - if expected != got { - t.Errorf("expected domains list: %q, got %q", expected, domainsUpdate) - } -} - func TestDNSPermanent_updateHostDNS_emptyUpstream(t *testing.T) { + skipUnlessAndroid(t) wgIFace, err := createWgInterfaceWithBind(t) if err != nil { t.Fatal("failed to initialize wg interface") @@ -748,6 +686,7 @@ func TestDNSPermanent_updateHostDNS_emptyUpstream(t *testing.T) { } func TestDNSPermanent_updateUpstream(t *testing.T) { + skipUnlessAndroid(t) wgIFace, err := createWgInterfaceWithBind(t) if err != nil { t.Fatal("failed to initialize wg interface") @@ -841,6 +780,7 @@ func TestDNSPermanent_updateUpstream(t *testing.T) { } func TestDNSPermanent_matchOnly(t *testing.T) { + skipUnlessAndroid(t) wgIFace, err := createWgInterfaceWithBind(t) if err != nil { t.Fatal("failed to initialize wg interface") @@ -913,6 +853,18 @@ func TestDNSPermanent_matchOnly(t *testing.T) { } } +// skipUnlessAndroid marks tests that exercise the mobile-permanent DNS path, +// which only matches a real production setup on android (NewDefaultServerPermanentUpstream +// + androidHostManager). On non-android the desktop host manager replaces it +// during Initialize and the assertion stops making sense. Skipped here until we +// have an android CI runner. +func skipUnlessAndroid(t *testing.T) { + t.Helper() + if runtime.GOOS != "android" { + t.Skip("requires android runner; mobile-permanent path doesn't match production on this OS") + } +} + func createWgInterfaceWithBind(t *testing.T) (*iface.WGIface, error) { t.Helper() ov := os.Getenv("NB_WG_KERNEL_DISABLED") @@ -1065,7 +1017,6 @@ type mockHandler struct { func (m *mockHandler) ServeDNS(dns.ResponseWriter, *dns.Msg) {} func (m *mockHandler) Stop() {} -func (m *mockHandler) ProbeAvailability(context.Context) {} func (m *mockHandler) ID() types.HandlerID { return types.HandlerID(m.Id) } type mockService struct{} @@ -2085,6 +2036,598 @@ func TestLocalResolverPriorityConstants(t *testing.T) { assert.Equal(t, "local.example.com", localMuxUpdates[0].domain) } +// TestBuildUpstreamHandler_MergesGroupsPerDomain verifies that multiple +// admin-defined nameserver groups targeting the same domain collapse into a +// single handler with each group preserved as a sequential inner list. +func TestBuildUpstreamHandler_MergesGroupsPerDomain(t *testing.T) { + wgInterface := &mocWGIface{} + service := NewServiceViaMemory(wgInterface) + server := &DefaultServer{ + ctx: context.Background(), + wgInterface: wgInterface, + service: service, + localResolver: local.NewResolver(), + handlerChain: NewHandlerChain(), + hostManager: &noopHostConfigurator{}, + dnsMuxMap: make(registeredHandlerMap), + } + + groups := []*nbdns.NameServerGroup{ + { + NameServers: []nbdns.NameServer{ + {IP: netip.MustParseAddr("192.0.2.1"), NSType: nbdns.UDPNameServerType, Port: 53}, + }, + Domains: []string{"example.com"}, + }, + { + NameServers: []nbdns.NameServer{ + {IP: netip.MustParseAddr("192.0.2.2"), NSType: nbdns.UDPNameServerType, Port: 53}, + {IP: netip.MustParseAddr("192.0.2.3"), NSType: nbdns.UDPNameServerType, Port: 53}, + }, + Domains: []string{"example.com"}, + }, + } + + muxUpdates, err := server.buildUpstreamHandlerUpdate(groups) + require.NoError(t, err) + require.Len(t, muxUpdates, 1, "same-domain groups should merge into one handler") + assert.Equal(t, "example.com", muxUpdates[0].domain) + assert.Equal(t, PriorityUpstream, muxUpdates[0].priority) + + handler := muxUpdates[0].handler.(*upstreamResolver) + require.Len(t, handler.upstreamServers, 2, "handler should have two groups") + assert.Equal(t, upstreamRace{netip.MustParseAddrPort("192.0.2.1:53")}, handler.upstreamServers[0]) + assert.Equal(t, upstreamRace{ + netip.MustParseAddrPort("192.0.2.2:53"), + netip.MustParseAddrPort("192.0.2.3:53"), + }, handler.upstreamServers[1]) +} + +// TestEvaluateNSGroupHealth covers the records-only verdict. The gate +// (overlay route selected-but-no-active-peer) is intentionally NOT an +// input to the evaluator anymore: the verdict drives the Enabled flag, +// which must always reflect what we actually observed. Gate-aware event +// suppression is tested separately in the projection test. +// +// Matrix per upstream: {no record, fresh Ok, fresh Fail, stale Fail, +// stale Ok, Ok newer than Fail, Fail newer than Ok}. +// Group verdict: any fresh-working → Healthy; any fresh-broken with no +// fresh-working → Unhealthy; otherwise Undecided. +func TestEvaluateNSGroupHealth(t *testing.T) { + now := time.Now() + a := netip.MustParseAddrPort("192.0.2.1:53") + b := netip.MustParseAddrPort("192.0.2.2:53") + + recentOk := UpstreamHealth{LastOk: now.Add(-2 * time.Second)} + recentFail := UpstreamHealth{LastFail: now.Add(-1 * time.Second), LastErr: "timeout"} + staleOk := UpstreamHealth{LastOk: now.Add(-10 * time.Minute)} + staleFail := UpstreamHealth{LastFail: now.Add(-10 * time.Minute), LastErr: "timeout"} + okThenFail := UpstreamHealth{ + LastOk: now.Add(-10 * time.Second), + LastFail: now.Add(-1 * time.Second), + LastErr: "timeout", + } + failThenOk := UpstreamHealth{ + LastOk: now.Add(-1 * time.Second), + LastFail: now.Add(-10 * time.Second), + LastErr: "timeout", + } + + tests := []struct { + name string + health map[netip.AddrPort]UpstreamHealth + servers []netip.AddrPort + wantVerdict nsGroupVerdict + wantErrSubst string + }{ + { + name: "no record, undecided", + servers: []netip.AddrPort{a}, + wantVerdict: nsVerdictUndecided, + }, + { + name: "fresh success, healthy", + health: map[netip.AddrPort]UpstreamHealth{a: recentOk}, + servers: []netip.AddrPort{a}, + wantVerdict: nsVerdictHealthy, + }, + { + name: "fresh failure, unhealthy", + health: map[netip.AddrPort]UpstreamHealth{a: recentFail}, + servers: []netip.AddrPort{a}, + wantVerdict: nsVerdictUnhealthy, + wantErrSubst: "timeout", + }, + { + name: "only stale success, undecided", + health: map[netip.AddrPort]UpstreamHealth{a: staleOk}, + servers: []netip.AddrPort{a}, + wantVerdict: nsVerdictUndecided, + }, + { + name: "only stale failure, undecided", + health: map[netip.AddrPort]UpstreamHealth{a: staleFail}, + servers: []netip.AddrPort{a}, + wantVerdict: nsVerdictUndecided, + }, + { + name: "both fresh, fail newer, unhealthy", + health: map[netip.AddrPort]UpstreamHealth{a: okThenFail}, + servers: []netip.AddrPort{a}, + wantVerdict: nsVerdictUnhealthy, + wantErrSubst: "timeout", + }, + { + name: "both fresh, ok newer, healthy", + health: map[netip.AddrPort]UpstreamHealth{a: failThenOk}, + servers: []netip.AddrPort{a}, + wantVerdict: nsVerdictHealthy, + }, + { + name: "two upstreams, one success wins", + health: map[netip.AddrPort]UpstreamHealth{ + a: recentFail, + b: recentOk, + }, + servers: []netip.AddrPort{a, b}, + wantVerdict: nsVerdictHealthy, + }, + { + name: "two upstreams, one fail one unseen, unhealthy", + health: map[netip.AddrPort]UpstreamHealth{ + a: recentFail, + }, + servers: []netip.AddrPort{a, b}, + wantVerdict: nsVerdictUnhealthy, + wantErrSubst: "timeout", + }, + { + name: "two upstreams, all recent failures, unhealthy", + health: map[netip.AddrPort]UpstreamHealth{ + a: {LastFail: now.Add(-5 * time.Second), LastErr: "timeout"}, + b: {LastFail: now.Add(-1 * time.Second), LastErr: "SERVFAIL"}, + }, + servers: []netip.AddrPort{a, b}, + wantVerdict: nsVerdictUnhealthy, + wantErrSubst: "SERVFAIL", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + verdict, err := evaluateNSGroupHealth(tc.health, tc.servers, now) + assert.Equal(t, tc.wantVerdict, verdict, "verdict mismatch") + if tc.wantErrSubst != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErrSubst) + } else { + assert.NoError(t, err) + } + }) + } +} + +// healthStubHandler is a minimal dnsMuxMap entry that exposes a fixed +// UpstreamHealth snapshot, letting tests drive recomputeNSGroupStates +// without spinning up real handlers. +type healthStubHandler struct { + health map[netip.AddrPort]UpstreamHealth +} + +func (h *healthStubHandler) ServeDNS(dns.ResponseWriter, *dns.Msg) {} +func (h *healthStubHandler) Stop() {} +func (h *healthStubHandler) ID() types.HandlerID { return "health-stub" } +func (h *healthStubHandler) UpstreamHealth() map[netip.AddrPort]UpstreamHealth { + return h.health +} + +// TestProjection_SteadyStateIsSilent guards against duplicate events: +// while a group stays Unhealthy tick after tick, only the first +// Unhealthy transition may emit. Same for staying Healthy. +func TestProjection_SteadyStateIsSilent(t *testing.T) { + fx := newProjTestFixture(t) + + fx.setHealth(UpstreamHealth{LastFail: time.Now(), LastErr: "timeout"}) + fx.tick() + fx.expectEvent("unreachable", "first fail emits warning") + + fx.setHealth(UpstreamHealth{LastFail: time.Now(), LastErr: "timeout"}) + fx.tick() + fx.tick() + fx.expectNoEvent("staying unhealthy must not re-emit") + + fx.setHealth(UpstreamHealth{LastOk: time.Now()}) + fx.tick() + fx.expectEvent("recovered", "recovery on transition") + + fx.tick() + fx.tick() + fx.expectNoEvent("staying healthy must not re-emit") +} + +// projTestFixture is the common setup for the projection tests: a +// single-upstream group whose route classification the test can flip by +// assigning to selected/active. Callers drive failures/successes by +// mutating stub.health and calling refreshHealth. +type projTestFixture struct { + t *testing.T + recorder *peer.Status + events <-chan *proto.SystemEvent + server *DefaultServer + stub *healthStubHandler + group *nbdns.NameServerGroup + srv netip.AddrPort + selected route.HAMap + active route.HAMap +} + +func newProjTestFixture(t *testing.T) *projTestFixture { + t.Helper() + recorder := peer.NewRecorder("mgm") + sub := recorder.SubscribeToEvents() + t.Cleanup(func() { recorder.UnsubscribeFromEvents(sub) }) + + srv := netip.MustParseAddrPort("100.64.0.1:53") + fx := &projTestFixture{ + t: t, + recorder: recorder, + events: sub.Events(), + stub: &healthStubHandler{health: map[netip.AddrPort]UpstreamHealth{}}, + srv: srv, + group: &nbdns.NameServerGroup{ + Domains: []string{"example.com"}, + NameServers: []nbdns.NameServer{{IP: srv.Addr(), NSType: nbdns.UDPNameServerType, Port: int(srv.Port())}}, + }, + } + fx.server = &DefaultServer{ + ctx: context.Background(), + wgInterface: &mocWGIface{}, + statusRecorder: recorder, + dnsMuxMap: make(registeredHandlerMap), + selectedRoutes: func() route.HAMap { return fx.selected }, + activeRoutes: func() route.HAMap { return fx.active }, + warningDelayBase: defaultWarningDelayBase, + } + fx.server.dnsMuxMap["example.com"] = handlerWrapper{domain: "example.com", handler: fx.stub, priority: PriorityUpstream} + + fx.server.mux.Lock() + fx.server.updateNSGroupStates([]*nbdns.NameServerGroup{fx.group}) + fx.server.mux.Unlock() + return fx +} + +func (f *projTestFixture) setHealth(h UpstreamHealth) { + f.stub.health = map[netip.AddrPort]UpstreamHealth{f.srv: h} +} + +func (f *projTestFixture) tick() []peer.NSGroupState { + f.server.refreshHealth() + return f.recorder.GetDNSStates() +} + +func (f *projTestFixture) expectNoEvent(why string) { + f.t.Helper() + select { + case evt := <-f.events: + f.t.Fatalf("unexpected event (%s): %+v", why, evt) + case <-time.After(100 * time.Millisecond): + } +} + +func (f *projTestFixture) expectEvent(substr, why string) *proto.SystemEvent { + f.t.Helper() + select { + case evt := <-f.events: + assert.Contains(f.t, evt.Message, substr, why) + return evt + case <-time.After(time.Second): + f.t.Fatalf("expected event (%s) with %q", why, substr) + return nil + } +} + +var overlayNetForTest = netip.MustParsePrefix("100.64.0.0/16") +var overlayMapForTest = route.HAMap{"overlay": {{Network: overlayNetForTest}}} + +// TestProjection_PublicFailEmitsImmediately covers rule 1: an upstream +// that is not inside any selected route (public DNS) fires the warning +// on the first Unhealthy tick, no grace period. +func TestProjection_PublicFailEmitsImmediately(t *testing.T) { + fx := newProjTestFixture(t) + + fx.setHealth(UpstreamHealth{LastFail: time.Now(), LastErr: "timeout"}) + states := fx.tick() + require.Len(t, states, 1) + assert.False(t, states[0].Enabled) + fx.expectEvent("unreachable", "public DNS failure") +} + +// TestProjection_OverlayConnectedFailEmitsImmediately covers rule 2: +// the upstream is inside a selected route AND the route has a Connected +// peer. Tunnel is up, failure is real, emit immediately. +func TestProjection_OverlayConnectedFailEmitsImmediately(t *testing.T) { + fx := newProjTestFixture(t) + fx.selected = overlayMapForTest + fx.active = overlayMapForTest + + fx.setHealth(UpstreamHealth{LastFail: time.Now(), LastErr: "timeout"}) + states := fx.tick() + require.Len(t, states, 1) + assert.False(t, states[0].Enabled) + fx.expectEvent("unreachable", "overlay + connected failure") +} + +// TestProjection_OverlayNotConnectedDelaysWarning covers rule 3: the +// upstream is routed but no peer is Connected (Connecting/Idle/missing). +// First tick: Unhealthy display, no warning. After the grace window +// elapses with no recovery, the warning fires. +func TestProjection_OverlayNotConnectedDelaysWarning(t *testing.T) { + grace := 50 * time.Millisecond + fx := newProjTestFixture(t) + fx.server.warningDelayBase = grace + fx.selected = overlayMapForTest + // active stays nil: routed but not connected. + + fx.setHealth(UpstreamHealth{LastFail: time.Now(), LastErr: "timeout"}) + states := fx.tick() + require.Len(t, states, 1) + assert.False(t, states[0].Enabled, "display must reflect failure even during grace window") + fx.expectNoEvent("first fail tick within grace window") + + time.Sleep(grace + 10*time.Millisecond) + fx.setHealth(UpstreamHealth{LastFail: time.Now(), LastErr: "timeout"}) + fx.tick() + fx.expectEvent("unreachable", "warning after grace window") +} + +// TestProjection_OverlayAddrNoRouteDelaysWarning covers an upstream +// whose address is inside the WireGuard overlay range but is not +// covered by any selected route (peer-to-peer DNS without an explicit +// route). Until a peer reports Connected for that address, startup +// failures must be held just like the routed case. +func TestProjection_OverlayAddrNoRouteDelaysWarning(t *testing.T) { + recorder := peer.NewRecorder("mgm") + sub := recorder.SubscribeToEvents() + t.Cleanup(func() { recorder.UnsubscribeFromEvents(sub) }) + + overlayPeer := netip.MustParseAddrPort("100.66.100.5:53") + server := &DefaultServer{ + ctx: context.Background(), + wgInterface: &mocWGIface{}, + statusRecorder: recorder, + dnsMuxMap: make(registeredHandlerMap), + selectedRoutes: func() route.HAMap { return nil }, + activeRoutes: func() route.HAMap { return nil }, + warningDelayBase: 50 * time.Millisecond, + } + group := &nbdns.NameServerGroup{ + Domains: []string{"example.com"}, + NameServers: []nbdns.NameServer{{IP: overlayPeer.Addr(), NSType: nbdns.UDPNameServerType, Port: int(overlayPeer.Port())}}, + } + stub := &healthStubHandler{health: map[netip.AddrPort]UpstreamHealth{ + overlayPeer: {LastFail: time.Now(), LastErr: "timeout"}, + }} + server.dnsMuxMap["example.com"] = handlerWrapper{domain: "example.com", handler: stub, priority: PriorityUpstream} + + server.mux.Lock() + server.updateNSGroupStates([]*nbdns.NameServerGroup{group}) + server.mux.Unlock() + server.refreshHealth() + + select { + case evt := <-sub.Events(): + t.Fatalf("unexpected event during grace window: %+v", evt) + case <-time.After(100 * time.Millisecond): + } + + time.Sleep(60 * time.Millisecond) + stub.health = map[netip.AddrPort]UpstreamHealth{overlayPeer: {LastFail: time.Now(), LastErr: "timeout"}} + server.refreshHealth() + + select { + case evt := <-sub.Events(): + assert.Contains(t, evt.Message, "unreachable") + case <-time.After(time.Second): + t.Fatal("expected warning after grace window") + } +} + +// TestProjection_StopClearsHealthState verifies that Stop wipes the +// per-group projection state so a subsequent Start doesn't inherit +// sticky flags (notably everHealthy) that would bypass the grace +// window during the next peer handshake. +func TestProjection_StopClearsHealthState(t *testing.T) { + wgIface := &mocWGIface{} + server := &DefaultServer{ + ctx: context.Background(), + wgInterface: wgIface, + service: NewServiceViaMemory(wgIface), + hostManager: &noopHostConfigurator{}, + extraDomains: map[domain.Domain]int{}, + dnsMuxMap: make(registeredHandlerMap), + statusRecorder: peer.NewRecorder("mgm"), + selectedRoutes: func() route.HAMap { return nil }, + activeRoutes: func() route.HAMap { return nil }, + warningDelayBase: defaultWarningDelayBase, + currentConfigHash: ^uint64(0), + } + server.ctx, server.ctxCancel = context.WithCancel(context.Background()) + + srv := netip.MustParseAddrPort("8.8.8.8:53") + group := &nbdns.NameServerGroup{ + Domains: []string{"example.com"}, + NameServers: []nbdns.NameServer{{IP: srv.Addr(), NSType: nbdns.UDPNameServerType, Port: int(srv.Port())}}, + } + stub := &healthStubHandler{health: map[netip.AddrPort]UpstreamHealth{srv: {LastOk: time.Now()}}} + server.dnsMuxMap["example.com"] = handlerWrapper{domain: "example.com", handler: stub, priority: PriorityUpstream} + + server.mux.Lock() + server.updateNSGroupStates([]*nbdns.NameServerGroup{group}) + server.mux.Unlock() + server.refreshHealth() + + server.healthProjectMu.Lock() + p, ok := server.nsGroupProj[generateGroupKey(group)] + server.healthProjectMu.Unlock() + require.True(t, ok, "projection state should exist after tick") + require.True(t, p.everHealthy, "tick with success must set everHealthy") + + server.Stop() + + server.healthProjectMu.Lock() + cleared := server.nsGroupProj == nil + server.healthProjectMu.Unlock() + assert.True(t, cleared, "Stop must clear nsGroupProj") +} + +// TestProjection_OverlayRecoversDuringGrace covers the happy path of +// rule 3: startup failures while the peer is handshaking, then the peer +// comes up and a query succeeds before the grace window elapses. No +// warning should ever have fired, and no recovery either. +func TestProjection_OverlayRecoversDuringGrace(t *testing.T) { + fx := newProjTestFixture(t) + fx.server.warningDelayBase = 200 * time.Millisecond + fx.selected = overlayMapForTest + + fx.setHealth(UpstreamHealth{LastFail: time.Now(), LastErr: "timeout"}) + fx.tick() + fx.expectNoEvent("fail within grace, warning suppressed") + + fx.active = overlayMapForTest + fx.setHealth(UpstreamHealth{LastOk: time.Now()}) + states := fx.tick() + require.Len(t, states, 1) + assert.True(t, states[0].Enabled) + fx.expectNoEvent("recovery without prior warning must not emit") +} + +// TestProjection_RecoveryOnlyAfterWarning enforces the invariant the +// whole design leans on: recovery events only appear when a warning +// event was actually emitted for the current streak. A Healthy verdict +// without a prior warning is silent, so the user never sees "recovered" +// out of thin air. +func TestProjection_RecoveryOnlyAfterWarning(t *testing.T) { + fx := newProjTestFixture(t) + + fx.setHealth(UpstreamHealth{LastOk: time.Now()}) + states := fx.tick() + require.Len(t, states, 1) + assert.True(t, states[0].Enabled) + fx.expectNoEvent("first healthy tick should not recover anything") + + fx.setHealth(UpstreamHealth{LastFail: time.Now(), LastErr: "timeout"}) + fx.tick() + fx.expectEvent("unreachable", "public fail emits immediately") + + fx.setHealth(UpstreamHealth{LastOk: time.Now()}) + fx.tick() + fx.expectEvent("recovered", "recovery follows real warning") + + fx.setHealth(UpstreamHealth{LastFail: time.Now(), LastErr: "timeout"}) + fx.tick() + fx.expectEvent("unreachable", "second cycle warning") + + fx.setHealth(UpstreamHealth{LastOk: time.Now()}) + fx.tick() + fx.expectEvent("recovered", "second cycle recovery") +} + +// TestProjection_EverHealthyOverridesDelay covers rule 4: once a group +// has ever been Healthy, subsequent failures skip the grace window even +// if classification says "routed + not connected". The system has +// proved it can work, so any new failure is real. +func TestProjection_EverHealthyOverridesDelay(t *testing.T) { + fx := newProjTestFixture(t) + // Large base so any emission must come from the everHealthy bypass, not elapsed time. + fx.server.warningDelayBase = time.Hour + fx.selected = overlayMapForTest + fx.active = overlayMapForTest + + // Establish "ever healthy". + fx.setHealth(UpstreamHealth{LastOk: time.Now()}) + fx.tick() + fx.expectNoEvent("first healthy tick") + + // Peer drops. Query fails. Routed + not connected → normally grace, + // but everHealthy flag bypasses it. + fx.active = nil + fx.setHealth(UpstreamHealth{LastFail: time.Now(), LastErr: "timeout"}) + fx.tick() + fx.expectEvent("unreachable", "failure after ever-healthy must be immediate") +} + +// TestProjection_ReconnectBlipEmitsPair covers the explicit tradeoff +// from the design discussion: once a group has been healthy, a brief +// reconnect that produces a failing tick will fire warning + recovery. +// This is by design: user-visible blips are accurate signal, not noise. +func TestProjection_ReconnectBlipEmitsPair(t *testing.T) { + fx := newProjTestFixture(t) + fx.selected = overlayMapForTest + fx.active = overlayMapForTest + + fx.setHealth(UpstreamHealth{LastOk: time.Now()}) + fx.tick() + + fx.setHealth(UpstreamHealth{LastFail: time.Now(), LastErr: "timeout"}) + fx.tick() + fx.expectEvent("unreachable", "blip warning") + + fx.setHealth(UpstreamHealth{LastOk: time.Now()}) + fx.tick() + fx.expectEvent("recovered", "blip recovery") +} + +// TestProjection_MixedGroupEmitsImmediately covers the multi-upstream +// rule: a group with at least one public upstream is in the "immediate" +// category regardless of the other upstreams' routing, because the +// public one has no peer-startup excuse. Prevents public-DNS failures +// from being hidden behind a routed sibling. +func TestProjection_MixedGroupEmitsImmediately(t *testing.T) { + recorder := peer.NewRecorder("mgm") + sub := recorder.SubscribeToEvents() + t.Cleanup(func() { recorder.UnsubscribeFromEvents(sub) }) + events := sub.Events() + + public := netip.MustParseAddrPort("8.8.8.8:53") + overlay := netip.MustParseAddrPort("100.64.0.1:53") + overlayMap := route.HAMap{"overlay": {{Network: netip.MustParsePrefix("100.64.0.0/16")}}} + + server := &DefaultServer{ + ctx: context.Background(), + statusRecorder: recorder, + dnsMuxMap: make(registeredHandlerMap), + selectedRoutes: func() route.HAMap { return overlayMap }, + activeRoutes: func() route.HAMap { return nil }, + warningDelayBase: time.Hour, + } + group := &nbdns.NameServerGroup{ + Domains: []string{"example.com"}, + NameServers: []nbdns.NameServer{ + {IP: public.Addr(), NSType: nbdns.UDPNameServerType, Port: int(public.Port())}, + {IP: overlay.Addr(), NSType: nbdns.UDPNameServerType, Port: int(overlay.Port())}, + }, + } + stub := &healthStubHandler{ + health: map[netip.AddrPort]UpstreamHealth{ + public: {LastFail: time.Now(), LastErr: "servfail"}, + overlay: {LastFail: time.Now(), LastErr: "timeout"}, + }, + } + server.dnsMuxMap["example.com"] = handlerWrapper{domain: "example.com", handler: stub, priority: PriorityUpstream} + + server.mux.Lock() + server.updateNSGroupStates([]*nbdns.NameServerGroup{group}) + server.mux.Unlock() + server.refreshHealth() + + select { + case evt := <-events: + assert.Contains(t, evt.Message, "unreachable") + case <-time.After(time.Second): + t.Fatal("expected immediate warning because group contains a public upstream") + } +} + func TestDNSLoopPrevention(t *testing.T) { wgInterface := &mocWGIface{} service := NewServiceViaMemory(wgInterface) @@ -2183,17 +2726,18 @@ func TestDNSLoopPrevention(t *testing.T) { if tt.expectedHandlers > 0 { handler := muxUpdates[0].handler.(*upstreamResolver) - assert.Len(t, handler.upstreamServers, len(tt.expectedServers)) + flat := handler.flatUpstreams() + assert.Len(t, flat, len(tt.expectedServers)) if tt.shouldFilterOwnIP { - for _, upstream := range handler.upstreamServers { + for _, upstream := range flat { assert.NotEqual(t, dnsServerIP, upstream.Addr()) } } for _, expected := range tt.expectedServers { found := false - for _, upstream := range handler.upstreamServers { + for _, upstream := range flat { if upstream.Addr() == expected { found = true break diff --git a/client/internal/dns/systemd_linux.go b/client/internal/dns/systemd_linux.go index 573dff540..bd301e177 100644 --- a/client/internal/dns/systemd_linux.go +++ b/client/internal/dns/systemd_linux.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "net/netip" + "slices" "time" "github.com/godbus/dbus/v5" @@ -40,10 +41,17 @@ const ( ) type systemdDbusConfigurator struct { - dbusLinkObject dbus.ObjectPath - ifaceName string + dbusLinkObject dbus.ObjectPath + ifaceName string + wgIndex int + origNameservers []netip.Addr } +const ( + systemdDbusLinkDNSProperty = systemdDbusLinkInterface + ".DNS" + systemdDbusLinkDefaultRouteProperty = systemdDbusLinkInterface + ".DefaultRoute" +) + // the types below are based on dbus specification, each field is mapped to a dbus type // see https://dbus.freedesktop.org/doc/dbus-specification.html#basic-types for more details on dbus types // see https://www.freedesktop.org/software/systemd/man/org.freedesktop.resolve1.html on resolve1 input types @@ -79,10 +87,145 @@ func newSystemdDbusConfigurator(wgInterface string) (*systemdDbusConfigurator, e log.Debugf("got dbus Link interface: %s from net interface %s and index %d", s, iface.Name, iface.Index) - return &systemdDbusConfigurator{ + c := &systemdDbusConfigurator{ dbusLinkObject: dbus.ObjectPath(s), ifaceName: wgInterface, - }, nil + wgIndex: iface.Index, + } + + origNameservers, err := c.captureOriginalNameservers() + switch { + case err != nil: + log.Warnf("capture original nameservers from systemd-resolved: %v", err) + case len(origNameservers) == 0: + log.Warnf("no original nameservers captured from systemd-resolved default-route links; DNS fallback will be empty") + default: + log.Debugf("captured %d original nameservers from systemd-resolved default-route links: %v", len(origNameservers), origNameservers) + } + c.origNameservers = origNameservers + return c, nil +} + +// captureOriginalNameservers reads per-link DNS from systemd-resolved for +// every default-route link except our own WG link. Non-default-route links +// (VPNs, docker bridges) are skipped because their upstreams wouldn't +// actually serve host queries. +func (s *systemdDbusConfigurator) captureOriginalNameservers() ([]netip.Addr, error) { + ifaces, err := net.Interfaces() + if err != nil { + return nil, fmt.Errorf("list interfaces: %w", err) + } + + seen := make(map[netip.Addr]struct{}) + var out []netip.Addr + for _, iface := range ifaces { + if !s.isCandidateLink(iface) { + continue + } + linkPath, err := getSystemdLinkPath(iface.Index) + if err != nil || !isSystemdLinkDefaultRoute(linkPath) { + continue + } + for _, addr := range readSystemdLinkDNS(linkPath) { + addr = normalizeSystemdAddr(addr, iface.Name) + if !addr.IsValid() { + continue + } + if _, dup := seen[addr]; dup { + continue + } + seen[addr] = struct{}{} + out = append(out, addr) + } + } + return out, nil +} + +func (s *systemdDbusConfigurator) isCandidateLink(iface net.Interface) bool { + if iface.Index == s.wgIndex { + return false + } + if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 { + return false + } + return true +} + +// normalizeSystemdAddr unmaps v4-mapped-v6, drops unspecified, and reattaches +// the link's iface name as zone for link-local v6 (Link.DNS strips it). +// Returns the zero Addr to signal "skip this entry". +func normalizeSystemdAddr(addr netip.Addr, ifaceName string) netip.Addr { + addr = addr.Unmap() + if !addr.IsValid() || addr.IsUnspecified() { + return netip.Addr{} + } + if addr.IsLinkLocalUnicast() { + return addr.WithZone(ifaceName) + } + return addr +} + +func getSystemdLinkPath(ifIndex int) (dbus.ObjectPath, error) { + obj, closeConn, err := getDbusObject(systemdResolvedDest, systemdDbusObjectNode) + if err != nil { + return "", fmt.Errorf("dbus resolve1: %w", err) + } + defer closeConn() + var p string + if err := obj.Call(systemdDbusGetLinkMethod, dbusDefaultFlag, int32(ifIndex)).Store(&p); err != nil { + return "", err + } + return dbus.ObjectPath(p), nil +} + +func isSystemdLinkDefaultRoute(linkPath dbus.ObjectPath) bool { + obj, closeConn, err := getDbusObject(systemdResolvedDest, linkPath) + if err != nil { + return false + } + defer closeConn() + v, err := obj.GetProperty(systemdDbusLinkDefaultRouteProperty) + if err != nil { + return false + } + b, ok := v.Value().(bool) + return ok && b +} + +func readSystemdLinkDNS(linkPath dbus.ObjectPath) []netip.Addr { + obj, closeConn, err := getDbusObject(systemdResolvedDest, linkPath) + if err != nil { + return nil + } + defer closeConn() + v, err := obj.GetProperty(systemdDbusLinkDNSProperty) + if err != nil { + return nil + } + entries, ok := v.Value().([][]any) + if !ok { + return nil + } + var out []netip.Addr + for _, entry := range entries { + if len(entry) < 2 { + continue + } + raw, ok := entry[1].([]byte) + if !ok { + continue + } + addr, ok := netip.AddrFromSlice(raw) + if !ok { + continue + } + out = append(out, addr) + } + return out +} + +func (s *systemdDbusConfigurator) getOriginalNameservers() []netip.Addr { + return slices.Clone(s.origNameservers) } func (s *systemdDbusConfigurator) supportCustomPort() bool { diff --git a/client/internal/dns/upstream.go b/client/internal/dns/upstream.go index 39064f26c..a4f713d68 100644 --- a/client/internal/dns/upstream.go +++ b/client/internal/dns/upstream.go @@ -1,3 +1,32 @@ +// Package dns implements the client-side DNS stack: listener/service on the +// peer's tunnel address, handler chain that routes questions by domain and +// priority, and upstream resolvers that forward what remains to configured +// nameservers. +// +// # Upstream resolution and the race model +// +// When two or more nameserver groups target the same domain, DefaultServer +// merges them into one upstream handler whose state is: +// +// upstreamResolverBase +// └── upstreamServers []upstreamRace // one entry per source NS group +// └── []netip.AddrPort // primary, fallback, ... +// +// Each source nameserver group contributes one upstreamRace. Within a race +// upstreams are tried in order: the next is used only on failure (timeout, +// SERVFAIL, REFUSED, no response). NXDOMAIN is a valid answer and stops +// the walk. When more than one race exists, ServeDNS fans out one +// goroutine per race and returns the first valid answer, cancelling the +// rest. A handler with a single race skips the fan-out. +// +// # Health projection +// +// Query outcomes are recorded per-upstream in UpstreamHealth. The server +// periodically merges these snapshots across handlers and projects them +// into peer.NSGroupState. There is no active probing: a group is marked +// unhealthy only when every seen upstream has a recent failure and none +// has a recent success. Healthy→unhealthy fires a single +// SystemEvent_WARNING; steady-state refreshes do not duplicate it. package dns import ( @@ -11,11 +40,8 @@ import ( "slices" "strings" "sync" - "sync/atomic" "time" - "github.com/cenkalti/backoff/v4" - "github.com/hashicorp/go-multierror" "github.com/miekg/dns" log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/tun/netstack" @@ -25,7 +51,8 @@ import ( "github.com/netbirdio/netbird/client/internal/dns/resutil" "github.com/netbirdio/netbird/client/internal/dns/types" "github.com/netbirdio/netbird/client/internal/peer" - "github.com/netbirdio/netbird/client/proto" + "github.com/netbirdio/netbird/route" + "github.com/netbirdio/netbird/shared/management/domain" ) var currentMTU uint16 = iface.DefaultMTU @@ -67,15 +94,17 @@ const ( // Set longer than UpstreamTimeout to ensure context timeout takes precedence ClientTimeout = 5 * time.Second - reactivatePeriod = 30 * time.Second - probeTimeout = 2 * time.Second - // ipv6HeaderSize + udpHeaderSize, used to derive the maximum DNS UDP // payload from the tunnel MTU. ipUDPHeaderSize = 60 + 8 -) -const testRecord = "com." + // raceMaxTotalTimeout caps the combined time spent walking all upstreams + // within one race, so a slow primary can't eat the whole race budget. + raceMaxTotalTimeout = 5 * time.Second + // raceMinPerUpstreamTimeout is the floor applied when dividing + // raceMaxTotalTimeout across upstreams within a race. + raceMinPerUpstreamTimeout = 2 * time.Second +) const ( protoUDP = "udp" @@ -84,6 +113,69 @@ const ( type dnsProtocolKey struct{} +type upstreamProtocolKey struct{} + +// upstreamProtocolResult holds the protocol used for the upstream exchange. +// Stored as a pointer in context so the exchange function can set it. +type upstreamProtocolResult struct { + protocol string +} + +type upstreamClient interface { + exchange(ctx context.Context, upstream string, r *dns.Msg) (*dns.Msg, time.Duration, error) +} + +type UpstreamResolver interface { + serveDNS(r *dns.Msg) (*dns.Msg, time.Duration, error) + upstreamExchange(upstream string, r *dns.Msg) (*dns.Msg, time.Duration, error) +} + +// upstreamRace is an ordered list of upstreams derived from one configured +// nameserver group. Order matters: the first upstream is tried first, the +// second only on failure, and so on. Multiple upstreamRace values coexist +// inside one resolver when overlapping nameserver groups target the same +// domain; those races run in parallel and the first valid answer wins. +type upstreamRace []netip.AddrPort + +// UpstreamHealth is the last query-path outcome for a single upstream, +// consumed by nameserver-group status projection. +type UpstreamHealth struct { + LastOk time.Time + LastFail time.Time + LastErr string +} + +type upstreamResolverBase struct { + ctx context.Context + cancel context.CancelFunc + upstreamClient upstreamClient + upstreamServers []upstreamRace + domain domain.Domain + upstreamTimeout time.Duration + + healthMu sync.RWMutex + health map[netip.AddrPort]*UpstreamHealth + + statusRecorder *peer.Status + // selectedRoutes returns the current set of client routes the admin + // has enabled. Called lazily from the query hot path when an upstream + // might need a tunnel-bound client (iOS) and from health projection. + selectedRoutes func() route.HAMap +} + +type upstreamFailure struct { + upstream netip.AddrPort + reason string +} + +type raceResult struct { + msg *dns.Msg + upstream netip.AddrPort + protocol string + ede string + failures []upstreamFailure +} + // contextWithDNSProtocol stores the inbound DNS protocol ("udp" or "tcp") in context. func contextWithDNSProtocol(ctx context.Context, network string) context.Context { return context.WithValue(ctx, dnsProtocolKey{}, network) @@ -100,16 +192,8 @@ func dnsProtocolFromContext(ctx context.Context) string { return "" } -type upstreamProtocolKey struct{} - -// upstreamProtocolResult holds the protocol used for the upstream exchange. -// Stored as a pointer in context so the exchange function can set it. -type upstreamProtocolResult struct { - protocol string -} - -// contextWithupstreamProtocolResult stores a mutable result holder in the context. -func contextWithupstreamProtocolResult(ctx context.Context) (context.Context, *upstreamProtocolResult) { +// contextWithUpstreamProtocolResult stores a mutable result holder in the context. +func contextWithUpstreamProtocolResult(ctx context.Context) (context.Context, *upstreamProtocolResult) { r := &upstreamProtocolResult{} return context.WithValue(ctx, upstreamProtocolKey{}, r), r } @@ -124,67 +208,37 @@ func setUpstreamProtocol(ctx context.Context, protocol string) { } } -type upstreamClient interface { - exchange(ctx context.Context, upstream string, r *dns.Msg) (*dns.Msg, time.Duration, error) -} - -type UpstreamResolver interface { - serveDNS(r *dns.Msg) (*dns.Msg, time.Duration, error) - upstreamExchange(upstream string, r *dns.Msg) (*dns.Msg, time.Duration, error) -} - -type upstreamResolverBase struct { - ctx context.Context - cancel context.CancelFunc - upstreamClient upstreamClient - upstreamServers []netip.AddrPort - domain string - disabled bool - successCount atomic.Int32 - mutex sync.Mutex - reactivatePeriod time.Duration - upstreamTimeout time.Duration - wg sync.WaitGroup - - deactivate func(error) - reactivate func() - statusRecorder *peer.Status - routeMatch func(netip.Addr) bool -} - -type upstreamFailure struct { - upstream netip.AddrPort - reason string -} - -func newUpstreamResolverBase(ctx context.Context, statusRecorder *peer.Status, domain string) *upstreamResolverBase { +func newUpstreamResolverBase(ctx context.Context, statusRecorder *peer.Status, d domain.Domain) *upstreamResolverBase { ctx, cancel := context.WithCancel(ctx) return &upstreamResolverBase{ - ctx: ctx, - cancel: cancel, - domain: domain, - upstreamTimeout: UpstreamTimeout, - reactivatePeriod: reactivatePeriod, - statusRecorder: statusRecorder, + ctx: ctx, + cancel: cancel, + domain: d, + upstreamTimeout: UpstreamTimeout, + statusRecorder: statusRecorder, } } // String returns a string representation of the upstream resolver func (u *upstreamResolverBase) String() string { - return fmt.Sprintf("Upstream %s", u.upstreamServers) + return fmt.Sprintf("Upstream %s", u.flatUpstreams()) } -// ID returns the unique handler ID +// ID returns the unique handler ID. Race groupings and within-race +// ordering are both part of the identity: [[A,B]] and [[A],[B]] query +// the same servers but with different semantics (serial fallback vs +// parallel race), so their handlers must not collide. func (u *upstreamResolverBase) ID() types.HandlerID { - servers := slices.Clone(u.upstreamServers) - slices.SortFunc(servers, func(a, b netip.AddrPort) int { return a.Compare(b) }) - hash := sha256.New() - hash.Write([]byte(u.domain + ":")) - for _, s := range servers { - hash.Write([]byte(s.String())) - hash.Write([]byte("|")) + hash.Write([]byte(u.domain.PunycodeString() + ":")) + for _, race := range u.upstreamServers { + hash.Write([]byte("[")) + for _, s := range race { + hash.Write([]byte(s.String())) + hash.Write([]byte("|")) + } + hash.Write([]byte("]")) } return types.HandlerID("upstream-" + hex.EncodeToString(hash.Sum(nil)[:8])) } @@ -194,13 +248,31 @@ func (u *upstreamResolverBase) MatchSubdomains() bool { } func (u *upstreamResolverBase) Stop() { - log.Debugf("stopping serving DNS for upstreams %s", u.upstreamServers) + log.Debugf("stopping serving DNS for upstreams %s", u.flatUpstreams()) u.cancel() +} - u.mutex.Lock() - u.wg.Wait() - u.mutex.Unlock() +// flatUpstreams is for logging and ID hashing only, not for dispatch. +func (u *upstreamResolverBase) flatUpstreams() []netip.AddrPort { + var out []netip.AddrPort + for _, g := range u.upstreamServers { + out = append(out, g...) + } + return out +} +// setSelectedRoutes swaps the accessor used to classify overlay-routed +// upstreams. Called when route sources are wired after the handler was +// built (permanent / iOS constructors). +func (u *upstreamResolverBase) setSelectedRoutes(selected func() route.HAMap) { + u.selectedRoutes = selected +} + +func (u *upstreamResolverBase) addRace(servers []netip.AddrPort) { + if len(servers) == 0 { + return + } + u.upstreamServers = append(u.upstreamServers, slices.Clone(servers)) } // ServeDNS handles a DNS request @@ -242,82 +314,201 @@ func (u *upstreamResolverBase) prepareRequest(r *dns.Msg) { } func (u *upstreamResolverBase) tryUpstreamServers(ctx context.Context, w dns.ResponseWriter, r *dns.Msg, logger *log.Entry) (bool, []upstreamFailure) { - timeout := u.upstreamTimeout - if len(u.upstreamServers) > 1 { - maxTotal := 5 * time.Second - minPerUpstream := 2 * time.Second - scaledTimeout := maxTotal / time.Duration(len(u.upstreamServers)) - if scaledTimeout > minPerUpstream { - timeout = scaledTimeout - } else { - timeout = minPerUpstream - } + groups := u.upstreamServers + switch len(groups) { + case 0: + return false, nil + case 1: + return u.tryOnlyRace(ctx, w, r, groups[0], logger) + default: + return u.raceAll(ctx, w, r, groups, logger) + } +} + +func (u *upstreamResolverBase) tryOnlyRace(ctx context.Context, w dns.ResponseWriter, r *dns.Msg, group upstreamRace, logger *log.Entry) (bool, []upstreamFailure) { + res := u.tryRace(ctx, r, group) + if res.msg == nil { + return false, res.failures + } + if res.ede != "" { + resutil.SetMeta(w, "ede", res.ede) + } + u.writeSuccessResponse(w, res.msg, res.upstream, r.Question[0].Name, res.protocol, logger) + return true, res.failures +} + +// raceAll runs one worker per group in parallel, taking the first valid +// answer and cancelling the rest. +func (u *upstreamResolverBase) raceAll(ctx context.Context, w dns.ResponseWriter, r *dns.Msg, groups []upstreamRace, logger *log.Entry) (bool, []upstreamFailure) { + raceCtx, cancel := context.WithCancel(ctx) + defer cancel() + + // Buffer sized to len(groups) so workers never block on send, even + // after the coordinator has returned. + results := make(chan raceResult, len(groups)) + for _, g := range groups { + // tryRace clones the request per attempt, so workers never share + // a *dns.Msg and concurrent EDNS0 mutations can't race. + go func(g upstreamRace) { + results <- u.tryRace(raceCtx, r, g) + }(g) } var failures []upstreamFailure - for _, upstream := range u.upstreamServers { - if failure := u.queryUpstream(ctx, w, r, upstream, timeout, logger); failure != nil { - failures = append(failures, *failure) - } else { - return true, failures + for range groups { + select { + case res := <-results: + failures = append(failures, res.failures...) + if res.msg != nil { + if res.ede != "" { + resutil.SetMeta(w, "ede", res.ede) + } + u.writeSuccessResponse(w, res.msg, res.upstream, r.Question[0].Name, res.protocol, logger) + return true, failures + } + case <-ctx.Done(): + return false, failures } } return false, failures } -// queryUpstream queries a single upstream server. Returns nil on success, or failure info to try next upstream. -func (u *upstreamResolverBase) queryUpstream(parentCtx context.Context, w dns.ResponseWriter, r *dns.Msg, upstream netip.AddrPort, timeout time.Duration, logger *log.Entry) *upstreamFailure { - var rm *dns.Msg - var t time.Duration - var err error +func (u *upstreamResolverBase) tryRace(ctx context.Context, r *dns.Msg, group upstreamRace) raceResult { + timeout := u.upstreamTimeout + if len(group) > 1 { + // Cap the whole walk at raceMaxTotalTimeout: per-upstream timeouts + // still honor raceMinPerUpstreamTimeout as a floor for correctness + // on slow links, but the outer context ensures the combined walk + // cannot exceed the cap regardless of group size. + timeout = max(raceMaxTotalTimeout/time.Duration(len(group)), raceMinPerUpstreamTimeout) + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, raceMaxTotalTimeout) + defer cancel() + } + + var failures []upstreamFailure + for _, upstream := range group { + if ctx.Err() != nil { + return raceResult{failures: failures} + } + // Clone the request per attempt: the exchange path mutates EDNS0 + // options in-place, so reusing the same *dns.Msg across sequential + // upstreams would carry those mutations (e.g. a reduced UDP size) + // into the next attempt. + res, failure := u.queryUpstream(ctx, r.Copy(), upstream, timeout) + if failure != nil { + failures = append(failures, *failure) + continue + } + res.failures = failures + return res + } + return raceResult{failures: failures} +} + +func (u *upstreamResolverBase) queryUpstream(parentCtx context.Context, r *dns.Msg, upstream netip.AddrPort, timeout time.Duration) (raceResult, *upstreamFailure) { + ctx, cancel := context.WithTimeout(parentCtx, timeout) + defer cancel() + ctx, upstreamProto := contextWithUpstreamProtocolResult(ctx) // Advertise EDNS0 so the upstream may include Extended DNS Errors // (RFC 8914) in failure responses; we use those to short-circuit // failover for definitive answers like DNSSEC validation failures. - // Operate on a copy so the inbound request is unchanged: a client that - // did not advertise EDNS0 must not see an OPT in the response. + // The caller already passed a per-attempt copy, so we can mutate r + // directly; hadEdns reflects the original client request's state and + // controls whether we strip the OPT from the response. hadEdns := r.IsEdns0() != nil - reqUp := r if !hadEdns { - reqUp = r.Copy() - reqUp.SetEdns0(upstreamUDPSize(), false) + r.SetEdns0(upstreamUDPSize(), false) } - var startTime time.Time - var upstreamProto *upstreamProtocolResult - func() { - ctx, cancel := context.WithTimeout(parentCtx, timeout) - defer cancel() - ctx, upstreamProto = contextWithupstreamProtocolResult(ctx) - startTime = time.Now() - rm, t, err = u.upstreamClient.exchange(ctx, upstream.String(), reqUp) - }() + startTime := time.Now() + rm, _, err := u.upstreamClient.exchange(ctx, upstream.String(), r) if err != nil { - return u.handleUpstreamError(err, upstream, startTime) + // A parent cancellation (e.g., another race won and the coordinator + // cancelled the losers) is not an upstream failure. Check both the + // error chain and the parent context: a transport may surface the + // cancellation as a read/deadline error rather than context.Canceled. + if errors.Is(err, context.Canceled) || errors.Is(parentCtx.Err(), context.Canceled) { + return raceResult{}, &upstreamFailure{upstream: upstream, reason: "canceled"} + } + failure := u.handleUpstreamError(err, upstream, startTime) + u.markUpstreamFail(upstream, failure.reason) + return raceResult{}, failure } if rm == nil || !rm.Response { - return &upstreamFailure{upstream: upstream, reason: "no response"} + u.markUpstreamFail(upstream, "no response") + return raceResult{}, &upstreamFailure{upstream: upstream, reason: "no response"} + } + + proto := "" + if upstreamProto != nil { + proto = upstreamProto.protocol } if rm.Rcode == dns.RcodeServerFailure || rm.Rcode == dns.RcodeRefused { if code, ok := nonRetryableEDE(rm); ok { - resutil.SetMeta(w, "ede", edeName(code)) if !hadEdns { stripOPT(rm) } - u.writeSuccessResponse(w, rm, upstream, r.Question[0].Name, t, upstreamProto, logger) - return nil + u.markUpstreamOk(upstream) + return raceResult{msg: rm, upstream: upstream, protocol: proto, ede: edeName(code)}, nil } - return &upstreamFailure{upstream: upstream, reason: dns.RcodeToString[rm.Rcode]} + reason := dns.RcodeToString[rm.Rcode] + u.markUpstreamFail(upstream, reason) + return raceResult{}, &upstreamFailure{upstream: upstream, reason: reason} } if !hadEdns { stripOPT(rm) } - u.writeSuccessResponse(w, rm, upstream, r.Question[0].Name, t, upstreamProto, logger) - return nil + + u.markUpstreamOk(upstream) + return raceResult{msg: rm, upstream: upstream, protocol: proto}, nil +} + +// healthEntry returns the mutable health record for addr, lazily creating +// the map and the entry. Caller must hold u.healthMu. +func (u *upstreamResolverBase) healthEntry(addr netip.AddrPort) *UpstreamHealth { + if u.health == nil { + u.health = make(map[netip.AddrPort]*UpstreamHealth) + } + h := u.health[addr] + if h == nil { + h = &UpstreamHealth{} + u.health[addr] = h + } + return h +} + +func (u *upstreamResolverBase) markUpstreamOk(addr netip.AddrPort) { + u.healthMu.Lock() + defer u.healthMu.Unlock() + h := u.healthEntry(addr) + h.LastOk = time.Now() + h.LastFail = time.Time{} + h.LastErr = "" +} + +func (u *upstreamResolverBase) markUpstreamFail(addr netip.AddrPort, reason string) { + u.healthMu.Lock() + defer u.healthMu.Unlock() + h := u.healthEntry(addr) + h.LastFail = time.Now() + h.LastErr = reason +} + +// UpstreamHealth returns a snapshot of per-upstream query outcomes. +func (u *upstreamResolverBase) UpstreamHealth() map[netip.AddrPort]UpstreamHealth { + u.healthMu.RLock() + defer u.healthMu.RUnlock() + out := make(map[netip.AddrPort]UpstreamHealth, len(u.health)) + for k, v := range u.health { + out[k] = *v + } + return out } // upstreamUDPSize returns the EDNS0 UDP buffer size we advertise to upstreams, @@ -358,12 +549,23 @@ func (u *upstreamResolverBase) handleUpstreamError(err error, upstream netip.Add return &upstreamFailure{upstream: upstream, reason: reason} } -func (u *upstreamResolverBase) writeSuccessResponse(w dns.ResponseWriter, rm *dns.Msg, upstream netip.AddrPort, domain string, t time.Duration, upstreamProto *upstreamProtocolResult, logger *log.Entry) bool { - u.successCount.Add(1) +func (u *upstreamResolverBase) debugUpstreamTimeout(upstream netip.AddrPort) string { + if u.statusRecorder == nil { + return "" + } + peerInfo := findPeerForIP(upstream.Addr(), u.statusRecorder) + if peerInfo == nil { + return "" + } + + return fmt.Sprintf("(routes through NetBird peer %s)", FormatPeerStatus(peerInfo)) +} + +func (u *upstreamResolverBase) writeSuccessResponse(w dns.ResponseWriter, rm *dns.Msg, upstream netip.AddrPort, domain string, proto string, logger *log.Entry) { resutil.SetMeta(w, "upstream", upstream.String()) - if upstreamProto != nil && upstreamProto.protocol != "" { - resutil.SetMeta(w, "upstream_protocol", upstreamProto.protocol) + if proto != "" { + resutil.SetMeta(w, "upstream_protocol", proto) } // Clear Zero bit from external responses to prevent upstream servers from @@ -372,14 +574,11 @@ func (u *upstreamResolverBase) writeSuccessResponse(w dns.ResponseWriter, rm *dn if err := w.WriteMsg(rm); err != nil { logger.Errorf("failed to write DNS response for question domain=%s: %s", domain, err) - return true } - - return true } func (u *upstreamResolverBase) logUpstreamFailures(domain string, failures []upstreamFailure, succeeded bool, logger *log.Entry) { - totalUpstreams := len(u.upstreamServers) + totalUpstreams := len(u.flatUpstreams()) failedCount := len(failures) failureSummary := formatFailures(failures) @@ -434,119 +633,6 @@ func edeName(code uint16) string { return fmt.Sprintf("EDE %d", code) } -// ProbeAvailability tests all upstream servers simultaneously and -// disables the resolver if none work -func (u *upstreamResolverBase) ProbeAvailability(ctx context.Context) { - u.mutex.Lock() - defer u.mutex.Unlock() - - // avoid probe if upstreams could resolve at least one query - if u.successCount.Load() > 0 { - return - } - - var success bool - var mu sync.Mutex - var wg sync.WaitGroup - - var errs *multierror.Error - for _, upstream := range u.upstreamServers { - wg.Add(1) - go func(upstream netip.AddrPort) { - defer wg.Done() - err := u.testNameserver(u.ctx, ctx, upstream, 500*time.Millisecond) - if err != nil { - mu.Lock() - errs = multierror.Append(errs, err) - mu.Unlock() - log.Warnf("probing upstream nameserver %s: %s", upstream, err) - return - } - - mu.Lock() - success = true - mu.Unlock() - }(upstream) - } - - wg.Wait() - - select { - case <-ctx.Done(): - return - case <-u.ctx.Done(): - return - default: - } - - // didn't find a working upstream server, let's disable and try later - if !success { - u.disable(errs.ErrorOrNil()) - - if u.statusRecorder == nil { - return - } - - u.statusRecorder.PublishEvent( - proto.SystemEvent_WARNING, - proto.SystemEvent_DNS, - "All upstream servers failed (probe failed)", - "Unable to reach one or more DNS servers. This might affect your ability to connect to some services.", - map[string]string{"upstreams": u.upstreamServersString()}, - ) - } -} - -// waitUntilResponse retries, in an exponential interval, querying the upstream servers until it gets a positive response -func (u *upstreamResolverBase) waitUntilResponse() { - exponentialBackOff := &backoff.ExponentialBackOff{ - InitialInterval: 500 * time.Millisecond, - RandomizationFactor: 0.5, - Multiplier: 1.1, - MaxInterval: u.reactivatePeriod, - MaxElapsedTime: 0, - Stop: backoff.Stop, - Clock: backoff.SystemClock, - } - - operation := func() error { - select { - case <-u.ctx.Done(): - return backoff.Permanent(fmt.Errorf("exiting upstream retry loop for upstreams %s: parent context has been canceled", u.upstreamServersString())) - default: - } - - for _, upstream := range u.upstreamServers { - if err := u.testNameserver(u.ctx, nil, upstream, probeTimeout); err != nil { - log.Tracef("upstream check for %s: %s", upstream, err) - } else { - // at least one upstream server is available, stop probing - return nil - } - } - - log.Tracef("checking connectivity with upstreams %s failed. Retrying in %s", u.upstreamServersString(), exponentialBackOff.NextBackOff()) - return fmt.Errorf("upstream check call error") - } - - err := backoff.Retry(operation, backoff.WithContext(exponentialBackOff, u.ctx)) - if err != nil { - if errors.Is(err, context.Canceled) { - log.Debugf("upstream retry loop exited for upstreams %s", u.upstreamServersString()) - } else { - log.Warnf("upstream retry loop exited for upstreams %s: %v", u.upstreamServersString(), err) - } - return - } - - log.Infof("upstreams %s are responsive again. Adding them back to system", u.upstreamServersString()) - u.successCount.Add(1) - u.reactivate() - u.mutex.Lock() - u.disabled = false - u.mutex.Unlock() -} - // isTimeout returns true if the given error is a network timeout error. // // Copied from k8s.io/apimachinery/pkg/util/net.IsTimeout @@ -558,45 +644,6 @@ func isTimeout(err error) bool { return false } -func (u *upstreamResolverBase) disable(err error) { - if u.disabled { - return - } - - log.Warnf("Upstream resolving is Disabled for %v", reactivatePeriod) - u.successCount.Store(0) - u.deactivate(err) - u.disabled = true - u.wg.Add(1) - go func() { - defer u.wg.Done() - u.waitUntilResponse() - }() -} - -func (u *upstreamResolverBase) upstreamServersString() string { - var servers []string - for _, server := range u.upstreamServers { - servers = append(servers, server.String()) - } - return strings.Join(servers, ", ") -} - -func (u *upstreamResolverBase) testNameserver(baseCtx context.Context, externalCtx context.Context, server netip.AddrPort, timeout time.Duration) error { - mergedCtx, cancel := context.WithTimeout(baseCtx, timeout) - defer cancel() - - if externalCtx != nil { - stop2 := context.AfterFunc(externalCtx, cancel) - defer stop2() - } - - r := new(dns.Msg).SetQuestion(testRecord, dns.TypeSOA) - - _, _, err := u.upstreamClient.exchange(mergedCtx, server.String(), r) - return err -} - // clientUDPMaxSize returns the maximum UDP response size the client accepts. func clientUDPMaxSize(r *dns.Msg) int { if opt := r.IsEdns0(); opt != nil { @@ -608,13 +655,10 @@ func clientUDPMaxSize(r *dns.Msg) int { // ExchangeWithFallback exchanges a DNS message with the upstream server. // It first tries to use UDP, and if it is truncated, it falls back to TCP. // If the inbound request came over TCP (via context), it skips the UDP attempt. -// If the passed context is nil, this will use Exchange instead of ExchangeContext. func ExchangeWithFallback(ctx context.Context, client *dns.Client, r *dns.Msg, upstream string) (*dns.Msg, time.Duration, error) { // If the request came in over TCP, go straight to TCP upstream. if dnsProtocolFromContext(ctx) == protoTCP { - tcpClient := *client - tcpClient.Net = protoTCP - rm, t, err := tcpClient.ExchangeContext(ctx, r, upstream) + rm, t, err := toTCPClient(client).ExchangeContext(ctx, r, upstream) if err != nil { return nil, t, fmt.Errorf("with tcp: %w", err) } @@ -634,18 +678,7 @@ func ExchangeWithFallback(ctx context.Context, client *dns.Client, r *dns.Msg, u opt.SetUDPSize(maxUDPPayload) } - var ( - rm *dns.Msg - t time.Duration - err error - ) - - if ctx == nil { - rm, t, err = client.Exchange(r, upstream) - } else { - rm, t, err = client.ExchangeContext(ctx, r, upstream) - } - + rm, t, err := client.ExchangeContext(ctx, r, upstream) if err != nil { return nil, t, fmt.Errorf("with udp: %w", err) } @@ -659,15 +692,7 @@ func ExchangeWithFallback(ctx context.Context, client *dns.Client, r *dns.Msg, u // data than the client's buffer, we could truncate locally and skip // the TCP retry. - tcpClient := *client - tcpClient.Net = protoTCP - - if ctx == nil { - rm, t, err = tcpClient.Exchange(r, upstream) - } else { - rm, t, err = tcpClient.ExchangeContext(ctx, r, upstream) - } - + rm, t, err = toTCPClient(client).ExchangeContext(ctx, r, upstream) if err != nil { return nil, t, fmt.Errorf("with tcp: %w", err) } @@ -681,6 +706,25 @@ func ExchangeWithFallback(ctx context.Context, client *dns.Client, r *dns.Msg, u return rm, t, nil } +// toTCPClient returns a copy of c configured for TCP. If c's Dialer has a +// *net.UDPAddr bound as LocalAddr (iOS does this to keep the source IP on +// the tunnel interface), it is converted to the equivalent *net.TCPAddr +// so net.Dialer doesn't reject the TCP dial with "mismatched local +// address type". +func toTCPClient(c *dns.Client) *dns.Client { + tcp := *c + tcp.Net = protoTCP + if tcp.Dialer == nil { + return &tcp + } + d := *tcp.Dialer + if ua, ok := d.LocalAddr.(*net.UDPAddr); ok { + d.LocalAddr = &net.TCPAddr{IP: ua.IP, Port: ua.Port, Zone: ua.Zone} + } + tcp.Dialer = &d + return &tcp +} + // ExchangeWithNetstack performs a DNS exchange using netstack for dialing. // This is needed when netstack is enabled to reach peer IPs through the tunnel. func ExchangeWithNetstack(ctx context.Context, nsNet *netstack.Net, r *dns.Msg, upstream string) (*dns.Msg, error) { @@ -822,15 +866,36 @@ func findPeerForIP(ip netip.Addr, statusRecorder *peer.Status) *peer.State { return bestMatch } -func (u *upstreamResolverBase) debugUpstreamTimeout(upstream netip.AddrPort) string { - if u.statusRecorder == nil { - return "" +// haMapRouteCount returns the total number of routes across all HA +// groups in the map. route.HAMap is keyed by HAUniqueID with slices of +// routes per key, so len(hm) is the number of HA groups, not routes. +func haMapRouteCount(hm route.HAMap) int { + total := 0 + for _, routes := range hm { + total += len(routes) } - - peerInfo := findPeerForIP(upstream.Addr(), u.statusRecorder) - if peerInfo == nil { - return "" - } - - return fmt.Sprintf("(routes through NetBird peer %s)", FormatPeerStatus(peerInfo)) + return total +} + +// haMapContains checks whether ip is covered by any concrete prefix in +// the HA map. haveDynamic is reported separately: dynamic (domain-based) +// routes carry a placeholder Network that can't be prefix-checked, so we +// can't know at this point whether ip is reached through one. Callers +// decide how to interpret the unknown: health projection treats it as +// "possibly routed" to avoid emitting false-positive warnings during +// startup, while iOS dial selection requires a concrete match before +// binding to the tunnel. +func haMapContains(hm route.HAMap, ip netip.Addr) (matched, haveDynamic bool) { + for _, routes := range hm { + for _, r := range routes { + if r.IsDynamic() { + haveDynamic = true + continue + } + if r.Network.Contains(ip) { + return true, haveDynamic + } + } + } + return false, haveDynamic } diff --git a/client/internal/dns/upstream_android.go b/client/internal/dns/upstream_android.go index 988adb7d2..f7ab48b10 100644 --- a/client/internal/dns/upstream_android.go +++ b/client/internal/dns/upstream_android.go @@ -11,6 +11,7 @@ import ( "github.com/netbirdio/netbird/client/internal/peer" nbnet "github.com/netbirdio/netbird/client/net" + "github.com/netbirdio/netbird/shared/management/domain" ) type upstreamResolver struct { @@ -26,9 +27,9 @@ func newUpstreamResolver( _ WGIface, statusRecorder *peer.Status, hostsDNSHolder *hostsDNSHolder, - domain string, + d domain.Domain, ) (*upstreamResolver, error) { - upstreamResolverBase := newUpstreamResolverBase(ctx, statusRecorder, domain) + upstreamResolverBase := newUpstreamResolverBase(ctx, statusRecorder, d) c := &upstreamResolver{ upstreamResolverBase: upstreamResolverBase, hostsDNSHolder: hostsDNSHolder, diff --git a/client/internal/dns/upstream_general.go b/client/internal/dns/upstream_general.go index 910c3779e..dc841757b 100644 --- a/client/internal/dns/upstream_general.go +++ b/client/internal/dns/upstream_general.go @@ -12,6 +12,7 @@ import ( "golang.zx2c4.com/wireguard/tun/netstack" "github.com/netbirdio/netbird/client/internal/peer" + "github.com/netbirdio/netbird/shared/management/domain" ) type upstreamResolver struct { @@ -24,9 +25,9 @@ func newUpstreamResolver( wgIface WGIface, statusRecorder *peer.Status, _ *hostsDNSHolder, - domain string, + d domain.Domain, ) (*upstreamResolver, error) { - upstreamResolverBase := newUpstreamResolverBase(ctx, statusRecorder, domain) + upstreamResolverBase := newUpstreamResolverBase(ctx, statusRecorder, d) nonIOS := &upstreamResolver{ upstreamResolverBase: upstreamResolverBase, nsNet: wgIface.GetNet(), diff --git a/client/internal/dns/upstream_ios.go b/client/internal/dns/upstream_ios.go index 0e04742a0..b989bf0f9 100644 --- a/client/internal/dns/upstream_ios.go +++ b/client/internal/dns/upstream_ios.go @@ -15,6 +15,7 @@ import ( "golang.org/x/sys/unix" "github.com/netbirdio/netbird/client/internal/peer" + "github.com/netbirdio/netbird/shared/management/domain" ) type upstreamResolverIOS struct { @@ -27,9 +28,9 @@ func newUpstreamResolver( wgIface WGIface, statusRecorder *peer.Status, _ *hostsDNSHolder, - domain string, + d domain.Domain, ) (*upstreamResolverIOS, error) { - upstreamResolverBase := newUpstreamResolverBase(ctx, statusRecorder, domain) + upstreamResolverBase := newUpstreamResolverBase(ctx, statusRecorder, d) ios := &upstreamResolverIOS{ upstreamResolverBase: upstreamResolverBase, @@ -62,9 +63,16 @@ func (u *upstreamResolverIOS) exchange(ctx context.Context, upstream string, r * upstreamIP = upstreamIP.Unmap() } addr := u.wgIface.Address() + var routed bool + if u.selectedRoutes != nil { + // Only a concrete prefix match binds to the tunnel: dialing + // through a private client for an upstream we can't prove is + // routed would break public resolvers. + routed, _ = haMapContains(u.selectedRoutes(), upstreamIP) + } needsPrivate := addr.Network.Contains(upstreamIP) || addr.IPv6Net.Contains(upstreamIP) || - (u.routeMatch != nil && u.routeMatch(upstreamIP)) + routed if needsPrivate { log.Debugf("using private client to query %s via upstream %s", r.Question[0].Name, upstream) client, err = GetClientPrivate(u.wgIface, upstreamIP, timeout) @@ -73,8 +81,7 @@ func (u *upstreamResolverIOS) exchange(ctx context.Context, upstream string, r * } } - // Cannot use client.ExchangeContext because it overwrites our Dialer - return ExchangeWithFallback(nil, client, r, upstream) + return ExchangeWithFallback(ctx, client, r, upstream) } // GetClientPrivate returns a new DNS client bound to the local IP of the Netbird interface. diff --git a/client/internal/dns/upstream_test.go b/client/internal/dns/upstream_test.go index d6aec05ca..8b3c589f1 100644 --- a/client/internal/dns/upstream_test.go +++ b/client/internal/dns/upstream_test.go @@ -6,6 +6,7 @@ import ( "net" "net/netip" "strings" + "sync/atomic" "testing" "time" @@ -73,7 +74,7 @@ func TestUpstreamResolver_ServeDNS(t *testing.T) { servers = append(servers, netip.AddrPortFrom(addrPort.Addr().Unmap(), addrPort.Port())) } } - resolver.upstreamServers = servers + resolver.addRace(servers) resolver.upstreamTimeout = testCase.timeout if testCase.cancelCTX { cancel() @@ -132,20 +133,10 @@ func (m *mockNetstackProvider) GetInterfaceGUIDString() (string, error) { return "", nil } -type mockUpstreamResolver struct { - r *dns.Msg - rtt time.Duration - err error -} - -// exchange mock implementation of exchange from upstreamResolver -func (c mockUpstreamResolver) exchange(_ context.Context, _ string, _ *dns.Msg) (*dns.Msg, time.Duration, error) { - return c.r, c.rtt, c.err -} - type mockUpstreamResponse struct { - msg *dns.Msg - err error + msg *dns.Msg + err error + delay time.Duration } type mockUpstreamResolverPerServer struct { @@ -153,63 +144,19 @@ type mockUpstreamResolverPerServer struct { rtt time.Duration } -func (c mockUpstreamResolverPerServer) exchange(_ context.Context, upstream string, _ *dns.Msg) (*dns.Msg, time.Duration, error) { - if r, ok := c.responses[upstream]; ok { - return r.msg, c.rtt, r.err +func (c mockUpstreamResolverPerServer) exchange(ctx context.Context, upstream string, _ *dns.Msg) (*dns.Msg, time.Duration, error) { + r, ok := c.responses[upstream] + if !ok { + return nil, c.rtt, fmt.Errorf("no mock response for %s", upstream) } - return nil, c.rtt, fmt.Errorf("no mock response for %s", upstream) -} - -func TestUpstreamResolver_DeactivationReactivation(t *testing.T) { - mockClient := &mockUpstreamResolver{ - err: dns.ErrTime, - r: new(dns.Msg), - rtt: time.Millisecond, - } - - resolver := &upstreamResolverBase{ - ctx: context.TODO(), - upstreamClient: mockClient, - upstreamTimeout: UpstreamTimeout, - reactivatePeriod: time.Microsecond * 100, - } - addrPort, _ := netip.ParseAddrPort("0.0.0.0:1") // Use valid port for parsing, test will still fail on connection - resolver.upstreamServers = []netip.AddrPort{netip.AddrPortFrom(addrPort.Addr().Unmap(), addrPort.Port())} - - failed := false - resolver.deactivate = func(error) { - failed = true - // After deactivation, make the mock client work again - mockClient.err = nil - } - - reactivated := false - resolver.reactivate = func() { - reactivated = true - } - - resolver.ProbeAvailability(context.TODO()) - - if !failed { - t.Errorf("expected that resolving was deactivated") - return - } - - if !resolver.disabled { - t.Errorf("resolver should be Disabled") - return - } - - time.Sleep(time.Millisecond * 200) - - if !reactivated { - t.Errorf("expected that resolving was reactivated") - return - } - - if resolver.disabled { - t.Errorf("should be enabled") + if r.delay > 0 { + select { + case <-time.After(r.delay): + case <-ctx.Done(): + return nil, c.rtt, ctx.Err() + } } + return r.msg, c.rtt, r.err } func TestUpstreamResolver_Failover(t *testing.T) { @@ -339,9 +286,9 @@ func TestUpstreamResolver_Failover(t *testing.T) { resolver := &upstreamResolverBase{ ctx: ctx, upstreamClient: trackingClient, - upstreamServers: []netip.AddrPort{upstream1, upstream2}, upstreamTimeout: UpstreamTimeout, } + resolver.addRace([]netip.AddrPort{upstream1, upstream2}) var responseMSG *dns.Msg responseWriter := &test.MockResponseWriter{ @@ -421,9 +368,9 @@ func TestUpstreamResolver_SingleUpstreamFailure(t *testing.T) { resolver := &upstreamResolverBase{ ctx: ctx, upstreamClient: mockClient, - upstreamServers: []netip.AddrPort{upstream}, upstreamTimeout: UpstreamTimeout, } + resolver.addRace([]netip.AddrPort{upstream}) var responseMSG *dns.Msg responseWriter := &test.MockResponseWriter{ @@ -440,6 +387,136 @@ func TestUpstreamResolver_SingleUpstreamFailure(t *testing.T) { assert.Equal(t, dns.RcodeServerFailure, responseMSG.Rcode, "single upstream SERVFAIL should return SERVFAIL") } +// TestUpstreamResolver_RaceAcrossGroups covers two nameserver groups +// configured for the same domain, with one broken group. The merge+race +// path should answer as fast as the working group and not pay the timeout +// of the broken one on every query. +func TestUpstreamResolver_RaceAcrossGroups(t *testing.T) { + broken := netip.MustParseAddrPort("192.0.2.1:53") + working := netip.MustParseAddrPort("192.0.2.2:53") + successAnswer := "192.0.2.100" + timeoutErr := &net.OpError{Op: "read", Err: fmt.Errorf("i/o timeout")} + + mockClient := &mockUpstreamResolverPerServer{ + responses: map[string]mockUpstreamResponse{ + // Force the broken upstream to only unblock via timeout / + // cancellation so the assertion below can't pass if races + // were run serially. + broken.String(): {err: timeoutErr, delay: 500 * time.Millisecond}, + working.String(): {msg: buildMockResponse(dns.RcodeSuccess, successAnswer)}, + }, + rtt: time.Millisecond, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + resolver := &upstreamResolverBase{ + ctx: ctx, + upstreamClient: mockClient, + upstreamTimeout: 250 * time.Millisecond, + } + resolver.addRace([]netip.AddrPort{broken}) + resolver.addRace([]netip.AddrPort{working}) + + var responseMSG *dns.Msg + responseWriter := &test.MockResponseWriter{ + WriteMsgFunc: func(m *dns.Msg) error { + responseMSG = m + return nil + }, + } + + inputMSG := new(dns.Msg).SetQuestion("example.com.", dns.TypeA) + start := time.Now() + resolver.ServeDNS(responseWriter, inputMSG) + elapsed := time.Since(start) + + require.NotNil(t, responseMSG, "should write a response") + assert.Equal(t, dns.RcodeSuccess, responseMSG.Rcode) + require.NotEmpty(t, responseMSG.Answer) + assert.Contains(t, responseMSG.Answer[0].String(), successAnswer) + // Working group answers in a single RTT; the broken group's + // timeout (100ms) must not block the response. + assert.Less(t, elapsed, 100*time.Millisecond, "race must not wait for broken group's timeout") +} + +// TestUpstreamResolver_AllGroupsFail checks that when every group fails the +// resolver returns SERVFAIL rather than leaking a partial response. +func TestUpstreamResolver_AllGroupsFail(t *testing.T) { + a := netip.MustParseAddrPort("192.0.2.1:53") + b := netip.MustParseAddrPort("192.0.2.2:53") + + mockClient := &mockUpstreamResolverPerServer{ + responses: map[string]mockUpstreamResponse{ + a.String(): {msg: buildMockResponse(dns.RcodeServerFailure, "")}, + b.String(): {msg: buildMockResponse(dns.RcodeServerFailure, "")}, + }, + rtt: time.Millisecond, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + resolver := &upstreamResolverBase{ + ctx: ctx, + upstreamClient: mockClient, + upstreamTimeout: UpstreamTimeout, + } + resolver.addRace([]netip.AddrPort{a}) + resolver.addRace([]netip.AddrPort{b}) + + var responseMSG *dns.Msg + responseWriter := &test.MockResponseWriter{ + WriteMsgFunc: func(m *dns.Msg) error { + responseMSG = m + return nil + }, + } + + resolver.ServeDNS(responseWriter, new(dns.Msg).SetQuestion("example.com.", dns.TypeA)) + require.NotNil(t, responseMSG) + assert.Equal(t, dns.RcodeServerFailure, responseMSG.Rcode) +} + +// TestUpstreamResolver_HealthTracking verifies that query-path results are +// recorded into per-upstream health, which is what projects back to +// NSGroupState for status reporting. +func TestUpstreamResolver_HealthTracking(t *testing.T) { + ok := netip.MustParseAddrPort("192.0.2.10:53") + bad := netip.MustParseAddrPort("192.0.2.11:53") + + mockClient := &mockUpstreamResolverPerServer{ + responses: map[string]mockUpstreamResponse{ + ok.String(): {msg: buildMockResponse(dns.RcodeSuccess, "192.0.2.100")}, + bad.String(): {msg: buildMockResponse(dns.RcodeServerFailure, "")}, + }, + rtt: time.Millisecond, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + resolver := &upstreamResolverBase{ + ctx: ctx, + upstreamClient: mockClient, + upstreamTimeout: UpstreamTimeout, + } + resolver.addRace([]netip.AddrPort{ok, bad}) + + responseWriter := &test.MockResponseWriter{WriteMsgFunc: func(m *dns.Msg) error { return nil }} + resolver.ServeDNS(responseWriter, new(dns.Msg).SetQuestion("example.com.", dns.TypeA)) + + health := resolver.UpstreamHealth() + require.Contains(t, health, ok) + assert.False(t, health[ok].LastOk.IsZero(), "ok upstream should have LastOk set") + assert.Empty(t, health[ok].LastErr) + + // bad upstream was never tried because ok answered first; its health + // should remain unset. + assert.NotContains(t, health, bad, "sibling upstream should not be queried when primary answers") +} + func TestFormatFailures(t *testing.T) { testCases := []struct { name string @@ -665,10 +742,10 @@ func TestExchangeWithFallback_EDNS0Capped(t *testing.T) { // Verify that a client EDNS0 larger than our MTU-derived limit gets // capped in the outgoing request so the upstream doesn't send a // response larger than our read buffer. - var receivedUDPSize uint16 + var receivedUDPSize atomic.Uint32 udpHandler := dns.HandlerFunc(func(w dns.ResponseWriter, r *dns.Msg) { if opt := r.IsEdns0(); opt != nil { - receivedUDPSize = opt.UDPSize() + receivedUDPSize.Store(uint32(opt.UDPSize())) } m := new(dns.Msg) m.SetReply(r) @@ -699,7 +776,7 @@ func TestExchangeWithFallback_EDNS0Capped(t *testing.T) { require.NotNil(t, rm) expectedMax := uint16(currentMTU - ipUDPHeaderSize) - assert.Equal(t, expectedMax, receivedUDPSize, + assert.Equal(t, expectedMax, uint16(receivedUDPSize.Load()), "upstream should see capped EDNS0, not the client's 4096") } @@ -874,7 +951,7 @@ func TestUpstreamResolver_NonRetryableEDEShortCircuits(t *testing.T) { resolver := &upstreamResolverBase{ ctx: ctx, upstreamClient: tracking, - upstreamServers: []netip.AddrPort{upstream1, upstream2}, + upstreamServers: []upstreamRace{{upstream1, upstream2}}, upstreamTimeout: UpstreamTimeout, } diff --git a/client/internal/engine.go b/client/internal/engine.go index 66fe6056b..3bd0d4621 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -512,16 +512,7 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL) e.routeManager.SetRouteChangeListener(e.mobileDep.NetworkChangeListener) - e.dnsServer.SetRouteChecker(func(ip netip.Addr) bool { - for _, routes := range e.routeManager.GetSelectedClientRoutes() { - for _, r := range routes { - if r.Network.Contains(ip) { - return true - } - } - } - return false - }) + e.dnsServer.SetRouteSources(e.routeManager.GetSelectedClientRoutes, e.routeManager.GetActiveClientRoutes) if err = e.wgInterfaceCreate(); err != nil { log.Errorf("failed creating tunnel interface %s: [%s]", e.config.WgIfaceName, err.Error()) @@ -1386,9 +1377,6 @@ func (e *Engine) updateNetworkMap(networkMap *mgmProto.NetworkMap) error { e.networkSerial = serial - // Test received (upstream) servers for availability right away instead of upon usage. - // If no server of a server group responds this will disable the respective handler and retry later. - go e.dnsServer.ProbeAvailability() return nil } @@ -1932,7 +1920,7 @@ func (e *Engine) newDnsServer(dnsConfig *nbdns.Config) (dns.Server, error) { return dnsServer, nil case "ios": - dnsServer := dns.NewDefaultServerIos(e.ctx, e.wgInterface, e.mobileDep.DnsManager, e.mobileDep.HostDNSAddresses, e.statusRecorder, e.config.DisableDNS) + dnsServer := dns.NewDefaultServerIos(e.ctx, e.wgInterface, e.mobileDep.DnsManager, e.statusRecorder, e.config.DisableDNS) return dnsServer, nil default: diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index 907f1f592..839ec14c0 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -53,6 +53,7 @@ type Manager interface { GetRouteSelector() *routeselector.RouteSelector GetClientRoutes() route.HAMap GetSelectedClientRoutes() route.HAMap + GetActiveClientRoutes() route.HAMap GetClientRoutesWithNetID() map[route.NetID][]*route.Route SetRouteChangeListener(listener listener.NetworkChangeListener) InitialRouteRange() []string @@ -485,6 +486,39 @@ func (m *DefaultManager) GetSelectedClientRoutes() route.HAMap { return m.routeSelector.FilterSelectedExitNodes(maps.Clone(m.clientRoutes)) } +// GetActiveClientRoutes returns the subset of selected client routes +// that are currently reachable: the route's peer is Connected and is +// the one actively carrying the route (not just an HA sibling). +func (m *DefaultManager) GetActiveClientRoutes() route.HAMap { + m.mux.Lock() + selected := m.routeSelector.FilterSelectedExitNodes(maps.Clone(m.clientRoutes)) + recorder := m.statusRecorder + m.mux.Unlock() + + if recorder == nil { + return selected + } + + out := make(route.HAMap, len(selected)) + for id, routes := range selected { + for _, r := range routes { + st, err := recorder.GetPeer(r.Peer) + if err != nil { + continue + } + if st.ConnStatus != peer.StatusConnected { + continue + } + if _, hasRoute := st.GetRoutes()[r.Network.String()]; !hasRoute { + continue + } + out[id] = routes + break + } + } + return out +} + // GetClientRoutesWithNetID returns the current routes from the route map, but the keys consist of the network ID only func (m *DefaultManager) GetClientRoutesWithNetID() map[route.NetID][]*route.Route { m.mux.Lock() diff --git a/client/internal/routemanager/mock.go b/client/internal/routemanager/mock.go index 66b5e30dd..937314995 100644 --- a/client/internal/routemanager/mock.go +++ b/client/internal/routemanager/mock.go @@ -19,6 +19,7 @@ type MockManager struct { GetRouteSelectorFunc func() *routeselector.RouteSelector GetClientRoutesFunc func() route.HAMap GetSelectedClientRoutesFunc func() route.HAMap + GetActiveClientRoutesFunc func() route.HAMap GetClientRoutesWithNetIDFunc func() map[route.NetID][]*route.Route StopFunc func(manager *statemanager.Manager) } @@ -78,6 +79,14 @@ func (m *MockManager) GetSelectedClientRoutes() route.HAMap { return nil } +// GetActiveClientRoutes mock implementation of GetActiveClientRoutes from the Manager interface +func (m *MockManager) GetActiveClientRoutes() route.HAMap { + if m.GetActiveClientRoutesFunc != nil { + return m.GetActiveClientRoutesFunc() + } + return nil +} + // GetClientRoutesWithNetID mock implementation of GetClientRoutesWithNetID from Manager interface func (m *MockManager) GetClientRoutesWithNetID() map[route.NetID][]*route.Route { if m.GetClientRoutesWithNetIDFunc != nil { diff --git a/client/ios/NetBirdSDK/client.go b/client/ios/NetBirdSDK/client.go index 33f5ab1b0..bafbb0031 100644 --- a/client/ios/NetBirdSDK/client.go +++ b/client/ios/NetBirdSDK/client.go @@ -162,11 +162,7 @@ func (c *Client) Run(fd int32, interfaceName string, envList *EnvList) error { cfg.WgIface = interfaceName c.connectClient = internal.NewConnectClient(ctx, cfg, c.recorder) - hostDNS := []netip.AddrPort{ - netip.MustParseAddrPort("9.9.9.9:53"), - netip.MustParseAddrPort("149.112.112.112:53"), - } - return c.connectClient.RunOniOS(fd, c.networkChangeListener, c.dnsManager, hostDNS, c.stateFile) + return c.connectClient.RunOniOS(fd, c.networkChangeListener, c.dnsManager, c.stateFile) } // Stop the internal client and free the resources From e916f12cca508dfea584e7b72cf99a135acebc2b Mon Sep 17 00:00:00 2001 From: Vlad <4941176+crn4@users.noreply.github.com> Date: Fri, 15 May 2026 19:13:44 +0200 Subject: [PATCH 18/29] [proxy] auth token generation on mapping (#6157) * [management / proxy] auth token generation on mapping * fix tests --- management/internals/shared/grpc/proxy.go | 15 +++--- .../shared/grpc/proxy_snapshot_test.go | 53 +++++++++++++++++++ .../shared/grpc/validate_session_test.go | 14 +++-- 3 files changed, 73 insertions(+), 9 deletions(-) diff --git a/management/internals/shared/grpc/proxy.go b/management/internals/shared/grpc/proxy.go index 9e5027547..eada2d86a 100644 --- a/management/internals/shared/grpc/proxy.go +++ b/management/internals/shared/grpc/proxy.go @@ -394,6 +394,13 @@ func (s *ProxyServiceServer) sendSnapshot(ctx context.Context, conn *proxyConnec if end > len(mappings) { end = len(mappings) } + for _, m := range mappings[i:end] { + token, err := s.tokenStore.GenerateToken(m.AccountId, m.Id, s.proxyTokenTTL()) + if err != nil { + return fmt.Errorf("generate auth token for service %s: %w", m.Id, err) + } + m.AuthToken = token + } if err := conn.stream.Send(&proto.GetMappingUpdateResponse{ Mapping: mappings[i:end], InitialSyncComplete: end == len(mappings), @@ -425,18 +432,14 @@ func (s *ProxyServiceServer) snapshotServiceMappings(ctx context.Context, conn * return nil, fmt.Errorf("get services from store: %w", err) } + oidcCfg := s.GetOIDCValidationConfig() var mappings []*proto.ProxyMapping for _, service := range services { if !service.Enabled || service.ProxyCluster == "" || service.ProxyCluster != conn.address { continue } - token, err := s.tokenStore.GenerateToken(service.AccountID, service.ID, s.proxyTokenTTL()) - if err != nil { - return nil, fmt.Errorf("generate auth token for service %s: %w", service.ID, err) - } - - m := service.ToProtoMapping(rpservice.Create, token, s.GetOIDCValidationConfig()) + m := service.ToProtoMapping(rpservice.Create, "", oidcCfg) if !proxyAcceptsMapping(conn, m) { continue } diff --git a/management/internals/shared/grpc/proxy_snapshot_test.go b/management/internals/shared/grpc/proxy_snapshot_test.go index e0c7425c5..68d2ecfd1 100644 --- a/management/internals/shared/grpc/proxy_snapshot_test.go +++ b/management/internals/shared/grpc/proxy_snapshot_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "testing" + "time" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -172,3 +173,55 @@ func TestSendSnapshot_EmptySnapshot(t *testing.T) { assert.Empty(t, stream.messages[0].Mapping) assert.True(t, stream.messages[0].InitialSyncComplete) } + +type hookingStream struct { + grpc.ServerStream + onSend func(*proto.GetMappingUpdateResponse) +} + +func (s *hookingStream) Send(m *proto.GetMappingUpdateResponse) error { + if s.onSend != nil { + s.onSend(m) + } + return nil +} + +func (s *hookingStream) Context() context.Context { return context.Background() } +func (s *hookingStream) SetHeader(metadata.MD) error { return nil } +func (s *hookingStream) SendHeader(metadata.MD) error { return nil } +func (s *hookingStream) SetTrailer(metadata.MD) {} +func (s *hookingStream) SendMsg(any) error { return nil } +func (s *hookingStream) RecvMsg(any) error { return nil } + +func TestSendSnapshot_TokensRemainValidUnderSlowSend(t *testing.T) { + const cluster = "cluster.example.com" + const batchSize = 2 + const totalServices = 6 + const ttl = 100 * time.Millisecond + const sendDelay = 200 * time.Millisecond + + ctrl := gomock.NewController(t) + mgr := rpservice.NewMockManager(ctrl) + mgr.EXPECT().GetGlobalServices(gomock.Any()).Return(makeServices(totalServices, cluster), nil) + + s := newSnapshotTestServer(t, batchSize) + s.serviceManager = mgr + s.tokenTTL = ttl + + var validateErrs []error + stream := &hookingStream{ + onSend: func(resp *proto.GetMappingUpdateResponse) { + for _, m := range resp.Mapping { + if err := s.tokenStore.ValidateAndConsume(m.AuthToken, m.AccountId, m.Id); err != nil { + validateErrs = append(validateErrs, fmt.Errorf("svc %s: %w", m.Id, err)) + } + } + time.Sleep(sendDelay) + }, + } + conn := &proxyConnection{proxyID: "proxy-a", address: cluster, stream: stream} + + require.NoError(t, s.sendSnapshot(context.Background(), conn)) + require.Empty(t, validateErrs, + "tokens must remain valid even when batches are sent slowly: lazy per-batch generation guarantees freshness") +} diff --git a/management/internals/shared/grpc/validate_session_test.go b/management/internals/shared/grpc/validate_session_test.go index 6cd95f988..7b7ffcfb2 100644 --- a/management/internals/shared/grpc/validate_session_test.go +++ b/management/internals/shared/grpc/validate_session_test.go @@ -326,17 +326,25 @@ func (m *testValidateSessionServiceManager) GetActiveClusters(_ context.Context, return nil, nil } +func (m *testValidateSessionServiceManager) DeleteAccountCluster(_ context.Context, _, _, _ string) error { + return nil +} + type testValidateSessionProxyManager struct{} -func (m *testValidateSessionProxyManager) Connect(_ context.Context, _, _, _ string, _ *string, _ *proxy.Capabilities) error { +func (m *testValidateSessionProxyManager) Connect(_ context.Context, _, _, _, _ string, _ *string, _ *proxy.Capabilities) (*proxy.Proxy, error) { + return nil, nil +} + +func (m *testValidateSessionProxyManager) Disconnect(_ context.Context, _, _ string) error { return nil } -func (m *testValidateSessionProxyManager) Disconnect(_ context.Context, _ string) error { +func (m *testValidateSessionProxyManager) Heartbeat(_ context.Context, _ *proxy.Proxy) error { return nil } -func (m *testValidateSessionProxyManager) Heartbeat(_ context.Context, _, _, _ string) error { +func (m *testValidateSessionProxyManager) DeleteAccountCluster(_ context.Context, _, _ string) error { return nil } From 22e2519d7113dffec718198e54474cc0a6d71c87 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Sat, 16 May 2026 22:51:48 +0900 Subject: [PATCH 19/29] [management] Avoid peer IP reallocation when account settings update preserves the network range (#6173) --- management/server/account.go | 37 +++++++++++-- management/server/account_test.go | 90 +++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/management/server/account.go b/management/server/account.go index 77a46a069..e7b4acaac 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -291,10 +291,15 @@ func (am *DefaultAccountManager) UpdateAccountSettings(ctx context.Context, acco return nil, status.NewPermissionDeniedError() } + // Canonicalize the incoming range so a caller-supplied prefix with host bits + // (e.g. 100.64.1.1/16) compares equal to the masked form stored on network.Net. + newSettings.NetworkRange = newSettings.NetworkRange.Masked() + var oldSettings *types.Settings var updateAccountPeers bool var groupChangesAffectPeers bool var reloadReverseProxy bool + var effectiveOldNetworkRange netip.Prefix err = am.Store.ExecuteInTransaction(ctx, func(transaction store.Store) error { var groupsUpdated bool @@ -308,6 +313,16 @@ func (am *DefaultAccountManager) UpdateAccountSettings(ctx context.Context, acco return err } + // No lock: the transaction already holds Settings(Update), and network.Net is + // only mutated by reallocateAccountPeerIPs, which is reachable only through + // this same code path. A Share lock here would extend an unnecessary row lock + // and complicate ordering against updatePeerIPv6InTransaction. + network, err := transaction.GetAccountNetwork(ctx, store.LockingStrengthNone, accountID) + if err != nil { + return fmt.Errorf("get account network: %w", err) + } + effectiveOldNetworkRange = prefixFromIPNet(network.Net) + if oldSettings.Extra != nil && newSettings.Extra != nil && oldSettings.Extra.PeerApprovalEnabled && !newSettings.Extra.PeerApprovalEnabled { approvedCount, err := transaction.ApproveAccountPeers(ctx, accountID) @@ -321,7 +336,7 @@ func (am *DefaultAccountManager) UpdateAccountSettings(ctx context.Context, acco } } - if oldSettings.NetworkRange != newSettings.NetworkRange { + if newSettings.NetworkRange.IsValid() && newSettings.NetworkRange != effectiveOldNetworkRange { if err = am.reallocateAccountPeerIPs(ctx, transaction, accountID, newSettings.NetworkRange); err != nil { return err } @@ -396,9 +411,9 @@ func (am *DefaultAccountManager) UpdateAccountSettings(ctx context.Context, acco } am.StoreEvent(ctx, userID, accountID, accountID, activity.AccountDNSDomainUpdated, eventMeta) } - if oldSettings.NetworkRange != newSettings.NetworkRange { + if newSettings.NetworkRange.IsValid() && newSettings.NetworkRange != effectiveOldNetworkRange { eventMeta := map[string]any{ - "old_network_range": oldSettings.NetworkRange.String(), + "old_network_range": effectiveOldNetworkRange.String(), "new_network_range": newSettings.NetworkRange.String(), } am.StoreEvent(ctx, userID, accountID, accountID, activity.AccountNetworkRangeUpdated, eventMeta) @@ -443,6 +458,22 @@ func ipv6SettingsChanged(old, updated *types.Settings) bool { return !slices.Equal(oldGroups, newGroups) } +// prefixFromIPNet returns the overlay prefix actually allocated on the account +// network, or an invalid prefix if none is set. Settings.NetworkRange is a +// user-facing override that is empty on legacy accounts, so the effective +// range must be read from network.Net to compare against an incoming update. +func prefixFromIPNet(ipNet net.IPNet) netip.Prefix { + if ipNet.IP == nil { + return netip.Prefix{} + } + addr, ok := netip.AddrFromSlice(ipNet.IP) + if !ok { + return netip.Prefix{} + } + ones, _ := ipNet.Mask.Size() + return netip.PrefixFrom(addr.Unmap(), ones) +} + func (am *DefaultAccountManager) validateSettingsUpdate(ctx context.Context, transaction store.Store, newSettings, oldSettings *types.Settings, userID, accountID string) error { halfYearLimit := 180 * 24 * time.Hour if newSettings.PeerLoginExpiration > halfYearLimit { diff --git a/management/server/account_test.go b/management/server/account_test.go index 65b27df49..60720faa6 100644 --- a/management/server/account_test.go +++ b/management/server/account_test.go @@ -3970,6 +3970,96 @@ func TestDefaultAccountManager_UpdateAccountSettings_NetworkRangeChange(t *testi } } +// TestDefaultAccountManager_UpdateAccountSettings_NetworkRangePreserved guards against +// peer IP reallocation when a settings update carries the network range that is already +// in use. Legacy accounts have Settings.NetworkRange unset in the DB while network.Net +// holds the actual allocated overlay; the dashboard backfills the GET response from +// network.Net and echoes the value back on PUT, so the diff must be against the +// effective range to avoid renumbering every peer on an unrelated settings change. +func TestDefaultAccountManager_UpdateAccountSettings_NetworkRangePreserved(t *testing.T) { + manager, _, account, peer1, peer2, peer3 := setupNetworkMapTest(t) + ctx := context.Background() + + settings, err := manager.Store.GetAccountSettings(ctx, store.LockingStrengthNone, account.Id) + require.NoError(t, err) + require.False(t, settings.NetworkRange.IsValid(), "precondition: new accounts leave Settings.NetworkRange unset") + + network, err := manager.Store.GetAccountNetwork(ctx, store.LockingStrengthNone, account.Id) + require.NoError(t, err) + require.NotNil(t, network.Net.IP, "precondition: network.Net should be allocated") + addr, ok := netip.AddrFromSlice(network.Net.IP) + require.True(t, ok) + ones, _ := network.Net.Mask.Size() + effective := netip.PrefixFrom(addr.Unmap(), ones) + require.True(t, effective.IsValid()) + + before := map[string]netip.Addr{peer1.ID: peer1.IP, peer2.ID: peer2.IP, peer3.ID: peer3.IP} + + // Round-trip the effective range as if the dashboard echoed back the GET-backfilled value. + _, err = manager.UpdateAccountSettings(ctx, account.Id, userID, &types.Settings{ + PeerLoginExpirationEnabled: true, + PeerLoginExpiration: types.DefaultPeerLoginExpiration, + NetworkRange: effective, + Extra: &types.ExtraSettings{}, + }) + require.NoError(t, err) + + peers, err := manager.Store.GetAccountPeers(ctx, store.LockingStrengthNone, account.Id, "", "") + require.NoError(t, err) + require.Len(t, peers, len(before)) + for _, p := range peers { + assert.Equal(t, before[p.ID], p.IP, "peer %s IP should not change when range matches effective", p.ID) + } + + // Carrying the same range with host bits set must also be a no-op once canonicalized. + hostBitsForm := netip.PrefixFrom(peer1.IP, ones) + require.NotEqual(t, effective, hostBitsForm, "precondition: host-bit form should differ before masking") + _, err = manager.UpdateAccountSettings(ctx, account.Id, userID, &types.Settings{ + PeerLoginExpirationEnabled: true, + PeerLoginExpiration: types.DefaultPeerLoginExpiration, + NetworkRange: hostBitsForm, + Extra: &types.ExtraSettings{}, + }) + require.NoError(t, err) + + peers, err = manager.Store.GetAccountPeers(ctx, store.LockingStrengthNone, account.Id, "", "") + require.NoError(t, err) + for _, p := range peers { + assert.Equal(t, before[p.ID], p.IP, "peer %s IP should not change for host-bit-set equivalent range", p.ID) + } + + // Omitting NetworkRange (invalid prefix) must also be a no-op. + _, err = manager.UpdateAccountSettings(ctx, account.Id, userID, &types.Settings{ + PeerLoginExpirationEnabled: true, + PeerLoginExpiration: types.DefaultPeerLoginExpiration, + Extra: &types.ExtraSettings{}, + }) + require.NoError(t, err) + + peers, err = manager.Store.GetAccountPeers(ctx, store.LockingStrengthNone, account.Id, "", "") + require.NoError(t, err) + for _, p := range peers { + assert.Equal(t, before[p.ID], p.IP, "peer %s IP should not change when NetworkRange omitted", p.ID) + } + + // Sanity: an actually different range still triggers reallocation. + newRange := netip.MustParsePrefix("100.99.0.0/16") + _, err = manager.UpdateAccountSettings(ctx, account.Id, userID, &types.Settings{ + PeerLoginExpirationEnabled: true, + PeerLoginExpiration: types.DefaultPeerLoginExpiration, + NetworkRange: newRange, + Extra: &types.ExtraSettings{}, + }) + require.NoError(t, err) + + peers, err = manager.Store.GetAccountPeers(ctx, store.LockingStrengthNone, account.Id, "", "") + require.NoError(t, err) + for _, p := range peers { + assert.True(t, newRange.Contains(p.IP), "peer %s should be in new range %s, got %s", p.ID, newRange, p.IP) + assert.NotEqual(t, before[p.ID], p.IP, "peer %s IP should change on real range update", p.ID) + } +} + func TestDefaultAccountManager_UpdateAccountSettings_IPv6EnabledGroups(t *testing.T) { manager, _, account, peer1, peer2, peer3 := setupNetworkMapTest(t) ctx := context.Background() From 347c5bf317794729a044ce9f866f29e357d386d9 Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Sat, 16 May 2026 16:29:01 +0200 Subject: [PATCH 20/29] Avoid context cancellation in `cancelPeerRoutines` (#6175) When closing go routines and handling peer disconnect, we should avoid canceling the flow due to parent gRPC context cancellation. This change triggers disconnection handling with a context that is not bound to the parent gRPC cancellation. --- management/internals/shared/grpc/server.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/management/internals/shared/grpc/server.go b/management/internals/shared/grpc/server.go index 70024bac6..1d8234304 100644 --- a/management/internals/shared/grpc/server.go +++ b/management/internals/shared/grpc/server.go @@ -522,10 +522,11 @@ func (s *Server) sendJob(ctx context.Context, peerKey wgtypes.Key, job *job.Even } func (s *Server) cancelPeerRoutines(ctx context.Context, accountID string, peer *nbpeer.Peer, streamStartTime time.Time) { - unlock := s.acquirePeerLockByUID(ctx, peer.Key) + uncanceledCTX := context.WithoutCancel(ctx) + unlock := s.acquirePeerLockByUID(uncanceledCTX, peer.Key) defer unlock() - s.cancelPeerRoutinesWithoutLock(ctx, accountID, peer, streamStartTime) + s.cancelPeerRoutinesWithoutLock(uncanceledCTX, accountID, peer, streamStartTime) } func (s *Server) cancelPeerRoutinesWithoutLock(ctx context.Context, accountID string, peer *nbpeer.Peer, streamStartTime time.Time) { From 3f91f49277e1841bdfccda06ae7baa0430e6de2e Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Sat, 16 May 2026 23:52:57 +0900 Subject: [PATCH 21/29] Clean up legacy 32-bit and HKCU registry entries on Windows install (#6176) --- client/installer.nsis | 23 ++++++++++++++++++----- client/netbird.wxs | 25 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/client/installer.nsis b/client/installer.nsis index 63bff1c5b..3e057df10 100644 --- a/client/installer.nsis +++ b/client/installer.nsis @@ -260,15 +260,23 @@ WriteRegStr ${REG_ROOT} "${UNINSTALL_PATH}" "Publisher" "${COMP_NAME}" WriteRegStr ${REG_ROOT} "${UI_REG_APP_PATH}" "" "$INSTDIR\${UI_APP_EXE}" -; Create autostart registry entry based on checkbox +; Drop Run, App Paths and Uninstall entries left in the 32-bit registry view +; or HKCU by legacy installers. +DetailPrint "Cleaning legacy 32-bit / HKCU entries..." +DeleteRegValue HKCU "${AUTOSTART_REG_KEY}" "${APP_NAME}" +SetRegView 32 +DeleteRegValue HKLM "${AUTOSTART_REG_KEY}" "${APP_NAME}" +DeleteRegKey HKLM "${REG_APP_PATH}" +DeleteRegKey HKLM "${UI_REG_APP_PATH}" +DeleteRegKey HKLM "${UNINSTALL_PATH}" +SetRegView 64 + DetailPrint "Autostart enabled: $AutostartEnabled" ${If} $AutostartEnabled == "1" WriteRegStr HKLM "${AUTOSTART_REG_KEY}" "${APP_NAME}" '"$INSTDIR\${UI_APP_EXE}.exe"' DetailPrint "Added autostart registry entry: $INSTDIR\${UI_APP_EXE}.exe" ${Else} DeleteRegValue HKLM "${AUTOSTART_REG_KEY}" "${APP_NAME}" - ; Legacy: pre-HKLM installs wrote to HKCU; clean that up too. - DeleteRegValue HKCU "${AUTOSTART_REG_KEY}" "${APP_NAME}" DetailPrint "Autostart not enabled by user" ${EndIf} @@ -299,11 +307,16 @@ ExecWait '"$INSTDIR\${MAIN_APP_EXE}" service uninstall' DetailPrint "Terminating Netbird UI process..." ExecWait `taskkill /im ${UI_APP_EXE}.exe /f` -; Remove autostart registry entry +; Remove autostart entries from every view a previous installer may have used. DetailPrint "Removing autostart registry entry if exists..." DeleteRegValue HKLM "${AUTOSTART_REG_KEY}" "${APP_NAME}" -; Legacy: pre-HKLM installs wrote to HKCU; clean that up too. DeleteRegValue HKCU "${AUTOSTART_REG_KEY}" "${APP_NAME}" +SetRegView 32 +DeleteRegValue HKLM "${AUTOSTART_REG_KEY}" "${APP_NAME}" +DeleteRegKey HKLM "${REG_APP_PATH}" +DeleteRegKey HKLM "${UI_REG_APP_PATH}" +DeleteRegKey HKLM "${UNINSTALL_PATH}" +SetRegView 64 ; Handle data deletion based on checkbox DetailPrint "Checking if user requested data deletion..." diff --git a/client/netbird.wxs b/client/netbird.wxs index 6f18b63b5..96814ce52 100644 --- a/client/netbird.wxs +++ b/client/netbird.wxs @@ -64,6 +64,13 @@ + + + + + @@ -76,10 +83,28 @@ + + + + + + + + + + + From 705f87fc20d4410fd8e21986725b2063f72d864d Mon Sep 17 00:00:00 2001 From: Nicolas Frati Date: Mon, 18 May 2026 12:57:59 +0200 Subject: [PATCH 22/29] [management] fix: device redirect uri wasn't registered (#6191) * fix: device redirect uri wasn't registered * fix lint --- management/server/idp/embedded.go | 27 ++++++++++++++++++++----- management/server/idp/embedded_test.go | 28 ++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/management/server/idp/embedded.go b/management/server/idp/embedded.go index a1852a8bc..821e6ff55 100644 --- a/management/server/idp/embedded.go +++ b/management/server/idp/embedded.go @@ -6,7 +6,9 @@ import ( "errors" "fmt" "net/http" + "net/url" "os" + "path" "strings" "github.com/dexidp/dex/storage" @@ -138,10 +140,13 @@ func (c *EmbeddedIdPConfig) ToYAMLConfig() (*dex.YAMLConfig, error) { return nil, fmt.Errorf("invalid IdP storage config: %w", err) } - // Build CLI redirect URIs including the device callback (both relative and absolute) + // Build CLI redirect URIs including the device callback. Dex uses the issuer-relative + // path (for example, /oauth2/device/callback) when completing the device flow, so + // include it explicitly in addition to the legacy bare path and absolute URL. cliRedirectURIs := c.CLIRedirectURIs cliRedirectURIs = append(cliRedirectURIs, "/device/callback") - cliRedirectURIs = append(cliRedirectURIs, c.Issuer+"/device/callback") + cliRedirectURIs = append(cliRedirectURIs, issuerRelativeDeviceCallback(c.Issuer)) + cliRedirectURIs = append(cliRedirectURIs, strings.TrimSuffix(c.Issuer, "/")+"/device/callback") // Build dashboard redirect URIs including the OAuth callback for proxy authentication dashboardRedirectURIs := c.DashboardRedirectURIs @@ -154,6 +159,10 @@ func (c *EmbeddedIdPConfig) ToYAMLConfig() (*dex.YAMLConfig, error) { // MGMT api and the dashboard, adding baseURL means less configuration for the instance admin dashboardPostLogoutRedirectURIs = append(dashboardPostLogoutRedirectURIs, baseURL) + redirectURIs := make([]string, 0) + redirectURIs = append(redirectURIs, cliRedirectURIs...) + redirectURIs = append(redirectURIs, dashboardRedirectURIs...) + cfg := &dex.YAMLConfig{ Issuer: c.Issuer, Storage: dex.Storage{ @@ -179,14 +188,14 @@ func (c *EmbeddedIdPConfig) ToYAMLConfig() (*dex.YAMLConfig, error) { ID: staticClientDashboard, Name: "NetBird Dashboard", Public: true, - RedirectURIs: dashboardRedirectURIs, + RedirectURIs: redirectURIs, PostLogoutRedirectURIs: sanitizePostLogoutRedirectURIs(dashboardPostLogoutRedirectURIs), }, { ID: staticClientCLI, Name: "NetBird CLI", Public: true, - RedirectURIs: cliRedirectURIs, + RedirectURIs: redirectURIs, }, }, StaticConnectors: c.StaticConnectors, @@ -217,6 +226,14 @@ func (c *EmbeddedIdPConfig) ToYAMLConfig() (*dex.YAMLConfig, error) { return cfg, nil } +func issuerRelativeDeviceCallback(issuer string) string { + u, err := url.Parse(issuer) + if err != nil || u.Path == "" { + return "/device/callback" + } + return path.Join(u.Path, "/device/callback") +} + // Due to how the frontend generates the logout, sometimes it appends a trailing slash // and because Dex only allows exact matches, we need to make sure we always have both // versions of each provided uri @@ -299,7 +316,7 @@ func resolveSessionCookieEncryptionKey(configuredKey string) (string, error) { } } - return "", fmt.Errorf("invalid embedded IdP session cookie encryption key: %s (or sessionCookieEncryptionKey) must be 16, 24, or 32 bytes as a raw string or base64-encoded to one of those lengths; got %d raw bytes", sessionCookieEncryptionKeyEnv, len([]byte(key))) + return "", fmt.Errorf("invalid embedded IdP session cookie encryption key:%s (or sessionCookieEncryptionKey) must be 16, 24, or 32 bytes as a raw string or base64-encoded to one of those lengths; got %d raw bytes", sessionCookieEncryptionKeyEnv, len([]byte(key))) } func validSessionCookieEncryptionKeyLength(length int) bool { diff --git a/management/server/idp/embedded_test.go b/management/server/idp/embedded_test.go index 09dc67614..91cd27aee 100644 --- a/management/server/idp/embedded_test.go +++ b/management/server/idp/embedded_test.go @@ -314,6 +314,34 @@ func TestEmbeddedIdPManager_UpdateUserPassword(t *testing.T) { }) } +func TestEmbeddedIdPConfig_ToYAMLConfig_IncludesDeviceCallbackRedirectURI(t *testing.T) { + config := &EmbeddedIdPConfig{ + Enabled: true, + Issuer: "https://example.com/oauth2", + Storage: EmbeddedStorageConfig{ + Type: "sqlite3", + Config: EmbeddedStorageTypeConfig{ + File: filepath.Join(t.TempDir(), "dex.db"), + }, + }, + } + + yamlConfig, err := config.ToYAMLConfig() + require.NoError(t, err) + + var cliRedirectURIs []string + for _, client := range yamlConfig.StaticClients { + if client.ID == staticClientCLI { + cliRedirectURIs = client.RedirectURIs + break + } + } + require.NotEmpty(t, cliRedirectURIs) + assert.Contains(t, cliRedirectURIs, "/device/callback") + assert.Contains(t, cliRedirectURIs, "/oauth2/device/callback") + assert.Contains(t, cliRedirectURIs, "https://example.com/oauth2/device/callback") +} + func TestEmbeddedIdPConfig_ToYAMLConfig_SessionCookieEncryptionKey(t *testing.T) { t.Setenv(sessionCookieEncryptionKeyEnv, "") From 13d32d274f74b700557f8f6a615f56be2ab9c6a5 Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Mon, 18 May 2026 20:25:12 +0200 Subject: [PATCH 23/29] [management] Fence peer status updates with a session token (#6193) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [management] Fence peer status updates with a session token The connect/disconnect path used a best-effort LastSeen-after-streamStart comparison to decide whether a status update should land. Under contention — a re-sync arriving while the previous stream's disconnect was still in flight, or two management replicas seeing the same peer at once — the check was a read-then-decide-then-write window: any UPDATE in between caused the wrong row to be written. The Go-side time.Now() that fed the comparison also drifted under lock contention, since it was captured seconds before the write actually committed. Replace it with an integer-nanosecond fencing token stored alongside the status. Every gRPC sync stream uses its open time (UnixNano) as its token. Connects only land when the incoming token is strictly greater than the stored one; disconnects only land when the incoming token equals the stored one (i.e. we're the stream that owns the current session). Both are single optimistic-locked UPDATEs — no read-then-write, no transaction wrapper. LastSeen is now written by the database itself (CURRENT_TIMESTAMP). The caller never supplies it, so the value always reflects the real moment of the UPDATE rather than the moment the caller queued the work — which was already off by minutes under heavy lock contention. Side effects (geo lookup, peer-login-expiration scheduling, network-map fan-out) are explicitly documented as running after the fence UPDATE commits, never inside it. Geo also skips the update when realIP equals the stored ConnectionIP, dropping a redundant SavePeerLocation call on same-IP reconnects. Tests cover the three semantic cases (matched disconnect lands, stale disconnect dropped, stale connect dropped) plus a 16-goroutine race test that asserts the highest token always wins. * [management] Add SessionStartedAt to peer status updates Stored `SessionStartedAt` for fencing token propagation across goroutines and updated database queries/functions to handle the new field. Removed outdated geolocation handling logic and adjusted tests for concurrency safety. * Rename `peer_status_required_approval` to `peer_status_requires_approval` in SQL store fields --- management/server/account.go | 29 ++-- management/server/account/manager.go | 3 +- management/server/account/manager_mock.go | 22 ++- management/server/account_test.go | 115 ++++++++++++--- management/server/mock_server/account_mock.go | 24 +++- management/server/peer.go | 131 +++++++++--------- management/server/peer/peer.go | 19 ++- management/server/store/sql_store.go | 84 ++++++++++- management/server/store/store.go | 15 ++ management/server/store/store_mock.go | 30 ++++ 10 files changed, 354 insertions(+), 118 deletions(-) diff --git a/management/server/account.go b/management/server/account.go index e7b4acaac..8e4e595f0 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -1868,35 +1868,32 @@ func domainIsUpToDate(domain string, domainCategory string, userAuth auth.UserAu return domainCategory == types.PrivateCategory || userAuth.DomainCategory != types.PrivateCategory || domain != userAuth.Domain } +// SyncAndMarkPeer is the per-Sync entry point: it refreshes the peer's +// network map and then marks the peer connected with a session token +// derived from syncTime (the moment the gRPC stream opened). Any +// concurrent stream that started earlier loses the optimistic-lock race +// in MarkPeerConnected and bails without writing. func (am *DefaultAccountManager) SyncAndMarkPeer(ctx context.Context, accountID string, peerPubKey string, meta nbpeer.PeerSystemMeta, realIP net.IP, syncTime time.Time) (*nbpeer.Peer, *types.NetworkMap, []*posture.Checks, int64, error) { peer, netMap, postureChecks, dnsfwdPort, err := am.SyncPeer(ctx, types.PeerSync{WireGuardPubKey: peerPubKey, Meta: meta}, accountID) if err != nil { return nil, nil, nil, 0, fmt.Errorf("error syncing peer: %w", err) } - err = am.MarkPeerConnected(ctx, peerPubKey, true, realIP, accountID, syncTime) - if err != nil { + if err := am.MarkPeerConnected(ctx, peerPubKey, realIP, accountID, syncTime.UnixNano()); err != nil { log.WithContext(ctx).Warnf("failed marking peer as connected %s %v", peerPubKey, err) } return peer, netMap, postureChecks, dnsfwdPort, nil } +// OnPeerDisconnected is invoked when a sync stream ends. It marks the +// peer disconnected only when the stored SessionStartedAt matches the +// nanosecond token derived from streamStartTime — i.e. only when this +// is the stream that currently owns the peer's session. A mismatch +// means a newer stream has already replaced us, so the disconnect is +// dropped. func (am *DefaultAccountManager) OnPeerDisconnected(ctx context.Context, accountID string, peerPubKey string, streamStartTime time.Time) error { - peer, err := am.Store.GetPeerByPeerPubKey(ctx, store.LockingStrengthNone, peerPubKey) - if err != nil { - log.WithContext(ctx).Warnf("failed to get peer %s for disconnect check: %v", peerPubKey, err) - return nil - } - - if peer.Status.LastSeen.After(streamStartTime) { - log.WithContext(ctx).Tracef("peer %s has newer activity (lastSeen=%s > streamStart=%s), skipping disconnect", - peerPubKey, peer.Status.LastSeen.Format(time.RFC3339), streamStartTime.Format(time.RFC3339)) - return nil - } - - err = am.MarkPeerConnected(ctx, peerPubKey, false, nil, accountID, time.Now().UTC()) - if err != nil { + if err := am.MarkPeerDisconnected(ctx, peerPubKey, accountID, streamStartTime.UnixNano()); err != nil { log.WithContext(ctx).Warnf("failed marking peer as disconnected %s %v", peerPubKey, err) } return nil diff --git a/management/server/account/manager.go b/management/server/account/manager.go index 71af0645c..ae3de8d79 100644 --- a/management/server/account/manager.go +++ b/management/server/account/manager.go @@ -61,7 +61,8 @@ type Manager interface { GetUserFromUserAuth(ctx context.Context, userAuth auth.UserAuth) (*types.User, error) ListUsers(ctx context.Context, accountID string) ([]*types.User, error) GetPeers(ctx context.Context, accountID, userID, nameFilter, ipFilter string) ([]*nbpeer.Peer, error) - MarkPeerConnected(ctx context.Context, peerKey string, connected bool, realIP net.IP, accountID string, syncTime time.Time) error + MarkPeerConnected(ctx context.Context, peerKey string, realIP net.IP, accountID string, sessionStartedAt int64) error + MarkPeerDisconnected(ctx context.Context, peerKey string, accountID string, sessionStartedAt int64) error DeletePeer(ctx context.Context, accountID, peerID, userID string) error UpdatePeer(ctx context.Context, accountID, userID string, p *nbpeer.Peer) (*nbpeer.Peer, error) UpdatePeerIP(ctx context.Context, accountID, userID, peerID string, newIP netip.Addr) error diff --git a/management/server/account/manager_mock.go b/management/server/account/manager_mock.go index 7ffc41d73..0486e63ec 100644 --- a/management/server/account/manager_mock.go +++ b/management/server/account/manager_mock.go @@ -1305,17 +1305,31 @@ func (mr *MockManagerMockRecorder) LoginPeer(ctx, login interface{}) *gomock.Cal } // MarkPeerConnected mocks base method. -func (m *MockManager) MarkPeerConnected(ctx context.Context, peerKey string, connected bool, realIP net.IP, accountID string, syncTime time.Time) error { +func (m *MockManager) MarkPeerConnected(ctx context.Context, peerKey string, realIP net.IP, accountID string, sessionStartedAt int64) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MarkPeerConnected", ctx, peerKey, connected, realIP, accountID, syncTime) + ret := m.ctrl.Call(m, "MarkPeerConnected", ctx, peerKey, realIP, accountID, sessionStartedAt) ret0, _ := ret[0].(error) return ret0 } // MarkPeerConnected indicates an expected call of MarkPeerConnected. -func (mr *MockManagerMockRecorder) MarkPeerConnected(ctx, peerKey, connected, realIP, accountID, syncTime interface{}) *gomock.Call { +func (mr *MockManagerMockRecorder) MarkPeerConnected(ctx, peerKey, realIP, accountID, sessionStartedAt interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkPeerConnected", reflect.TypeOf((*MockManager)(nil).MarkPeerConnected), ctx, peerKey, connected, realIP, accountID, syncTime) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkPeerConnected", reflect.TypeOf((*MockManager)(nil).MarkPeerConnected), ctx, peerKey, realIP, accountID, sessionStartedAt) +} + +// MarkPeerDisconnected mocks base method. +func (m *MockManager) MarkPeerDisconnected(ctx context.Context, peerKey string, accountID string, sessionStartedAt int64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkPeerDisconnected", ctx, peerKey, accountID, sessionStartedAt) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkPeerDisconnected indicates an expected call of MarkPeerDisconnected. +func (mr *MockManagerMockRecorder) MarkPeerDisconnected(ctx, peerKey, accountID, sessionStartedAt interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkPeerDisconnected", reflect.TypeOf((*MockManager)(nil).MarkPeerDisconnected), ctx, peerKey, accountID, sessionStartedAt) } // OnPeerDisconnected mocks base method. diff --git a/management/server/account_test.go b/management/server/account_test.go index 60720faa6..ba621030c 100644 --- a/management/server/account_test.go +++ b/management/server/account_test.go @@ -1813,7 +1813,7 @@ func TestDefaultAccountManager_UpdatePeer_PeerLoginExpiration(t *testing.T) { accountID, err := manager.GetAccountIDByUserID(context.Background(), auth.UserAuth{UserId: userID}) require.NoError(t, err, "unable to get the account") - err = manager.MarkPeerConnected(context.Background(), key.PublicKey().String(), true, nil, accountID, time.Now().UTC()) + err = manager.MarkPeerConnected(context.Background(), key.PublicKey().String(), nil, accountID, time.Now().UTC().UnixNano()) require.NoError(t, err, "unable to mark peer connected") _, err = manager.UpdateAccountSettings(context.Background(), accountID, userID, &types.Settings{ @@ -1884,7 +1884,7 @@ func TestDefaultAccountManager_MarkPeerConnected_PeerLoginExpiration(t *testing. require.NoError(t, err, "unable to get the account") // when we mark peer as connected, the peer login expiration routine should trigger - err = manager.MarkPeerConnected(context.Background(), key.PublicKey().String(), true, nil, accountID, time.Now().UTC()) + err = manager.MarkPeerConnected(context.Background(), key.PublicKey().String(), nil, accountID, time.Now().UTC().UnixNano()) require.NoError(t, err, "unable to mark peer connected") failed := waitTimeout(wg, time.Second) @@ -1910,15 +1910,16 @@ func TestDefaultAccountManager_OnPeerDisconnected_LastSeenCheck(t *testing.T) { }, false) require.NoError(t, err, "unable to add peer") - t.Run("disconnect peer when streamStartTime is after LastSeen", func(t *testing.T) { - err = manager.MarkPeerConnected(context.Background(), peerPubKey, true, nil, accountID, time.Now().UTC()) + t.Run("disconnect peer when session token matches", func(t *testing.T) { + streamStartTime := time.Now().UTC() + err = manager.MarkPeerConnected(context.Background(), peerPubKey, nil, accountID, streamStartTime.UnixNano()) require.NoError(t, err, "unable to mark peer connected") peer, err := manager.Store.GetPeerByPeerPubKey(context.Background(), store.LockingStrengthNone, peerPubKey) require.NoError(t, err, "unable to get peer") require.True(t, peer.Status.Connected, "peer should be connected") - - streamStartTime := time.Now().UTC() + require.Equal(t, streamStartTime.UnixNano(), peer.Status.SessionStartedAt, + "SessionStartedAt should equal the token we passed in") err = manager.OnPeerDisconnected(context.Background(), accountID, peerPubKey, streamStartTime) require.NoError(t, err) @@ -1926,49 +1927,127 @@ func TestDefaultAccountManager_OnPeerDisconnected_LastSeenCheck(t *testing.T) { peer, err = manager.Store.GetPeerByPeerPubKey(context.Background(), store.LockingStrengthNone, peerPubKey) require.NoError(t, err) require.False(t, peer.Status.Connected, "peer should be disconnected") + require.Equal(t, int64(0), peer.Status.SessionStartedAt, "SessionStartedAt should be reset to 0") }) - t.Run("skip disconnect when LastSeen is after streamStartTime (zombie stream protection)", func(t *testing.T) { - err = manager.MarkPeerConnected(context.Background(), peerPubKey, true, nil, accountID, time.Now().UTC()) + t.Run("skip disconnect when stored session is newer (zombie stream protection)", func(t *testing.T) { + // Newer stream wins on connect (sets SessionStartedAt = now ns). + streamStartTime := time.Now().UTC() + err = manager.MarkPeerConnected(context.Background(), peerPubKey, nil, accountID, streamStartTime.UnixNano()) require.NoError(t, err, "unable to mark peer connected") peer, err := manager.Store.GetPeerByPeerPubKey(context.Background(), store.LockingStrengthNone, peerPubKey) require.NoError(t, err) require.True(t, peer.Status.Connected, "peer should be connected") - streamStartTime := peer.Status.LastSeen.Add(-1 * time.Hour) + // Older stream tries to mark disconnect with its own (older) session token — + // fencing kicks in and the write is dropped. + staleStreamStartTime := streamStartTime.Add(-1 * time.Hour) - err = manager.OnPeerDisconnected(context.Background(), accountID, peerPubKey, streamStartTime) + err = manager.OnPeerDisconnected(context.Background(), accountID, peerPubKey, staleStreamStartTime) require.NoError(t, err) peer, err = manager.Store.GetPeerByPeerPubKey(context.Background(), store.LockingStrengthNone, peerPubKey) require.NoError(t, err) require.True(t, peer.Status.Connected, - "peer should remain connected because LastSeen > streamStartTime (zombie stream protection)") + "peer should remain connected because the stored session is newer than the disconnect token") + require.Equal(t, streamStartTime.UnixNano(), peer.Status.SessionStartedAt, + "SessionStartedAt should still hold the winning stream's token") }) - t.Run("skip stale connect when peer already has newer LastSeen (blocked goroutine protection)", func(t *testing.T) { + t.Run("skip stale connect when stored session is newer (blocked goroutine protection)", func(t *testing.T) { node2SyncTime := time.Now().UTC() - err = manager.MarkPeerConnected(context.Background(), peerPubKey, true, nil, accountID, node2SyncTime) + err = manager.MarkPeerConnected(context.Background(), peerPubKey, nil, accountID, node2SyncTime.UnixNano()) require.NoError(t, err, "node 2 should connect peer") peer, err := manager.Store.GetPeerByPeerPubKey(context.Background(), store.LockingStrengthNone, peerPubKey) require.NoError(t, err) require.True(t, peer.Status.Connected, "peer should be connected") - require.Equal(t, node2SyncTime.Unix(), peer.Status.LastSeen.Unix(), "LastSeen should be node2SyncTime") + require.Equal(t, node2SyncTime.UnixNano(), peer.Status.SessionStartedAt, + "SessionStartedAt should equal node2SyncTime token") node1StaleSyncTime := node2SyncTime.Add(-1 * time.Minute) - err = manager.MarkPeerConnected(context.Background(), peerPubKey, true, nil, accountID, node1StaleSyncTime) + err = manager.MarkPeerConnected(context.Background(), peerPubKey, nil, accountID, node1StaleSyncTime.UnixNano()) require.NoError(t, err, "stale connect should not return error") peer, err = manager.Store.GetPeerByPeerPubKey(context.Background(), store.LockingStrengthNone, peerPubKey) require.NoError(t, err) require.True(t, peer.Status.Connected, "peer should still be connected") - require.Equal(t, node2SyncTime.Unix(), peer.Status.LastSeen.Unix(), - "LastSeen should NOT be overwritten by stale syncTime from blocked goroutine") + require.Equal(t, node2SyncTime.UnixNano(), peer.Status.SessionStartedAt, + "SessionStartedAt should NOT be overwritten by stale token from blocked goroutine") }) } +// TestDefaultAccountManager_MarkPeerConnected_ConcurrentRace exercises the +// fencing protocol under contention: many goroutines race to mark the +// same peer connected with distinct session tokens at the same time. +// The contract is that the highest token always wins and is what remains +// in the store, regardless of execution order. +func TestDefaultAccountManager_MarkPeerConnected_ConcurrentRace(t *testing.T) { + manager, _, err := createManager(t) + require.NoError(t, err, "unable to create account manager") + + accountID, err := manager.GetAccountIDByUserID(context.Background(), auth.UserAuth{UserId: userID}) + require.NoError(t, err, "unable to get account") + + key, err := wgtypes.GenerateKey() + require.NoError(t, err, "unable to generate WireGuard key") + peerPubKey := key.PublicKey().String() + + _, _, _, err = manager.AddPeer(context.Background(), "", "", userID, &nbpeer.Peer{ + Key: peerPubKey, + Meta: nbpeer.PeerSystemMeta{Hostname: "race-peer"}, + }, false) + require.NoError(t, err, "unable to add peer") + + const workers = 16 + base := time.Now().UTC().UnixNano() + tokens := make([]int64, workers) + for i := range tokens { + // Spread tokens by 1ms so the comparison is unambiguous; the + // largest is index workers-1. + tokens[i] = base + int64(i)*int64(time.Millisecond) + } + expected := tokens[workers-1] + + var ready sync.WaitGroup + ready.Add(workers) + var start sync.WaitGroup + start.Add(1) + var done sync.WaitGroup + done.Add(workers) + + // require.* calls t.FailNow which is documented as unsafe from + // non-test goroutines (it calls runtime.Goexit on the wrong stack and + // races with the WaitGroup). Collect errors here and assert from the + // main goroutine after done.Wait(). + errs := make(chan error, workers) + + for i := 0; i < workers; i++ { + token := tokens[i] + go func() { + defer done.Done() + ready.Done() + start.Wait() + errs <- manager.MarkPeerConnected(context.Background(), peerPubKey, nil, accountID, token) + }() + } + + ready.Wait() + start.Done() + done.Wait() + close(errs) + for err := range errs { + require.NoError(t, err, "MarkPeerConnected must not error under contention") + } + + peer, err := manager.Store.GetPeerByPeerPubKey(context.Background(), store.LockingStrengthNone, peerPubKey) + require.NoError(t, err) + require.True(t, peer.Status.Connected, "peer should be connected after the race") + require.Equal(t, expected, peer.Status.SessionStartedAt, + "the largest token must win regardless of execution order") +} + func TestDefaultAccountManager_UpdateAccountSettings_PeerLoginExpiration(t *testing.T) { manager, _, err := createManager(t) require.NoError(t, err, "unable to create account manager") @@ -1991,7 +2070,7 @@ func TestDefaultAccountManager_UpdateAccountSettings_PeerLoginExpiration(t *test account, err := manager.Store.GetAccount(context.Background(), accountID) require.NoError(t, err, "unable to get the account") - err = manager.MarkPeerConnected(context.Background(), key.PublicKey().String(), true, nil, accountID, time.Now().UTC()) + err = manager.MarkPeerConnected(context.Background(), key.PublicKey().String(), nil, accountID, time.Now().UTC().UnixNano()) require.NoError(t, err, "unable to mark peer connected") wg := &sync.WaitGroup{} diff --git a/management/server/mock_server/account_mock.go b/management/server/mock_server/account_mock.go index 08091d4b7..aba408184 100644 --- a/management/server/mock_server/account_mock.go +++ b/management/server/mock_server/account_mock.go @@ -38,7 +38,8 @@ type MockAccountManager struct { GetUserFromUserAuthFunc func(ctx context.Context, userAuth auth.UserAuth) (*types.User, error) ListUsersFunc func(ctx context.Context, accountID string) ([]*types.User, error) GetPeersFunc func(ctx context.Context, accountID, userID, nameFilter, ipFilter string) ([]*nbpeer.Peer, error) - MarkPeerConnectedFunc func(ctx context.Context, peerKey string, connected bool, realIP net.IP, syncTime time.Time) error + MarkPeerConnectedFunc func(ctx context.Context, peerKey string, realIP net.IP, accountID string, sessionStartedAt int64) error + MarkPeerDisconnectedFunc func(ctx context.Context, peerKey string, accountID string, sessionStartedAt int64) error SyncAndMarkPeerFunc func(ctx context.Context, accountID string, peerPubKey string, meta nbpeer.PeerSystemMeta, realIP net.IP, syncTime time.Time) (*nbpeer.Peer, *types.NetworkMap, []*posture.Checks, int64, error) DeletePeerFunc func(ctx context.Context, accountID, peerKey, userID string) error GetNetworkMapFunc func(ctx context.Context, peerKey string) (*types.NetworkMap, error) @@ -227,7 +228,14 @@ func (am *MockAccountManager) SyncAndMarkPeer(ctx context.Context, accountID str return nil, nil, nil, 0, status.Errorf(codes.Unimplemented, "method MarkPeerConnected is not implemented") } -func (am *MockAccountManager) OnPeerDisconnected(_ context.Context, accountID string, peerPubKey string, streamStartTime time.Time) error { +func (am *MockAccountManager) OnPeerDisconnected(ctx context.Context, accountID string, peerPubKey string, streamStartTime time.Time) error { + // Mirror DefaultAccountManager.OnPeerDisconnected: drive the fencing + // hook so tests that inject MarkPeerDisconnectedFunc actually observe + // disconnect events. Falls through to nil when no hook is set, which + // is the original behaviour. + if am.MarkPeerDisconnectedFunc != nil { + return am.MarkPeerDisconnectedFunc(ctx, peerPubKey, accountID, streamStartTime.UnixNano()) + } return nil } @@ -328,13 +336,21 @@ func (am *MockAccountManager) GetAccountIDByUserID(ctx context.Context, userAuth } // MarkPeerConnected mock implementation of MarkPeerConnected from server.AccountManager interface -func (am *MockAccountManager) MarkPeerConnected(ctx context.Context, peerKey string, connected bool, realIP net.IP, accountID string, syncTime time.Time) error { +func (am *MockAccountManager) MarkPeerConnected(ctx context.Context, peerKey string, realIP net.IP, accountID string, sessionStartedAt int64) error { if am.MarkPeerConnectedFunc != nil { - return am.MarkPeerConnectedFunc(ctx, peerKey, connected, realIP, syncTime) + return am.MarkPeerConnectedFunc(ctx, peerKey, realIP, accountID, sessionStartedAt) } return status.Errorf(codes.Unimplemented, "method MarkPeerConnected is not implemented") } +// MarkPeerDisconnected mock implementation of MarkPeerDisconnected from server.AccountManager interface +func (am *MockAccountManager) MarkPeerDisconnected(ctx context.Context, peerKey string, accountID string, sessionStartedAt int64) error { + if am.MarkPeerDisconnectedFunc != nil { + return am.MarkPeerDisconnectedFunc(ctx, peerKey, accountID, sessionStartedAt) + } + return status.Errorf(codes.Unimplemented, "method MarkPeerDisconnected is not implemented") +} + // DeleteAccount mock implementation of DeleteAccount from server.AccountManager interface func (am *MockAccountManager) DeleteAccount(ctx context.Context, accountID, userID string) error { if am.DeleteAccountFunc != nil { diff --git a/management/server/peer.go b/management/server/peer.go index c3b130ba2..4790a5aab 100644 --- a/management/server/peer.go +++ b/management/server/peer.go @@ -16,7 +16,6 @@ import ( "golang.org/x/exp/maps" nbdns "github.com/netbirdio/netbird/dns" - "github.com/netbirdio/netbird/management/server/geolocation" "github.com/netbirdio/netbird/management/server/idp" routerTypes "github.com/netbirdio/netbird/management/server/networks/routers/types" "github.com/netbirdio/netbird/management/server/permissions/modules" @@ -63,56 +62,51 @@ func (am *DefaultAccountManager) GetPeers(ctx context.Context, accountID, userID return am.Store.GetUserPeers(ctx, store.LockingStrengthNone, accountID, userID) } -// MarkPeerConnected marks peer as connected (true) or disconnected (false) -// syncTime is used as the LastSeen timestamp and for stale request detection -func (am *DefaultAccountManager) MarkPeerConnected(ctx context.Context, peerPubKey string, connected bool, realIP net.IP, accountID string, syncTime time.Time) error { - var peer *nbpeer.Peer - var settings *types.Settings - var expired bool - var err error - var skipped bool - - err = am.Store.ExecuteInTransaction(ctx, func(transaction store.Store) error { - peer, err = transaction.GetPeerByPeerPubKey(ctx, store.LockingStrengthUpdate, peerPubKey) - if err != nil { - return err - } - - if connected && !syncTime.After(peer.Status.LastSeen) { - log.WithContext(ctx).Tracef("peer %s has newer activity (lastSeen=%s >= syncTime=%s), skipping connect", - peer.ID, peer.Status.LastSeen.Format(time.RFC3339), syncTime.Format(time.RFC3339)) - skipped = true - return nil - } - - expired, err = updatePeerStatusAndLocation(ctx, am.geo, transaction, peer, connected, realIP, accountID, syncTime) - return err - }) - if skipped { - return nil - } +// MarkPeerConnected marks a peer as connected with optimistic-locked +// fencing on PeerStatus.SessionStartedAt. The sessionStartedAt argument +// is the start time of the gRPC sync stream that owns this update, +// expressed as Unix nanoseconds — only the call whose token is greater +// than what's stored wins. LastSeen is written by the database itself; +// we never pass it down. +// +// Disconnects use MarkPeerDisconnected and require the session to match +// exactly; see PeerStatus.SessionStartedAt for the protocol. +func (am *DefaultAccountManager) MarkPeerConnected(ctx context.Context, peerPubKey string, realIP net.IP, accountID string, sessionStartedAt int64) error { + peer, err := am.Store.GetPeerByPeerPubKey(ctx, store.LockingStrengthNone, peerPubKey) if err != nil { return err } + updated, err := am.Store.MarkPeerConnectedIfNewerSession(ctx, accountID, peer.ID, sessionStartedAt) + if err != nil { + return err + } + if !updated { + log.WithContext(ctx).Tracef("peer %s already has a newer session in store, skipping connect", peer.ID) + return nil + } + + if am.geo != nil && realIP != nil { + am.updatePeerLocationIfChanged(ctx, accountID, peer, realIP) + } + + expired := peer.Status != nil && peer.Status.LoginExpired + if peer.AddedWithSSOLogin() { - settings, err = am.Store.GetAccountSettings(ctx, store.LockingStrengthNone, accountID) + settings, err := am.Store.GetAccountSettings(ctx, store.LockingStrengthNone, accountID) if err != nil { return err } - if peer.LoginExpirationEnabled && settings.PeerLoginExpirationEnabled { am.schedulePeerLoginExpiration(ctx, accountID) } - if peer.InactivityExpirationEnabled && settings.PeerInactivityExpirationEnabled { am.checkAndSchedulePeerInactivityExpiration(ctx, accountID) } } if expired { - err = am.networkMapController.OnPeersUpdated(ctx, accountID, []string{peer.ID}) - if err != nil { + if err = am.networkMapController.OnPeersUpdated(ctx, accountID, []string{peer.ID}); err != nil { return fmt.Errorf("notify network map controller of peer update: %w", err) } } @@ -120,41 +114,46 @@ func (am *DefaultAccountManager) MarkPeerConnected(ctx context.Context, peerPubK return nil } -func updatePeerStatusAndLocation(ctx context.Context, geo geolocation.Geolocation, transaction store.Store, peer *nbpeer.Peer, connected bool, realIP net.IP, accountID string, syncTime time.Time) (bool, error) { - oldStatus := peer.Status.Copy() - newStatus := oldStatus - newStatus.LastSeen = syncTime - newStatus.Connected = connected - // whenever peer got connected that means that it logged in successfully - if newStatus.Connected { - newStatus.LoginExpired = false - } - peer.Status = newStatus - - if geo != nil && realIP != nil { - location, err := geo.Lookup(realIP) - if err != nil { - log.WithContext(ctx).Warnf("failed to get location for peer %s realip: [%s]: %v", peer.ID, realIP.String(), err) - } else { - peer.Location.ConnectionIP = realIP - peer.Location.CountryCode = location.Country.ISOCode - peer.Location.CityName = location.City.Names.En - peer.Location.GeoNameID = location.City.GeonameID - err = transaction.SavePeerLocation(ctx, accountID, peer) - if err != nil { - log.WithContext(ctx).Warnf("could not store location for peer %s: %s", peer.ID, err) - } - } - } - - log.WithContext(ctx).Debugf("saving peer status for peer %s is connected: %t", peer.ID, connected) - - err := transaction.SavePeerStatus(ctx, accountID, peer.ID, *newStatus) +// MarkPeerDisconnected marks a peer as disconnected, but only when the +// stored session token matches the one passed in. A mismatch means a +// newer stream has already taken ownership of the peer — disconnects from +// the older stream are ignored. LastSeen is written by the database. +func (am *DefaultAccountManager) MarkPeerDisconnected(ctx context.Context, peerPubKey string, accountID string, sessionStartedAt int64) error { + peer, err := am.Store.GetPeerByPeerPubKey(ctx, store.LockingStrengthNone, peerPubKey) if err != nil { - return false, err + return err } - return oldStatus.LoginExpired, nil + updated, err := am.Store.MarkPeerDisconnectedIfSameSession(ctx, accountID, peer.ID, sessionStartedAt) + if err != nil { + return err + } + if !updated { + log.WithContext(ctx).Tracef("peer %s session token mismatch on disconnect (token=%d), skipping", + peer.ID, sessionStartedAt) + } + return nil +} + +// updatePeerLocationIfChanged refreshes the geolocation on a separate +// row update, only when the connection IP actually changed. Geo lookups +// are expensive so we skip same-IP reconnects. +func (am *DefaultAccountManager) updatePeerLocationIfChanged(ctx context.Context, accountID string, peer *nbpeer.Peer, realIP net.IP) { + if peer.Location.ConnectionIP != nil && peer.Location.ConnectionIP.Equal(realIP) { + return + } + location, err := am.geo.Lookup(realIP) + if err != nil { + log.WithContext(ctx).Warnf("failed to get location for peer %s realip: [%s]: %v", peer.ID, realIP.String(), err) + return + } + peer.Location.ConnectionIP = realIP + peer.Location.CountryCode = location.Country.ISOCode + peer.Location.CityName = location.City.Names.En + peer.Location.GeoNameID = location.City.GeonameID + if err := am.Store.SavePeerLocation(ctx, accountID, peer); err != nil { + log.WithContext(ctx).Warnf("could not store location for peer %s: %s", peer.ID, err) + } } // UpdatePeer updates peer. Only Peer.Name, Peer.SSHEnabled, Peer.LoginExpirationEnabled and Peer.InactivityExpirationEnabled can be updated. diff --git a/management/server/peer/peer.go b/management/server/peer/peer.go index 17df761a1..2963dfcbd 100644 --- a/management/server/peer/peer.go +++ b/management/server/peer/peer.go @@ -74,8 +74,19 @@ type ProxyMeta struct { } type PeerStatus struct { //nolint:revive - // LastSeen is the last time peer was connected to the management service + // LastSeen is the last time the peer status was updated (i.e. the last + // time we observed the peer being alive on a sync stream). Written by + // the database (CURRENT_TIMESTAMP) — callers do not supply it. LastSeen time.Time + // SessionStartedAt records when the currently-active sync stream began, + // stored as Unix nanoseconds. It acts as the optimistic-locking token + // for status updates: a stream is only allowed to mutate the peer's + // status when its own token strictly exceeds the stored token (when connecting) + // or matches it exactly (for disconnects). Zero means "no + // active session". Integer nanoseconds are used so equality is + // precision-safe across drivers, and so the predicates compose to a + // single bigint comparison. + SessionStartedAt int64 // Connected indicates whether peer is connected to the management service or not Connected bool // LoginExpired @@ -375,10 +386,14 @@ func (p *Peer) EventMeta(dnsDomain string) map[string]any { return meta } -// Copy PeerStatus +// Copy PeerStatus. SessionStartedAt must be propagated so clone-based +// callers (Peer.Copy, MarkLoginExpired, UpdateLastLogin) don't silently +// reset the fencing token to zero — that would let any subsequent +// SavePeerStatus write reopen the optimistic-lock window. func (p *PeerStatus) Copy() *PeerStatus { return &PeerStatus{ LastSeen: p.LastSeen, + SessionStartedAt: p.SessionStartedAt, Connected: p.Connected, LoginExpired: p.LoginExpired, RequiresApproval: p.RequiresApproval, diff --git a/management/server/store/sql_store.go b/management/server/store/sql_store.go index 893ee2168..8cf37de56 100644 --- a/management/server/store/sql_store.go +++ b/management/server/store/sql_store.go @@ -498,8 +498,9 @@ func (s *SqlStore) SavePeerStatus(ctx context.Context, accountID, peerID string, peerCopy.Status = &peerStatus fieldsToUpdate := []string{ - "peer_status_last_seen", "peer_status_connected", - "peer_status_login_expired", "peer_status_required_approval", + "peer_status_last_seen", "peer_status_session_started_at", + "peer_status_connected", "peer_status_login_expired", + "peer_status_requires_approval", } result := s.db.Model(&nbpeer.Peer{}). Select(fieldsToUpdate). @@ -516,6 +517,69 @@ func (s *SqlStore) SavePeerStatus(ctx context.Context, accountID, peerID string, return nil } +// MarkPeerConnectedIfNewerSession is an atomic optimistic-locked update. +// The peer is marked connected with the given session token only when +// the stored SessionStartedAt is strictly smaller than the incoming +// one — equivalently, when no newer stream has already taken ownership. +// The sentinel zero (set on peer creation or after a disconnect) counts +// as the smallest possible token. This is the write half of the +// fencing protocol described on PeerStatus.SessionStartedAt. +// +// The post-write side effects in the caller — geo lookup, +// schedulePeerLoginExpiration, checkAndSchedulePeerInactivityExpiration, +// OnPeersUpdated — all run AFTER this method returns and are deliberately +// outside the database write so they cannot extend the row-lock window. +// +// LastSeen is set to the database's clock (CURRENT_TIMESTAMP) at the +// moment the row is written. The caller never supplies LastSeen because +// the value would otherwise drift under lock contention — a Go-side +// time.Now() taken before the write can land minutes later than the +// actual UPDATE under load, which previously caused real ordering bugs. +func (s *SqlStore) MarkPeerConnectedIfNewerSession(ctx context.Context, accountID, peerID string, newSessionStartedAt int64) (bool, error) { + result := s.db.WithContext(ctx). + Model(&nbpeer.Peer{}). + Where(accountAndIDQueryCondition, accountID, peerID). + Where("peer_status_session_started_at < ?", newSessionStartedAt). + Updates(map[string]any{ + "peer_status_connected": true, + "peer_status_last_seen": gorm.Expr("CURRENT_TIMESTAMP"), + "peer_status_session_started_at": newSessionStartedAt, + "peer_status_login_expired": false, + }) + if result.Error != nil { + return false, status.Errorf(status.Internal, "mark peer connected: %v", result.Error) + } + return result.RowsAffected > 0, nil +} + +// MarkPeerDisconnectedIfSameSession is an atomic optimistic-locked update. +// The peer is marked disconnected only when the stored SessionStartedAt +// matches the incoming token — meaning the stream that owns the current +// session is the one ending. If a newer stream has already replaced the +// session, the update is skipped. LastSeen is set to CURRENT_TIMESTAMP at +// write time; see MarkPeerConnectedIfNewerSession for the rationale. +// +// A zero sessionStartedAt is rejected at the call site; the underlying +// WHERE on equality would otherwise match every never-connected peer. +func (s *SqlStore) MarkPeerDisconnectedIfSameSession(ctx context.Context, accountID, peerID string, sessionStartedAt int64) (bool, error) { + if sessionStartedAt == 0 { + return false, nil + } + result := s.db.WithContext(ctx). + Model(&nbpeer.Peer{}). + Where(accountAndIDQueryCondition, accountID, peerID). + Where("peer_status_session_started_at = ?", sessionStartedAt). + Updates(map[string]any{ + "peer_status_connected": false, + "peer_status_last_seen": gorm.Expr("CURRENT_TIMESTAMP"), + "peer_status_session_started_at": int64(0), + }) + if result.Error != nil { + return false, status.Errorf(status.Internal, "mark peer disconnected: %v", result.Error) + } + return result.RowsAffected > 0, nil +} + func (s *SqlStore) SavePeerLocation(ctx context.Context, accountID string, peerWithLocation *nbpeer.Peer) error { // To maintain data integrity, we create a copy of the peer's location to prevent unintended updates to other fields. var peerCopy nbpeer.Peer @@ -1723,9 +1787,10 @@ func (s *SqlStore) getPeers(ctx context.Context, accountID string) ([]nbpeer.Pee inactivity_expiration_enabled, last_login, created_at, ephemeral, extra_dns_labels, allow_extra_dns_labels, meta_hostname, meta_go_os, meta_kernel, meta_core, meta_platform, meta_os, meta_os_version, meta_wt_version, meta_ui_version, meta_kernel_version, meta_network_addresses, meta_system_serial_number, meta_system_product_name, meta_system_manufacturer, - meta_environment, meta_flags, meta_files, meta_capabilities, peer_status_last_seen, peer_status_connected, peer_status_login_expired, - peer_status_requires_approval, location_connection_ip, location_country_code, location_city_name, - location_geo_name_id, proxy_meta_embedded, proxy_meta_cluster, ipv6 FROM peers WHERE account_id = $1` + meta_environment, meta_flags, meta_files, meta_capabilities, peer_status_last_seen, peer_status_session_started_at, + peer_status_connected, peer_status_login_expired, peer_status_requires_approval, location_connection_ip, + location_country_code, location_city_name, location_geo_name_id, proxy_meta_embedded, proxy_meta_cluster, ipv6 + FROM peers WHERE account_id = $1` rows, err := s.pool.Query(ctx, query, accountID) if err != nil { return nil, err @@ -1738,6 +1803,7 @@ func (s *SqlStore) getPeers(ctx context.Context, accountID string) ([]nbpeer.Pee lastLogin, createdAt sql.NullTime sshEnabled, loginExpirationEnabled, inactivityExpirationEnabled, ephemeral, allowExtraDNSLabels sql.NullBool peerStatusLastSeen sql.NullTime + peerStatusSessionStartedAt sql.NullInt64 peerStatusConnected, peerStatusLoginExpired, peerStatusRequiresApproval, proxyEmbedded sql.NullBool ip, extraDNS, netAddr, env, flags, files, capabilities, connIP, ipv6 []byte metaHostname, metaGoOS, metaKernel, metaCore, metaPlatform sql.NullString @@ -1752,8 +1818,9 @@ func (s *SqlStore) getPeers(ctx context.Context, accountID string) ([]nbpeer.Pee &allowExtraDNSLabels, &metaHostname, &metaGoOS, &metaKernel, &metaCore, &metaPlatform, &metaOS, &metaOSVersion, &metaWtVersion, &metaUIVersion, &metaKernelVersion, &netAddr, &metaSystemSerialNumber, &metaSystemProductName, &metaSystemManufacturer, &env, &flags, &files, &capabilities, - &peerStatusLastSeen, &peerStatusConnected, &peerStatusLoginExpired, &peerStatusRequiresApproval, &connIP, - &locationCountryCode, &locationCityName, &locationGeoNameID, &proxyEmbedded, &proxyCluster, &ipv6) + &peerStatusLastSeen, &peerStatusSessionStartedAt, &peerStatusConnected, &peerStatusLoginExpired, + &peerStatusRequiresApproval, &connIP, &locationCountryCode, &locationCityName, &locationGeoNameID, + &proxyEmbedded, &proxyCluster, &ipv6) if err == nil { if lastLogin.Valid { @@ -1780,6 +1847,9 @@ func (s *SqlStore) getPeers(ctx context.Context, accountID string) ([]nbpeer.Pee if peerStatusLastSeen.Valid { p.Status.LastSeen = peerStatusLastSeen.Time } + if peerStatusSessionStartedAt.Valid { + p.Status.SessionStartedAt = peerStatusSessionStartedAt.Int64 + } if peerStatusConnected.Valid { p.Status.Connected = peerStatusConnected.Bool } diff --git a/management/server/store/store.go b/management/server/store/store.go index aa601c33f..a723c1fc3 100644 --- a/management/server/store/store.go +++ b/management/server/store/store.go @@ -167,6 +167,21 @@ type Store interface { GetAllEphemeralPeers(ctx context.Context, lockStrength LockingStrength) ([]*nbpeer.Peer, error) SavePeer(ctx context.Context, accountID string, peer *nbpeer.Peer) error SavePeerStatus(ctx context.Context, accountID, peerID string, status nbpeer.PeerStatus) error + // MarkPeerConnectedIfNewerSession sets the peer to connected with the + // given session token, but only when the stored SessionStartedAt is + // strictly less than newSessionStartedAt (the sentinel zero counts as + // "older"). LastSeen is recorded by the database at the moment the + // row is updated — never by the caller — so it always reflects the + // real write time even under lock contention. + // Returns true when the update happened, false when this stream lost + // the race against a newer session. + MarkPeerConnectedIfNewerSession(ctx context.Context, accountID, peerID string, newSessionStartedAt int64) (bool, error) + // MarkPeerDisconnectedIfSameSession sets the peer to disconnected and + // resets SessionStartedAt to zero, but only when the stored + // SessionStartedAt equals the given sessionStartedAt. LastSeen is + // recorded by the database. Returns true when the update happened, + // false when a newer session has taken over. + MarkPeerDisconnectedIfSameSession(ctx context.Context, accountID, peerID string, sessionStartedAt int64) (bool, error) SavePeerLocation(ctx context.Context, accountID string, peer *nbpeer.Peer) error ApproveAccountPeers(ctx context.Context, accountID string) (int, error) DeletePeer(ctx context.Context, accountID string, peerID string) error diff --git a/management/server/store/store_mock.go b/management/server/store/store_mock.go index 9780c521e..d51629606 100644 --- a/management/server/store/store_mock.go +++ b/management/server/store/store_mock.go @@ -2878,6 +2878,36 @@ func (mr *MockStoreMockRecorder) SavePeerStatus(ctx, accountID, peerID, status i return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SavePeerStatus", reflect.TypeOf((*MockStore)(nil).SavePeerStatus), ctx, accountID, peerID, status) } +// MarkPeerConnectedIfNewerSession mocks base method. +func (m *MockStore) MarkPeerConnectedIfNewerSession(ctx context.Context, accountID, peerID string, newSessionStartedAt int64) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkPeerConnectedIfNewerSession", ctx, accountID, peerID, newSessionStartedAt) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MarkPeerConnectedIfNewerSession indicates an expected call of MarkPeerConnectedIfNewerSession. +func (mr *MockStoreMockRecorder) MarkPeerConnectedIfNewerSession(ctx, accountID, peerID, newSessionStartedAt interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkPeerConnectedIfNewerSession", reflect.TypeOf((*MockStore)(nil).MarkPeerConnectedIfNewerSession), ctx, accountID, peerID, newSessionStartedAt) +} + +// MarkPeerDisconnectedIfSameSession mocks base method. +func (m *MockStore) MarkPeerDisconnectedIfSameSession(ctx context.Context, accountID, peerID string, sessionStartedAt int64) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkPeerDisconnectedIfSameSession", ctx, accountID, peerID, sessionStartedAt) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MarkPeerDisconnectedIfSameSession indicates an expected call of MarkPeerDisconnectedIfSameSession. +func (mr *MockStoreMockRecorder) MarkPeerDisconnectedIfSameSession(ctx, accountID, peerID, sessionStartedAt interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkPeerDisconnectedIfSameSession", reflect.TypeOf((*MockStore)(nil).MarkPeerDisconnectedIfSameSession), ctx, accountID, peerID, sessionStartedAt) +} + // SavePolicy mocks base method. func (m *MockStore) SavePolicy(ctx context.Context, policy *types2.Policy) error { m.ctrl.T.Helper() From af24fd779640538c05c5f261a1e9fdf20fe7773f Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Mon, 18 May 2026 22:55:19 +0200 Subject: [PATCH 24/29] [management] Add metrics for peer status updates and ephemeral cleanup (#6196) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [management] Add metrics for peer status updates and ephemeral cleanup The session-fenced MarkPeerConnected / MarkPeerDisconnected path and the ephemeral peer cleanup loop both run silently today: when fencing rejects a stale stream, when a cleanup tick deletes peers, or when a batch delete fails, we have no operational signal beyond log lines. Add OpenTelemetry counters and a histogram so the same SLO-style dashboards that already exist for the network-map controller can cover peer connect/disconnect and ephemeral cleanup too. All new attributes are bounded enums: operation in {connect,disconnect} and outcome in {applied,stale,error,peer_not_found}. No account, peer, or user ID is ever written as a metric label — total cardinality is fixed at compile time (8 counter series, 2 histogram series, 4 unlabeled ephemeral series). Metric methods are nil-receiver safe so test composition that doesn't wire telemetry (the bulk of the existing tests) works unchanged. The ephemeral manager exposes a SetMetrics setter rather than taking the collector through its constructor, keeping the constructor signature stable across all test call sites. * [management] Add OpenTelemetry metrics for ephemeral peer cleanup Introduce counters for tracking ephemeral peer cleanup, including peers pending deletion, cleanup runs, successful deletions, and failed batches. Metrics are nil-receiver safe to ensure compatibility with test setups without telemetry. --- .../peers/ephemeral/manager/ephemeral.go | 45 ++++++- management/internals/server/controllers.go | 6 +- management/server/peer.go | 28 +++++ .../telemetry/accountmanager_metrics.go | 65 ++++++++++ management/server/telemetry/app_metrics.go | 28 +++++ .../server/telemetry/ephemeral_metrics.go | 115 ++++++++++++++++++ 6 files changed, 281 insertions(+), 6 deletions(-) create mode 100644 management/server/telemetry/ephemeral_metrics.go diff --git a/management/internals/modules/peers/ephemeral/manager/ephemeral.go b/management/internals/modules/peers/ephemeral/manager/ephemeral.go index 758f643d0..0f902ea70 100644 --- a/management/internals/modules/peers/ephemeral/manager/ephemeral.go +++ b/management/internals/modules/peers/ephemeral/manager/ephemeral.go @@ -11,6 +11,7 @@ import ( "github.com/netbirdio/netbird/management/internals/modules/peers/ephemeral" "github.com/netbirdio/netbird/management/server/activity" nbpeer "github.com/netbirdio/netbird/management/server/peer" + "github.com/netbirdio/netbird/management/server/telemetry" "github.com/netbirdio/netbird/management/server/store" ) @@ -47,6 +48,11 @@ type EphemeralManager struct { lifeTime time.Duration cleanupWindow time.Duration + + // metrics is nil-safe; methods on telemetry.EphemeralPeersMetrics + // no-op when the receiver is nil so deployments without an app + // metrics provider work unchanged. + metrics *telemetry.EphemeralPeersMetrics } // NewEphemeralManager instantiate new EphemeralManager @@ -60,6 +66,15 @@ func NewEphemeralManager(store store.Store, peersManager peers.Manager) *Ephemer } } +// SetMetrics attaches a metrics collector. Safe to call once before +// LoadInitialPeers; later attachment is fine but earlier loads won't be +// reflected in the gauge. Pass nil to detach. +func (e *EphemeralManager) SetMetrics(m *telemetry.EphemeralPeersMetrics) { + e.peersLock.Lock() + e.metrics = m + e.peersLock.Unlock() +} + // LoadInitialPeers load from the database the ephemeral type of peers and schedule a cleanup procedure to the head // of the linked list (to the most deprecated peer). At the end of cleanup it schedules the next cleanup to the new // head. @@ -97,7 +112,9 @@ func (e *EphemeralManager) OnPeerConnected(ctx context.Context, peer *nbpeer.Pee e.peersLock.Lock() defer e.peersLock.Unlock() - e.removePeer(peer.ID) + if e.removePeer(peer.ID) { + e.metrics.DecPending(1) + } // stop the unnecessary timer if e.headPeer == nil && e.timer != nil { @@ -123,6 +140,7 @@ func (e *EphemeralManager) OnPeerDisconnected(ctx context.Context, peer *nbpeer. } e.addPeer(peer.AccountID, peer.ID, e.newDeadLine()) + e.metrics.IncPending() if e.timer == nil { delay := e.headPeer.deadline.Sub(timeNow()) + e.cleanupWindow if delay < 0 { @@ -145,6 +163,7 @@ func (e *EphemeralManager) loadEphemeralPeers(ctx context.Context) { for _, p := range peers { e.addPeer(p.AccountID, p.ID, t) } + e.metrics.AddPending(int64(len(peers))) log.WithContext(ctx).Debugf("loaded ephemeral peer(s): %d", len(peers)) } @@ -181,6 +200,15 @@ func (e *EphemeralManager) cleanup(ctx context.Context) { e.peersLock.Unlock() + // Drop the gauge by the number of entries we just took off the list, + // regardless of whether the subsequent DeletePeers call succeeds. The + // list invariant is what the gauge tracks; failed delete batches are + // counted separately via CountCleanupError so we can still see them. + if len(deletePeers) > 0 { + e.metrics.CountCleanupRun() + e.metrics.DecPending(int64(len(deletePeers))) + } + peerIDsPerAccount := make(map[string][]string) for id, p := range deletePeers { peerIDsPerAccount[p.accountID] = append(peerIDsPerAccount[p.accountID], id) @@ -191,7 +219,10 @@ func (e *EphemeralManager) cleanup(ctx context.Context) { err := e.peersManager.DeletePeers(ctx, accountID, peerIDs, activity.SystemInitiator, true) if err != nil { log.WithContext(ctx).Errorf("failed to delete ephemeral peers: %s", err) + e.metrics.CountCleanupError() + continue } + e.metrics.CountPeersCleaned(int64(len(peerIDs))) } } @@ -211,9 +242,12 @@ func (e *EphemeralManager) addPeer(accountID string, peerID string, deadline tim e.tailPeer = ep } -func (e *EphemeralManager) removePeer(id string) { +// removePeer drops the entry from the linked list. Returns true if a +// matching entry was found and removed so callers can keep the pending +// metric gauge in sync. +func (e *EphemeralManager) removePeer(id string) bool { if e.headPeer == nil { - return + return false } if e.headPeer.id == id { @@ -221,7 +255,7 @@ func (e *EphemeralManager) removePeer(id string) { if e.tailPeer.id == id { e.tailPeer = nil } - return + return true } for p := e.headPeer; p.next != nil; p = p.next { @@ -231,9 +265,10 @@ func (e *EphemeralManager) removePeer(id string) { e.tailPeer = p } p.next = p.next.next - return + return true } } + return false } func (e *EphemeralManager) isPeerOnList(id string) bool { diff --git a/management/internals/server/controllers.go b/management/internals/server/controllers.go index 89bdf0abe..794c3ebe0 100644 --- a/management/internals/server/controllers.go +++ b/management/internals/server/controllers.go @@ -112,7 +112,11 @@ func (s *BaseServer) AuthManager() auth.Manager { func (s *BaseServer) EphemeralManager() ephemeral.Manager { return Create(s, func() ephemeral.Manager { - return manager.NewEphemeralManager(s.Store(), s.PeersManager()) + em := manager.NewEphemeralManager(s.Store(), s.PeersManager()) + if metrics := s.Metrics(); metrics != nil { + em.SetMetrics(metrics.EphemeralPeersMetrics()) + } + return em }) } diff --git a/management/server/peer.go b/management/server/peer.go index 4790a5aab..34b681f51 100644 --- a/management/server/peer.go +++ b/management/server/peer.go @@ -28,6 +28,7 @@ import ( "github.com/netbirdio/netbird/management/server/activity" nbpeer "github.com/netbirdio/netbird/management/server/peer" + "github.com/netbirdio/netbird/management/server/telemetry" "github.com/netbirdio/netbird/shared/management/status" ) @@ -72,19 +73,32 @@ func (am *DefaultAccountManager) GetPeers(ctx context.Context, accountID, userID // Disconnects use MarkPeerDisconnected and require the session to match // exactly; see PeerStatus.SessionStartedAt for the protocol. func (am *DefaultAccountManager) MarkPeerConnected(ctx context.Context, peerPubKey string, realIP net.IP, accountID string, sessionStartedAt int64) error { + start := time.Now() + defer func() { + am.metrics.AccountManagerMetrics().RecordPeerStatusUpdateDuration(telemetry.PeerStatusConnect, time.Since(start)) + }() + peer, err := am.Store.GetPeerByPeerPubKey(ctx, store.LockingStrengthNone, peerPubKey) if err != nil { + outcome := telemetry.PeerStatusError + if s, ok := status.FromError(err); ok && s.Type() == status.NotFound { + outcome = telemetry.PeerStatusPeerNotFound + } + am.metrics.AccountManagerMetrics().CountPeerStatusUpdate(telemetry.PeerStatusConnect, outcome) return err } updated, err := am.Store.MarkPeerConnectedIfNewerSession(ctx, accountID, peer.ID, sessionStartedAt) if err != nil { + am.metrics.AccountManagerMetrics().CountPeerStatusUpdate(telemetry.PeerStatusConnect, telemetry.PeerStatusError) return err } if !updated { + am.metrics.AccountManagerMetrics().CountPeerStatusUpdate(telemetry.PeerStatusConnect, telemetry.PeerStatusStale) log.WithContext(ctx).Tracef("peer %s already has a newer session in store, skipping connect", peer.ID) return nil } + am.metrics.AccountManagerMetrics().CountPeerStatusUpdate(telemetry.PeerStatusConnect, telemetry.PeerStatusApplied) if am.geo != nil && realIP != nil { am.updatePeerLocationIfChanged(ctx, accountID, peer, realIP) @@ -119,19 +133,33 @@ func (am *DefaultAccountManager) MarkPeerConnected(ctx context.Context, peerPubK // newer stream has already taken ownership of the peer — disconnects from // the older stream are ignored. LastSeen is written by the database. func (am *DefaultAccountManager) MarkPeerDisconnected(ctx context.Context, peerPubKey string, accountID string, sessionStartedAt int64) error { + start := time.Now() + defer func() { + am.metrics.AccountManagerMetrics().RecordPeerStatusUpdateDuration(telemetry.PeerStatusDisconnect, time.Since(start)) + }() + peer, err := am.Store.GetPeerByPeerPubKey(ctx, store.LockingStrengthNone, peerPubKey) if err != nil { + outcome := telemetry.PeerStatusError + if s, ok := status.FromError(err); ok && s.Type() == status.NotFound { + outcome = telemetry.PeerStatusPeerNotFound + } + am.metrics.AccountManagerMetrics().CountPeerStatusUpdate(telemetry.PeerStatusDisconnect, outcome) return err } updated, err := am.Store.MarkPeerDisconnectedIfSameSession(ctx, accountID, peer.ID, sessionStartedAt) if err != nil { + am.metrics.AccountManagerMetrics().CountPeerStatusUpdate(telemetry.PeerStatusDisconnect, telemetry.PeerStatusError) return err } if !updated { + am.metrics.AccountManagerMetrics().CountPeerStatusUpdate(telemetry.PeerStatusDisconnect, telemetry.PeerStatusStale) log.WithContext(ctx).Tracef("peer %s session token mismatch on disconnect (token=%d), skipping", peer.ID, sessionStartedAt) + return nil } + am.metrics.AccountManagerMetrics().CountPeerStatusUpdate(telemetry.PeerStatusDisconnect, telemetry.PeerStatusApplied) return nil } diff --git a/management/server/telemetry/accountmanager_metrics.go b/management/server/telemetry/accountmanager_metrics.go index 518aae7eb..bb6fb7e12 100644 --- a/management/server/telemetry/accountmanager_metrics.go +++ b/management/server/telemetry/accountmanager_metrics.go @@ -16,6 +16,8 @@ type AccountManagerMetrics struct { getPeerNetworkMapDurationMs metric.Float64Histogram networkMapObjectCount metric.Int64Histogram peerMetaUpdateCount metric.Int64Counter + peerStatusUpdateCounter metric.Int64Counter + peerStatusUpdateDurationMs metric.Float64Histogram } // NewAccountManagerMetrics creates an instance of AccountManagerMetrics @@ -64,6 +66,24 @@ func NewAccountManagerMetrics(ctx context.Context, meter metric.Meter) (*Account return nil, err } + // peerStatusUpdateCounter records every attempt to mark a peer as connected or disconnected + peerStatusUpdateCounter, err := meter.Int64Counter("management.account.peer.status.update.counter", + metric.WithUnit("1"), + metric.WithDescription("Number of peer status update attempts, labeled by operation (connect|disconnect) and outcome (applied|stale|error|peer_not_found)")) + if err != nil { + return nil, err + } + + peerStatusUpdateDurationMs, err := meter.Float64Histogram("management.account.peer.status.update.duration.ms", + metric.WithUnit("milliseconds"), + metric.WithExplicitBucketBoundaries( + 1, 5, 15, 25, 50, 100, 250, 500, 1000, 2000, 5000, + ), + metric.WithDescription("Duration of a peer status update (fence UPDATE + post-write side effects), labeled by operation")) + if err != nil { + return nil, err + } + return &AccountManagerMetrics{ ctx: ctx, getPeerNetworkMapDurationMs: getPeerNetworkMapDurationMs, @@ -71,10 +91,35 @@ func NewAccountManagerMetrics(ctx context.Context, meter metric.Meter) (*Account updateAccountPeersCounter: updateAccountPeersCounter, networkMapObjectCount: networkMapObjectCount, peerMetaUpdateCount: peerMetaUpdateCount, + peerStatusUpdateCounter: peerStatusUpdateCounter, + peerStatusUpdateDurationMs: peerStatusUpdateDurationMs, }, nil } +// PeerStatusOperation labels the kind of fence-locked peer status write. +type PeerStatusOperation string + +// PeerStatusOutcome labels how a fence-locked peer status write resolved. +type PeerStatusOutcome string + +const ( + PeerStatusConnect PeerStatusOperation = "connect" + PeerStatusDisconnect PeerStatusOperation = "disconnect" + + // PeerStatusApplied — the fence WHERE matched and the UPDATE landed. + PeerStatusApplied PeerStatusOutcome = "applied" + // PeerStatusStale — the fence WHERE rejected the write because a + // newer session has already taken ownership (connect: stored token + // >= incoming; disconnect: stored token != incoming). + PeerStatusStale PeerStatusOutcome = "stale" + // PeerStatusError — the store returned a non-NotFound error. + PeerStatusError PeerStatusOutcome = "error" + // PeerStatusPeerNotFound — the peer lookup failed (the peer was + // deleted between the gRPC sync handshake and the status write). + PeerStatusPeerNotFound PeerStatusOutcome = "peer_not_found" +) + // CountUpdateAccountPeersDuration counts the duration of updating account peers func (metrics *AccountManagerMetrics) CountUpdateAccountPeersDuration(duration time.Duration) { metrics.updateAccountPeersDurationMs.Record(metrics.ctx, float64(duration.Nanoseconds())/1e6) @@ -104,3 +149,23 @@ func (metrics *AccountManagerMetrics) CountUpdateAccountPeersTriggered(resource, func (metrics *AccountManagerMetrics) CountPeerMetUpdate() { metrics.peerMetaUpdateCount.Add(metrics.ctx, 1) } + +// CountPeerStatusUpdate increments the connect/disconnect counter, +// labeled by operation and outcome. Both labels are bounded enums. +func (metrics *AccountManagerMetrics) CountPeerStatusUpdate(op PeerStatusOperation, outcome PeerStatusOutcome) { + metrics.peerStatusUpdateCounter.Add(metrics.ctx, 1, + metric.WithAttributes( + attribute.String("operation", string(op)), + attribute.String("outcome", string(outcome)), + ), + ) +} + +// RecordPeerStatusUpdateDuration records the wall-clock time spent +// running a peer status update (including post-write side effects), +// labeled by operation. +func (metrics *AccountManagerMetrics) RecordPeerStatusUpdateDuration(op PeerStatusOperation, d time.Duration) { + metrics.peerStatusUpdateDurationMs.Record(metrics.ctx, float64(d.Nanoseconds())/1e6, + metric.WithAttributes(attribute.String("operation", string(op))), + ) +} diff --git a/management/server/telemetry/app_metrics.go b/management/server/telemetry/app_metrics.go index 1fd78bc3a..fd9087a96 100644 --- a/management/server/telemetry/app_metrics.go +++ b/management/server/telemetry/app_metrics.go @@ -29,6 +29,7 @@ type MockAppMetrics struct { StoreMetricsFunc func() *StoreMetrics UpdateChannelMetricsFunc func() *UpdateChannelMetrics AddAccountManagerMetricsFunc func() *AccountManagerMetrics + EphemeralPeersMetricsFunc func() *EphemeralPeersMetrics } // GetMeter mocks the GetMeter function of the AppMetrics interface @@ -103,6 +104,14 @@ func (mock *MockAppMetrics) AccountManagerMetrics() *AccountManagerMetrics { return nil } +// EphemeralPeersMetrics mocks the MockAppMetrics function of the EphemeralPeersMetrics interface +func (mock *MockAppMetrics) EphemeralPeersMetrics() *EphemeralPeersMetrics { + if mock.EphemeralPeersMetricsFunc != nil { + return mock.EphemeralPeersMetricsFunc() + } + return nil +} + // AppMetrics is metrics interface type AppMetrics interface { GetMeter() metric2.Meter @@ -114,6 +123,7 @@ type AppMetrics interface { StoreMetrics() *StoreMetrics UpdateChannelMetrics() *UpdateChannelMetrics AccountManagerMetrics() *AccountManagerMetrics + EphemeralPeersMetrics() *EphemeralPeersMetrics } // defaultAppMetrics are core application metrics based on OpenTelemetry https://opentelemetry.io/ @@ -129,6 +139,7 @@ type defaultAppMetrics struct { storeMetrics *StoreMetrics updateChannelMetrics *UpdateChannelMetrics accountManagerMetrics *AccountManagerMetrics + ephemeralMetrics *EphemeralPeersMetrics } // IDPMetrics returns metrics for the idp package @@ -161,6 +172,11 @@ func (appMetrics *defaultAppMetrics) AccountManagerMetrics() *AccountManagerMetr return appMetrics.accountManagerMetrics } +// EphemeralPeersMetrics returns metrics for the ephemeral peer cleanup loop +func (appMetrics *defaultAppMetrics) EphemeralPeersMetrics() *EphemeralPeersMetrics { + return appMetrics.ephemeralMetrics +} + // Close stop application metrics HTTP handler and closes listener. func (appMetrics *defaultAppMetrics) Close() error { if appMetrics.listener == nil { @@ -245,6 +261,11 @@ func NewDefaultAppMetrics(ctx context.Context) (AppMetrics, error) { return nil, fmt.Errorf("failed to initialize account manager metrics: %w", err) } + ephemeralMetrics, err := NewEphemeralPeersMetrics(ctx, meter) + if err != nil { + return nil, fmt.Errorf("failed to initialize ephemeral peers metrics: %w", err) + } + return &defaultAppMetrics{ Meter: meter, ctx: ctx, @@ -254,6 +275,7 @@ func NewDefaultAppMetrics(ctx context.Context) (AppMetrics, error) { storeMetrics: storeMetrics, updateChannelMetrics: updateChannelMetrics, accountManagerMetrics: accountManagerMetrics, + ephemeralMetrics: ephemeralMetrics, }, nil } @@ -290,6 +312,11 @@ func NewAppMetricsWithMeter(ctx context.Context, meter metric2.Meter) (AppMetric return nil, fmt.Errorf("failed to initialize account manager metrics: %w", err) } + ephemeralMetrics, err := NewEphemeralPeersMetrics(ctx, meter) + if err != nil { + return nil, fmt.Errorf("failed to initialize ephemeral peers metrics: %w", err) + } + return &defaultAppMetrics{ Meter: meter, ctx: ctx, @@ -300,5 +327,6 @@ func NewAppMetricsWithMeter(ctx context.Context, meter metric2.Meter) (AppMetric storeMetrics: storeMetrics, updateChannelMetrics: updateChannelMetrics, accountManagerMetrics: accountManagerMetrics, + ephemeralMetrics: ephemeralMetrics, }, nil } diff --git a/management/server/telemetry/ephemeral_metrics.go b/management/server/telemetry/ephemeral_metrics.go new file mode 100644 index 000000000..a7fb432f8 --- /dev/null +++ b/management/server/telemetry/ephemeral_metrics.go @@ -0,0 +1,115 @@ +package telemetry + +import ( + "context" + + "go.opentelemetry.io/otel/metric" +) + +// EphemeralPeersMetrics tracks the ephemeral peer cleanup pipeline: how +// many peers are currently scheduled for deletion, how many tick runs +// the cleaner has performed, how many peers it has removed, and how +// many delete batches failed. +type EphemeralPeersMetrics struct { + ctx context.Context + + pending metric.Int64UpDownCounter + cleanupRuns metric.Int64Counter + peersCleaned metric.Int64Counter + errors metric.Int64Counter +} + +// NewEphemeralPeersMetrics constructs the ephemeral cleanup counters. +func NewEphemeralPeersMetrics(ctx context.Context, meter metric.Meter) (*EphemeralPeersMetrics, error) { + pending, err := meter.Int64UpDownCounter("management.ephemeral.peers.pending", + metric.WithUnit("1"), + metric.WithDescription("Number of ephemeral peers currently waiting to be cleaned up")) + if err != nil { + return nil, err + } + + cleanupRuns, err := meter.Int64Counter("management.ephemeral.cleanup.runs.counter", + metric.WithUnit("1"), + metric.WithDescription("Number of ephemeral cleanup ticks that processed at least one peer")) + if err != nil { + return nil, err + } + + peersCleaned, err := meter.Int64Counter("management.ephemeral.peers.cleaned.counter", + metric.WithUnit("1"), + metric.WithDescription("Total number of ephemeral peers deleted by the cleanup loop")) + if err != nil { + return nil, err + } + + errors, err := meter.Int64Counter("management.ephemeral.cleanup.errors.counter", + metric.WithUnit("1"), + metric.WithDescription("Number of ephemeral cleanup batches (per account) that failed to delete")) + if err != nil { + return nil, err + } + + return &EphemeralPeersMetrics{ + ctx: ctx, + pending: pending, + cleanupRuns: cleanupRuns, + peersCleaned: peersCleaned, + errors: errors, + }, nil +} + +// All methods are nil-receiver safe so callers that haven't wired metrics +// (tests, self-hosted with metrics off) can invoke them unconditionally. + +// IncPending bumps the pending gauge when a peer is added to the cleanup list. +func (m *EphemeralPeersMetrics) IncPending() { + if m == nil { + return + } + m.pending.Add(m.ctx, 1) +} + +// AddPending bumps the pending gauge by n — used at startup when the +// initial set of ephemeral peers is loaded from the store. +func (m *EphemeralPeersMetrics) AddPending(n int64) { + if m == nil || n <= 0 { + return + } + m.pending.Add(m.ctx, n) +} + +// DecPending decreases the pending gauge — used both when a peer reconnects +// before its deadline (removed from the list) and when a cleanup tick +// actually deletes it. +func (m *EphemeralPeersMetrics) DecPending(n int64) { + if m == nil || n <= 0 { + return + } + m.pending.Add(m.ctx, -n) +} + +// CountCleanupRun records one cleanup pass that processed >0 peers. Idle +// ticks (nothing to do) deliberately don't increment so the rate +// reflects useful work. +func (m *EphemeralPeersMetrics) CountCleanupRun() { + if m == nil { + return + } + m.cleanupRuns.Add(m.ctx, 1) +} + +// CountPeersCleaned records the number of peers a single tick deleted. +func (m *EphemeralPeersMetrics) CountPeersCleaned(n int64) { + if m == nil || n <= 0 { + return + } + m.peersCleaned.Add(m.ctx, n) +} + +// CountCleanupError records a failed delete batch. +func (m *EphemeralPeersMetrics) CountCleanupError() { + if m == nil { + return + } + m.errors.Add(m.ctx, 1) +} From 80966ab1b09bd86b7a526d9402b6a47438bc0943 Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Wed, 20 May 2026 08:25:30 +0200 Subject: [PATCH 25/29] [management] Ensure SessionStartedAt has a default value (#6211) * [management] Ensure SessionStartedAt has a default value Avoid null values for the new column * [management] Add PeerStatus with LastSeen in peer_test * [management] Add migration for PeerStatusSessionStartedAt default value * [management] Add PeerStatus with LastSeen in migration tests --- management/server/migration/migration_test.go | 6 +++++- management/server/peer/peer.go | 2 +- management/server/peer_test.go | 3 +++ management/server/store/store.go | 3 +++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/management/server/migration/migration_test.go b/management/server/migration/migration_test.go index 5e00976c2..cc97c2dff 100644 --- a/management/server/migration/migration_test.go +++ b/management/server/migration/migration_test.go @@ -198,7 +198,11 @@ func TestMigrateNetIPFieldFromBlobToJSON_WithJSONData(t *testing.T) { require.NoError(t, err, "Failed to insert account") account.PeersG = []nbpeer.Peer{ - {AccountID: "1234", Location: nbpeer.Location{ConnectionIP: net.IP{10, 0, 0, 1}}}, + { + AccountID: "1234", + Location: nbpeer.Location{ConnectionIP: net.IP{10, 0, 0, 1}}, + Status: &nbpeer.PeerStatus{LastSeen: time.Now()}, + }, } err = db.Save(account).Error diff --git a/management/server/peer/peer.go b/management/server/peer/peer.go index 2963dfcbd..6294d1c0a 100644 --- a/management/server/peer/peer.go +++ b/management/server/peer/peer.go @@ -86,7 +86,7 @@ type PeerStatus struct { //nolint:revive // active session". Integer nanoseconds are used so equality is // precision-safe across drivers, and so the predicates compose to a // single bigint comparison. - SessionStartedAt int64 + SessionStartedAt int64 `gorm:"not null;default:0"` // Connected indicates whether peer is connected to the management service or not Connected bool // LoginExpired diff --git a/management/server/peer_test.go b/management/server/peer_test.go index 07acf865f..9d6856740 100644 --- a/management/server/peer_test.go +++ b/management/server/peer_test.go @@ -2218,6 +2218,9 @@ func Test_IsUniqueConstraintError(t *testing.T) { ID: "test-peer-id", AccountID: "bf1c8084-ba50-4ce7-9439-34653001fc3b", DNSLabel: "test-peer-dns-label", + Status: &nbpeer.PeerStatus{ + LastSeen: time.Now(), + }, } for _, tt := range tests { diff --git a/management/server/store/store.go b/management/server/store/store.go index a723c1fc3..045f1576a 100644 --- a/management/server/store/store.go +++ b/management/server/store/store.go @@ -471,6 +471,9 @@ func getMigrationsPreAuto(ctx context.Context) []migrationFunc { func(db *gorm.DB) error { return migration.MigrateNewField[types.User](ctx, db, "email", "") }, + func(db *gorm.DB) error { + return migration.MigrateNewField[nbpeer.Peer](ctx, db, "peer_status_session_started_at", int64(0)) + }, func(db *gorm.DB) error { return migration.RemoveDuplicatePeerKeys(ctx, db) }, From d250f92c435bac83fd55f00fad3ee2c292eee910 Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Wed, 20 May 2026 10:08:34 +0200 Subject: [PATCH 26/29] feat(reverse-proxy): clusters API surfaces type, online status, and capability flags (#6148) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cluster listing now answers three questions in one round-trip instead of forcing the dashboard to cross-reference the domains API: which clusters can this account see, are they currently up, and what do they support. The ProxyCluster wire type drops the boolean self_hosted in favour of a `type` enum (`account` / `shared`) plus explicit `online`, `supports_custom_ports`, `require_subdomain`, and `supports_crowdsec` fields. Store query reworked so offline clusters still appear (no last_seen WHERE), with online and connected_proxies both derived from the existing 2-min active window via portable CASE expressions; the 1-hour heartbeat reaper still removes long-stale rows. Service manager enriches each cluster with the capability flags via the existing per-cluster lookups (CapabilityProvider now also exposes ClusterSupportsCrowdSec). GetActiveClusterAddresses* keep their tight 2-min filter so service routing and domain enumeration aren't pulled into the wider window. The hard cut removes self_hosted from the response — the dashboard is the only consumer and is updated in the matching PR; no transitional field is shipped. Adds a cross-engine regression test asserting offline clusters surface, connected_proxies counts only fresh proxies, and account-scoped BYOP clusters never leak across accounts. --- .../reverseproxy/proxy/manager/manager.go | 2 +- .../proxy/manager/manager_test.go | 2 +- .../modules/reverseproxy/proxy/proxy.go | 27 ++++- .../modules/reverseproxy/service/interface.go | 2 +- .../reverseproxy/service/interface_mock.go | 72 ++++++------ .../reverseproxy/service/manager/api.go | 14 ++- .../reverseproxy/service/manager/manager.go | 22 +++- .../shared/grpc/proxy_group_access_test.go | 2 +- .../shared/grpc/validate_session_test.go | 2 +- .../proxy/auth_callback_integration_test.go | 2 +- management/server/store/sql_store.go | 64 ++++++++-- .../store/sql_store_proxy_clusters_test.go | 109 ++++++++++++++++++ management/server/store/store.go | 2 +- management/server/store/store_mock.go | 86 +++++++------- proxy/management_integration_test.go | 2 +- shared/management/http/api/openapi.yml | 32 ++++- shared/management/http/api/types.gen.go | 73 +++++++++--- 17 files changed, 393 insertions(+), 122 deletions(-) create mode 100644 management/server/store/sql_store_proxy_clusters_test.go diff --git a/management/internals/modules/reverseproxy/proxy/manager/manager.go b/management/internals/modules/reverseproxy/proxy/manager/manager.go index b72a6ebe5..510500e0c 100644 --- a/management/internals/modules/reverseproxy/proxy/manager/manager.go +++ b/management/internals/modules/reverseproxy/proxy/manager/manager.go @@ -17,7 +17,7 @@ type store interface { UpdateProxyHeartbeat(ctx context.Context, p *proxy.Proxy) error GetActiveProxyClusterAddresses(ctx context.Context) ([]string, error) GetActiveProxyClusterAddressesForAccount(ctx context.Context, accountID string) ([]string, error) - GetActiveProxyClusters(ctx context.Context, accountID string) ([]proxy.Cluster, error) + GetProxyClusters(ctx context.Context, accountID string) ([]proxy.Cluster, error) GetClusterSupportsCustomPorts(ctx context.Context, clusterAddr string) *bool GetClusterRequireSubdomain(ctx context.Context, clusterAddr string) *bool GetClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool diff --git a/management/internals/modules/reverseproxy/proxy/manager/manager_test.go b/management/internals/modules/reverseproxy/proxy/manager/manager_test.go index 3c53fe684..3436216b4 100644 --- a/management/internals/modules/reverseproxy/proxy/manager/manager_test.go +++ b/management/internals/modules/reverseproxy/proxy/manager/manager_test.go @@ -57,7 +57,7 @@ func (m *mockStore) GetActiveProxyClusterAddressesForAccount(ctx context.Context } return nil, nil } -func (m *mockStore) GetActiveProxyClusters(_ context.Context, _ string) ([]proxy.Cluster, error) { +func (m *mockStore) GetProxyClusters(_ context.Context, _ string) ([]proxy.Cluster, error) { return nil, nil } func (m *mockStore) CleanupStaleProxies(ctx context.Context, d time.Duration) error { diff --git a/management/internals/modules/reverseproxy/proxy/proxy.go b/management/internals/modules/reverseproxy/proxy/proxy.go index 64394799e..9da7910df 100644 --- a/management/internals/modules/reverseproxy/proxy/proxy.go +++ b/management/internals/modules/reverseproxy/proxy/proxy.go @@ -42,10 +42,35 @@ func (Proxy) TableName() string { return "proxies" } +// ClusterType is the source of a proxy cluster. +type ClusterType string + +const ( + // ClusterTypeAccount is a cluster operated by the account itself (BYOP) — + // at least one proxy row in the cluster carries a non-NULL account_id. + ClusterTypeAccount ClusterType = "account" + // ClusterTypeShared is a cluster operated by NetBird and shared across + // accounts — all proxy rows in the cluster have account_id IS NULL. + ClusterTypeShared ClusterType = "shared" +) + // Cluster represents a group of proxy nodes serving the same address. +// +// Online and ConnectedProxies derive from the same 2-min active window +// the rest of the module uses, but Cluster rows are not gated on it — +// the cluster listing surfaces offline clusters too so operators can +// see and clean them up. The 1-hour heartbeat reaper still bounds the +// table eventually. type Cluster struct { ID string Address string + Type ClusterType + Online bool ConnectedProxies int - SelfHosted bool + // Capability flags. *bool because nil means "no proxy reported a + // capability for this cluster" — the dashboard renders these as + // unknown rather than false. + SupportsCustomPorts *bool + RequireSubdomain *bool + SupportsCrowdSec *bool } diff --git a/management/internals/modules/reverseproxy/service/interface.go b/management/internals/modules/reverseproxy/service/interface.go index 6a94aa32b..dddf6ae8a 100644 --- a/management/internals/modules/reverseproxy/service/interface.go +++ b/management/internals/modules/reverseproxy/service/interface.go @@ -9,7 +9,7 @@ import ( ) type Manager interface { - GetActiveClusters(ctx context.Context, accountID, userID string) ([]proxy.Cluster, error) + GetClusters(ctx context.Context, accountID, userID string) ([]proxy.Cluster, error) DeleteAccountCluster(ctx context.Context, accountID, userID, clusterAddress string) error GetAllServices(ctx context.Context, accountID, userID string) ([]*Service, error) GetService(ctx context.Context, accountID, userID, serviceID string) (*Service, error) diff --git a/management/internals/modules/reverseproxy/service/interface_mock.go b/management/internals/modules/reverseproxy/service/interface_mock.go index 83b2162ed..24963fe30 100644 --- a/management/internals/modules/reverseproxy/service/interface_mock.go +++ b/management/internals/modules/reverseproxy/service/interface_mock.go @@ -65,20 +65,6 @@ func (mr *MockManagerMockRecorder) CreateServiceFromPeer(ctx, accountID, peerID, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateServiceFromPeer", reflect.TypeOf((*MockManager)(nil).CreateServiceFromPeer), ctx, accountID, peerID, req) } -// DeleteAllServices mocks base method. -func (m *MockManager) DeleteAllServices(ctx context.Context, accountID, userID string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteAllServices", ctx, accountID, userID) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteAllServices indicates an expected call of DeleteAllServices. -func (mr *MockManagerMockRecorder) DeleteAllServices(ctx, accountID, userID interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllServices", reflect.TypeOf((*MockManager)(nil).DeleteAllServices), ctx, accountID, userID) -} - // DeleteAccountCluster mocks base method. func (m *MockManager) DeleteAccountCluster(ctx context.Context, accountID, userID, clusterAddress string) error { m.ctrl.T.Helper() @@ -93,6 +79,20 @@ func (mr *MockManagerMockRecorder) DeleteAccountCluster(ctx, accountID, userID, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAccountCluster", reflect.TypeOf((*MockManager)(nil).DeleteAccountCluster), ctx, accountID, userID, clusterAddress) } +// DeleteAllServices mocks base method. +func (m *MockManager) DeleteAllServices(ctx context.Context, accountID, userID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAllServices", ctx, accountID, userID) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAllServices indicates an expected call of DeleteAllServices. +func (mr *MockManagerMockRecorder) DeleteAllServices(ctx, accountID, userID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllServices", reflect.TypeOf((*MockManager)(nil).DeleteAllServices), ctx, accountID, userID) +} + // DeleteService mocks base method. func (m *MockManager) DeleteService(ctx context.Context, accountID, userID, serviceID string) error { m.ctrl.T.Helper() @@ -122,21 +122,6 @@ func (mr *MockManagerMockRecorder) GetAccountServices(ctx, accountID interface{} return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAccountServices", reflect.TypeOf((*MockManager)(nil).GetAccountServices), ctx, accountID) } -// GetActiveClusters mocks base method. -func (m *MockManager) GetActiveClusters(ctx context.Context, accountID, userID string) ([]proxy.Cluster, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetActiveClusters", ctx, accountID, userID) - ret0, _ := ret[0].([]proxy.Cluster) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetActiveClusters indicates an expected call of GetActiveClusters. -func (mr *MockManagerMockRecorder) GetActiveClusters(ctx, accountID, userID interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveClusters", reflect.TypeOf((*MockManager)(nil).GetActiveClusters), ctx, accountID, userID) -} - // GetAllServices mocks base method. func (m *MockManager) GetAllServices(ctx context.Context, accountID, userID string) ([]*Service, error) { m.ctrl.T.Helper() @@ -152,19 +137,19 @@ func (mr *MockManagerMockRecorder) GetAllServices(ctx, accountID, userID interfa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllServices", reflect.TypeOf((*MockManager)(nil).GetAllServices), ctx, accountID, userID) } -// GetServiceByDomain mocks base method. -func (m *MockManager) GetServiceByDomain(ctx context.Context, domain string) (*Service, error) { +// GetClusters mocks base method. +func (m *MockManager) GetClusters(ctx context.Context, accountID, userID string) ([]proxy.Cluster, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetServiceByDomain", ctx, domain) - ret0, _ := ret[0].(*Service) + ret := m.ctrl.Call(m, "GetClusters", ctx, accountID, userID) + ret0, _ := ret[0].([]proxy.Cluster) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetServiceByDomain indicates an expected call of GetServiceByDomain. -func (mr *MockManagerMockRecorder) GetServiceByDomain(ctx, domain interface{}) *gomock.Call { +// GetClusters indicates an expected call of GetClusters. +func (mr *MockManagerMockRecorder) GetClusters(ctx, accountID, userID interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceByDomain", reflect.TypeOf((*MockManager)(nil).GetServiceByDomain), ctx, domain) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusters", reflect.TypeOf((*MockManager)(nil).GetClusters), ctx, accountID, userID) } // GetGlobalServices mocks base method. @@ -197,6 +182,21 @@ func (mr *MockManagerMockRecorder) GetService(ctx, accountID, userID, serviceID return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetService", reflect.TypeOf((*MockManager)(nil).GetService), ctx, accountID, userID, serviceID) } +// GetServiceByDomain mocks base method. +func (m *MockManager) GetServiceByDomain(ctx context.Context, domain string) (*Service, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetServiceByDomain", ctx, domain) + ret0, _ := ret[0].(*Service) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetServiceByDomain indicates an expected call of GetServiceByDomain. +func (mr *MockManagerMockRecorder) GetServiceByDomain(ctx, domain interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServiceByDomain", reflect.TypeOf((*MockManager)(nil).GetServiceByDomain), ctx, domain) +} + // GetServiceByID mocks base method. func (m *MockManager) GetServiceByID(ctx context.Context, accountID, serviceID string) (*Service, error) { m.ctrl.T.Helper() diff --git a/management/internals/modules/reverseproxy/service/manager/api.go b/management/internals/modules/reverseproxy/service/manager/api.go index 08272077c..9d93d52ee 100644 --- a/management/internals/modules/reverseproxy/service/manager/api.go +++ b/management/internals/modules/reverseproxy/service/manager/api.go @@ -187,7 +187,7 @@ func (h *handler) getClusters(w http.ResponseWriter, r *http.Request) { return } - clusters, err := h.manager.GetActiveClusters(r.Context(), userAuth.AccountId, userAuth.UserId) + clusters, err := h.manager.GetClusters(r.Context(), userAuth.AccountId, userAuth.UserId) if err != nil { util.WriteError(r.Context(), err, w) return @@ -196,10 +196,14 @@ func (h *handler) getClusters(w http.ResponseWriter, r *http.Request) { apiClusters := make([]api.ProxyCluster, 0, len(clusters)) for _, c := range clusters { apiClusters = append(apiClusters, api.ProxyCluster{ - Id: c.ID, - Address: c.Address, - ConnectedProxies: c.ConnectedProxies, - SelfHosted: c.SelfHosted, + Id: c.ID, + Address: c.Address, + Type: api.ProxyClusterType(c.Type), + Online: c.Online, + ConnectedProxies: c.ConnectedProxies, + SupportsCustomPorts: c.SupportsCustomPorts, + RequireSubdomain: c.RequireSubdomain, + SupportsCrowdsec: c.SupportsCrowdSec, }) } diff --git a/management/internals/modules/reverseproxy/service/manager/manager.go b/management/internals/modules/reverseproxy/service/manager/manager.go index 4a8598afb..ca0c5540f 100644 --- a/management/internals/modules/reverseproxy/service/manager/manager.go +++ b/management/internals/modules/reverseproxy/service/manager/manager.go @@ -81,6 +81,7 @@ type ClusterDeriver interface { type CapabilityProvider interface { ClusterSupportsCustomPorts(ctx context.Context, clusterAddr string) *bool ClusterRequireSubdomain(ctx context.Context, clusterAddr string) *bool + ClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool } type Manager struct { @@ -112,8 +113,12 @@ func (m *Manager) StartExposeReaper(ctx context.Context) { m.exposeReaper.StartExposeReaper(ctx) } -// GetActiveClusters returns all active proxy clusters with their connected proxy count. -func (m *Manager) GetActiveClusters(ctx context.Context, accountID, userID string) ([]proxy.Cluster, error) { +// GetClusters returns every proxy cluster visible to the account +// (shared + its own BYOP), regardless of whether any proxy in the +// cluster is currently heartbeating. Each cluster is enriched with the +// capability flags reported by its active proxies so the dashboard can +// render feature support without a second round-trip. +func (m *Manager) GetClusters(ctx context.Context, accountID, userID string) ([]proxy.Cluster, error) { ok, err := m.permissionsManager.ValidateUserPermissions(ctx, accountID, userID, modules.Services, operations.Read) if err != nil { return nil, status.NewPermissionValidationError(err) @@ -122,7 +127,18 @@ func (m *Manager) GetActiveClusters(ctx context.Context, accountID, userID strin return nil, status.NewPermissionDeniedError() } - return m.store.GetActiveProxyClusters(ctx, accountID) + clusters, err := m.store.GetProxyClusters(ctx, accountID) + if err != nil { + return nil, err + } + + for i := range clusters { + clusters[i].SupportsCustomPorts = m.capabilities.ClusterSupportsCustomPorts(ctx, clusters[i].Address) + clusters[i].RequireSubdomain = m.capabilities.ClusterRequireSubdomain(ctx, clusters[i].Address) + clusters[i].SupportsCrowdSec = m.capabilities.ClusterSupportsCrowdSec(ctx, clusters[i].Address) + } + + return clusters, nil } // DeleteAccountCluster removes all proxy registrations for the given cluster address diff --git a/management/internals/shared/grpc/proxy_group_access_test.go b/management/internals/shared/grpc/proxy_group_access_test.go index 46dad5b56..5980f8a30 100644 --- a/management/internals/shared/grpc/proxy_group_access_test.go +++ b/management/internals/shared/grpc/proxy_group_access_test.go @@ -109,7 +109,7 @@ func (m *mockReverseProxyManager) GetServiceByDomain(_ context.Context, domain s return nil, errors.New("service not found for domain: " + domain) } -func (m *mockReverseProxyManager) GetActiveClusters(_ context.Context, _, _ string) ([]proxy.Cluster, error) { +func (m *mockReverseProxyManager) GetClusters(_ context.Context, _, _ string) ([]proxy.Cluster, error) { return nil, nil } diff --git a/management/internals/shared/grpc/validate_session_test.go b/management/internals/shared/grpc/validate_session_test.go index 7b7ffcfb2..774c5d1d3 100644 --- a/management/internals/shared/grpc/validate_session_test.go +++ b/management/internals/shared/grpc/validate_session_test.go @@ -322,7 +322,7 @@ func (m *testValidateSessionServiceManager) GetServiceByDomain(ctx context.Conte return m.store.GetServiceByDomain(ctx, domain) } -func (m *testValidateSessionServiceManager) GetActiveClusters(_ context.Context, _, _ string) ([]proxy.Cluster, error) { +func (m *testValidateSessionServiceManager) GetClusters(_ context.Context, _, _ string) ([]proxy.Cluster, error) { return nil, nil } diff --git a/management/server/http/handlers/proxy/auth_callback_integration_test.go b/management/server/http/handlers/proxy/auth_callback_integration_test.go index 30d8aa0e7..f08d5daf1 100644 --- a/management/server/http/handlers/proxy/auth_callback_integration_test.go +++ b/management/server/http/handlers/proxy/auth_callback_integration_test.go @@ -444,7 +444,7 @@ func (m *testServiceManager) GetServiceByDomain(ctx context.Context, domain stri return m.store.GetServiceByDomain(ctx, domain) } -func (m *testServiceManager) GetActiveClusters(_ context.Context, _, _ string) ([]nbproxy.Cluster, error) { +func (m *testServiceManager) GetClusters(_ context.Context, _, _ string) ([]nbproxy.Cluster, error) { return nil, nil } diff --git a/management/server/store/sql_store.go b/management/server/store/sql_store.go index 8cf37de56..f3c6b741b 100644 --- a/management/server/store/sql_store.go +++ b/management/server/store/sql_store.go @@ -5736,19 +5736,67 @@ func (s *SqlStore) DeleteAccountCluster(ctx context.Context, clusterAddress, acc return nil } -func (s *SqlStore) GetActiveProxyClusters(ctx context.Context, accountID string) ([]proxy.Cluster, error) { - var clusters []proxy.Cluster +// GetProxyClusters returns every cluster the account can see (shared +// plus its own BYOP), regardless of whether any proxy in the cluster +// is currently heartbeating. Online and ConnectedProxies are derived +// from the 2-min active window so the dashboard can render offline +// clusters distinctly; the 1-hour heartbeat reaper still removes rows +// that go quiet for too long. +// +// AccountOwned is determined by whether any proxy row in the group +// carries a non-NULL account_id; the caller maps that to Cluster.Type. +// Capability flags are NOT filled here — the handler enriches them via +// the per-cluster capability lookups. +func (s *SqlStore) GetProxyClusters(ctx context.Context, accountID string) ([]proxy.Cluster, error) { + activeCutoff := time.Now().Add(-proxyActiveThreshold) + type clusterRow struct { + ID string + Address string + ConnectedProxies int + Online bool + AccountOwned bool + } + + var rows []clusterRow result := s.db.Model(&proxy.Proxy{}). - Select("MIN(id) as id, cluster_address as address, COUNT(*) as connected_proxies, COUNT(account_id) > 0 as self_hosted"). - Where("status = ? AND last_seen > ? AND (account_id IS NULL OR account_id = ?)", - proxy.StatusConnected, time.Now().Add(-proxyActiveThreshold), accountID). + Select( + "MIN(id) AS id, "+ + "cluster_address AS address, "+ + // COUNT(CASE WHEN ... THEN 1 END) counts only non-NULL — i.e. only + // rows that satisfy the predicate — so it works portably across + // sqlite/postgres/mysql without dialect-specific FILTER syntax. + "COUNT(CASE WHEN status = ? AND last_seen > ? THEN 1 END) AS connected_proxies, "+ + // MAX(CASE …) > 0 expresses BOOL_OR in a way Postgres tolerates + // (Postgres can't MAX a boolean column). + "MAX(CASE WHEN status = ? AND last_seen > ? THEN 1 ELSE 0 END) > 0 AS online, "+ + "MAX(CASE WHEN account_id IS NOT NULL THEN 1 ELSE 0 END) > 0 AS account_owned", + proxy.StatusConnected, activeCutoff, + proxy.StatusConnected, activeCutoff, + ). + Where("account_id IS NULL OR account_id = ?", accountID). Group("cluster_address"). - Scan(&clusters) + Scan(&rows) if result.Error != nil { - log.WithContext(ctx).Errorf("failed to get active proxy clusters: %v", result.Error) - return nil, status.Errorf(status.Internal, "get active proxy clusters") + log.WithContext(ctx).Errorf("failed to get proxy clusters: %v", result.Error) + return nil, status.Errorf(status.Internal, "get proxy clusters") + } + + clusters := make([]proxy.Cluster, 0, len(rows)) + for _, r := range rows { + c := proxy.Cluster{ + ID: r.ID, + Address: r.Address, + Online: r.Online, + ConnectedProxies: r.ConnectedProxies, + } + if r.AccountOwned { + c.Type = proxy.ClusterTypeAccount + } else { + c.Type = proxy.ClusterTypeShared + } + clusters = append(clusters, c) } return clusters, nil diff --git a/management/server/store/sql_store_proxy_clusters_test.go b/management/server/store/sql_store_proxy_clusters_test.go new file mode 100644 index 000000000..cdacfedae --- /dev/null +++ b/management/server/store/sql_store_proxy_clusters_test.go @@ -0,0 +1,109 @@ +package store + +import ( + "context" + "os" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + rpproxy "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy" +) + +// TestSqlStore_GetProxyClusters_DerivesOnlineAndType guards the +// account-visible cluster list against silent regressions in two +// dimensions: +// +// 1. Online derivation: a cluster with one stale and one fresh proxy +// is online and counts only the fresh proxy; a cluster whose +// proxies all heartbeated outside the 2-min window appears offline +// with connected_proxies = 0 (rather than disappearing, which is +// what the old query did). +// 2. Type derivation: a cluster scoped to the calling account is +// reported as `account`; a cluster with account_id IS NULL is +// reported as `shared`. Clusters scoped to other accounts must not +// leak into the result. +// +// Capability flags are intentionally not asserted here — they're filled +// by the manager (handler) layer from the per-cluster capability +// lookups, not by the store query. +func TestSqlStore_GetProxyClusters_DerivesOnlineAndType(t *testing.T) { + if (os.Getenv("CI") == "true" && runtime.GOOS == "darwin") || runtime.GOOS == "windows" { + t.Skip("skip CI tests on darwin and windows") + } + + runTestForAllEngines(t, "", func(t *testing.T, store Store) { + ctx := context.Background() + accountID := "acct-clusters" + require.NoError(t, store.SaveAccount(ctx, newAccountWithId(ctx, accountID, "user-1", ""))) + + otherAccountID := "acct-other" + require.NoError(t, store.SaveAccount(ctx, newAccountWithId(ctx, otherAccountID, "user-2", ""))) + + acctID := accountID + otherID := otherAccountID + + fresh := time.Now().Add(-30 * time.Second) + stale := time.Now().Add(-30 * time.Minute) + + mustSave := func(id, cluster string, accID *string, status string, lastSeen time.Time) { + require.NoError(t, store.SaveProxy(ctx, &rpproxy.Proxy{ + ID: id, + SessionID: id + "-sess", + ClusterAddress: cluster, + IPAddress: "10.0.0.1", + AccountID: accID, + LastSeen: lastSeen, + Status: status, + })) + } + + // shared-mixed: one fresh + one stale proxy → online, connected=1 + mustSave("p-shared-fresh", "shared-mixed.netbird.io", nil, rpproxy.StatusConnected, fresh) + mustSave("p-shared-stale", "shared-mixed.netbird.io", nil, rpproxy.StatusConnected, stale) + + // shared-offline: only stale proxies → offline, connected=0, + // but row must still appear (this is the new semantic — old + // query would have dropped it entirely). + mustSave("p-shared-off", "shared-offline.netbird.io", nil, rpproxy.StatusConnected, stale) + + // account-online: BYOP cluster owned by acctID, fresh + mustSave("p-acct-fresh", "byop.acct.example", &acctID, rpproxy.StatusConnected, fresh) + + // other-account: must not surface for acctID + mustSave("p-other", "byop.other.example", &otherID, rpproxy.StatusConnected, fresh) + + clusters, err := store.GetProxyClusters(ctx, accountID) + require.NoError(t, err) + + byAddr := map[string]rpproxy.Cluster{} + for _, c := range clusters { + byAddr[c.Address] = c + } + + assert.NotContains(t, byAddr, "byop.other.example", + "another account's BYOP cluster must not leak into this account's listing") + + require.Contains(t, byAddr, "shared-mixed.netbird.io") + mixed := byAddr["shared-mixed.netbird.io"] + assert.Equal(t, rpproxy.ClusterTypeShared, mixed.Type, "shared cluster (account_id IS NULL) must be reported as Type=shared") + assert.True(t, mixed.Online, "cluster with a fresh proxy must be online") + assert.Equal(t, 1, mixed.ConnectedProxies, "connected_proxies must count only fresh proxies; the stale one should not bump the count") + + require.Contains(t, byAddr, "shared-offline.netbird.io", + "offline clusters must still appear so the dashboard can render them — the old GetActiveProxyClusters would have dropped this row, which is the regression this test guards against") + offline := byAddr["shared-offline.netbird.io"] + assert.Equal(t, rpproxy.ClusterTypeShared, offline.Type) + assert.False(t, offline.Online, "no fresh heartbeat → offline") + assert.Equal(t, 0, offline.ConnectedProxies, "no fresh proxies → connected_proxies=0") + + require.Contains(t, byAddr, "byop.acct.example") + acct := byAddr["byop.acct.example"] + assert.Equal(t, rpproxy.ClusterTypeAccount, acct.Type, "BYOP cluster owned by the account must be reported as Type=account") + assert.True(t, acct.Online) + assert.Equal(t, 1, acct.ConnectedProxies) + }) +} diff --git a/management/server/store/store.go b/management/server/store/store.go index 045f1576a..42cdcf36d 100644 --- a/management/server/store/store.go +++ b/management/server/store/store.go @@ -307,7 +307,7 @@ type Store interface { UpdateProxyHeartbeat(ctx context.Context, p *proxy.Proxy) error GetActiveProxyClusterAddresses(ctx context.Context) ([]string, error) GetActiveProxyClusterAddressesForAccount(ctx context.Context, accountID string) ([]string, error) - GetActiveProxyClusters(ctx context.Context, accountID string) ([]proxy.Cluster, error) + GetProxyClusters(ctx context.Context, accountID string) ([]proxy.Cluster, error) GetClusterSupportsCustomPorts(ctx context.Context, clusterAddr string) *bool GetClusterRequireSubdomain(ctx context.Context, clusterAddr string) *bool GetClusterSupportsCrowdSec(ctx context.Context, clusterAddr string) *bool diff --git a/management/server/store/store_mock.go b/management/server/store/store_mock.go index d51629606..4f9d875d2 100644 --- a/management/server/store/store_mock.go +++ b/management/server/store/store_mock.go @@ -380,6 +380,20 @@ func (mr *MockStoreMockRecorder) DeleteAccount(ctx, account interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAccount", reflect.TypeOf((*MockStore)(nil).DeleteAccount), ctx, account) } +// DeleteAccountCluster mocks base method. +func (m *MockStore) DeleteAccountCluster(ctx context.Context, clusterAddress, accountID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAccountCluster", ctx, clusterAddress, accountID) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAccountCluster indicates an expected call of DeleteAccountCluster. +func (mr *MockStoreMockRecorder) DeleteAccountCluster(ctx, clusterAddress, accountID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAccountCluster", reflect.TypeOf((*MockStore)(nil).DeleteAccountCluster), ctx, clusterAddress, accountID) +} + // DeleteCustomDomain mocks base method. func (m *MockStore) DeleteCustomDomain(ctx context.Context, accountID, domainID string) error { m.ctrl.T.Helper() @@ -577,20 +591,6 @@ func (mr *MockStoreMockRecorder) DeletePostureChecks(ctx, accountID, postureChec return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeletePostureChecks", reflect.TypeOf((*MockStore)(nil).DeletePostureChecks), ctx, accountID, postureChecksID) } -// DeleteAccountCluster mocks base method. -func (m *MockStore) DeleteAccountCluster(ctx context.Context, clusterAddress, accountID string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteAccountCluster", ctx, clusterAddress, accountID) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteAccountCluster indicates an expected call of DeleteAccountCluster. -func (mr *MockStoreMockRecorder) DeleteAccountCluster(ctx, clusterAddress, accountID interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAccountCluster", reflect.TypeOf((*MockStore)(nil).DeleteAccountCluster), ctx, clusterAddress, accountID) -} - // DeleteRoute mocks base method. func (m *MockStore) DeleteRoute(ctx context.Context, accountID, routeID string) error { m.ctrl.T.Helper() @@ -731,6 +731,20 @@ func (mr *MockStoreMockRecorder) DeleteZoneDNSRecords(ctx, accountID, zoneID int return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteZoneDNSRecords", reflect.TypeOf((*MockStore)(nil).DeleteZoneDNSRecords), ctx, accountID, zoneID) } +// DisconnectProxy mocks base method. +func (m *MockStore) DisconnectProxy(ctx context.Context, proxyID, sessionID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DisconnectProxy", ctx, proxyID, sessionID) + ret0, _ := ret[0].(error) + return ret0 +} + +// DisconnectProxy indicates an expected call of DisconnectProxy. +func (mr *MockStoreMockRecorder) DisconnectProxy(ctx, proxyID, sessionID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DisconnectProxy", reflect.TypeOf((*MockStore)(nil).DisconnectProxy), ctx, proxyID, sessionID) +} + // EphemeralServiceExists mocks base method. func (m *MockStore) EphemeralServiceExists(ctx context.Context, lockStrength LockingStrength, accountID, peerID, domain string) (bool, error) { m.ctrl.T.Helper() @@ -1332,21 +1346,6 @@ func (mr *MockStoreMockRecorder) GetActiveProxyClusterAddressesForAccount(ctx, a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveProxyClusterAddressesForAccount", reflect.TypeOf((*MockStore)(nil).GetActiveProxyClusterAddressesForAccount), ctx, accountID) } -// GetActiveProxyClusters mocks base method. -func (m *MockStore) GetActiveProxyClusters(ctx context.Context, accountID string) ([]proxy.Cluster, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetActiveProxyClusters", ctx, accountID) - ret0, _ := ret[0].([]proxy.Cluster) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetActiveProxyClusters indicates an expected call of GetActiveProxyClusters. -func (mr *MockStoreMockRecorder) GetActiveProxyClusters(ctx, accountID interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveProxyClusters", reflect.TypeOf((*MockStore)(nil).GetActiveProxyClusters), ctx, accountID) -} - // GetAllAccounts mocks base method. func (m *MockStore) GetAllAccounts(ctx context.Context) []*types2.Account { m.ctrl.T.Helper() @@ -2048,6 +2047,21 @@ func (mr *MockStoreMockRecorder) GetProxyByAccountID(ctx, accountID interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProxyByAccountID", reflect.TypeOf((*MockStore)(nil).GetProxyByAccountID), ctx, accountID) } +// GetProxyClusters mocks base method. +func (m *MockStore) GetProxyClusters(ctx context.Context, accountID string) ([]proxy.Cluster, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetProxyClusters", ctx, accountID) + ret0, _ := ret[0].([]proxy.Cluster) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetProxyClusters indicates an expected call of GetProxyClusters. +func (mr *MockStoreMockRecorder) GetProxyClusters(ctx, accountID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProxyClusters", reflect.TypeOf((*MockStore)(nil).GetProxyClusters), ctx, accountID) +} + // GetResourceGroups mocks base method. func (m *MockStore) GetResourceGroups(ctx context.Context, lockStrength LockingStrength, accountID, resourceID string) ([]*types2.Group, error) { m.ctrl.T.Helper() @@ -2950,20 +2964,6 @@ func (mr *MockStoreMockRecorder) SaveProxy(ctx, proxy interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveProxy", reflect.TypeOf((*MockStore)(nil).SaveProxy), ctx, proxy) } -// DisconnectProxy mocks base method. -func (m *MockStore) DisconnectProxy(ctx context.Context, proxyID, sessionID string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DisconnectProxy", ctx, proxyID, sessionID) - ret0, _ := ret[0].(error) - return ret0 -} - -// DisconnectProxy indicates an expected call of DisconnectProxy. -func (mr *MockStoreMockRecorder) DisconnectProxy(ctx, proxyID, sessionID interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DisconnectProxy", reflect.TypeOf((*MockStore)(nil).DisconnectProxy), ctx, proxyID, sessionID) -} - // SaveProxyAccessToken mocks base method. func (m *MockStore) SaveProxyAccessToken(ctx context.Context, token *types2.ProxyAccessToken) error { m.ctrl.T.Helper() diff --git a/proxy/management_integration_test.go b/proxy/management_integration_test.go index 9fd3d2ce9..d7e891801 100644 --- a/proxy/management_integration_test.go +++ b/proxy/management_integration_test.go @@ -366,7 +366,7 @@ func (m *storeBackedServiceManager) GetServiceByDomain(ctx context.Context, doma return m.store.GetServiceByDomain(ctx, domain) } -func (m *storeBackedServiceManager) GetActiveClusters(_ context.Context, _, _ string) ([]nbproxy.Cluster, error) { +func (m *storeBackedServiceManager) GetClusters(_ context.Context, _, _ string) ([]nbproxy.Cluster, error) { return nil, nil } diff --git a/shared/management/http/api/openapi.yml b/shared/management/http/api/openapi.yml index 942f3aa45..353aff72d 100644 --- a/shared/management/http/api/openapi.yml +++ b/shared/management/http/api/openapi.yml @@ -3417,19 +3417,43 @@ components: type: string description: Cluster address used for CNAME targets example: "eu.proxy.netbird.io" + type: + $ref: '#/components/schemas/ProxyClusterType' + online: + type: boolean + description: Whether at least one proxy in the cluster has heartbeated within the active window + example: true connected_proxies: type: integer - description: Number of proxy nodes connected in this cluster + description: Number of proxy nodes currently connected (heartbeat within the active window) example: 3 - self_hosted: + supports_custom_ports: type: boolean - description: Whether this cluster is a self-hosted (BYOP) proxy managed by the account owner + description: Whether the cluster supports binding arbitrary TCP/UDP ports + example: true + require_subdomain: + type: boolean + description: Whether services on this cluster must include a subdomain label + example: false + supports_crowdsec: + type: boolean + description: Whether all active proxies in the cluster have CrowdSec configured example: false required: - id - address + - type + - online - connected_proxies - - self_hosted + ProxyClusterType: + type: string + description: | + Source of the proxy cluster. `account` clusters are owned and operated by the account (BYOP); + `shared` clusters are operated by NetBird and shared across accounts. + enum: + - account + - shared + example: shared ReverseProxyDomainType: type: string description: Type of Reverse Proxy Domain diff --git a/shared/management/http/api/types.gen.go b/shared/management/http/api/types.gen.go index b3bb475a9..16e765f8c 100644 --- a/shared/management/http/api/types.gen.go +++ b/shared/management/http/api/types.gen.go @@ -1,6 +1,6 @@ // Package api provides primitives to interact with the openapi HTTP API. // -// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.6.0 DO NOT EDIT. +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.7.0 DO NOT EDIT. package api import ( @@ -13,8 +13,8 @@ import ( ) const ( - BearerAuthScopes = "BearerAuth.Scopes" - TokenAuthScopes = "TokenAuth.Scopes" + BearerAuthScopes bearerAuthContextKey = "BearerAuth.Scopes" + TokenAuthScopes tokenAuthContextKey = "TokenAuth.Scopes" ) // Defines values for AccessRestrictionsCrowdsecMode. @@ -511,6 +511,7 @@ func (e GroupMinimumIssued) Valid() bool { // Defines values for IdentityProviderType. const ( + IdentityProviderTypeAdfs IdentityProviderType = "adfs" IdentityProviderTypeEntra IdentityProviderType = "entra" IdentityProviderTypeGoogle IdentityProviderType = "google" IdentityProviderTypeMicrosoft IdentityProviderType = "microsoft" @@ -518,12 +519,13 @@ const ( IdentityProviderTypeOkta IdentityProviderType = "okta" IdentityProviderTypePocketid IdentityProviderType = "pocketid" IdentityProviderTypeZitadel IdentityProviderType = "zitadel" - IdentityProviderTypeAdfs IdentityProviderType = "adfs" ) // Valid indicates whether the value is a known member of the IdentityProviderType enum. func (e IdentityProviderType) Valid() bool { switch e { + case IdentityProviderTypeAdfs: + return true case IdentityProviderTypeEntra: return true case IdentityProviderTypeGoogle: @@ -538,8 +540,6 @@ func (e IdentityProviderType) Valid() bool { return true case IdentityProviderTypeZitadel: return true - case IdentityProviderTypeAdfs: - return true default: return false } @@ -878,6 +878,24 @@ func (e PolicyRuleUpdateProtocol) Valid() bool { } } +// Defines values for ProxyClusterType. +const ( + ProxyClusterTypeAccount ProxyClusterType = "account" + ProxyClusterTypeShared ProxyClusterType = "shared" +) + +// Valid indicates whether the value is a known member of the ProxyClusterType enum. +func (e ProxyClusterType) Valid() bool { + switch e { + case ProxyClusterTypeAccount: + return true + case ProxyClusterTypeShared: + return true + default: + return false + } +} + // Defines values for ResourceType. const ( ResourceTypeDomain ResourceType = "domain" @@ -1638,7 +1656,9 @@ type Checks struct { // OsVersionCheck Posture check for the version of operating system OsVersionCheck *OSVersionCheck `json:"os_version_check,omitempty"` - // PeerNetworkRangeCheck Posture check for allow or deny access based on the peer's IP addresses. A range matches when it contains any of the peer's local network interface IPs or its public connection (NAT egress) IP, so ranges may target private subnets, public CIDRs, or single hosts via a /32 or /128. + // PeerNetworkRangeCheck Posture check for allow or deny access based on the peer's IP addresses. A range matches when it + // contains any of the peer's local network interface IPs or its public connection (NAT egress) IP, + // so ranges may target private subnets, public CIDRs, or single hosts via a /32 or /128. PeerNetworkRangeCheck *PeerNetworkRangeCheck `json:"peer_network_range_check,omitempty"` // ProcessCheck Posture Check for binaries exist and are running in the peer’s system @@ -3330,7 +3350,9 @@ type PeerMinimum struct { Name string `json:"name"` } -// PeerNetworkRangeCheck Posture check for allow or deny access based on the peer's IP addresses. A range matches when it contains any of the peer's local network interface IPs or its public connection (NAT egress) IP, so ranges may target private subnets, public CIDRs, or single hosts via a /32 or /128. +// PeerNetworkRangeCheck Posture check for allow or deny access based on the peer's IP addresses. A range matches when it +// contains any of the peer's local network interface IPs or its public connection (NAT egress) IP, +// so ranges may target private subnets, public CIDRs, or single hosts via a /32 or /128. type PeerNetworkRangeCheck struct { // Action Action to take upon policy match Action PeerNetworkRangeCheckAction `json:"action"` @@ -3785,19 +3807,36 @@ type ProxyAccessLogsResponse struct { // ProxyCluster A proxy cluster represents a group of proxy nodes serving the same address type ProxyCluster struct { - // Id Unique identifier of a proxy in this cluster - Id string `json:"id"` - // Address Cluster address used for CNAME targets Address string `json:"address"` - // ConnectedProxies Number of proxy nodes connected in this cluster + // ConnectedProxies Number of proxy nodes currently connected (heartbeat within the active window) ConnectedProxies int `json:"connected_proxies"` - // SelfHosted Whether this cluster is a self-hosted (BYOP) proxy managed by the account owner - SelfHosted bool `json:"self_hosted"` + // Id Unique identifier of a proxy in this cluster + Id string `json:"id"` + + // Online Whether at least one proxy in the cluster has heartbeated within the active window + Online bool `json:"online"` + + // RequireSubdomain Whether services on this cluster must include a subdomain label + RequireSubdomain *bool `json:"require_subdomain,omitempty"` + + // SupportsCrowdsec Whether all active proxies in the cluster have CrowdSec configured + SupportsCrowdsec *bool `json:"supports_crowdsec,omitempty"` + + // SupportsCustomPorts Whether the cluster supports binding arbitrary TCP/UDP ports + SupportsCustomPorts *bool `json:"supports_custom_ports,omitempty"` + + // Type Source of the proxy cluster. `account` clusters are owned and operated by the account (BYOP); + // `shared` clusters are operated by NetBird and shared across accounts. + Type ProxyClusterType `json:"type"` } +// ProxyClusterType Source of the proxy cluster. `account` clusters are owned and operated by the account (BYOP); +// `shared` clusters are operated by NetBird and shared across accounts. +type ProxyClusterType string + // ProxyToken defines model for ProxyToken. type ProxyToken struct { CreatedAt time.Time `json:"created_at"` @@ -4820,6 +4859,12 @@ type ZoneRequest struct { // Conflict Standard error response. Note: The exact structure of this error response is inferred from `util.WriteErrorResponse` and `util.WriteError` usage in the provided Go code, as a specific Go struct for errors was not provided. type Conflict = ErrorResponse +// bearerAuthContextKey is the context key for BearerAuth security scheme +type bearerAuthContextKey string + +// tokenAuthContextKey is the context key for TokenAuth security scheme +type tokenAuthContextKey string + // GetApiEventsNetworkTrafficParams defines parameters for GetApiEventsNetworkTraffic. type GetApiEventsNetworkTrafficParams struct { // Page Page number From c784b0255063b9cbfde830c78670de2400e46c1c Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Wed, 20 May 2026 12:21:03 +0200 Subject: [PATCH 27/29] [misc] Update contribution guidelines (#6219) Update contribution guidelines and PR template to require discussing impactful changes with the team --- .github/pull_request_template.md | 1 + CONTRIBUTING.md | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9d6bc96eb..8e68054bd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -12,6 +12,7 @@ - [ ] Is a feature enhancement - [ ] It is a refactor - [ ] Created tests that fail without the change (if possible) +- [ ] This change does **not** modify the public API, gRPC protocols, functionality behavior, CLI / service flags, or introduce a new feature — **OR** I have discussed it with the NetBird team beforehand (link the issue / Slack thread in the description). See [CONTRIBUTING.md](https://github.com/netbirdio/netbird/blob/main/CONTRIBUTING.md#discuss-changes-with-the-netbird-team-first). > By submitting this pull request, you confirm that you have read and agree to the terms of the [Contributor License Agreement](https://github.com/netbirdio/netbird/blob/main/CONTRIBUTOR_LICENSE_AGREEMENT.md). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 960cd30e9..cd1c087bb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,6 +15,7 @@ If you haven't already, join our slack workspace [here](https://docs.netbird.io/ - [Contributing to NetBird](#contributing-to-netbird) - [Contents](#contents) - [Code of conduct](#code-of-conduct) + - [Discuss changes with the NetBird team first](#discuss-changes-with-the-netbird-team-first) - [Directory structure](#directory-structure) - [Development setup](#development-setup) - [Requirements](#requirements) @@ -33,6 +34,14 @@ Conduct which can be found in the file [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to community@netbird.io. +## Discuss changes with the NetBird team first + +Changes to the **public API**, **gRPC protocols**, **functionality behavior**, **CLI / service flags**, or **new features** should be discussed with the NetBird team before you start the work. These surfaces are part of NetBird's contract with operators, self-hosters, and downstream integrators, and changes to them have compatibility, security, and release-planning implications that benefit from an early conversation. + +Open an issue or reach out on [Slack](https://docs.netbird.io/slack-url) to talk through what you have in mind. We'll help shape the change, flag any constraints we know about, and confirm the direction so the PR review can focus on implementation rather than design. + +Typical bug fixes, internal refactors, documentation updates, and tests do not need pre-discussion — open the PR directly. + ## Directory structure The NetBird project monorepo is organized to maintain most of its individual dependencies code within their directories, except for a few auxiliary or shared packages. From 9192b4f029f8f0eeaad77fff8625c34ba9849668 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Wed, 20 May 2026 20:09:22 +0900 Subject: [PATCH 28/29] [client] Bump macOS sleep callback timeout to 20s (#6220) --- client/internal/sleep/detector_darwin.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/internal/sleep/detector_darwin.go b/client/internal/sleep/detector_darwin.go index ef495bded..fc4713b21 100644 --- a/client/internal/sleep/detector_darwin.go +++ b/client/internal/sleep/detector_darwin.go @@ -188,7 +188,9 @@ func (d *Detector) triggerCallback(event EventType, cb func(event EventType), do } doneChan := make(chan struct{}) - timeout := time.NewTimer(500 * time.Millisecond) + // macOS forces sleep ~30s after kIOMessageSystemWillSleep, so block long + // enough for teardown to finish while staying under that deadline. + timeout := time.NewTimer(20 * time.Second) defer timeout.Stop() go func() { From 4955c345d53f63394266305744841c4e1bff8123 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Wed, 20 May 2026 23:25:56 +0900 Subject: [PATCH 29/29] Clean up README header, key features table, and self-hosted quickstart (#6178) --- README.md | 153 +++++++++++++++++++++++++----------------------------- 1 file changed, 70 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index dc84af2fd..cc27e2d28 100644 --- a/README.md +++ b/README.md @@ -1,147 +1,134 @@
-
-
-

- -

-

- - - - - - -
+

+ NetBird logo +

+

+ + SonarCloud alert status + + + BSD-3 License + - - + NetBird Slack + - - -
+ Community forum + - - + Gurubase: Ask NetBird Guru +

-

- - Start using NetBird at netbird.io + + Start using NetBird at netbird.io +
+ See Documentation +
+ Join our Slack channel or our Community forum +
- See Documentation
- Join our Slack channel or our Community forum -
- - -
- - 🚀 We are hiring! Join us at careers.netbird.io - -
-
- - New: NetBird terraform provider - + + 🚀 We are hiring! Join us at careers.netbird.io +

-
- **NetBird combines a configuration-free peer-to-peer private network and a centralized access control system in a single platform, making it easy to create secure private networks for your organization or home.** **Connect.** NetBird creates a WireGuard-based overlay network that automatically connects your machines over an encrypted tunnel, leaving behind the hassle of opening ports, complex firewall rules, VPN gateways, and so forth. **Secure.** NetBird enables secure remote access by applying granular access policies while allowing you to manage them intuitively from a single place. Works universally on any infrastructure. -### Open Source Network Security in a Single Platform - https://github.com/user-attachments/assets/10cec749-bb56-4ab3-97af-4e38850108d2 -### Self-Host NetBird (Video) +### Self-host NetBird (video) + [![Watch the video](https://img.youtube.com/vi/bZAgpT6nzaQ/0.jpg)](https://youtu.be/bZAgpT6nzaQ) ### Key features -| Connectivity | Management | Security | Automation| Platforms | -|----|----|----|----|----| -|
  • - \[x] Kernel WireGuard
|
  • - \[x] [Admin Web UI](https://github.com/netbirdio/dashboard)
|
  • - \[x] [SSO & MFA support](https://docs.netbird.io/how-to/installation#running-net-bird-with-sso-login)
|
  • - \[x] [Public API](https://docs.netbird.io/api)
|
  • - \[x] Linux
| -|
  • - \[x] Peer-to-peer connections
|
  • - \[x] Auto peer discovery and configuration
    • |
    • - \[x] [Access control - groups & rules](https://docs.netbird.io/how-to/manage-network-access)
      • |
      • - \[x] [Setup keys for bulk network provisioning](https://docs.netbird.io/how-to/register-machines-using-setup-keys)
        • |
        • - \[x] Mac
          • | -|
          • - \[x] Connection relay fallback
            • |
            • - \[x] [IdP integrations](https://docs.netbird.io/selfhosted/identity-providers)
              • |
              • - \[x] [Activity logging](https://docs.netbird.io/how-to/audit-events-logging)
                • |
                • - \[x] [Self-hosting quickstart script](https://docs.netbird.io/selfhosted/selfhosted-quickstart)
                  • |
                  • - \[x] Windows
                    • | -|
                    • - \[x] [Routes to external networks](https://docs.netbird.io/how-to/routing-traffic-to-private-networks)
                      • |
                      • - \[x] [Private DNS](https://docs.netbird.io/how-to/manage-dns-in-your-network)
                        • |
                        • - \[x] [Device posture checks](https://docs.netbird.io/how-to/manage-posture-checks)
                          • |
                          • - \[x] IdP groups sync with JWT
                            • |
                            • - \[x] Android
                              • | -|
                              • - \[x] NAT traversal with BPF
                                • |
                                • - \[x] [Multiuser support](https://docs.netbird.io/how-to/add-users-to-your-network)
                                  • |
                                  • - \[x] Peer-to-peer encryption
                                    • ||
                                    • - \[x] iOS
                                      • | -|||
                                      • - \[x] [Quantum-resistance with Rosenpass](https://netbird.io/knowledge-hub/the-first-quantum-resistant-mesh-vpn)
                                        • ||
                                        • - \[x] OpenWRT
                                          • | -|||
                                          • - \[x] [Periodic re-authentication](https://docs.netbird.io/how-to/enforce-periodic-user-authentication)
                                            • ||
                                            • - \[x] [Serverless](https://docs.netbird.io/how-to/netbird-on-faas)
                                              • | -|||||
                                              • - \[x] Docker
                                                • | +| Connectivity | Management | Security | Automation | Platforms | +|---|---|---|---|---| +| ✓ [Kernel WireGuard](https://docs.netbird.io/about-netbird/why-wireguard-with-netbird) | ✓ [Admin Web UI](https://github.com/netbirdio/dashboard) | ✓ [SSO & MFA support](https://docs.netbird.io/how-to/installation#running-net-bird-with-sso-login) | ✓ [Public API](https://docs.netbird.io/api) | ✓ [Linux](https://docs.netbird.io/get-started/install/linux) | +| ✓ [Peer-to-peer connections](https://docs.netbird.io/about-netbird/how-netbird-works) | ✓ Auto peer discovery and configuration | ✓ [Access control: groups & rules](https://docs.netbird.io/how-to/manage-network-access) | ✓ [Setup keys for bulk provisioning](https://docs.netbird.io/how-to/register-machines-using-setup-keys) | ✓ [macOS](https://docs.netbird.io/get-started/install/macos) | +| ✓ Connection relay fallback | ✓ [IdP integrations](https://docs.netbird.io/selfhosted/identity-providers) | ✓ [Activity logging](https://docs.netbird.io/how-to/audit-events-logging) | ✓ [Self-hosting quickstart script](https://docs.netbird.io/selfhosted/selfhosted-quickstart) | ✓ [Windows](https://docs.netbird.io/get-started/install/windows) | +| ✓ [Routes to external networks](https://docs.netbird.io/how-to/routing-traffic-to-private-networks) | ✓ [Private DNS](https://docs.netbird.io/how-to/manage-dns-in-your-network) | ✓ [Traffic events](https://docs.netbird.io/manage/activity/traffic-events-logging) | ✓ [IdP groups sync with JWT](https://docs.netbird.io/manage/team/idp-sync) | ✓ [Android](https://docs.netbird.io/get-started/install/android) | +| ✓ [Domain-based DNS routes](https://docs.netbird.io/manage/dns/dns-aliases-for-routed-networks) | ✓ [Custom DNS zones](https://docs.netbird.io/manage/dns/custom-zones) | ✓ [Device posture checks](https://docs.netbird.io/how-to/manage-posture-checks) | ✓ [Terraform provider](https://registry.terraform.io/providers/netbirdio/netbird/latest) | ✓ [Android TV](https://docs.netbird.io/get-started/install/android-tv) | +| ✓ [Exit nodes](https://docs.netbird.io/manage/network-routes/use-cases/exit-nodes) | ✓ [Multiuser support](https://docs.netbird.io/how-to/add-users-to-your-network) | ✓ Peer-to-peer encryption | ✓ [Ansible collection](https://github.com/netbirdio/ansible-netbird) | ✓ [iOS](https://docs.netbird.io/get-started/install/ios) | +| ✓ [IPv6 dual-stack overlay](https://docs.netbird.io/manage/settings/ipv6) | ✓ [Multi-account profile switching](https://docs.netbird.io/client/profiles) | ✓ [SSH with central access policies](https://docs.netbird.io/manage/peers/ssh) | | ✓ [Apple TV](https://docs.netbird.io/get-started/install/tvos) | +| ✓ [Browser SSH & RDP](https://docs.netbird.io/manage/peers/browser-client) | | ✓ [Quantum-resistance with Rosenpass](https://netbird.io/knowledge-hub/the-first-quantum-resistant-mesh-vpn) | | ✓ FreeBSD | +| ✓ [Reverse proxy with auto-TLS](https://docs.netbird.io/manage/reverse-proxy) | | ✓ [Periodic re-authentication](https://docs.netbird.io/how-to/enforce-periodic-user-authentication) | | ✓ [pfSense](https://docs.netbird.io/get-started/install/pfsense) | +| | | | | ✓ [OPNsense](https://docs.netbird.io/get-started/install/opnsense) | +| | | | | ✓ [MikroTik RouterOS](https://docs.netbird.io/use-cases/homelab/client-on-mikrotik-router) | +| | | | | ✓ OpenWRT | +| | | | | ✓ [Synology](https://docs.netbird.io/get-started/install/synology) | +| | | | | ✓ [TrueNAS](https://docs.netbird.io/get-started/install/truenas) | +| | | | | ✓ [Proxmox](https://docs.netbird.io/get-started/install/proxmox-ve) | +| | | | | ✓ [Raspberry Pi](https://docs.netbird.io/get-started/install/raspberrypi) | +| | | | | ✓ [Serverless](https://docs.netbird.io/how-to/netbird-on-faas) | +| | | | | ✓ [Container](https://docs.netbird.io/get-started/install/docker) | ### Quickstart with NetBird Cloud -- Download and install NetBird at [https://app.netbird.io/install](https://app.netbird.io/install) -- Follow the steps to sign-up with Google, Microsoft, GitHub or your email address. -- Check NetBird [admin UI](https://app.netbird.io/). -- Add more machines. +- Download and install NetBird at [https://app.netbird.io/install](https://app.netbird.io/install). +- Follow the steps to sign up with Google, Microsoft, GitHub or your email address. +- Check the NetBird [admin UI](https://app.netbird.io/). ### Quickstart with self-hosted NetBird -> This is the quickest way to try self-hosted NetBird. It should take around 5 minutes to get started if you already have a public domain and a VM. -Follow the [Advanced guide with a custom identity provider](https://docs.netbird.io/selfhosted/selfhosted-guide#advanced-guide-with-a-custom-identity-provider) for installations with different IDPs. +This is the quickest way to try self-hosted NetBird. It should take around 5 minutes to get started if you already have a public domain and a VM. Follow the [Advanced guide with a custom identity provider](https://docs.netbird.io/selfhosted/selfhosted-guide#advanced-guide-with-a-custom-identity-provider) for installations with different IdPs. **Infrastructure requirements:** -- A Linux VM with at least **1CPU** and **2GB** of memory. -- The VM should be publicly accessible on TCP ports **80** and **443** and UDP port: **3478**. -- **Public domain** name pointing to the VM. +- A Linux VM with at least **1 CPU** and **2 GB** of memory. +- The VM should be publicly accessible on TCP ports **80** and **443** and UDP port **3478**. +- A **public domain** name pointing to the VM. **Software requirements:** -- Docker installed on the VM with the docker-compose plugin ([Docker installation guide](https://docs.docker.com/engine/install/)) or docker with docker-compose in version 2 or higher. -- [jq](https://jqlang.github.io/jq/) installed. In most distributions - Usually available in the official repositories and can be installed with `sudo apt install jq` or `sudo yum install jq` -- [curl](https://curl.se/) installed. - Usually available in the official repositories and can be installed with `sudo apt install curl` or `sudo yum install curl` +- Docker with the Compose plugin (Compose v2 or higher). See the [Docker installation guide](https://docs.docker.com/engine/install/). **Steps** - Download and run the installation script: ```bash export NETBIRD_DOMAIN=netbird.example.com; curl -fsSL https://github.com/netbirdio/netbird/releases/latest/download/getting-started.sh | bash ``` -- Once finished, you can manage the resources via `docker-compose` ### A bit on NetBird internals -- Every machine in the network runs [NetBird Agent (or Client)](client/) that manages WireGuard. -- Every agent connects to [Management Service](management/) that holds network state, manages peer IPs, and distributes network updates to agents (peers). -- NetBird agent uses WebRTC ICE implemented in [pion/ice library](https://github.com/pion/ice) to discover connection candidates when establishing a peer-to-peer connection between machines. -- Connection candidates are discovered with the help of [STUN](https://en.wikipedia.org/wiki/STUN) servers. -- Agents negotiate a connection through [Signal Service](signal/) passing p2p encrypted messages with candidates. -- Sometimes the NAT traversal is unsuccessful due to strict NATs (e.g. mobile carrier-grade NAT) and a p2p connection isn't possible. When this occurs the system falls back to a relay server called [TURN](https://en.wikipedia.org/wiki/Traversal_Using_Relays_around_NAT), and a secure WireGuard tunnel is established via the TURN server. - -[Coturn](https://github.com/coturn/coturn) is the one that has been successfully used for STUN and TURN in NetBird setups. +- Every machine in the network runs the [NetBird agent](client/), which manages WireGuard. +- Every agent connects to the [Management Service](management/), which holds network state, manages peer IPs, and distributes updates to agents. +- Agents use ICE (via [pion/ice](https://github.com/pion/ice)) to discover connection candidates for peer-to-peer connections. +- Candidates are discovered with the help of [STUN](https://en.wikipedia.org/wiki/STUN) servers. +- Agents negotiate a connection through the [Signal Service](signal/), exchanging end-to-end encrypted messages with candidates. +- When NAT traversal fails (e.g. mobile carrier-grade NAT) and a direct p2p connection isn't possible, the system falls back to a [Relay Service](relay/) and a secure WireGuard tunnel is established through it.

                                                - + NetBird high-level architecture diagram

                                                See a complete [architecture overview](https://docs.netbird.io/about-netbird/how-netbird-works#architecture) for details. ### Community projects -- [NetBird installer script](https://github.com/physk/netbird-installer) -- [NetBird ansible collection by Dominion Solutions](https://galaxy.ansible.com/ui/repo/published/dominion_solutions/netbird/) -- [netbird-tui](https://github.com/n0pashkov/netbird-tui) — terminal UI for managing NetBird peers, routes, and settings +- [NetBird installer script](https://github.com/physk/netbird-installer) +- [netbird-tui](https://github.com/n0pashkov/netbird-tui) - terminal UI for managing NetBird peers, routes, and settings +- [caddy-netbird](https://github.com/lixmal/caddy-netbird) - Caddy plugin that embeds a NetBird client for proxying HTTP and TCP/UDP traffic through NetBird networks **Note**: The `main` branch may be in an *unstable or even broken state* during development. For stable versions, see [releases](https://github.com/netbirdio/netbird/releases). ### Support acknowledgement -In November 2022, NetBird joined the [StartUpSecure program](https://www.forschung-it-sicherheit-kommunikationssysteme.de/foerderung/bekanntmachungen/startup-secure) sponsored by The Federal Ministry of Education and Research of The Federal Republic of Germany. Together with [CISPA Helmholtz Center for Information Security](https://cispa.de/en) NetBird brings the security best practices and simplicity to private networking. +In November 2022, NetBird joined the [StartUpSecure program](https://www.forschung-it-sicherheit-kommunikationssysteme.de/foerderung/bekanntmachungen/startup-secure) sponsored by the Federal Ministry of Education and Research of the Federal Republic of Germany. Together with the [CISPA Helmholtz Center for Information Security](https://cispa.de/en), NetBird brings security best practices and simplicity to private networking. ![CISPA_Logo_BLACK_EN_RZ_RGB (1)](https://user-images.githubusercontent.com/700848/203091324-c6d311a0-22b5-4b05-a288-91cbc6cdcc46.png) -### Testimonials -We use open-source technologies like [WireGuard®](https://www.wireguard.com/), [Pion ICE (WebRTC)](https://github.com/pion/ice), [Coturn](https://github.com/coturn/coturn), and [Rosenpass](https://rosenpass.eu). We very much appreciate the work these guys are doing and we'd greatly appreciate if you could support them in any way (e.g., by giving a star or a contribution). +### Acknowledgements +We build on open-source technologies like [WireGuard®](https://www.wireguard.com/), [Pion ICE](https://github.com/pion/ice), and [Rosenpass](https://rosenpass.eu). We greatly appreciate the work these projects are doing, and we'd love it if you could support them too (e.g., by starring or contributing). ### Legal -This repository is licensed under BSD-3-Clause license that applies to all parts of the repository except for the directories management/, signal/ and relay/. +This repository is licensed under the BSD-3-Clause license, which applies to all parts of the repository except for the directories management/, signal/ and relay/. Those directories are licensed under the GNU Affero General Public License version 3.0 (AGPLv3). See the respective LICENSE files inside each directory. _WireGuard_ and the _WireGuard_ logo are [registered trademarks](https://www.wireguard.com/trademark-policy/) of Jason A. Donenfeld.