diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml index 4427c22f1..0af506bba 100644 --- a/.github/workflows/golang-test-linux.yml +++ b/.github/workflows/golang-test-linux.yml @@ -166,7 +166,7 @@ jobs: # resolve; the grep then drops the broken package by path. Without -e, # go list aborts with empty stdout and `go test` falls back to the repo # root, which has no Go files. - run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} CI=true go test -coverprofile=coverage.txt -tags devcert -timeout 10m -p 1 $(go list -e ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui) + run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} CI=true go test -coverprofile=coverage.txt -tags 'devcert privileged' -exec 'sudo --preserve-env=CI,CGO_ENABLED' -timeout 10m -p 1 $(go list -e ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui -e /client/testutil/privileged) - name: Upload coverage reports to Codecov if: matrix.arch == 'amd64' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 67fa92c0f..76a5b36ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -293,8 +293,11 @@ jobs: ${{ steps.goreleaser.outputs.artifacts }} JSON + # dockers_v2 artifacts have no top-level goarch field, so match the + # per-platform -amd64 tag suffix instead; it works for both the old + # dockers and the new dockers_v2 image naming. mapfile -t src_images < <( - jq -r '.[] | select(.type == "Docker Image") | select(.goarch == "amd64") | .name | select(startswith("ghcr.io/"))' /tmp/goreleaser-artifacts.json + jq -r '.[] | select(.type == "Docker Image") | .name | select(startswith("ghcr.io/") and endswith("-amd64"))' /tmp/goreleaser-artifacts.json ) for src in "${src_images[@]}"; do diff --git a/.gitignore b/.gitignore index 783fe77f3..305f3cb50 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.claude .idea .run *.iml diff --git a/client/android/env_list.go b/client/android/env_list.go index a0a4d7040..d0e0a1e78 100644 --- a/client/android/env_list.go +++ b/client/android/env_list.go @@ -10,7 +10,7 @@ var ( EnvKeyNBForceRelay = peer.EnvKeyNBForceRelay // EnvKeyNBLazyConn Exported for Android java client to configure lazy connection - EnvKeyNBLazyConn = lazyconn.EnvEnableLazyConn + EnvKeyNBLazyConn = lazyconn.EnvLazyConn // EnvKeyNBInactivityThreshold Exported for Android java client to configure connection inactivity threshold EnvKeyNBInactivityThreshold = lazyconn.EnvInactivityThreshold diff --git a/client/cmd/root.go b/client/cmd/root.go index f3fde2f1c..f1ef32717 100644 --- a/client/cmd/root.go +++ b/client/cmd/root.go @@ -71,12 +71,14 @@ var ( extraIFaceBlackList []string anonymizeFlag bool dnsRouteInterval time.Duration - lazyConnEnabled bool - mtu uint16 - profilesDisabled bool - updateSettingsDisabled bool - captureEnabled bool - networksDisabled bool + // lazyConnEnabled is the parse target for the deprecated --enable-lazy-connection + // flag. The flag is inert; the value is no longer read (use NB_LAZY_CONN instead). + lazyConnEnabled bool + mtu uint16 + profilesDisabled bool + updateSettingsDisabled bool + captureEnabled bool + networksDisabled bool rootCmd = &cobra.Command{ Use: "netbird", @@ -210,7 +212,8 @@ func init() { upCmd.PersistentFlags().BoolVar(&rosenpassEnabled, enableRosenpassFlag, false, "[Experimental] Enable Rosenpass feature. If enabled, the connection will be post-quantum secured via Rosenpass.") upCmd.PersistentFlags().BoolVar(&rosenpassPermissive, rosenpassPermissiveFlag, false, "[Experimental] Enable Rosenpass in permissive mode to allow this peer to accept WireGuard connections without requiring Rosenpass functionality from peers that do not have Rosenpass enabled.") upCmd.PersistentFlags().BoolVar(&autoConnectDisabled, disableAutoConnectFlag, false, "Disables auto-connect feature. If enabled, then the client won't connect automatically when the service starts.") - upCmd.PersistentFlags().BoolVar(&lazyConnEnabled, enableLazyConnectionFlag, false, "[Experimental] Enable the lazy connection feature. If enabled, the client will establish connections on-demand. Note: this setting may be overridden by management configuration.") + upCmd.PersistentFlags().BoolVar(&lazyConnEnabled, enableLazyConnectionFlag, false, "Deprecated: no longer used. Lazy connections are controlled by the server and the NB_LAZY_CONN environment variable.") + _ = upCmd.PersistentFlags().MarkDeprecated(enableLazyConnectionFlag, "no longer used; lazy connections are controlled by the server and the NB_LAZY_CONN environment variable") } diff --git a/client/cmd/up.go b/client/cmd/up.go index 0506bc65b..8b3de3c66 100644 --- a/client/cmd/up.go +++ b/client/cmd/up.go @@ -479,10 +479,6 @@ func setupSetConfigReq(customDNSAddressConverted []byte, cmd *cobra.Command, pro req.DisableIpv6 = &disableIPv6 } - if cmd.Flag(enableLazyConnectionFlag).Changed { - req.LazyConnectionEnabled = &lazyConnEnabled - } - return &req } @@ -600,9 +596,6 @@ func setupConfig(customDNSAddressConverted []byte, cmd *cobra.Command, configFil ic.DisableIPv6 = &disableIPv6 } - if cmd.Flag(enableLazyConnectionFlag).Changed { - ic.LazyConnectionEnabled = &lazyConnEnabled - } return &ic, nil } @@ -718,9 +711,6 @@ func setupLoginRequest(providedSetupKey string, customDNSAddressConverted []byte loginRequest.DisableIpv6 = &disableIPv6 } - if cmd.Flag(enableLazyConnectionFlag).Changed { - loginRequest.LazyConnectionEnabled = &lazyConnEnabled - } return &loginRequest, nil } diff --git a/client/iface/configurer/kernel_unix.go b/client/iface/configurer/kernel_unix.go index a29fe181a..da69c2a35 100644 --- a/client/iface/configurer/kernel_unix.go +++ b/client/iface/configurer/kernel_unix.go @@ -17,12 +17,15 @@ import ( type KernelConfigurer struct { deviceName string + statsCache *statsCache } func NewKernelConfigurer(deviceName string) *KernelConfigurer { - return &KernelConfigurer{ + c := &KernelConfigurer{ deviceName: deviceName, } + c.statsCache = newStatsCache(statsCacheTTL, c.fetchStats) + return c } func (c *KernelConfigurer) ConfigureInterface(privateKey string, port int) error { @@ -246,12 +249,6 @@ func (c *KernelConfigurer) configure(config wgtypes.Config) error { } }() - // validate if device with name exists - _, err = wg.Device(c.deviceName) - if err != nil { - return err - } - return wg.ConfigureDevice(c.deviceName, config) } @@ -300,6 +297,14 @@ func (c *KernelConfigurer) FullStats() (*Stats, error) { } func (c *KernelConfigurer) GetStats() (map[string]WGStats, error) { + return c.statsCache.get() +} + +func (c *KernelConfigurer) LastActivities() map[string]monotime.Time { + return nil +} + +func (c *KernelConfigurer) fetchStats() (map[string]WGStats, error) { stats := make(map[string]WGStats) wg, err := wgctrl.New() if err != nil { @@ -326,7 +331,3 @@ func (c *KernelConfigurer) GetStats() (map[string]WGStats, error) { } return stats, nil } - -func (c *KernelConfigurer) LastActivities() map[string]monotime.Time { - return nil -} diff --git a/client/iface/configurer/stats_cache.go b/client/iface/configurer/stats_cache.go new file mode 100644 index 000000000..71a4e88fc --- /dev/null +++ b/client/iface/configurer/stats_cache.go @@ -0,0 +1,52 @@ +package configurer + +import ( + "sync" + "time" + + "golang.org/x/sync/singleflight" +) + +const statsCacheTTL = 1 * time.Second + +type statsCache struct { + ttl time.Duration + fetch func() (map[string]WGStats, error) + + mu sync.RWMutex + value map[string]WGStats + expireAt time.Time + + sf singleflight.Group +} + +func newStatsCache(ttl time.Duration, fetch func() (map[string]WGStats, error)) *statsCache { + return &statsCache{ttl: ttl, fetch: fetch} +} + +func (c *statsCache) get() (map[string]WGStats, error) { + c.mu.RLock() + if c.value != nil && time.Now().Before(c.expireAt) { + value := c.value + c.mu.RUnlock() + return value, nil + } + c.mu.RUnlock() + + value, err, _ := c.sf.Do("stats", func() (interface{}, error) { + res, err := c.fetch() + if err != nil { + return nil, err + } + + c.mu.Lock() + c.value = res + c.expireAt = time.Now().Add(c.ttl) + c.mu.Unlock() + return res, nil + }) + if err != nil { + return nil, err + } + return value.(map[string]WGStats), nil +} diff --git a/client/iface/configurer/stats_cache_test.go b/client/iface/configurer/stats_cache_test.go new file mode 100644 index 000000000..bcee5cd52 --- /dev/null +++ b/client/iface/configurer/stats_cache_test.go @@ -0,0 +1,70 @@ +package configurer + +import ( + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestStatsCache_CachesWithinTTL(t *testing.T) { + var calls atomic.Int64 + c := newStatsCache(50*time.Millisecond, func() (map[string]WGStats, error) { + calls.Add(1) + return map[string]WGStats{"p": {}}, nil + }) + + for i := 0; i < 10; i++ { + _, err := c.get() + require.NoError(t, err) + } + require.Equal(t, int64(1), calls.Load(), "within TTL only one underlying fetch") + + time.Sleep(60 * time.Millisecond) + _, err := c.get() + require.NoError(t, err) + require.Equal(t, int64(2), calls.Load(), "after TTL expiry a fresh fetch happens") +} + +func TestStatsCache_SingleFlight(t *testing.T) { + var calls atomic.Int64 + release := make(chan struct{}) + c := newStatsCache(time.Minute, func() (map[string]WGStats, error) { + calls.Add(1) + <-release + return map[string]WGStats{}, nil + }) + + const n = 50 + var wg sync.WaitGroup + wg.Add(n) + for i := 0; i < n; i++ { + go func() { + defer wg.Done() + _, _ = c.get() + }() + } + time.Sleep(20 * time.Millisecond) + close(release) + wg.Wait() + + require.Equal(t, int64(1), calls.Load(), "concurrent misses collapse into one fetch") +} + +func TestStatsCache_ErrorNotCached(t *testing.T) { + var calls atomic.Int64 + wantErr := errors.New("dump failed") + c := newStatsCache(time.Minute, func() (map[string]WGStats, error) { + calls.Add(1) + return nil, wantErr + }) + + _, err := c.get() + require.ErrorIs(t, err, wantErr) + _, err = c.get() + require.ErrorIs(t, err, wantErr) + require.Equal(t, int64(2), calls.Load(), "errors are not cached; each call retries") +} diff --git a/client/iface/configurer/usp.go b/client/iface/configurer/usp.go index 9b070aab8..0a25c55bc 100644 --- a/client/iface/configurer/usp.go +++ b/client/iface/configurer/usp.go @@ -40,6 +40,7 @@ type WGUSPConfigurer struct { device *device.Device deviceName string activityRecorder *bind.ActivityRecorder + statsCache *statsCache uapiListener net.Listener } @@ -50,16 +51,19 @@ func NewUSPConfigurer(device *device.Device, deviceName string, activityRecorder deviceName: deviceName, activityRecorder: activityRecorder, } + wgCfg.statsCache = newStatsCache(statsCacheTTL, wgCfg.fetchStats) wgCfg.startUAPI() return wgCfg } func NewUSPConfigurerNoUAPI(device *device.Device, deviceName string, activityRecorder *bind.ActivityRecorder) *WGUSPConfigurer { - return &WGUSPConfigurer{ + wgCfg := &WGUSPConfigurer{ device: device, deviceName: deviceName, activityRecorder: activityRecorder, } + wgCfg.statsCache = newStatsCache(statsCacheTTL, wgCfg.fetchStats) + return wgCfg } func (c *WGUSPConfigurer) ConfigureInterface(privateKey string, port int) error { @@ -348,6 +352,10 @@ func (t *WGUSPConfigurer) Close() { } func (t *WGUSPConfigurer) GetStats() (map[string]WGStats, error) { + return t.statsCache.get() +} + +func (t *WGUSPConfigurer) fetchStats() (map[string]WGStats, error) { ipc, err := t.device.IpcGet() if err != nil { return nil, fmt.Errorf("ipc get: %w", err) diff --git a/client/internal/auth/auth.go b/client/internal/auth/auth.go index 72d63e34e..51f56b644 100644 --- a/client/internal/auth/auth.go +++ b/client/internal/auth/auth.go @@ -351,7 +351,6 @@ func (a *Auth) setSystemInfoFlags(info *system.Info) { a.config.BlockLANAccess, a.config.BlockInbound, a.config.DisableIPv6, - a.config.LazyConnectionEnabled, a.config.EnableSSHRoot, a.config.EnableSSHSFTP, a.config.EnableSSHLocalPortForwarding, diff --git a/client/internal/conn_mgr.go b/client/internal/conn_mgr.go index 5d7308706..33b19808c 100644 --- a/client/internal/conn_mgr.go +++ b/client/internal/conn_mgr.go @@ -16,6 +16,16 @@ import ( "github.com/netbirdio/netbird/route" ) +// lazyForce is the resolved local decision for lazy connections, layered above the +// management feature flag. lazyForceNone defers to management. +type lazyForce int + +const ( + lazyForceNone lazyForce = iota + lazyForceOn + lazyForceOff +) + // ConnMgr coordinates both lazy connections (established on-demand) and permanent peer connections. // // The connection manager is responsible for: @@ -28,7 +38,7 @@ type ConnMgr struct { peerStore *peerstore.Store statusRecorder *peer.Status iface lazyconn.WGIface - enabledLocally bool + force lazyForce rosenpassEnabled bool lazyConnMgr *manager.Manager @@ -43,28 +53,34 @@ func NewConnMgr(engineConfig *EngineConfig, statusRecorder *peer.Status, peerSto peerStore: peerStore, statusRecorder: statusRecorder, iface: iface, + force: resolveLazyForce(engineConfig.LazyConnection), rosenpassEnabled: engineConfig.RosenpassEnabled, } - if engineConfig.LazyConnectionEnabled || lazyconn.IsLazyConnEnabledByEnv() { - e.enabledLocally = true - } return e } -// Start initializes the connection manager and starts the lazy connection manager if enabled by env var or cmd line option. +// Start initializes the connection manager. It starts the lazy connection manager when a +// local override forces it on; with no local override it waits for the management feature flag. func (e *ConnMgr) Start(ctx context.Context) { if e.lazyConnMgr != nil { log.Errorf("lazy connection manager is already started") return } - if !e.enabledLocally { - log.Infof("lazy connection manager is disabled") + switch e.force { + case lazyForceOff: + log.Infof("lazy connection manager is disabled by local override (%s or MDM policy)", lazyconn.EnvLazyConn) + e.statusRecorder.UpdateLazyConnection(false) + return + case lazyForceNone: + log.Infof("lazy connection manager is managed by the management feature flag") + e.statusRecorder.UpdateLazyConnection(false) return } if e.rosenpassEnabled { log.Warnf("rosenpass connection manager is enabled, lazy connection manager will not be started") + e.statusRecorder.UpdateLazyConnection(false) return } @@ -76,8 +92,8 @@ func (e *ConnMgr) Start(ctx context.Context) { // If enabled, it initializes the lazy connection manager and start it. Do not need to call Start() again. // If disabled, then it closes the lazy connection manager and open the connections to all peers. func (e *ConnMgr) UpdatedRemoteFeatureFlag(ctx context.Context, enabled bool) error { - // do not disable lazy connection manager if it was enabled by env var - if e.enabledLocally { + // a local override (NB_LAZY_CONN or local config) takes precedence over management + if e.force != lazyForceNone { return nil } @@ -89,6 +105,7 @@ func (e *ConnMgr) UpdatedRemoteFeatureFlag(ctx context.Context, enabled bool) er if e.rosenpassEnabled { log.Infof("rosenpass connection manager is enabled, lazy connection manager will not be started") + e.statusRecorder.UpdateLazyConnection(false) return nil } @@ -98,6 +115,7 @@ func (e *ConnMgr) UpdatedRemoteFeatureFlag(ctx context.Context, enabled bool) er return e.addPeersToLazyConnManager() } else { if e.lazyConnMgr == nil { + e.statusRecorder.UpdateLazyConnection(false) return nil } log.Infof("lazy connection manager is disabled by management feature flag") @@ -108,10 +126,15 @@ func (e *ConnMgr) UpdatedRemoteFeatureFlag(ctx context.Context, enabled bool) er } // SetLocalLazyConn applies a local lazy connection override (UI / CLI / env). -// While enabledLocally is true, UpdatedRemoteFeatureFlag (management sync) is a -// no-op, so the local setting wins until it is turned off again. +// While the local override pins the setting (force != lazyForceNone), +// UpdatedRemoteFeatureFlag (management sync) is a no-op, so the local setting +// wins until it is turned off again, which returns control to management. func (e *ConnMgr) SetLocalLazyConn(ctx context.Context, enabled bool) error { - e.enabledLocally = enabled + if enabled { + e.force = lazyForceOn + } else { + e.force = lazyForceNone + } if enabled { if e.lazyConnMgr != nil { @@ -340,6 +363,25 @@ func (e *ConnMgr) isStartedWithLazyMgr() bool { return e.lazyConnMgr != nil && e.lazyCtxCancel != nil } +// resolveLazyForce determines the local override. NB_LAZY_CONN takes precedence; when it +// is unset the MDM policy override (mdmState) applies. Either wins in both directions over +// the management feature flag; StateUnset for both defers to management. +func resolveLazyForce(mdmState lazyconn.State) lazyForce { + state := lazyconn.EnvState() + if state == lazyconn.StateUnset { + state = mdmState + } + + switch state { + case lazyconn.StateOn: + return lazyForceOn + case lazyconn.StateOff: + return lazyForceOff + default: + return lazyForceNone + } +} + func inactivityThresholdEnv() *time.Duration { envValue := os.Getenv(lazyconn.EnvInactivityThreshold) if envValue == "" { diff --git a/client/internal/conn_mgr_test.go b/client/internal/conn_mgr_test.go new file mode 100644 index 000000000..5e2c53e35 --- /dev/null +++ b/client/internal/conn_mgr_test.go @@ -0,0 +1,40 @@ +package internal + +import ( + "os" + "testing" + + "github.com/netbirdio/netbird/client/internal/lazyconn" +) + +func TestResolveLazyForce(t *testing.T) { + tests := []struct { + name string + env string + envSet bool + mdm lazyconn.State + want lazyForce + }{ + {name: "env unset, mdm unset -> defer to management", mdm: lazyconn.StateUnset, want: lazyForceNone}, + {name: "env on -> force on", env: "on", envSet: true, mdm: lazyconn.StateUnset, want: lazyForceOn}, + {name: "env off -> force off", env: "off", envSet: true, mdm: lazyconn.StateUnset, want: lazyForceOff}, + {name: "env unset, mdm on -> force on", mdm: lazyconn.StateOn, want: lazyForceOn}, + {name: "env unset, mdm off -> force off", mdm: lazyconn.StateOff, want: lazyForceOff}, + {name: "env on beats mdm off", env: "on", envSet: true, mdm: lazyconn.StateOff, want: lazyForceOn}, + {name: "env off beats mdm on", env: "off", envSet: true, mdm: lazyconn.StateOn, want: lazyForceOff}, + {name: "unrecognized env, mdm on -> mdm wins", env: "auto", envSet: true, mdm: lazyconn.StateOn, want: lazyForceOn}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(lazyconn.EnvLazyConn, tt.env) + if !tt.envSet { + os.Unsetenv(lazyconn.EnvLazyConn) + } + + if got := resolveLazyForce(tt.mdm); got != tt.want { + t.Fatalf("resolveLazyForce(%v) = %v, want %v", tt.mdm, got, tt.want) + } + }) + } +} diff --git a/client/internal/connect.go b/client/internal/connect.go index 26e798175..c2fc2fd73 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -27,6 +27,7 @@ import ( "github.com/netbirdio/netbird/client/iface/device" "github.com/netbirdio/netbird/client/iface/netstack" "github.com/netbirdio/netbird/client/internal/dns" + "github.com/netbirdio/netbird/client/internal/lazyconn" "github.com/netbirdio/netbird/client/internal/listener" "github.com/netbirdio/netbird/client/internal/metrics" "github.com/netbirdio/netbird/client/internal/peer" @@ -618,7 +619,7 @@ func createEngineConfig(key wgtypes.Key, config *profilemanager.Config, peerConf BlockInbound: config.BlockInbound, DisableIPv6: config.DisableIPv6, - LazyConnectionEnabled: config.LazyConnectionEnabled, + LazyConnection: lazyconn.ParseState(config.LazyConnection), MTU: selectMTU(config.MTU, peerConfig.Mtu), LogPath: logPath, @@ -692,7 +693,6 @@ func loginToManagement(ctx context.Context, client mgm.Client, pubSSHKey []byte, config.BlockLANAccess, config.BlockInbound, config.DisableIPv6, - config.LazyConnectionEnabled, config.EnableSSHRoot, config.EnableSSHSFTP, config.EnableSSHLocalPortForwarding, diff --git a/client/internal/debug/debug.go b/client/internal/debug/debug.go index 99f13b0b8..3a7c0ebff 100644 --- a/client/internal/debug/debug.go +++ b/client/internal/debug/debug.go @@ -695,7 +695,7 @@ func (g *BundleGenerator) addCommonConfigFields(configContent *strings.Builder) configContent.WriteString(fmt.Sprintf("ClientCertKeyPath: %s\n", g.internalConfig.ClientCertKeyPath)) } - configContent.WriteString(fmt.Sprintf("LazyConnectionEnabled: %v\n", g.internalConfig.LazyConnectionEnabled)) + configContent.WriteString(fmt.Sprintf("LazyConnection: %q\n", g.internalConfig.LazyConnection)) configContent.WriteString(fmt.Sprintf("MTU: %d\n", g.internalConfig.MTU)) } diff --git a/client/internal/debug/debug_test.go b/client/internal/debug/debug_test.go index ca7785d35..8286f6852 100644 --- a/client/internal/debug/debug_test.go +++ b/client/internal/debug/debug_test.go @@ -885,7 +885,7 @@ func TestAddConfig_AllFieldsCovered(t *testing.T) { DNSRouteInterval: 5 * time.Second, ClientCertPath: "/tmp/cert", ClientCertKeyPath: "/tmp/key", - LazyConnectionEnabled: true, + LazyConnection: "on", MTU: 1280, } diff --git a/client/internal/engine.go b/client/internal/engine.go index eccfb2059..4367f68b0 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -40,6 +40,7 @@ import ( "github.com/netbirdio/netbird/client/internal/dnsfwd" "github.com/netbirdio/netbird/client/internal/expose" "github.com/netbirdio/netbird/client/internal/ingressgw" + "github.com/netbirdio/netbird/client/internal/lazyconn" "github.com/netbirdio/netbird/client/internal/metrics" "github.com/netbirdio/netbird/client/internal/netflow" nftypes "github.com/netbirdio/netbird/client/internal/netflow/types" @@ -147,7 +148,9 @@ type EngineConfig struct { BlockInbound bool DisableIPv6 bool - LazyConnectionEnabled bool + // LazyConnection is the MDM-sourced lazy-connection override; StateUnset defers to + // the env var and management feature flag. + LazyConnection lazyconn.State MTU uint16 @@ -1161,7 +1164,6 @@ func (e *Engine) applyInfoFlags(info *system.Info) { e.config.BlockLANAccess, e.config.BlockInbound, e.config.DisableIPv6, - e.config.LazyConnectionEnabled, e.config.EnableSSHRoot, e.config.EnableSSHSFTP, e.config.EnableSSHLocalPortForwarding, @@ -2030,7 +2032,6 @@ func (e *Engine) readInitialSettings() ([]*route.Route, *nbdns.Config, bool, err e.config.BlockLANAccess, e.config.BlockInbound, e.config.DisableIPv6, - e.config.LazyConnectionEnabled, e.config.EnableSSHRoot, e.config.EnableSSHSFTP, e.config.EnableSSHLocalPortForwarding, diff --git a/client/internal/lazyconn/env.go b/client/internal/lazyconn/env.go index 649d1cd65..d408083e7 100644 --- a/client/internal/lazyconn/env.go +++ b/client/internal/lazyconn/env.go @@ -3,24 +3,57 @@ package lazyconn import ( "os" "strconv" + "strings" log "github.com/sirupsen/logrus" ) const ( - EnvEnableLazyConn = "NB_ENABLE_EXPERIMENTAL_LAZY_CONN" + EnvLazyConn = "NB_LAZY_CONN" EnvInactivityThreshold = "NB_LAZY_CONN_INACTIVITY_THRESHOLD" ) -func IsLazyConnEnabledByEnv() bool { - val := os.Getenv(EnvEnableLazyConn) - if val == "" { - return false - } - enabled, err := strconv.ParseBool(val) - if err != nil { - log.Warnf("failed to parse %s: %v", EnvEnableLazyConn, err) - return false - } - return enabled +// State is the tri-state local override for lazy connections read from the environment. +type State int + +const ( + // StateUnset means no local override; defer to the management feature flag. + StateUnset State = iota + // StateOn forces lazy connections on, overriding management. + StateOn + // StateOff forces lazy connections off, overriding management. + StateOff +) + +// EnvState reads NB_LAZY_CONN and returns the local override state. +func EnvState() State { + return ParseState(os.Getenv(EnvLazyConn)) +} + +// ParseState interprets a lazy-connection override value (from the environment or an MDM +// policy). It accepts the on/off aliases plus any value strconv.ParseBool understands +// (true/false/1/0). An empty or unrecognized value returns StateUnset so that the +// management feature flag remains in control. +func ParseState(raw string) State { + if raw == "" { + return StateUnset + } + + normalized := strings.ToLower(strings.TrimSpace(raw)) + switch normalized { + case "on": + return StateOn + case "off": + return StateOff + } + + enabled, err := strconv.ParseBool(normalized) + if err != nil { + log.Warnf("failed to parse lazy connection value %q (from %s env or MDM policy): %v", raw, EnvLazyConn, err) + return StateUnset + } + if enabled { + return StateOn + } + return StateOff } diff --git a/client/internal/lazyconn/env_test.go b/client/internal/lazyconn/env_test.go new file mode 100644 index 000000000..59ee40c4b --- /dev/null +++ b/client/internal/lazyconn/env_test.go @@ -0,0 +1,45 @@ +package lazyconn + +import ( + "os" + "testing" +) + +func TestEnvState(t *testing.T) { + tests := []struct { + value string + set bool + want State + }{ + {set: false, want: StateUnset}, + {value: "", set: true, want: StateUnset}, + {value: "on", set: true, want: StateOn}, + {value: "ON", set: true, want: StateOn}, + {value: "true", set: true, want: StateOn}, + {value: "1", set: true, want: StateOn}, + {value: " on ", set: true, want: StateOn}, + {value: "off", set: true, want: StateOff}, + {value: "OFF", set: true, want: StateOff}, + {value: "false", set: true, want: StateOff}, + {value: "0", set: true, want: StateOff}, + {value: "auto", set: true, want: StateUnset}, + {value: "garbage", set: true, want: StateUnset}, + } + + for _, tt := range tests { + name := tt.value + if !tt.set { + name = "unset" + } + t.Run(name, func(t *testing.T) { + t.Setenv(EnvLazyConn, tt.value) + if !tt.set { + os.Unsetenv(EnvLazyConn) + } + + if got := EnvState(); got != tt.want { + t.Fatalf("EnvState() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/client/internal/profilemanager/config.go b/client/internal/profilemanager/config.go index 8ffcb16f2..ed2f21999 100644 --- a/client/internal/profilemanager/config.go +++ b/client/internal/profilemanager/config.go @@ -101,8 +101,6 @@ type ConfigInput struct { DNSLabels domain.List - LazyConnectionEnabled *bool - MTU *uint16 } @@ -180,7 +178,9 @@ type Config struct { ClientCertKeyPair *tls.Certificate `json:"-"` - LazyConnectionEnabled bool + // LazyConnection is the MDM-managed lazy-connection override ("on"/"off"/""). + // Runtime-only: re-derived from MDM policy on each load, never persisted. + LazyConnection string `json:"-"` MTU uint16 @@ -632,12 +632,6 @@ func (config *Config) apply(input ConfigInput) (updated bool, err error) { updated = true } - if input.LazyConnectionEnabled != nil && *input.LazyConnectionEnabled != config.LazyConnectionEnabled { - log.Infof("switching lazy connection to %t", *input.LazyConnectionEnabled) - config.LazyConnectionEnabled = *input.LazyConnectionEnabled - updated = true - } - if input.MTU != nil && *input.MTU != config.MTU { log.Infof("updating MTU to %d (old value %d)", *input.MTU, config.MTU) config.MTU = *input.MTU @@ -728,6 +722,15 @@ func (config *Config) applyMDMPolicy(policy *mdm.Policy) { log.Warnf("MDM wireguard port %d out of range [1,65535]; keeping previous value", v) } } + + if v, ok := policy.GetBool(mdm.KeyLazyConnection); ok { + state := "off" + if v { + state = "on" + } + config.LazyConnection = state + logApplied(mdm.KeyLazyConnection, state) + } } // parseURL parses and validates the URL for the named service. The URL diff --git a/client/internal/profilemanager/config_mdm_test.go b/client/internal/profilemanager/config_mdm_test.go index 6a201235e..c6a688ab2 100644 --- a/client/internal/profilemanager/config_mdm_test.go +++ b/client/internal/profilemanager/config_mdm_test.go @@ -130,6 +130,37 @@ func TestApply_MDMBoolKeysOverrideOnDiskValue(t *testing.T) { assert.True(t, cfg.Policy().HasKey(mdm.KeyRosenpassEnabled)) } +func TestApply_MDMLazyConnection(t *testing.T) { + cases := []struct { + name string + raw any + want string + }{ + {"native true", true, "on"}, + {"native false", false, "off"}, + {"string on", "on", "on"}, + {"string off", "off", "off"}, + {"string yes", "yes", "on"}, + {"string no", "no", "off"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + withMDMPolicy(t, mdm.NewPolicy(map[string]any{ + mdm.KeyLazyConnection: c.raw, + })) + + cfg, err := UpdateOrCreateConfig(ConfigInput{ + ConfigPath: filepath.Join(t.TempDir(), "config.json"), + }) + require.NoError(t, err) + require.NotNil(t, cfg) + + assert.Equal(t, c.want, cfg.LazyConnection) + assert.True(t, cfg.Policy().HasKey(mdm.KeyLazyConnection)) + }) + } +} + func TestApply_MDMPreSharedKeyRedactionSentinelRejected(t *testing.T) { const maskSentinel = "**********" diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index 62933dbf1..16f8d48fa 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -836,29 +836,12 @@ func pickPreferredExitNode(info exitNodeInfo) route.NetID { // enforceSingleExitNode makes preferred the only selected exit node: every other // available exit node is deselected and preferred (if any) is selected, without -// disturbing non-exit route selections. A global deselect-all is left untouched -// so the user's "all off" stays in effect. +// disturbing non-exit route selections. The whole reconciliation runs under a +// single RouteSelector lock (SetExclusiveExitNode) so a concurrent deselect-all +// cannot interleave and get undone; a global deselect-all is left untouched so +// the user's "all off" stays in effect. func (m *DefaultManager) enforceSingleExitNode(preferred route.NetID, allIDs []route.NetID) { - if m.routeSelector.IsDeselectAll() { - return - } - - others := make([]route.NetID, 0, len(allIDs)) - for _, id := range allIDs { - if id != preferred { - others = append(others, id) - } - } - if len(others) > 0 { - if err := m.routeSelector.DeselectRoutes(others, allIDs); err != nil { - log.Warnf("deselect other exit nodes: %v", err) - } - } - if preferred != "" { - if err := m.routeSelector.SelectRoutes([]route.NetID{preferred}, true, allIDs); err != nil { - log.Warnf("select preferred exit node %q: %v", preferred, err) - } - } + m.routeSelector.SetExclusiveExitNode(preferred, allIDs) } func (m *DefaultManager) logExitNodeUpdate(info exitNodeInfo, preferred route.NetID) { diff --git a/client/internal/routeselector/routeselector.go b/client/internal/routeselector/routeselector.go index 3d2b19ba7..1254b384d 100644 --- a/client/internal/routeselector/routeselector.go +++ b/client/internal/routeselector/routeselector.go @@ -115,6 +115,35 @@ func (rs *RouteSelector) DeselectAllRoutes() { clear(rs.selectedRoutes) } +// SetExclusiveExitNode atomically makes preferred the only selected exit node +// among exitIDs: every other ID in exitIDs is deselected and preferred (when +// non-empty) is selected, all under a single lock. Holding the lock across the +// whole reconciliation prevents a concurrent DeselectAllRoutes from interleaving +// between the deselect and select steps and being silently undone. A global +// deselect-all is left untouched so the user's "all off" stays in effect; +// non-exit routes are never referenced, so their selection is preserved. +func (rs *RouteSelector) SetExclusiveExitNode(preferred route.NetID, exitIDs []route.NetID) { + rs.mu.Lock() + defer rs.mu.Unlock() + + if rs.deselectAll { + return + } + + for _, id := range exitIDs { + if id == preferred { + continue + } + rs.deselectedRoutes[id] = struct{}{} + delete(rs.selectedRoutes, id) + } + + if preferred != "" { + delete(rs.deselectedRoutes, preferred) + rs.selectedRoutes[preferred] = struct{}{} + } +} + // IsDeselectAll reports whether the global "deselect all" flag is set, i.e. the // user explicitly disabled every route. Callers enforcing per-route invariants // (e.g. single exit node) should leave the selection untouched when it is. diff --git a/client/ios/NetBirdSDK/env_list.go b/client/ios/NetBirdSDK/env_list.go index 88ac97957..a3ffa0ebe 100644 --- a/client/ios/NetBirdSDK/env_list.go +++ b/client/ios/NetBirdSDK/env_list.go @@ -38,7 +38,7 @@ func GetEnvKeyNBForceRelay() string { // GetEnvKeyNBLazyConn Exports the environment variable for the iOS client func GetEnvKeyNBLazyConn() string { - return lazyconn.EnvEnableLazyConn + return lazyconn.EnvLazyConn } // GetEnvKeyNBInactivityThreshold Exports the environment variable for the iOS client diff --git a/client/mdm/canonical_loaders.go b/client/mdm/canonical_loaders.go index d4a8234b2..cb9af9ccb 100644 --- a/client/mdm/canonical_loaders.go +++ b/client/mdm/canonical_loaders.go @@ -28,6 +28,7 @@ var allKeys = []string{ KeyWireguardPort, KeySplitTunnelMode, KeySplitTunnelApps, + KeyLazyConnection, } // canonicalKey maps the lowercase form of a managed-config value name to diff --git a/client/mdm/policy.go b/client/mdm/policy.go index e7da6e7f1..b76c70a75 100644 --- a/client/mdm/policy.go +++ b/client/mdm/policy.go @@ -11,6 +11,7 @@ package mdm import ( "sort" "strconv" + "strings" log "github.com/sirupsen/logrus" ) @@ -48,6 +49,11 @@ const ( // construction — only one mode can be set at a time. KeySplitTunnelMode = "splitTunnelMode" KeySplitTunnelApps = "splitTunnelApps" + + // KeyLazyConnection forces the lazy-connection feature on or off, overriding + // the management feature flag. Read as a bool (native bool, or on/off, + // true/false, 1/0, yes/no); absent = defer to management. + KeyLazyConnection = "lazyConnection" ) // Split-tunnel mode literals (KeySplitTunnelMode values). @@ -69,12 +75,13 @@ var boolStringLiterals = map[string]bool{ "true": true, "1": true, "yes": true, + "on": true, "false": false, "0": false, "no": false, + "off": false, } - // Policy holds MDM-managed settings read from the platform source. A nil or // empty Policy means no enforcement is active. type Policy struct { @@ -157,7 +164,8 @@ func (p *Policy) GetString(key string) (string, bool) { } // GetBool returns the managed value for key coerced to bool, and whether the -// key was set. Accepts native bool and string literals "true"/"false"/"1"/"0". +// key was set. Accepts native bool and string literals (true/false, 1/0, +// yes/no, on/off), case-insensitively and trimmed of surrounding whitespace. func (p *Policy) GetBool(key string) (bool, bool) { if p == nil { return false, false @@ -170,7 +178,7 @@ func (p *Policy) GetBool(key string) (bool, bool) { case bool: return t, true case string: - b, known := boolStringLiterals[t] + b, known := boolStringLiterals[strings.ToLower(strings.TrimSpace(t))] return b, known case int: return t != 0, true diff --git a/client/mdm/policy_test.go b/client/mdm/policy_test.go index 47a6ed2c9..6cbe69776 100644 --- a/client/mdm/policy_test.go +++ b/client/mdm/policy_test.go @@ -31,8 +31,8 @@ func TestPolicy_Empty(t *testing.T) { func TestPolicy_HasKey(t *testing.T) { p := NewPolicy(map[string]any{ - KeyManagementURL: "https://corp.example.com", - KeyDisableProfiles: true, + KeyManagementURL: "https://corp.example.com", + KeyDisableProfiles: true, }) assert.False(t, p.IsEmpty()) assert.True(t, p.HasKey(KeyManagementURL)) @@ -53,8 +53,8 @@ func TestPolicy_ManagedKeysSorted(t *testing.T) { func TestPolicy_GetString(t *testing.T) { p := NewPolicy(map[string]any{ KeyManagementURL: "https://corp.example.com", - KeyDisableProfiles: true, // wrong type for GetString - KeyPreSharedKey: "", // empty rejected + KeyDisableProfiles: true, // wrong type for GetString + KeyPreSharedKey: "", // empty rejected }) v, ok := p.GetString(KeyManagementURL) assert.True(t, ok) @@ -85,6 +85,11 @@ func TestPolicy_GetBool(t *testing.T) { {"string 0", "0", false, true}, {"string yes", "yes", true, true}, {"string no", "no", false, true}, + {"string on", "on", true, true}, + {"string off", "off", false, true}, + {"mixed case On", "On", true, true}, + {"upper TRUE", "TRUE", true, true}, + {"padded yes", " yes ", true, true}, {"int nonzero", 1, true, true}, {"int zero", 0, false, true}, {"int64 nonzero", int64(2), true, true}, diff --git a/client/server/mdm.go b/client/server/mdm.go index 547baf8b8..b33cacd7f 100644 --- a/client/server/mdm.go +++ b/client/server/mdm.go @@ -155,7 +155,6 @@ func (s *Server) restartEngineForMDMLocked() error { s.config = config s.statusRecorder.UpdateManagementAddress(config.ManagementURL.String()) s.statusRecorder.UpdateRosenpass(config.RosenpassEnabled, config.RosenpassPermissive) - s.statusRecorder.UpdateLazyConnection(config.LazyConnectionEnabled) ctx, cancel := context.WithCancel(s.rootCtx) s.actCancel = cancel @@ -308,7 +307,6 @@ func setConfigRequestHasConfigOverrides(msg *proto.SetConfigRequest) bool { msg.DisableFirewall != nil || msg.BlockLanAccess != nil || msg.DisableNotifications != nil || - msg.LazyConnectionEnabled != nil || msg.BlockInbound != nil || msg.DisableIpv6 != nil || msg.EnableSSHRoot != nil || @@ -351,7 +349,6 @@ func loginRequestHasConfigOverrides(msg *proto.LoginRequest) bool { msg.BlockLanAccess != nil || msg.DisableNotifications != nil || len(msg.DnsLabels) > 0 || msg.CleanDNSLabels || - msg.LazyConnectionEnabled != nil || msg.BlockInbound != nil } diff --git a/client/server/server.go b/client/server/server.go index 672b7299a..13489db67 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -238,7 +238,6 @@ func (s *Server) Start() error { s.statusRecorder.UpdateManagementAddress(config.ManagementURL.String()) s.statusRecorder.UpdateRosenpass(config.RosenpassEnabled, config.RosenpassPermissive) - s.statusRecorder.UpdateLazyConnection(config.LazyConnectionEnabled) if s.sessionWatcher == nil { s.sessionWatcher = internal.NewSessionWatcher(s.rootCtx, s.statusRecorder) @@ -510,7 +509,6 @@ func (s *Server) setConfigInputFromRequest(msg *proto.SetConfigRequest) (profile config.DisableFirewall = msg.DisableFirewall config.BlockLANAccess = msg.BlockLanAccess config.DisableNotifications = msg.DisableNotifications - config.LazyConnectionEnabled = msg.LazyConnectionEnabled config.BlockInbound = msg.BlockInbound config.DisableIPv6 = msg.DisableIpv6 config.EnableSSHRoot = msg.EnableSSHRoot @@ -1981,7 +1979,6 @@ func (s *Server) GetConfig(ctx context.Context, req *proto.GetConfigRequest) (*p ServerSSHAllowed: *cfg.ServerSSHAllowed, RosenpassEnabled: cfg.RosenpassEnabled, RosenpassPermissive: cfg.RosenpassPermissive, - LazyConnectionEnabled: cfg.LazyConnectionEnabled, BlockInbound: cfg.BlockInbound, DisableNotifications: disableNotifications, NetworkMonitor: networkMonitor, diff --git a/client/server/setconfig_test.go b/client/server/setconfig_test.go index 7c85d16ce..0e55257a9 100644 --- a/client/server/setconfig_test.go +++ b/client/server/setconfig_test.go @@ -69,43 +69,41 @@ func TestSetConfig_AllFieldsSaved(t *testing.T) { disableFirewall := true blockLANAccess := true disableNotifications := true - lazyConnectionEnabled := true blockInbound := true disableIPv6 := true mtu := int64(1280) sshJWTCacheTTL := int32(300) req := &proto.SetConfigRequest{ - ProfileName: profName, - Username: currUser.Username, - ManagementUrl: "https://new-api.netbird.io:443", - AdminURL: "https://new-admin.netbird.io", - RosenpassEnabled: &rosenpassEnabled, - RosenpassPermissive: &rosenpassPermissive, - ServerSSHAllowed: &serverSSHAllowed, - InterfaceName: &interfaceName, - WireguardPort: &wireguardPort, - OptionalPreSharedKey: &preSharedKey, - DisableAutoConnect: &disableAutoConnect, - NetworkMonitor: &networkMonitor, - DisableClientRoutes: &disableClientRoutes, - DisableServerRoutes: &disableServerRoutes, - DisableDns: &disableDNS, - DisableFirewall: &disableFirewall, - BlockLanAccess: &blockLANAccess, - DisableNotifications: &disableNotifications, - LazyConnectionEnabled: &lazyConnectionEnabled, - BlockInbound: &blockInbound, - DisableIpv6: &disableIPv6, - NatExternalIPs: []string{"1.2.3.4", "5.6.7.8"}, - CleanNATExternalIPs: false, - CustomDNSAddress: []byte("1.1.1.1:53"), - ExtraIFaceBlacklist: []string{"eth1", "eth2"}, - DnsLabels: []string{"label1", "label2"}, - CleanDNSLabels: false, - DnsRouteInterval: durationpb.New(2 * time.Minute), - Mtu: &mtu, - SshJWTCacheTTL: &sshJWTCacheTTL, + ProfileName: profName, + Username: currUser.Username, + ManagementUrl: "https://new-api.netbird.io:443", + AdminURL: "https://new-admin.netbird.io", + RosenpassEnabled: &rosenpassEnabled, + RosenpassPermissive: &rosenpassPermissive, + ServerSSHAllowed: &serverSSHAllowed, + InterfaceName: &interfaceName, + WireguardPort: &wireguardPort, + OptionalPreSharedKey: &preSharedKey, + DisableAutoConnect: &disableAutoConnect, + NetworkMonitor: &networkMonitor, + DisableClientRoutes: &disableClientRoutes, + DisableServerRoutes: &disableServerRoutes, + DisableDns: &disableDNS, + DisableFirewall: &disableFirewall, + BlockLanAccess: &blockLANAccess, + DisableNotifications: &disableNotifications, + BlockInbound: &blockInbound, + DisableIpv6: &disableIPv6, + NatExternalIPs: []string{"1.2.3.4", "5.6.7.8"}, + CleanNATExternalIPs: false, + CustomDNSAddress: []byte("1.1.1.1:53"), + ExtraIFaceBlacklist: []string{"eth1", "eth2"}, + DnsLabels: []string{"label1", "label2"}, + CleanDNSLabels: false, + DnsRouteInterval: durationpb.New(2 * time.Minute), + Mtu: &mtu, + SshJWTCacheTTL: &sshJWTCacheTTL, } _, err = s.SetConfig(ctx, req) @@ -140,7 +138,6 @@ func TestSetConfig_AllFieldsSaved(t *testing.T) { require.Equal(t, blockLANAccess, cfg.BlockLANAccess) require.NotNil(t, cfg.DisableNotifications) require.Equal(t, disableNotifications, *cfg.DisableNotifications) - require.Equal(t, lazyConnectionEnabled, cfg.LazyConnectionEnabled) require.Equal(t, blockInbound, cfg.BlockInbound) require.Equal(t, disableIPv6, cfg.DisableIPv6) require.Equal(t, []string{"1.2.3.4", "5.6.7.8"}, cfg.NATExternalIPs) @@ -164,13 +161,14 @@ func verifyAllFieldsCovered(t *testing.T, req *proto.SetConfigRequest) { t.Helper() metadataFields := map[string]bool{ - "state": true, // protobuf internal - "sizeCache": true, // protobuf internal - "unknownFields": true, // protobuf internal - "Username": true, // metadata - "ProfileName": true, // metadata - "CleanNATExternalIPs": true, // control flag for clearing - "CleanDNSLabels": true, // control flag for clearing + "state": true, // protobuf internal + "sizeCache": true, // protobuf internal + "unknownFields": true, // protobuf internal + "Username": true, // metadata + "ProfileName": true, // metadata + "CleanNATExternalIPs": true, // control flag for clearing + "CleanDNSLabels": true, // control flag for clearing + "LazyConnectionEnabled": true, // deprecated: proto field retained for compat, no longer applied } expectedFields := map[string]bool{ @@ -190,7 +188,6 @@ func verifyAllFieldsCovered(t *testing.T, req *proto.SetConfigRequest) { "DisableFirewall": true, "BlockLanAccess": true, "DisableNotifications": true, - "LazyConnectionEnabled": true, "BlockInbound": true, "DisableIpv6": true, "NatExternalIPs": true, @@ -252,7 +249,6 @@ func TestCLIFlags_MappedToSetConfig(t *testing.T) { "block-lan-access": "BlockLanAccess", "block-inbound": "BlockInbound", "disable-ipv6": "DisableIpv6", - "enable-lazy-connection": "LazyConnectionEnabled", "external-ip-map": "NatExternalIPs", "dns-resolver-address": "CustomDNSAddress", "extra-iface-blacklist": "ExtraIFaceBlacklist", @@ -269,7 +265,8 @@ func TestCLIFlags_MappedToSetConfig(t *testing.T) { // SetConfigRequest fields that don't have CLI flags (settable only via UI or other means). fieldsWithoutCLIFlags := map[string]bool{ - "DisableNotifications": true, // Only settable via UI + "DisableNotifications": true, // Only settable via UI + "LazyConnectionEnabled": true, // deprecated: no longer settable (managed by server + NB_LAZY_CONN) } // Get all SetConfigRequest fields to verify our map is complete. diff --git a/client/system/info.go b/client/system/info.go index 496b478a3..1838204b8 100644 --- a/client/system/info.go +++ b/client/system/info.go @@ -74,8 +74,6 @@ type Info struct { BlockInbound bool DisableIPv6 bool - LazyConnectionEnabled bool - EnableSSHRoot bool EnableSSHSFTP bool EnableSSHLocalPortForwarding bool @@ -87,7 +85,7 @@ func (i *Info) SetFlags( rosenpassEnabled, rosenpassPermissive bool, serverSSHAllowed *bool, disableClientRoutes, disableServerRoutes, - disableDNS, disableFirewall, blockLANAccess, blockInbound, disableIPv6, lazyConnectionEnabled bool, + disableDNS, disableFirewall, blockLANAccess, blockInbound, disableIPv6 bool, enableSSHRoot, enableSSHSFTP, enableSSHLocalPortForwarding, enableSSHRemotePortForwarding *bool, disableSSHAuth *bool, ) { @@ -105,8 +103,6 @@ func (i *Info) SetFlags( i.BlockInbound = blockInbound i.DisableIPv6 = disableIPv6 - i.LazyConnectionEnabled = lazyConnectionEnabled - if enableSSHRoot != nil { i.EnableSSHRoot = *enableSSHRoot } diff --git a/e2e/agentnetwork/vllm_test.go b/e2e/agentnetwork/vllm_test.go new file mode 100644 index 000000000..329994ca9 --- /dev/null +++ b/e2e/agentnetwork/vllm_test.go @@ -0,0 +1,171 @@ +//go:build e2e + +package agentnetwork + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/e2e/harness" + "github.com/netbirdio/netbird/shared/management/http/api" +) + +// TestVLLMProvider proves the proxy supports a self-hosted vLLM backend. vLLM is +// OpenAI-compatible, so it uses the "vllm" catalog entry (KindCustom) and is +// reached over plain HTTP — no TLS anywhere on the path: +// +// client --tunnel--> netbird proxy --http--> vllm (:8000, OpenAI-compatible) +// +// The mock vLLM server answers /v1/chat/completions with an OpenAI-shaped +// completion carrying a non-zero usage block. The test asserts the chat returns +// 200 with the completion, that the request is recorded in the access log by its +// session id, and that vLLM's usage block is metered into a consumption row — +// which together prove request routing, response parsing, and token accounting +// all work for a self-hosted OpenAI-compatible provider. +// +// It needs no external credentials (the mock ignores auth), so it always runs. +func TestVLLMProvider(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + + vllm, err := harness.StartVLLM(ctx, srv) + require.NoError(t, err, "start mock vLLM server") + t.Cleanup(func() { _ = vllm.Terminate(context.Background()) }) + + grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-vllm"}) + require.NoError(t, err, "create group") + t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) }) + + ephemeral := false + sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{ + Name: "e2e-vllm-client", + Type: "reusable", + ExpiresIn: 86400, + UsageLimit: 0, + AutoGroups: []string{grp.Id}, + Ephemeral: &ephemeral, + }) + require.NoError(t, err, "mint setup key") + require.NotEmpty(t, sk.Key, "setup key plaintext") + + // vLLM provider pointed at the mock over plain HTTP. The mock ignores auth, + // so a dummy key satisfies the "Bearer ${API_KEY}" template. The served model + // is enumerated so the router dispatches this model string to this provider. + dummyKey := "sk-vllm-e2e" + prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ + Name: "vllm", + ProviderId: "vllm", + UpstreamUrl: vllm.URL, + ApiKey: &dummyKey, + Enabled: ptr(true), + BootstrapCluster: ptr(harness.AgentNetworkCluster), + Models: &[]api.AgentNetworkProviderModel{ + {Id: harness.VLLMModel, InputPer1k: 0.001, OutputPer1k: 0.002}, + }, + }) + require.NoError(t, err, "create vllm provider") + t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) + + // Token limit far above the handful of tokens this test drives, so it never + // blocks but switches on usage metering — the switch that makes consumption + // rows get recorded. + enabled := true + pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{ + Name: "e2e-vllm-allow", + Enabled: &enabled, + SourceGroups: []string{grp.Id}, + DestinationProviderIds: []string{prov.Id}, + Limits: &api.AgentNetworkPolicyLimits{ + TokenLimit: api.AgentNetworkPolicyTokenLimit{ + Enabled: true, + GroupCap: 10_000_000, + UserCap: 10_000_000, + WindowSeconds: 60, + }, + }, + }) + require.NoError(t, err, "create policy") + t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) }) + + settings, err := srv.GetSettings(ctx) + require.NoError(t, err, "read settings") + require.NotEmpty(t, settings.Endpoint, "endpoint must be assigned") + + proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-vllm-proxy") + require.NoError(t, err, "mint proxy token") + px, err := harness.StartProxy(ctx, srv, proxyToken) + require.NoError(t, err, "start proxy") + t.Cleanup(func() { _ = px.Terminate(context.Background()) }) + + cl, err := harness.StartClient(ctx, srv, sk.Key) + require.NoError(t, err, "start client") + t.Cleanup(func() { _ = cl.Terminate(context.Background()) }) + + require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management") + if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil { + t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background())) + } + proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint) + require.NoError(t, err, "resolve endpoint to proxy IP") + + before, _ := srv.ListAccessLogs(ctx) + sessionID := "e2e-session-vllm" + + // Retry to absorb tunnel/DNS jitter on the first call. + var code int + var body string + deadline := time.Now().Add(90 * time.Second) + for time.Now().Before(deadline) { + c, b, cerr := cl.Chat(ctx, settings.Endpoint, proxyIP, harness.WireChat, harness.VLLMModel, "Reply with exactly: pong", sessionID) + if cerr == nil { + code, body = c, b + if code == 200 { + break + } + } + time.Sleep(5 * time.Second) + } + require.Equal(t, 200, code, + "chat through the vLLM provider must return 200; body: %s\n=== vllm logs ===\n%s\n=== proxy logs ===\n%s", + body, vllm.Logs(context.Background()), px.Logs(context.Background())) + require.True(t, strings.Contains(body, "chat.completion"), + "body should be an OpenAI-compatible chat completion; got: %s", body) + + // The request must surface as an access-log row carrying our session id. + require.Eventually(t, func() bool { + logs, lerr := srv.ListAccessLogs(ctx) + return lerr == nil && logs.TotalRecords > before.TotalRecords + }, 30*time.Second, 2*time.Second, "an access-log row should be ingested for the vLLM provider") + + require.Eventually(t, func() bool { + logs, lerr := srv.ListAccessLogs(ctx) + if lerr != nil { + return false + } + for _, r := range logs.Data { + if r.SessionId != nil && *r.SessionId == sessionID { + return true + } + } + return false + }, 30*time.Second, 2*time.Second, "session id %q must be recorded in an access-log row", sessionID) + + // vLLM's usage block (prompt_tokens=11, completion_tokens=2) must be parsed + // and metered into a consumption row with positive token counts. + require.Eventually(t, func() bool { + rows, lerr := srv.ListConsumption(ctx) + if lerr != nil { + return false + } + for _, r := range rows { + if r.TokensInput > 0 && r.TokensOutput > 0 { + return true + } + } + return false + }, 60*time.Second, 3*time.Second, "vLLM usage must be metered into a consumption row") +} diff --git a/e2e/harness/vllm.go b/e2e/harness/vllm.go new file mode 100644 index 000000000..2f3d306cc --- /dev/null +++ b/e2e/harness/vllm.go @@ -0,0 +1,113 @@ +//go:build e2e + +package harness + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +const ( + vllmImage = "nginx:alpine" + vllmAlias = "vllm" + vllmPort = "8000/tcp" + // VLLMModel is the served model id the mock advertises and echoes back. It + // matches a real small model commonly served by vLLM so the provider's + // enumerated model and the client's request line up. + VLLMModel = "Qwen/Qwen2.5-0.5B-Instruct" +) + +// vllmNginxConf emulates a vLLM OpenAI-compatible server over plain HTTP (vLLM's +// default: no TLS, port 8000). It answers /v1/models with a one-model list and +// any chat/completions path with a canned OpenAI-shaped chat completion carrying +// a non-zero usage block, so the proxy's OpenAI parser records real token +// consumption. Running actual vLLM in CI is infeasible (GPU + multi-GB model +// download), so this stands in for the wire contract the proxy depends on. +const vllmNginxConf = `pid /tmp/nginx.pid; +events {} +http { + server { + listen 8000; + location = /v1/models { + default_type application/json; + return 200 '{"object":"list","data":[{"id":"Qwen/Qwen2.5-0.5B-Instruct","object":"model","owned_by":"vllm"}]}'; + } + location / { + default_type application/json; + return 200 '{"id":"chatcmpl-e2e-vllm","object":"chat.completion","created":1700000000,"model":"Qwen/Qwen2.5-0.5B-Instruct","choices":[{"index":0,"message":{"role":"assistant","content":"pong"},"finish_reason":"stop"}],"usage":{"prompt_tokens":11,"completion_tokens":2,"total_tokens":13}}'; + } + } +} +` + +// VLLM is a mock vLLM OpenAI-compatible server on the combined server's network, +// reachable at http://vllm:8000. A "vllm" provider points at it to exercise the +// proxy's support for self-hosted OpenAI-compatible backends. +type VLLM struct { + container testcontainers.Container + workDir string + // URL is the upstream URL the vllm provider points at (http://:8000). + URL string +} + +// StartVLLM runs the mock vLLM server on the shared network over plain HTTP. +func StartVLLM(ctx context.Context, c *Combined) (*VLLM, error) { + workDir, err := os.MkdirTemp("/tmp", "nb-e2e-vllm-*") + if err != nil { + return nil, fmt.Errorf("create vllm work dir: %w", err) + } + // Widen so the (non-root worker) nginx container can traverse the bind mount. + if err := os.Chmod(workDir, 0o755); err != nil { //nolint:gosec // throwaway e2e config dir + return nil, fmt.Errorf("chmod vllm dir: %w", err) + } + if err := os.WriteFile(filepath.Join(workDir, "nginx.conf"), []byte(vllmNginxConf), 0o644); err != nil { //nolint:gosec // non-secret e2e config + return nil, fmt.Errorf("write nginx conf: %w", err) + } + + req := testcontainers.ContainerRequest{ + Image: vllmImage, + ExposedPorts: []string{vllmPort}, + Networks: []string{c.network.Name}, + NetworkAliases: map[string][]string{c.network.Name: {vllmAlias}}, + Cmd: []string{"nginx", "-c", "/conf/nginx.conf", "-g", "daemon off;"}, + HostConfigModifier: func(hc *container.HostConfig) { + hc.Binds = append(hc.Binds, workDir+":/conf:ro") + }, + WaitingFor: wait.ForListeningPort(vllmPort).WithStartupTimeout(60 * time.Second), + } + + ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + _ = os.RemoveAll(workDir) + return nil, fmt.Errorf("start vllm container: %w", err) + } + + return &VLLM{container: ctr, workDir: workDir, URL: "http://" + vllmAlias + ":8000"}, nil +} + +// Logs returns the vLLM container logs, for diagnostics on failure. +func (v *VLLM) Logs(ctx context.Context) string { + return containerLogs(ctx, v.container) +} + +// Terminate stops the vLLM container and cleans its work dir. +func (v *VLLM) Terminate(ctx context.Context) error { + var err error + if v.container != nil { + err = v.container.Terminate(ctx) + } + if v.workDir != "" { + _ = os.RemoveAll(v.workDir) + } + return err +} diff --git a/infrastructure_files/getting-started.sh b/infrastructure_files/getting-started.sh index 46bef5a1f..837cc42e6 100755 --- a/infrastructure_files/getting-started.sh +++ b/infrastructure_files/getting-started.sh @@ -351,11 +351,6 @@ initialize_default_values() { NETBIRD_STUN_PORT=3478 # Docker images - # Record whether the operator explicitly pinned the server/proxy images via - # env vars, so the agent-network preset can pick its own defaults without - # clobbering an explicit override. - NETBIRD_SERVER_IMAGE_EXPLICIT=${NETBIRD_SERVER_IMAGE:+true} - NETBIRD_PROXY_IMAGE_EXPLICIT=${NETBIRD_PROXY_IMAGE:+true} DASHBOARD_IMAGE=${DASHBOARD_IMAGE:-"netbirdio/dashboard:latest"} # Combined server replaces separate signal, relay, and management containers NETBIRD_SERVER_IMAGE=${NETBIRD_SERVER_IMAGE:-"netbirdio/netbird-server:latest"} @@ -415,15 +410,6 @@ apply_agent_network_preset() { ENABLE_PROXY="true" ENABLE_CROWDSEC="false" - # Agent-network ships dedicated server/proxy images. Honor an explicit - # env override; otherwise pin the agent-network builds. - if [[ "${NETBIRD_SERVER_IMAGE_EXPLICIT}" != "true" ]]; then - NETBIRD_SERVER_IMAGE="netbirdio/netbird-server:0.74.0-rc.2" - fi - if [[ "${NETBIRD_PROXY_IMAGE_EXPLICIT}" != "true" ]]; then - NETBIRD_PROXY_IMAGE="netbirdio/reverse-proxy:0.74.0-rc.2" - fi - if [[ -n "${NETBIRD_LETSENCRYPT_EMAIL}" ]]; then TRAEFIK_ACME_EMAIL="${NETBIRD_LETSENCRYPT_EMAIL}" else diff --git a/management/internals/modules/agentnetwork/catalog/catalog.go b/management/internals/modules/agentnetwork/catalog/catalog.go index baf622778..962b30250 100644 --- a/management/internals/modules/agentnetwork/catalog/catalog.go +++ b/management/internals/modules/agentnetwork/catalog/catalog.go @@ -627,6 +627,21 @@ var providers = []Provider{ }, Models: []Model{}, }, + { + // vLLM is an OpenAI-compatible self-hosted server. It behaves like + // the generic custom entry; it gets its own catalog id purely so it + // surfaces as a named "vLLM" choice in the provider picker. + ID: "vllm", + Kind: KindCustom, + Name: "vLLM", + Description: "Self-hosted vLLM (OpenAI-compatible)", + DefaultHost: "", + AuthHeaderName: "Authorization", + AuthHeaderTemplate: "Bearer ${API_KEY}", + DefaultContentType: "application/json", + BrandColor: "#30A2FF", + Models: []Model{}, + }, { ID: "custom", Kind: KindCustom, diff --git a/management/internals/shared/grpc/conversion.go b/management/internals/shared/grpc/conversion.go index 973749eb0..bdb4c8cf4 100644 --- a/management/internals/shared/grpc/conversion.go +++ b/management/internals/shared/grpc/conversion.go @@ -47,16 +47,13 @@ func init() { precomputedDeprecatedRemotePeersConstraint = constraint } +// toNetbirdConfig converts the server configuration to the wire representation. It returns +// nil when no server config is set (the fan-out network-map path) because clients treat any +// non-nil config as authoritative: a config without a relay section is interpreted as relay +// disabled and wipes the clients' relay URLs. func toNetbirdConfig(config *nbconfig.Config, turnCredentials *Token, relayToken *Token, extraSettings *types.ExtraSettings, settings *types.Settings) *proto.NetbirdConfig { if config == nil { - if settings == nil { - return nil - } - return &proto.NetbirdConfig{ - Metrics: &proto.MetricsConfig{ - Enabled: settings.MetricsPushEnabled, - }, - } + return nil } var stuns []*proto.HostConfig diff --git a/management/internals/shared/grpc/conversion_test.go b/management/internals/shared/grpc/conversion_test.go index 01a67e4fa..c81bef25c 100644 --- a/management/internals/shared/grpc/conversion_test.go +++ b/management/internals/shared/grpc/conversion_test.go @@ -8,11 +8,13 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" nbdns "github.com/netbirdio/netbird/dns" "github.com/netbirdio/netbird/management/internals/controllers/network_map" "github.com/netbirdio/netbird/management/internals/controllers/network_map/controller/cache" nbconfig "github.com/netbirdio/netbird/management/internals/server/config" + "github.com/netbirdio/netbird/management/server/types" ) func TestToProtocolDNSConfigWithCache(t *testing.T) { @@ -263,3 +265,39 @@ func TestEncodeSessionExpiresAt(t *testing.T) { assert.True(t, got.AsTime().Equal(deadline)) }) } + +// TestToNetbirdConfig_RelayInvariant guards against the v0.74.0 relay-wipe regression. +// Clients treat any non-nil NetbirdConfig as authoritative and interpret a missing relay +// section as relay disabled, wiping their relay URLs. toNetbirdConfig must therefore +// return nil when no server config is set (the fan-out network-map path) instead of a +// partial config, and a result built from a relay-enabled config must carry the relay +// section. +func TestToNetbirdConfig_RelayInvariant(t *testing.T) { + settings := &types.Settings{MetricsPushEnabled: true} + + t.Run("nil server config returns nil config", func(t *testing.T) { + nbCfg := toNetbirdConfig(nil, nil, nil, nil, settings) + assert.Nil(t, nbCfg, "fan-out updates must not carry a partial NetbirdConfig even when settings are present") + }) + + t.Run("relay-enabled config carries relay section", func(t *testing.T) { + cfg := &nbconfig.Config{ + Stuns: []*nbconfig.Host{{Proto: nbconfig.UDP, URI: "stun:stun.example.com:3478"}}, + TURNConfig: &nbconfig.TURNConfig{ + Turns: []*nbconfig.Host{{Proto: nbconfig.UDP, URI: "turn:turn.example.com:3478", Username: "user", Password: "pass"}}, + }, + Relay: &nbconfig.Relay{Addresses: []string{"rels://relay.example.com:443"}}, + Signal: &nbconfig.Host{Proto: nbconfig.HTTP, URI: "signal.example.com:10000"}, + } + relayToken := &Token{Payload: "token-payload", Signature: "token-signature"} + + nbCfg := toNetbirdConfig(cfg, nil, relayToken, nil, settings) + require.NotNil(t, nbCfg) + require.NotNil(t, nbCfg.Relay, "non-nil NetbirdConfig must include the relay section") + assert.Equal(t, cfg.Relay.Addresses, nbCfg.Relay.Urls, "relay URLs should match the server config") + assert.Equal(t, relayToken.Payload, nbCfg.Relay.TokenPayload, "relay token payload should be set") + assert.Equal(t, relayToken.Signature, nbCfg.Relay.TokenSignature, "relay token signature should be set") + require.NotNil(t, nbCfg.Metrics) + assert.True(t, nbCfg.Metrics.Enabled, "metrics flag should carry the settings value") + }) +} diff --git a/management/server/peer_test.go b/management/server/peer_test.go index 6c243c4c7..d471a1302 100644 --- a/management/server/peer_test.go +++ b/management/server/peer_test.go @@ -1048,11 +1048,7 @@ func testUpdateAccountPeers(t *testing.T) { for _, channel := range peerChannels { update := <-channel - assert.NotNil(t, update.Update.NetbirdConfig) - assert.Nil(t, update.Update.NetbirdConfig.Stuns) - assert.Nil(t, update.Update.NetbirdConfig.Turns) - assert.Nil(t, update.Update.NetbirdConfig.Signal) - assert.Nil(t, update.Update.NetbirdConfig.Relay) + assert.Nil(t, update.Update.NetbirdConfig, "fan-out updates must not carry a NetbirdConfig; clients treat a config without relay as relay disabled and wipe their relay URLs") assert.Equal(t, tc.peers, len(update.Update.NetworkMap.RemotePeers)) assert.Equal(t, tc.peers*2, len(update.Update.NetworkMap.FirewallRules)) } diff --git a/shared/management/client/grpc.go b/shared/management/client/grpc.go index 01296b4e6..0735a15b9 100644 --- a/shared/management/client/grpc.go +++ b/shared/management/client/grpc.go @@ -33,10 +33,15 @@ const ConnectTimeout = 10 * time.Second const healthCheckTimeout = 5 * time.Second const ( - // EnvMaxRecvMsgSize overrides the default gRPC max receive message size (4 MB) + // EnvMaxRecvMsgSize overrides the default gRPC max receive message size // for the management client connection. Value is in bytes. EnvMaxRecvMsgSize = "NB_MANAGEMENT_GRPC_MAX_MSG_SIZE" + // defaultMaxRecvMsgSize is the max gRPC receive message size used for the + // management client connection when EnvMaxRecvMsgSize is unset or invalid. + // It overrides the gRPC library default of 4 MB. + defaultMaxRecvMsgSize = 1024 * 1024 * 16 + errMsgMgmtPublicKey = "failed getting Management Service public key: %s" errMsgNoMgmtConnection = "no connection to management" ) @@ -84,22 +89,22 @@ type ExposeResponse struct { } // MaxRecvMsgSize returns the configured max gRPC receive message size from -// the environment, or 0 if unset (which uses the gRPC default of 4 MB). +// the environment, or defaultMaxRecvMsgSize (16 MB) if unset or invalid. func MaxRecvMsgSize() int { val := os.Getenv(EnvMaxRecvMsgSize) if val == "" { - return 0 + return defaultMaxRecvMsgSize } size, err := strconv.Atoi(val) if err != nil { log.Warnf("invalid %s value %q, using default: %v", EnvMaxRecvMsgSize, val, err) - return 0 + return defaultMaxRecvMsgSize } if size <= 0 { log.Warnf("invalid %s value %d, must be positive, using default", EnvMaxRecvMsgSize, size) - return 0 + return defaultMaxRecvMsgSize } return size @@ -1018,8 +1023,6 @@ func infoToMetaData(info *system.Info) *proto.PeerSystemMeta { BlockLANAccess: info.BlockLANAccess, BlockInbound: info.BlockInbound, DisableIPv6: info.DisableIPv6, - - LazyConnectionEnabled: info.LazyConnectionEnabled, }, Capabilities: peerCapabilities(*info), diff --git a/shared/management/client/grpc_test.go b/shared/management/client/grpc_test.go index 462cc43af..c947130fd 100644 --- a/shared/management/client/grpc_test.go +++ b/shared/management/client/grpc_test.go @@ -21,11 +21,11 @@ func TestMaxRecvMsgSize(t *testing.T) { envValue string expected int }{ - {name: "unset returns 0", envValue: "", expected: 0}, + {name: "unset returns default", envValue: "", expected: defaultMaxRecvMsgSize}, {name: "valid value", envValue: "10485760", expected: 10485760}, - {name: "non-numeric returns 0", envValue: "abc", expected: 0}, - {name: "negative returns 0", envValue: "-1", expected: 0}, - {name: "zero returns 0", envValue: "0", expected: 0}, + {name: "non-numeric returns default", envValue: "abc", expected: defaultMaxRecvMsgSize}, + {name: "negative returns default", envValue: "-1", expected: defaultMaxRecvMsgSize}, + {name: "zero returns default", envValue: "0", expected: defaultMaxRecvMsgSize}, } for _, tt := range tests {