[management, client] Add management-controlled client metrics push (#5886)

* [management, client] Add management-controlled client metrics push

Allow enabling/disabling client metrics push from the dashboard via
account settings instead of requiring env vars on every client.

- Add MetricsConfig proto message to NetbirdConfig
- Add MetricsPushEnabled to account Settings (DB-persisted)
- Expose metrics_push_enabled in OpenAPI and dashboard API handler
- Populate MetricsConfig in sync and login responses
- Client dynamically starts/stops push based on management config
- NB_METRICS_PUSH_ENABLED env var overrides management when explicitly set
- Add activity events for metrics push enable/disable

* Remove log line

* [management] Fix peer update test for MetricsConfig in NetbirdConfig

Update TestUpdateAccountPeers assertions: NetbirdConfig is no longer
nil in peer update responses since it now carries MetricsConfig even
when STUN/TURN config is absent.

* Regenerate proto files with protoc v7.34.1

* [management] Read metrics push setting in Postgres account query

getAccountPgx omitted settings_metrics_push_enabled from its hand-written
SELECT and Scan, so the toggle was always read back as false on Postgres
and never reached clients.

* [client] Fix metrics push getting stuck off after engine restart

Engine restarts (backoff retries within the same login session) cancel
e.ctx, which the push goroutine's lifetime was tied to. The goroutine
died silently but ClientMetrics.push stayed non-nil since only an
explicit stop clears it, so the next UpdatePushFromMgm call saw a
"push already running" state and never restarted it.

Give the Engine its own metricsCtx sourced from ConnectClient.ctx,
which outlives engine restarts, so handleMetricsUpdate stops tying the
push to the wrong-scoped context. Additionally make ClientMetrics.push
an atomic.Pointer that the push goroutine clears via CompareAndSwap on
exit, so the tracked state can never drift from the goroutine's actual
lifetime regardless of which context a future caller passes in.

* [management] Regenerate OpenAPI types with oapi-codegen v2.7.1

types.gen.go was regenerated with a stale local v2.6.0 binary,
causing the CI git-diff check against generate.sh's pinned v2.7.1
to fail.
This commit is contained in:
Zoltan Papp
2026-07-01 14:50:25 +02:00
committed by GitHub
parent 92a66cdd20
commit 980598ed4a
17 changed files with 526 additions and 316 deletions

View File

@@ -314,6 +314,10 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan
c.clientMetrics.RecordLoginDuration(engineCtx, time.Since(loginStarted), true)
c.statusRecorder.MarkManagementConnected()
if metricsConfig := loginResp.GetNetbirdConfig().GetMetrics(); metricsConfig != nil {
c.clientMetrics.UpdatePushFromMgm(c.ctx, metricsConfig.GetEnabled())
}
localPeerState := peer.LocalPeerState{
IP: loginResp.GetPeerConfig().GetAddress(),
PubKey: myPrivateKey.PublicKey().String(),
@@ -399,6 +403,7 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan
StateManager: stateManager,
UpdateManager: c.updateManager,
ClientMetrics: c.clientMetrics,
MetricsCtx: c.ctx,
}, mobileDependency)
engine.SetSyncResponsePersistence(c.persistSyncResponse)
c.engine = engine

View File

@@ -172,6 +172,7 @@ type EngineServices struct {
StateManager *statemanager.Manager
UpdateManager *updater.Manager
ClientMetrics *metrics.ClientMetrics
MetricsCtx context.Context
}
// Engine is a mechanism responsible for reacting on Signal and Management stream events and managing connections to the remote peers.
@@ -264,6 +265,7 @@ type Engine struct {
// clientMetrics collects and pushes metrics
clientMetrics *metrics.ClientMetrics
metricsCtx context.Context
jobExecutor *jobexec.Executor
jobExecutorWG sync.WaitGroup
@@ -316,6 +318,7 @@ func NewEngine(
probeStunTurn: relay.NewStunTurnProbe(relay.DefaultCacheTTL),
jobExecutor: jobexec.NewExecutor(),
clientMetrics: services.ClientMetrics,
metricsCtx: services.MetricsCtx,
updateManager: services.UpdateManager,
syncStoreDir: config.StateDir,
}
@@ -997,6 +1000,8 @@ func (e *Engine) updateNetbirdConfig(wCfg *mgmProto.NetbirdConfig) error {
return fmt.Errorf("handle the flow configuration: %w", err)
}
e.handleMetricsUpdate(wCfg.GetMetrics())
if err := e.PopulateNetbirdConfig(wCfg, nil); err != nil {
log.Warnf("Failed to update DNS server config: %v", err)
}
@@ -1066,6 +1071,14 @@ func (e *Engine) handleFlowUpdate(config *mgmProto.FlowConfig) error {
return e.flowManager.Update(flowConfig)
}
func (e *Engine) handleMetricsUpdate(config *mgmProto.MetricsConfig) {
if config == nil {
return
}
log.Infof("received metrics configuration from management: enabled=%v", config.GetEnabled())
e.clientMetrics.UpdatePushFromMgm(e.metricsCtx, config.GetEnabled())
}
func toFlowLoggerConfig(config *mgmProto.FlowConfig) (*nftypes.FlowConfig, error) {
if config.GetInterval() == nil {
return nil, errors.New("flow interval is nil")

View File

@@ -60,6 +60,13 @@ func getMetricsInterval() time.Duration {
return interval
}
// isMetricsPushEnvSet returns true if NB_METRICS_PUSH_ENABLED is explicitly set (to any value).
// When set, the env var takes full precedence over management server configuration.
func isMetricsPushEnvSet() bool {
_, set := os.LookupEnv(EnvMetricsPushEnabled)
return set
}
func isForceSending() bool {
force, _ := strconv.ParseBool(os.Getenv(EnvMetricsForceSending))
return force

View File

@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"sync"
"sync/atomic"
"time"
log "github.com/sirupsen/logrus"
@@ -75,7 +76,7 @@ type ClientMetrics struct {
agentInfo AgentInfo
mu sync.RWMutex
push *Push
push atomic.Pointer[Push]
pushMu sync.Mutex
wg sync.WaitGroup
pushCancel context.CancelFunc
@@ -167,10 +168,7 @@ func (c *ClientMetrics) UpdateAgentInfo(agentInfo AgentInfo, publicKey string) {
c.agentInfo = agentInfo
c.mu.Unlock()
c.pushMu.Lock()
push := c.push
c.pushMu.Unlock()
if push != nil {
if push := c.push.Load(); push != nil {
push.SetPeerID(agentInfo.peerID)
}
}
@@ -184,7 +182,7 @@ func (c *ClientMetrics) Export(w io.Writer) error {
return c.impl.Export(w)
}
// StartPush starts periodic pushing of metrics with the given configuration
// StartPush starts periodic pushing of metrics with the given configuration.
// Precedence: PushConfig.ServerAddress > remote config server_url
func (c *ClientMetrics) StartPush(ctx context.Context, config PushConfig) {
if c == nil {
@@ -194,11 +192,58 @@ func (c *ClientMetrics) StartPush(ctx context.Context, config PushConfig) {
c.pushMu.Lock()
defer c.pushMu.Unlock()
if c.push != nil {
if c.push.Load() != nil {
log.Warnf("metrics push already running")
return
}
c.startPushLocked(ctx, config)
}
// StopPush stops the periodic metrics push.
func (c *ClientMetrics) StopPush() {
if c == nil {
return
}
c.pushMu.Lock()
defer c.pushMu.Unlock()
c.stopPushLocked()
}
// UpdatePushFromMgm updates metrics push based on management server configuration.
// If NB_METRICS_PUSH_ENABLED is explicitly set (true or false), management config is ignored.
// When unset, management controls whether push is enabled.
func (c *ClientMetrics) UpdatePushFromMgm(ctx context.Context, enabled bool) {
if c == nil {
return
}
if isMetricsPushEnvSet() {
log.Debugf("ignoring management config, env var is explicitly set: %s", EnvMetricsPushEnabled)
return
}
c.pushMu.Lock()
defer c.pushMu.Unlock()
if enabled {
if c.push.Load() != nil {
return
}
log.Infof("enabled metrics push by management")
c.startPushLocked(ctx, PushConfigFromEnv())
} else {
if c.push.Load() == nil {
return
}
log.Infof("disabled metrics push by management")
c.stopPushLocked()
}
}
// startPushLocked starts push. Caller must hold pushMu.
func (c *ClientMetrics) startPushLocked(ctx context.Context, config PushConfig) {
c.mu.RLock()
agentVersion := c.agentInfo.Version
peerID := c.agentInfo.peerID
@@ -214,26 +259,23 @@ func (c *ClientMetrics) StartPush(ctx context.Context, config PushConfig) {
ctx, cancel := context.WithCancel(ctx)
c.pushCancel = cancel
c.push.Store(push)
c.wg.Add(1)
go func() {
defer c.wg.Done()
push.Start(ctx)
c.push.CompareAndSwap(push, nil)
}()
c.push = push
}
func (c *ClientMetrics) StopPush() {
if c == nil {
return
}
c.pushMu.Lock()
defer c.pushMu.Unlock()
if c.push == nil {
// stopPushLocked stops push. Caller must hold pushMu.
func (c *ClientMetrics) stopPushLocked() {
if c.push.Load() == nil {
return
}
c.pushCancel()
c.wg.Wait()
c.push = nil
c.push.Store(nil)
}

View File

@@ -47,9 +47,16 @@ func init() {
precomputedDeprecatedRemotePeersConstraint = constraint
}
func toNetbirdConfig(config *nbconfig.Config, turnCredentials *Token, relayToken *Token, extraSettings *types.ExtraSettings) *proto.NetbirdConfig {
func toNetbirdConfig(config *nbconfig.Config, turnCredentials *Token, relayToken *Token, extraSettings *types.ExtraSettings, settings *types.Settings) *proto.NetbirdConfig {
if config == nil {
return nil
if settings == nil {
return nil
}
return &proto.NetbirdConfig{
Metrics: &proto.MetricsConfig{
Enabled: settings.MetricsPushEnabled,
},
}
}
var stuns []*proto.HostConfig
@@ -110,6 +117,12 @@ func toNetbirdConfig(config *nbconfig.Config, turnCredentials *Token, relayToken
Relay: relayCfg,
}
if settings != nil {
nbConfig.Metrics = &proto.MetricsConfig{
Enabled: settings.MetricsPushEnabled,
}
}
return nbConfig
}
@@ -166,7 +179,7 @@ func ToSyncResponse(ctx context.Context, config *nbconfig.Config, httpConfig *nb
Checks: toProtocolChecks(ctx, checks),
}
nbConfig := toNetbirdConfig(config, turnCredentials, relayCredentials, extraSettings)
nbConfig := toNetbirdConfig(config, turnCredentials, relayCredentials, extraSettings, settings)
extendedConfig := integrationsConfig.ExtendNetBirdConfig(peer.ID, peerGroups, nbConfig, extraSettings)
response.NetbirdConfig = extendedConfig

View File

@@ -917,7 +917,7 @@ func (s *Server) prepareLoginResponse(ctx context.Context, peer *nbpeer.Peer, ne
// if peer has reached this point then it has logged in
loginResp := &proto.LoginResponse{
NetbirdConfig: toNetbirdConfig(s.config, nil, relayToken, nil),
NetbirdConfig: toNetbirdConfig(s.config, nil, relayToken, nil, settings),
PeerConfig: toPeerConfig(peer, network, s.networkMapController.GetDNSDomain(settings), settings, s.config.HttpConfig, s.config.DeviceAuthorizationFlow, enableSSH),
Checks: toProtocolChecks(ctx, postureChecks),
}

View File

@@ -358,7 +358,8 @@ func (am *DefaultAccountManager) UpdateAccountSettings(ctx context.Context, acco
oldSettings.AutoUpdateVersion != newSettings.AutoUpdateVersion ||
oldSettings.AutoUpdateAlways != newSettings.AutoUpdateAlways ||
oldSettings.PeerLoginExpirationEnabled != newSettings.PeerLoginExpirationEnabled ||
oldSettings.PeerLoginExpiration != newSettings.PeerLoginExpiration {
oldSettings.PeerLoginExpiration != newSettings.PeerLoginExpiration ||
oldSettings.MetricsPushEnabled != newSettings.MetricsPushEnabled {
// Session deadline is derived from LastLogin + PeerLoginExpiration
// on every Login/Sync response. Without a fan-out push, connected
// peers keep the deadline they received at login time and only see
@@ -409,6 +410,7 @@ func (am *DefaultAccountManager) UpdateAccountSettings(ctx context.Context, acco
am.handleAutoUpdateVersionSettings(ctx, oldSettings, newSettings, userID, accountID)
am.handleAutoUpdateAlwaysSettings(ctx, oldSettings, newSettings, userID, accountID)
am.handlePeerExposeSettings(ctx, oldSettings, newSettings, userID, accountID)
am.handleMetricsPushSettings(ctx, oldSettings, newSettings, userID, accountID)
if err = am.handleInactivityExpirationSettings(ctx, oldSettings, newSettings, userID, accountID); err != nil {
return nil, err
}
@@ -563,6 +565,16 @@ func (am *DefaultAccountManager) handleLazyConnectionSettings(ctx context.Contex
}
}
func (am *DefaultAccountManager) handleMetricsPushSettings(ctx context.Context, oldSettings, newSettings *types.Settings, userID, accountID string) {
if oldSettings.MetricsPushEnabled != newSettings.MetricsPushEnabled {
if newSettings.MetricsPushEnabled {
am.StoreEvent(ctx, userID, accountID, accountID, activity.AccountMetricsPushEnabled, nil)
} else {
am.StoreEvent(ctx, userID, accountID, accountID, activity.AccountMetricsPushDisabled, nil)
}
}
}
func (am *DefaultAccountManager) handlePeerLoginExpirationSettings(ctx context.Context, oldSettings, newSettings *types.Settings, userID, accountID string) {
if oldSettings.PeerLoginExpirationEnabled != newSettings.PeerLoginExpirationEnabled {
event := activity.AccountPeerLoginExpirationEnabled

View File

@@ -276,6 +276,11 @@ const (
// AgentNetworkSettingsUpdated indicates that a user updated Agent Network account settings
AgentNetworkSettingsUpdated Activity = 139
// AccountMetricsPushEnabled indicates that a user enabled metrics push for the account
AccountMetricsPushEnabled Activity = 140
// AccountMetricsPushDisabled indicates that a user disabled metrics push for the account
AccountMetricsPushDisabled Activity = 141
AccountDeleted Activity = 99999
)
@@ -449,6 +454,9 @@ var activityMap = map[Activity]Code{
AgentNetworkSettingsUpdated: {"Agent Network settings updated", "agent_network.settings.update"},
AccountMetricsPushEnabled: {"Account metrics push enabled", "account.setting.metrics.push.enable"},
AccountMetricsPushDisabled: {"Account metrics push disabled", "account.setting.metrics.push.disable"},
DomainAdded: {"Domain added", "domain.add"},
DomainDeleted: {"Domain deleted", "domain.delete"},
DomainValidated: {"Domain validated", "domain.validate"},

View File

@@ -283,6 +283,9 @@ func (h *handler) updateAccountRequestSettings(req api.PutApiAccountsAccountIdJS
if req.Settings.Ipv6EnabledGroups != nil {
returnSettings.IPv6EnabledGroups = *req.Settings.Ipv6EnabledGroups
}
if req.Settings.MetricsPushEnabled != nil {
returnSettings.MetricsPushEnabled = *req.Settings.MetricsPushEnabled
}
return returnSettings, nil
}
@@ -413,6 +416,7 @@ func toAccountResponse(accountID string, settings *types.Settings, meta *types.A
AutoUpdateVersion: &settings.AutoUpdateVersion,
AutoUpdateAlways: &settings.AutoUpdateAlways,
Ipv6EnabledGroups: &settings.IPv6EnabledGroups,
MetricsPushEnabled: &settings.MetricsPushEnabled,
EmbeddedIdpEnabled: &settings.EmbeddedIdpEnabled,
LocalAuthDisabled: &settings.LocalAuthDisabled,
LocalMfaEnabled: &settings.LocalMfaEnabled,

View File

@@ -129,6 +129,7 @@ func TestAccounts_AccountsHandler(t *testing.T) {
DnsDomain: sr(""),
AutoUpdateAlways: br(false),
AutoUpdateVersion: sr(""),
MetricsPushEnabled: br(false),
EmbeddedIdpEnabled: br(false),
LocalAuthDisabled: br(false),
LocalMfaEnabled: br(false),
@@ -156,6 +157,7 @@ func TestAccounts_AccountsHandler(t *testing.T) {
DnsDomain: sr(""),
AutoUpdateAlways: br(false),
AutoUpdateVersion: sr(""),
MetricsPushEnabled: br(false),
EmbeddedIdpEnabled: br(false),
LocalAuthDisabled: br(false),
LocalMfaEnabled: br(false),
@@ -183,6 +185,7 @@ func TestAccounts_AccountsHandler(t *testing.T) {
DnsDomain: sr(""),
AutoUpdateAlways: br(false),
AutoUpdateVersion: sr("latest"),
MetricsPushEnabled: br(false),
EmbeddedIdpEnabled: br(false),
LocalAuthDisabled: br(false),
LocalMfaEnabled: br(false),
@@ -210,6 +213,7 @@ func TestAccounts_AccountsHandler(t *testing.T) {
DnsDomain: sr(""),
AutoUpdateAlways: br(false),
AutoUpdateVersion: sr(""),
MetricsPushEnabled: br(false),
EmbeddedIdpEnabled: br(false),
LocalAuthDisabled: br(false),
LocalMfaEnabled: br(false),
@@ -237,6 +241,7 @@ func TestAccounts_AccountsHandler(t *testing.T) {
DnsDomain: sr(""),
AutoUpdateAlways: br(false),
AutoUpdateVersion: sr(""),
MetricsPushEnabled: br(false),
EmbeddedIdpEnabled: br(false),
LocalAuthDisabled: br(false),
LocalMfaEnabled: br(false),
@@ -264,6 +269,7 @@ func TestAccounts_AccountsHandler(t *testing.T) {
DnsDomain: sr(""),
AutoUpdateAlways: br(false),
AutoUpdateVersion: sr(""),
MetricsPushEnabled: br(false),
EmbeddedIdpEnabled: br(false),
LocalAuthDisabled: br(false),
LocalMfaEnabled: br(false),

View File

@@ -1048,7 +1048,11 @@ func testUpdateAccountPeers(t *testing.T) {
for _, channel := range peerChannels {
update := <-channel
assert.Nil(t, update.Update.NetbirdConfig)
assert.NotNil(t, update.Update.NetbirdConfig)
assert.Nil(t, update.Update.NetbirdConfig.Stuns)
assert.Nil(t, update.Update.NetbirdConfig.Turns)
assert.Nil(t, update.Update.NetbirdConfig.Signal)
assert.Nil(t, update.Update.NetbirdConfig.Relay)
assert.Equal(t, tc.peers, len(update.Update.NetworkMap.RemotePeers))
assert.Equal(t, tc.peers*2, len(update.Update.NetworkMap.FirewallRules))
}

View File

@@ -1605,7 +1605,7 @@ func (s *SqlStore) getAccount(ctx context.Context, accountID string) (*types.Acc
settings_jwt_groups_enabled, settings_jwt_groups_claim_name, settings_jwt_allow_groups,
settings_routing_peer_dns_resolution_enabled, settings_dns_domain, settings_network_range,
settings_network_range_v6, settings_ipv6_enabled_groups, settings_lazy_connection_enabled,
settings_local_mfa_enabled,
settings_local_mfa_enabled, settings_metrics_push_enabled,
-- Embedded ExtraSettings
settings_extra_peer_approval_enabled, settings_extra_user_approval_required,
settings_extra_integrated_validator, settings_extra_integrated_validator_groups
@@ -1628,6 +1628,7 @@ func (s *SqlStore) getAccount(ctx context.Context, accountID string) (*types.Acc
sIPv6EnabledGroups sql.NullString
sLazyConnectionEnabled sql.NullBool
sLocalMFAEnabled sql.NullBool
sMetricsPushEnabled sql.NullBool
sExtraPeerApprovalEnabled sql.NullBool
sExtraUserApprovalRequired sql.NullBool
sExtraIntegratedValidator sql.NullString
@@ -1650,7 +1651,7 @@ func (s *SqlStore) getAccount(ctx context.Context, accountID string) (*types.Acc
&sJWTGroupsEnabled, &sJWTGroupsClaimName, &sJWTAllowGroups,
&sRoutingPeerDNSResolutionEnabled, &sDNSDomain, &sNetworkRange,
&sNetworkRangeV6, &sIPv6EnabledGroups, &sLazyConnectionEnabled,
&sLocalMFAEnabled,
&sLocalMFAEnabled, &sMetricsPushEnabled,
&sExtraPeerApprovalEnabled, &sExtraUserApprovalRequired,
&sExtraIntegratedValidator, &sExtraIntegratedValidatorGroups,
)
@@ -1716,6 +1717,9 @@ func (s *SqlStore) getAccount(ctx context.Context, accountID string) (*types.Acc
if sLocalMFAEnabled.Valid {
account.Settings.LocalMfaEnabled = sLocalMFAEnabled.Bool
}
if sMetricsPushEnabled.Valid {
account.Settings.MetricsPushEnabled = sMetricsPushEnabled.Bool
}
if sJWTAllowGroups.Valid {
_ = json.Unmarshal([]byte(sJWTAllowGroups.String), &account.Settings.JWTAllowGroups)
}

View File

@@ -73,6 +73,9 @@ type Settings struct {
// For new accounts this defaults to the All group.
IPv6EnabledGroups []string `gorm:"serializer:json"`
// MetricsPushEnabled globally enables or disables client metrics push for the account
MetricsPushEnabled bool `gorm:"default:false"`
// EmbeddedIdpEnabled indicates if the embedded identity provider is enabled.
// This is a runtime-only field, not stored in the database.
EmbeddedIdpEnabled bool `gorm:"-"`
@@ -110,6 +113,7 @@ func (s *Settings) Copy() *Settings {
AutoUpdateVersion: s.AutoUpdateVersion,
AutoUpdateAlways: s.AutoUpdateAlways,
IPv6EnabledGroups: slices.Clone(s.IPv6EnabledGroups),
MetricsPushEnabled: s.MetricsPushEnabled,
EmbeddedIdpEnabled: s.EmbeddedIdpEnabled,
LocalAuthDisabled: s.LocalAuthDisabled,
LocalMfaEnabled: s.LocalMfaEnabled,

View File

@@ -371,6 +371,10 @@ components:
description: When true, updates are installed automatically in the background. When false, updates require user interaction from the UI.
type: boolean
example: false
metrics_push_enabled:
description: Enables or disables client metrics push for all peers in the account
type: boolean
example: false
embedded_idp_enabled:
description: Indicates whether the embedded identity provider (Dex) is enabled for this account. This is a read-only field.
type: boolean

View File

@@ -1684,6 +1684,9 @@ type AccountSettings struct {
// LocalMfaEnabled Enables or disables TOTP multi-factor authentication for local users. Only applicable when the embedded identity provider is enabled.
LocalMfaEnabled *bool `json:"local_mfa_enabled,omitempty"`
// MetricsPushEnabled Enables or disables client metrics push for all peers in the account
MetricsPushEnabled *bool `json:"metrics_push_enabled,omitempty"`
// NetworkRange Allows to define a custom network range for the account in CIDR format
NetworkRange *string `json:"network_range,omitempty"`

File diff suppressed because it is too large Load Diff

View File

@@ -312,6 +312,8 @@ message NetbirdConfig {
RelayConfig relay = 4;
FlowConfig flow = 5;
MetricsConfig metrics = 6;
}
// HostConfig describes connection properties of some server (e.g. STUN, Signal, Management)
@@ -350,6 +352,10 @@ message FlowConfig {
bool dnsCollection = 8;
}
message MetricsConfig {
bool enabled = 1;
}
// JWTConfig represents JWT authentication configuration for validating tokens.
message JWTConfig {
string issuer = 1;