From 63c26be72faf2d4b69f268fd69e5fb4955b1d04c Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:53:06 +0900 Subject: [PATCH] [client] Add local Prometheus metrics endpoint (#6689) --- client/cmd/root.go | 7 + client/cmd/up.go | 128 +- client/internal/debug/debug.go | 2 + client/internal/localmetrics/localmetrics.go | 274 ++++ .../localmetrics/localmetrics_test.go | 151 +++ client/internal/metrics/influxdb.go | 23 +- client/internal/metrics/metrics.go | 23 + client/internal/metrics/metrics_default.go | 16 +- client/internal/metrics/prometheus.go | 119 ++ client/internal/peer/status.go | 12 + client/internal/peer/status_test.go | 22 + client/internal/profilemanager/config.go | 26 + .../profilemanager/config_mdm_test.go | 26 + client/mdm/canonical_loaders.go | 2 + client/mdm/canonical_loaders_test.go | 52 + client/mdm/policy.go | 2 + client/proto/daemon.pb.go | 52 +- client/proto/daemon.proto | 6 + client/server/mdm.go | 30 +- client/server/server.go | 39 +- client/server/setconfig_mdm_test.go | 45 + client/server/setconfig_test.go | 10 + client/server/ssh_gate.go | 81 +- client/server/ssh_gate_test.go | 98 ++ go.mod | 3 +- .../grafana/dashboards/client.json | 1107 +++++++++++++++++ 26 files changed, 2265 insertions(+), 91 deletions(-) create mode 100644 client/internal/localmetrics/localmetrics.go create mode 100644 client/internal/localmetrics/localmetrics_test.go create mode 100644 client/internal/metrics/prometheus.go create mode 100644 client/mdm/canonical_loaders_test.go create mode 100644 infrastructure_files/observability/grafana/dashboards/client.json diff --git a/client/cmd/root.go b/client/cmd/root.go index ccad78942..be6479440 100644 --- a/client/cmd/root.go +++ b/client/cmd/root.go @@ -23,6 +23,7 @@ import ( "github.com/netbirdio/netbird/client/anonymize" daddr "github.com/netbirdio/netbird/client/internal/daemonaddr" + "github.com/netbirdio/netbird/client/internal/localmetrics" "github.com/netbirdio/netbird/client/internal/profilemanager" ) @@ -31,6 +32,8 @@ const ( dnsResolverAddress = "dns-resolver-address" enableRosenpassFlag = "enable-rosenpass" rosenpassPermissiveFlag = "rosenpass-permissive" + enableLocalMetricsFlag = "enable-local-metrics" + localMetricsAddressFlag = "local-metrics-address" preSharedKeyFlag = "preshared-key" interfaceNameFlag = "interface-name" wireguardPortFlag = "wireguard-port" @@ -80,6 +83,8 @@ var ( updateSettingsDisabled bool captureEnabled bool networksDisabled bool + localMetricsEnabled bool + localMetricsAddr string rootCmd = &cobra.Command{ Use: "netbird", @@ -215,6 +220,8 @@ func init() { upCmd.PersistentFlags().BoolVar(&rosenpassEnabled, enableRosenpassFlag, false, "[Experimental] Enable Rosenpass feature. If enabled, the connection will be post-quantum secured via Rosenpass.") upCmd.PersistentFlags().BoolVar(&rosenpassPermissive, rosenpassPermissiveFlag, false, "[Experimental] Enable Rosenpass in permissive mode to allow this peer to accept WireGuard connections without requiring Rosenpass functionality from peers that do not have Rosenpass enabled.") upCmd.PersistentFlags().BoolVar(&autoConnectDisabled, disableAutoConnectFlag, false, "Disables auto-connect feature. If enabled, then the client won't connect automatically when the service starts.") + upCmd.PersistentFlags().BoolVar(&localMetricsEnabled, enableLocalMetricsFlag, false, "Enables a local Prometheus /metrics endpoint exposing connection state (peers, latency, P2P vs relay).") + upCmd.PersistentFlags().StringVar(&localMetricsAddr, localMetricsAddressFlag, localmetrics.DefaultListenAddress, "Listen address of the local Prometheus /metrics endpoint.") upCmd.PersistentFlags().BoolVar(&lazyConnEnabled, enableLazyConnectionFlag, false, "Deprecated: no longer used. Lazy connections are controlled by the server and the NB_LAZY_CONN environment variable.") _ = upCmd.PersistentFlags().MarkDeprecated(enableLazyConnectionFlag, "no longer used; lazy connections are controlled by the server and the NB_LAZY_CONN environment variable") diff --git a/client/cmd/up.go b/client/cmd/up.go index 9f4fa8c33..5bc41a964 100644 --- a/client/cmd/up.go +++ b/client/cmd/up.go @@ -398,26 +398,10 @@ func doDaemonUp(ctx context.Context, cmd *cobra.Command, client proto.DaemonServ return nil } -func setupSetConfigReq(customDNSAddressConverted []byte, cmd *cobra.Command, profileName, username string) *proto.SetConfigRequest { - var req proto.SetConfigRequest - req.ProfileName = profileName - req.Username = username - - req.ManagementUrl = managementURL - req.AdminURL = adminURL - req.NatExternalIPs = natExternalIPs - req.CustomDNSAddress = customDNSAddressConverted - req.ExtraIFaceBlacklist = extraIFaceBlackList - req.DnsLabels = dnsLabelsValidated.ToPunycodeList() - req.CleanDNSLabels = dnsLabels != nil && len(dnsLabels) == 0 - req.CleanNATExternalIPs = natExternalIPs != nil && len(natExternalIPs) == 0 - - if cmd.Flag(enableRosenpassFlag).Changed { - req.RosenpassEnabled = &rosenpassEnabled - } - if cmd.Flag(rosenpassPermissiveFlag).Changed { - req.RosenpassPermissive = &rosenpassPermissive - } +// setSSHSetConfigFields copies the SSH server flags the user actually +// passed into req, leaving the rest unset so the daemon keeps the +// persisted values. +func setSSHSetConfigFields(req *proto.SetConfigRequest, cmd *cobra.Command) { if cmd.Flag(serverSSHAllowedFlag).Changed { req.ServerSSHAllowed = &serverSSHAllowed } @@ -440,6 +424,30 @@ func setupSetConfigReq(customDNSAddressConverted []byte, cmd *cobra.Command, pro sshJWTCacheTTL32 := int32(sshJWTCacheTTL) req.SshJWTCacheTTL = &sshJWTCacheTTL32 } +} + +func setupSetConfigReq(customDNSAddressConverted []byte, cmd *cobra.Command, profileName, username string) *proto.SetConfigRequest { + var req proto.SetConfigRequest + req.ProfileName = profileName + req.Username = username + + req.ManagementUrl = managementURL + req.AdminURL = adminURL + req.NatExternalIPs = natExternalIPs + req.CustomDNSAddress = customDNSAddressConverted + req.ExtraIFaceBlacklist = extraIFaceBlackList + req.DnsLabels = dnsLabelsValidated.ToPunycodeList() + req.CleanDNSLabels = dnsLabels != nil && len(dnsLabels) == 0 + req.CleanNATExternalIPs = natExternalIPs != nil && len(natExternalIPs) == 0 + + if cmd.Flag(enableRosenpassFlag).Changed { + req.RosenpassEnabled = &rosenpassEnabled + } + if cmd.Flag(rosenpassPermissiveFlag).Changed { + req.RosenpassPermissive = &rosenpassPermissive + } + setSSHSetConfigFields(&req, cmd) + if cmd.Flag(interfaceNameFlag).Changed { if err := parseInterfaceName(interfaceName); err != nil { log.Errorf("parse interface name: %v", err) @@ -499,6 +507,13 @@ func setupSetConfigReq(customDNSAddressConverted []byte, cmd *cobra.Command, pro req.DisableIpv6 = &disableIPv6 } + if cmd.Flag(enableLocalMetricsFlag).Changed { + req.EnableLocalMetrics = &localMetricsEnabled + } + if cmd.Flag(localMetricsAddressFlag).Changed { + req.LocalMetricsAddress = &localMetricsAddr + } + return &req } @@ -616,9 +631,45 @@ func setupConfig(customDNSAddressConverted []byte, cmd *cobra.Command, configFil ic.DisableIPv6 = &disableIPv6 } + if cmd.Flag(enableLocalMetricsFlag).Changed { + ic.LocalMetricsEnabled = &localMetricsEnabled + } + + if cmd.Flag(localMetricsAddressFlag).Changed { + ic.LocalMetricsAddress = &localMetricsAddr + } + return &ic, nil } +// setSSHLoginFields copies the SSH server flags the user actually passed +// into req, leaving the rest unset so the daemon keeps the persisted +// values. +func setSSHLoginFields(req *proto.LoginRequest, cmd *cobra.Command) { + if cmd.Flag(serverSSHAllowedFlag).Changed { + req.ServerSSHAllowed = &serverSSHAllowed + } + if cmd.Flag(enableSSHRootFlag).Changed { + req.EnableSSHRoot = &enableSSHRoot + } + if cmd.Flag(enableSSHSFTPFlag).Changed { + req.EnableSSHSFTP = &enableSSHSFTP + } + if cmd.Flag(enableSSHLocalPortForwardFlag).Changed { + req.EnableSSHLocalPortForwarding = &enableSSHLocalPortForward + } + if cmd.Flag(enableSSHRemotePortForwardFlag).Changed { + req.EnableSSHRemotePortForwarding = &enableSSHRemotePortForward + } + if cmd.Flag(disableSSHAuthFlag).Changed { + req.DisableSSHAuth = &disableSSHAuth + } + if cmd.Flag(sshJWTCacheTTLFlag).Changed { + sshJWTCacheTTL32 := int32(sshJWTCacheTTL) + req.SshJWTCacheTTL = &sshJWTCacheTTL32 + } +} + func setupLoginRequest(providedSetupKey string, customDNSAddressConverted []byte, cmd *cobra.Command) (*proto.LoginRequest, error) { loginRequest := proto.LoginRequest{ SetupKey: providedSetupKey, @@ -645,39 +696,20 @@ func setupLoginRequest(providedSetupKey string, customDNSAddressConverted []byte loginRequest.RosenpassPermissive = &rosenpassPermissive } - if cmd.Flag(serverSSHAllowedFlag).Changed { - loginRequest.ServerSSHAllowed = &serverSSHAllowed - } - - if cmd.Flag(enableSSHRootFlag).Changed { - loginRequest.EnableSSHRoot = &enableSSHRoot - } - - if cmd.Flag(enableSSHSFTPFlag).Changed { - loginRequest.EnableSSHSFTP = &enableSSHSFTP - } - - if cmd.Flag(enableSSHLocalPortForwardFlag).Changed { - loginRequest.EnableSSHLocalPortForwarding = &enableSSHLocalPortForward - } - - if cmd.Flag(enableSSHRemotePortForwardFlag).Changed { - loginRequest.EnableSSHRemotePortForwarding = &enableSSHRemotePortForward - } - - if cmd.Flag(disableSSHAuthFlag).Changed { - loginRequest.DisableSSHAuth = &disableSSHAuth - } - - if cmd.Flag(sshJWTCacheTTLFlag).Changed { - sshJWTCacheTTL32 := int32(sshJWTCacheTTL) - loginRequest.SshJWTCacheTTL = &sshJWTCacheTTL32 - } + setSSHLoginFields(&loginRequest, cmd) if cmd.Flag(disableAutoConnectFlag).Changed { loginRequest.DisableAutoConnect = &autoConnectDisabled } + if cmd.Flag(enableLocalMetricsFlag).Changed { + loginRequest.EnableLocalMetrics = &localMetricsEnabled + } + + if cmd.Flag(localMetricsAddressFlag).Changed { + loginRequest.LocalMetricsAddress = &localMetricsAddr + } + if cmd.Flag(interfaceNameFlag).Changed { if err := parseInterfaceName(interfaceName); err != nil { return nil, err diff --git a/client/internal/debug/debug.go b/client/internal/debug/debug.go index 1d31c75ca..7bb71c53b 100644 --- a/client/internal/debug/debug.go +++ b/client/internal/debug/debug.go @@ -737,6 +737,8 @@ func (g *BundleGenerator) addCommonConfigFields(configContent *strings.Builder) configContent.WriteString(fmt.Sprintf("BlockLANAccess: %v\n", g.internalConfig.BlockLANAccess)) configContent.WriteString(fmt.Sprintf("BlockInbound: %v\n", g.internalConfig.BlockInbound)) configContent.WriteString(fmt.Sprintf("DisableIPv6: %v\n", g.internalConfig.DisableIPv6)) + configContent.WriteString(fmt.Sprintf("LocalMetricsEnabled: %v\n", g.internalConfig.LocalMetricsEnabled)) + configContent.WriteString(fmt.Sprintf("LocalMetricsAddress: %s\n", g.internalConfig.LocalMetricsAddress)) configContent.WriteString(fmt.Sprintf("SyncMessageVersion: %v\n", g.internalConfig.SyncMessageVersion)) if g.internalConfig.DisableNotifications != nil { diff --git a/client/internal/localmetrics/localmetrics.go b/client/internal/localmetrics/localmetrics.go new file mode 100644 index 000000000..f829fa132 --- /dev/null +++ b/client/internal/localmetrics/localmetrics.go @@ -0,0 +1,274 @@ +// Package localmetrics exposes client connection state as a local +// Prometheus /metrics endpoint. +package localmetrics + +import ( + "context" + "errors" + "net" + "net/http" + "net/netip" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + dto "github.com/prometheus/client_model/go" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/peer" +) + +// DefaultListenAddress is used when local metrics are enabled without an explicit address. +const DefaultListenAddress = "127.0.0.1:9191" + +const ( + shutdownTimeout = 3 * time.Second + readHeaderTimeout = 5 * time.Second + readTimeout = 10 * time.Second + writeTimeout = 30 * time.Second + idleTimeout = time.Minute +) + +// statusSource provides the connection state snapshots the collector reads on scrape. +type statusSource interface { + GetPeerStates() []peer.State + GetManagementState() peer.ManagementState + GetSignalState() peer.SignalState +} + +// GathererProvider returns the current client metrics gatherer, or nil when +// no engine is running. It is called on every scrape. +type GathererProvider func() prometheus.Gatherer + +// Manager runs the local /metrics HTTP endpoint according to the active +// client configuration. Reconcile is safe to call on every config change. +type Manager struct { + status statusSource + clientMetrics GathererProvider + + mu sync.Mutex + srv *http.Server + addr string +} + +// NewManager creates a manager that serves metrics from status and +// clientMetrics and shuts down when ctx is canceled. +func NewManager(ctx context.Context, status statusSource, clientMetrics GathererProvider) *Manager { + m := &Manager{status: status, clientMetrics: clientMetrics} + go func() { + <-ctx.Done() + m.Stop() + }() + return m +} + +// Reconcile starts, stops, or restarts the metrics endpoint to match the +// desired state. An empty addr falls back to DefaultListenAddress. +func (m *Manager) Reconcile(enabled bool, addr string) { + if addr == "" { + addr = DefaultListenAddress + } + warnIfNotLoopback(addr) + + m.mu.Lock() + defer m.mu.Unlock() + + if !enabled { + m.stop() + return + } + if m.srv != nil && m.addr == addr { + return + } + m.stop() + + registry := prometheus.NewRegistry() + registry.MustRegister(newCollector(m.status)) + + gatherers := prometheus.Gatherers{registry, prometheus.GathererFunc(func() ([]*dto.MetricFamily, error) { + if m.clientMetrics == nil { + return nil, nil + } + g := m.clientMetrics() + if g == nil { + return nil, nil + } + return g.Gather() + })} + + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{})) + + srv := &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: readHeaderTimeout, + ReadTimeout: readTimeout, + WriteTimeout: writeTimeout, + IdleTimeout: idleTimeout, + } + m.srv = srv + m.addr = addr + + log.Infof("serving local metrics on http://%s/metrics", addr) + go func() { + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Errorf("failed to serve local metrics on %s: %v", addr, err) + m.clear(srv) + } + }() +} + +// clear drops the reference to srv so a later Reconcile with the same +// address restarts it. A newer server may already have replaced it, in +// which case the reference must stay. +func (m *Manager) clear(srv *http.Server) { + m.mu.Lock() + defer m.mu.Unlock() + + if m.srv != srv { + return + } + m.srv = nil + m.addr = "" +} + +// Stop shuts down the metrics endpoint if it is running. +func (m *Manager) Stop() { + m.mu.Lock() + defer m.mu.Unlock() + m.stop() +} + +// stop shuts down the running server. Callers must hold m.mu. +func (m *Manager) stop() { + if m.srv == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + if err := m.srv.Shutdown(ctx); err != nil { + log.Debugf("failed to shut down local metrics server: %v", err) + } + m.srv = nil + m.addr = "" +} + +// collector converts status recorder snapshots into Prometheus metrics at scrape time. +type collector struct { + status statusSource + + managementConnected *prometheus.Desc + signalConnected *prometheus.Desc + peersTotal *prometheus.Desc + peersConnected *prometheus.Desc + peerLatency *prometheus.Desc +} + +func newCollector(status statusSource) *collector { + return &collector{ + status: status, + managementConnected: prometheus.NewDesc( + "netbird_management_connected", + "Whether the client is connected to the management service (1 connected, 0 disconnected).", + nil, nil, + ), + signalConnected: prometheus.NewDesc( + "netbird_signal_connected", + "Whether the client is connected to the signal service (1 connected, 0 disconnected).", + nil, nil, + ), + peersTotal: prometheus.NewDesc( + "netbird_peers", + "Number of peers known to this client.", + nil, nil, + ), + peersConnected: prometheus.NewDesc( + "netbird_peers_connected", + "Number of connected peers by connection type.", + []string{"connection_type"}, nil, + ), + peerLatency: prometheus.NewDesc( + "netbird_peer_latency_seconds", + "Round-trip latency per directly connected peer; relayed connections have no latency measurement.", + []string{"peer"}, nil, + ), + } +} + +// Describe implements prometheus.Collector. +func (c *collector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.managementConnected + ch <- c.signalConnected + ch <- c.peersTotal + ch <- c.peersConnected + ch <- c.peerLatency +} + +// Collect implements prometheus.Collector. +func (c *collector) Collect(ch chan<- prometheus.Metric) { + ch <- prometheus.MustNewConstMetric(c.managementConnected, prometheus.GaugeValue, boolToFloat(c.status.GetManagementState().Connected)) + ch <- prometheus.MustNewConstMetric(c.signalConnected, prometheus.GaugeValue, boolToFloat(c.status.GetSignalState().Connected)) + + peers := c.status.GetPeerStates() + ch <- prometheus.MustNewConstMetric(c.peersTotal, prometheus.GaugeValue, float64(len(peers))) + + var p2p, relayed float64 + for _, p := range peers { + if p.ConnStatus != peer.StatusConnected { + continue + } + if p.Relayed { + relayed++ + continue + } + p2p++ + + if latency := p.Latency.Seconds(); latency > 0 { + ch <- prometheus.MustNewConstMetric(c.peerLatency, prometheus.GaugeValue, latency, p.FQDN) + } + } + ch <- prometheus.MustNewConstMetric(c.peersConnected, prometheus.GaugeValue, p2p, "p2p") + ch <- prometheus.MustNewConstMetric(c.peersConnected, prometheus.GaugeValue, relayed, "relay") +} + +func boolToFloat(b bool) float64 { + if b { + return 1 + } + return 0 +} + +// IsLoopback reports whether addr binds the endpoint to the local host only. +// An empty address means DefaultListenAddress. It fails closed: an address +// that cannot be confirmed loopback, including an unparseable one, is not. +func IsLoopback(addr string) bool { + if addr == "" { + addr = DefaultListenAddress + } + + host, _, err := net.SplitHostPort(addr) + if err != nil { + return false + } + if host == "localhost" { + return true + } + + ip, err := netip.ParseAddr(host) + if err != nil { + return false + } + return ip.Unmap().IsLoopback() +} + +// warnIfNotLoopback logs a warning when the listen address cannot be +// confirmed to be local-only, since the endpoint exposes peer and +// connectivity details without authentication. +func warnIfNotLoopback(addr string) { + if IsLoopback(addr) { + return + } + log.Warnf("local metrics endpoint listens on non-loopback address %s and is reachable from the network without authentication", addr) +} diff --git a/client/internal/localmetrics/localmetrics_test.go b/client/internal/localmetrics/localmetrics_test.go new file mode 100644 index 000000000..727137077 --- /dev/null +++ b/client/internal/localmetrics/localmetrics_test.go @@ -0,0 +1,151 @@ +package localmetrics + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/internal/peer" +) + +type stubStatus struct { + peers []peer.State + management peer.ManagementState + signal peer.SignalState +} + +func (s *stubStatus) GetPeerStates() []peer.State { return s.peers } +func (s *stubStatus) GetManagementState() peer.ManagementState { return s.management } +func (s *stubStatus) GetSignalState() peer.SignalState { return s.signal } + +func testStatus() *stubStatus { + return &stubStatus{ + management: peer.ManagementState{Connected: true}, + signal: peer.SignalState{Connected: true}, + peers: []peer.State{ + {FQDN: "peer-a.netbird.cloud", IP: "100.90.0.1", ConnStatus: peer.StatusConnected, Relayed: false, Latency: 12 * time.Millisecond}, + {FQDN: "peer-b.netbird.cloud", IP: "100.90.0.2", ConnStatus: peer.StatusConnected, Relayed: false, Latency: 36 * time.Millisecond}, + {FQDN: "peer-c.netbird.cloud", IP: "100.90.0.3", ConnStatus: peer.StatusConnected, Relayed: true}, + {FQDN: "peer-d.netbird.cloud", IP: "100.90.0.4", ConnStatus: peer.StatusIdle}, + }, + } +} + +func TestCollector(t *testing.T) { + c := newCollector(testStatus()) + + expected := ` +# HELP netbird_management_connected Whether the client is connected to the management service (1 connected, 0 disconnected). +# TYPE netbird_management_connected gauge +netbird_management_connected 1 +# HELP netbird_peer_latency_seconds Round-trip latency per directly connected peer; relayed connections have no latency measurement. +# TYPE netbird_peer_latency_seconds gauge +netbird_peer_latency_seconds{peer="peer-a.netbird.cloud"} 0.012 +netbird_peer_latency_seconds{peer="peer-b.netbird.cloud"} 0.036 +# HELP netbird_peers Number of peers known to this client. +# TYPE netbird_peers gauge +netbird_peers 4 +# HELP netbird_peers_connected Number of connected peers by connection type. +# TYPE netbird_peers_connected gauge +netbird_peers_connected{connection_type="p2p"} 2 +netbird_peers_connected{connection_type="relay"} 1 +# HELP netbird_signal_connected Whether the client is connected to the signal service (1 connected, 0 disconnected). +# TYPE netbird_signal_connected gauge +netbird_signal_connected 1 +` + require.NoError(t, testutil.CollectAndCompare(c, strings.NewReader(expected))) +} + +func TestServe(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err, "must find a free port") + addr := ln.Addr().String() + require.NoError(t, ln.Close()) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + m := NewManager(ctx, testStatus(), nil) + m.Reconcile(true, addr) + + var body string + require.Eventually(t, func() bool { + resp, err := http.Get(fmt.Sprintf("http://%s/metrics", addr)) + if err != nil { + return false + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil || resp.StatusCode != http.StatusOK { + return false + } + body = string(data) + return true + }, 2*time.Second, 50*time.Millisecond, "metrics endpoint should come up") + + assert.Contains(t, body, "netbird_peers 4") + assert.Contains(t, body, `netbird_peers_connected{connection_type="relay"} 1`) + assert.Contains(t, body, `netbird_peer_latency_seconds{peer="peer-a.netbird.cloud"} 0.012`) +} + +// A server that never came up must not be remembered, otherwise reconciling the +// same address again is a no-op and the endpoint never recovers. +func TestReconcileForgetsAFailedServer(t *testing.T) { + blocker, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err, "must find a free port") + t.Cleanup(func() { _ = blocker.Close() }) + addr := blocker.Addr().String() + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + m := NewManager(ctx, testStatus(), nil) + m.Reconcile(true, addr) + + require.Eventually(t, func() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.srv == nil && m.addr == "" + }, 2*time.Second, 20*time.Millisecond, "the failed server should be dropped") + + require.NoError(t, blocker.Close()) + m.Reconcile(true, addr) + + require.Eventually(t, func() bool { + resp, err := http.Get(fmt.Sprintf("http://%s/metrics", addr)) + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK + }, 2*time.Second, 50*time.Millisecond, "reconciling the same address should retry the bind") +} + +func TestIsLoopback(t *testing.T) { + tests := map[string]bool{ + "": true, + "127.0.0.1:9191": true, + "127.9.9.9:9191": true, + "[::1]:9191": true, + "[::ffff:127.0.0.1]:9191": true, + "localhost:9191": true, + "0.0.0.0:9191": false, + "[::]:9191": false, + "192.168.1.10:9191": false, + "not-an-address": false, + "example.com:9191": false, + } + + for addr, want := range tests { + t.Run(addr, func(t *testing.T) { + assert.Equal(t, want, IsLoopback(addr), "loopback verdict for %q", addr) + }) + } +} diff --git a/client/internal/metrics/influxdb.go b/client/internal/metrics/influxdb.go index 4ba14bf44..717544f6a 100644 --- a/client/internal/metrics/influxdb.go +++ b/client/internal/metrics/influxdb.go @@ -45,30 +45,13 @@ func (m *influxDBMetrics) RecordConnectionStages( isReconnection bool, timestamps ConnectionStageTimestamps, ) { - var signalingReceivedToConnection, connectionToWgHandshake, totalDuration float64 - - if !timestamps.SignalingReceived.IsZero() && !timestamps.ConnectionReady.IsZero() { - signalingReceivedToConnection = timestamps.ConnectionReady.Sub(timestamps.SignalingReceived).Seconds() - } - - if !timestamps.ConnectionReady.IsZero() && !timestamps.WgHandshakeSuccess.IsZero() { - connectionToWgHandshake = timestamps.WgHandshakeSuccess.Sub(timestamps.ConnectionReady).Seconds() - } - - if !timestamps.SignalingReceived.IsZero() && !timestamps.WgHandshakeSuccess.IsZero() { - totalDuration = timestamps.WgHandshakeSuccess.Sub(timestamps.SignalingReceived).Seconds() - } - - attemptType := "initial" - if isReconnection { - attemptType = "reconnection" - } + signalingReceivedToConnection, connectionToWgHandshake, totalDuration := timestamps.Durations() connTypeStr := connectionType.String() tags := fmt.Sprintf("deployment_type=%s,connection_type=%s,attempt_type=%s,version=%s,os=%s,arch=%s,peer_id=%s,connection_pair_id=%s", agentInfo.DeploymentType.String(), connTypeStr, - attemptType, + attemptType(isReconnection), agentInfo.Version, agentInfo.OS, agentInfo.Arch, @@ -94,7 +77,7 @@ func (m *influxDBMetrics) RecordConnectionStages( m.trimLocked() log.Tracef("peer connection metrics [%s, %s, %s]: signalingReceived→connection: %.3fs, connection→wg_handshake: %.3fs, total: %.3fs", - agentInfo.DeploymentType.String(), connTypeStr, attemptType, signalingReceivedToConnection, connectionToWgHandshake, totalDuration) + agentInfo.DeploymentType.String(), connTypeStr, attemptType(isReconnection), signalingReceivedToConnection, connectionToWgHandshake, totalDuration) } func (m *influxDBMetrics) RecordSyncDuration(_ context.Context, agentInfo AgentInfo, duration time.Duration) { diff --git a/client/internal/metrics/metrics.go b/client/internal/metrics/metrics.go index cfe477107..5edf1d9c7 100644 --- a/client/internal/metrics/metrics.go +++ b/client/internal/metrics/metrics.go @@ -89,6 +89,21 @@ type ConnectionStageTimestamps struct { WgHandshakeSuccess time.Time } +// Durations returns the stage durations in seconds. A duration is zero when +// either of its timestamps is missing. +func (c ConnectionStageTimestamps) Durations() (signalingToConnection, connectionToWgHandshake, total float64) { + if !c.SignalingReceived.IsZero() && !c.ConnectionReady.IsZero() { + signalingToConnection = c.ConnectionReady.Sub(c.SignalingReceived).Seconds() + } + if !c.ConnectionReady.IsZero() && !c.WgHandshakeSuccess.IsZero() { + connectionToWgHandshake = c.WgHandshakeSuccess.Sub(c.ConnectionReady).Seconds() + } + if !c.SignalingReceived.IsZero() && !c.WgHandshakeSuccess.IsZero() { + total = c.WgHandshakeSuccess.Sub(c.SignalingReceived).Seconds() + } + return signalingToConnection, connectionToWgHandshake, total +} + // String returns a human-readable representation of the connection stage timestamps func (c ConnectionStageTimestamps) String() string { return fmt.Sprintf("ConnectionStageTimestamps{SignalingReceived=%v, ConnectionReady=%v, WgHandshakeSuccess=%v}", @@ -279,3 +294,11 @@ func (c *ClientMetrics) stopPushLocked() { c.wg.Wait() c.push.Store(nil) } + +// attemptType returns the metric label for an initial vs reconnection attempt. +func attemptType(isReconnection bool) string { + if isReconnection { + return "reconnection" + } + return "initial" +} diff --git a/client/internal/metrics/metrics_default.go b/client/internal/metrics/metrics_default.go index 927ab51d1..3798adab6 100644 --- a/client/internal/metrics/metrics_default.go +++ b/client/internal/metrics/metrics_default.go @@ -2,10 +2,24 @@ package metrics +import "github.com/prometheus/client_golang/prometheus" + // NewClientMetrics creates a new ClientMetrics instance func NewClientMetrics(agentInfo AgentInfo) *ClientMetrics { return &ClientMetrics{ - impl: newInfluxDBMetrics(), + impl: newPrometheusMetrics(newInfluxDBMetrics()), agentInfo: agentInfo, } } + +// PrometheusGatherer returns the registry with the mirrored Prometheus +// metrics, or nil when unavailable. +func (c *ClientMetrics) PrometheusGatherer() prometheus.Gatherer { + if c == nil { + return nil + } + if pm, ok := c.impl.(*prometheusMetrics); ok { + return pm.Gatherer() + } + return nil +} diff --git a/client/internal/metrics/prometheus.go b/client/internal/metrics/prometheus.go new file mode 100644 index 000000000..7f5020ea9 --- /dev/null +++ b/client/internal/metrics/prometheus.go @@ -0,0 +1,119 @@ +//go:build !js + +package metrics + +import ( + "context" + "io" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// prometheusMetrics mirrors recorded client metrics into a Prometheus +// registry for the local /metrics endpoint, then delegates to the wrapped +// implementation. Export and Reset pass through untouched: Prometheus +// metrics are cumulative and pull-based. +type prometheusMetrics struct { + next metricsImplementation + registry *prometheus.Registry + + connectionStages *prometheus.HistogramVec + syncDuration prometheus.Histogram + syncPhaseDuration *prometheus.HistogramVec + loginDuration *prometheus.HistogramVec +} + +func newPrometheusMetrics(next metricsImplementation) *prometheusMetrics { + connectionBuckets := []float64{.05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60} + + m := &prometheusMetrics{ + next: next, + registry: prometheus.NewRegistry(), + connectionStages: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "netbird_peer_connection_stage_duration_seconds", + Help: "Duration of peer connection establishment stages.", + Buckets: connectionBuckets, + }, []string{"stage", "connection_type", "attempt_type"}), + syncDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "netbird_sync_duration_seconds", + Help: "Duration of management sync message processing.", + Buckets: prometheus.DefBuckets, + }), + syncPhaseDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "netbird_sync_phase_duration_seconds", + Help: "Duration of individual sync processing phases.", + Buckets: prometheus.DefBuckets, + }, []string{"phase"}), + loginDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "netbird_login_duration_seconds", + Help: "Duration of logins to the management service.", + Buckets: prometheus.DefBuckets, + }, []string{"success"}), + } + + m.registry.MustRegister(m.connectionStages, m.syncDuration, m.syncPhaseDuration, m.loginDuration) + return m +} + +// Gatherer returns the registry holding the mirrored metrics. +func (m *prometheusMetrics) Gatherer() prometheus.Gatherer { + return m.registry +} + +// RecordConnectionStages implements metricsImplementation. +func (m *prometheusMetrics) RecordConnectionStages( + ctx context.Context, + agentInfo AgentInfo, + connectionPairID string, + connectionType ConnectionType, + isReconnection bool, + timestamps ConnectionStageTimestamps, +) { + attempt := attemptType(isReconnection) + connType := connectionType.String() + + signalingToConnection, connectionToWgHandshake, total := timestamps.Durations() + if signalingToConnection > 0 { + m.connectionStages.WithLabelValues("signaling_to_connection", connType, attempt).Observe(signalingToConnection) + } + if connectionToWgHandshake > 0 { + m.connectionStages.WithLabelValues("connection_to_wg_handshake", connType, attempt).Observe(connectionToWgHandshake) + } + if total > 0 { + m.connectionStages.WithLabelValues("total", connType, attempt).Observe(total) + } + + m.next.RecordConnectionStages(ctx, agentInfo, connectionPairID, connectionType, isReconnection, timestamps) +} + +// RecordSyncDuration implements metricsImplementation. +func (m *prometheusMetrics) RecordSyncDuration(ctx context.Context, agentInfo AgentInfo, duration time.Duration) { + m.syncDuration.Observe(duration.Seconds()) + m.next.RecordSyncDuration(ctx, agentInfo, duration) +} + +// RecordSyncPhase implements metricsImplementation. +func (m *prometheusMetrics) RecordSyncPhase(ctx context.Context, agentInfo AgentInfo, phase string, duration time.Duration) { + m.syncPhaseDuration.WithLabelValues(phase).Observe(duration.Seconds()) + m.next.RecordSyncPhase(ctx, agentInfo, phase, duration) +} + +// RecordLoginDuration implements metricsImplementation. +func (m *prometheusMetrics) RecordLoginDuration(ctx context.Context, agentInfo AgentInfo, duration time.Duration, success bool) { + m.loginDuration.WithLabelValues(strconv.FormatBool(success)).Observe(duration.Seconds()) + m.next.RecordLoginDuration(ctx, agentInfo, duration, success) +} + +// Export implements metricsImplementation by delegating to the wrapped +// implementation; Prometheus metrics are pulled via the registry instead. +func (m *prometheusMetrics) Export(w io.Writer) error { + return m.next.Export(w) +} + +// Reset implements metricsImplementation by delegating to the wrapped +// implementation; Prometheus metrics must not be cleared on push. +func (m *prometheusMetrics) Reset() { + m.next.Reset() +} diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index 24e3e7fac..bf36b944b 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -1167,6 +1167,18 @@ func (d *Status) GetResolvedDomainsStates() map[domain.Domain]ResolvedDomainInfo return maps.Clone(d.resolvedDomainsStates) } +// GetPeerStates returns a snapshot of all known peer states, including offline peers. +func (d *Status) GetPeerStates() []State { + d.mux.RLock() + defer d.mux.RUnlock() + + states := make([]State, 0, d.numOfPeers()) + for _, state := range d.peers { + states = append(states, state) + } + return append(states, d.offlinePeers...) +} + // GetFullStatus gets full status func (d *Status) GetFullStatus() FullStatus { fullStatus := FullStatus{ diff --git a/client/internal/peer/status_test.go b/client/internal/peer/status_test.go index 29404d413..82dff0d6f 100644 --- a/client/internal/peer/status_test.go +++ b/client/internal/peer/status_test.go @@ -129,6 +129,28 @@ func TestStatus_PeerStateByIP_RemovedPeer(t *testing.T) { req.False(ok, "removed peer must not resolve by IPv6 tunnel address") } +// TestStatus_GetPeerStates_IncludesOfflinePeers keeps the snapshot in line with +// GetFullStatus: offline peers are known peers, so a consumer counting peers +// must see the same total the status command reports. +func TestStatus_GetPeerStates_IncludesOfflinePeers(t *testing.T) { + status := NewRecorder("https://mgm") + req := require.New(t) + + req.NoError(status.AddPeer("pk-online", "online.netbird", "100.64.0.10", "fd00::1")) + status.ReplaceOfflinePeers([]State{ + {PubKey: "pk-offline", FQDN: "offline.netbird", IP: "100.64.0.20", ConnStatus: StatusIdle}, + }) + + states := status.GetPeerStates() + req.Len(states, 2, "snapshot must carry both the online and the offline peer") + + keys := make([]string, 0, len(states)) + for _, s := range states { + keys = append(keys, s.PubKey) + } + req.ElementsMatch([]string{"pk-online", "pk-offline"}, keys, "snapshot must carry both peers") +} + func TestStatus_UpdatePeerFQDN(t *testing.T) { key := "abc" fqdn := "peer-a.netbird.local" diff --git a/client/internal/profilemanager/config.go b/client/internal/profilemanager/config.go index e1668238e..eacc6fd5f 100644 --- a/client/internal/profilemanager/config.go +++ b/client/internal/profilemanager/config.go @@ -103,6 +103,9 @@ type ConfigInput struct { DNSLabels domain.List MTU *uint16 + + LocalMetricsEnabled *bool + LocalMetricsAddress *string } // Config Configuration type @@ -144,6 +147,11 @@ type Config struct { DNSLabels domain.List + // LocalMetricsEnabled enables the local Prometheus /metrics endpoint. + LocalMetricsEnabled bool + // LocalMetricsAddress is the listen address of the local /metrics endpoint. + LocalMetricsAddress string + // SSHKey is a private SSH key in a PEM format SSHKey string @@ -388,6 +396,18 @@ func (config *Config) apply(input ConfigInput) (updated bool, err error) { updated = true } + if input.LocalMetricsEnabled != nil && *input.LocalMetricsEnabled != config.LocalMetricsEnabled { + log.Infof("switching local metrics to %t", *input.LocalMetricsEnabled) + config.LocalMetricsEnabled = *input.LocalMetricsEnabled + updated = true + } + + if input.LocalMetricsAddress != nil && *input.LocalMetricsAddress != config.LocalMetricsAddress { + log.Infof("switching local metrics address to %s", *input.LocalMetricsAddress) + config.LocalMetricsAddress = *input.LocalMetricsAddress + updated = true + } + if input.NetworkMonitor != nil && (config.NetworkMonitor == nil || *input.NetworkMonitor != *config.NetworkMonitor) { log.Infof("switching Network Monitor to %t", *input.NetworkMonitor) config.NetworkMonitor = input.NetworkMonitor @@ -718,6 +738,12 @@ func (config *Config) applyMDMPolicy(policy *mdm.Policy) { applyBool(mdm.KeyDisableAutoConnect, func(v bool) { config.DisableAutoConnect = v }) applyBool(mdm.KeyRosenpassEnabled, func(v bool) { config.RosenpassEnabled = v }) applyBool(mdm.KeyRosenpassPermissive, func(v bool) { config.RosenpassPermissive = v }) + applyBool(mdm.KeyEnableLocalMetrics, func(v bool) { config.LocalMetricsEnabled = v }) + + if v, ok := policy.GetString(mdm.KeyLocalMetricsAddress); ok { + config.LocalMetricsAddress = v + logApplied(mdm.KeyLocalMetricsAddress, v) + } if v, ok := policy.GetInt(mdm.KeyWireguardPort); ok { // REG_DWORD is 32-bit; UDP port range is 1-65535. Clamp at the diff --git a/client/internal/profilemanager/config_mdm_test.go b/client/internal/profilemanager/config_mdm_test.go index c6a688ab2..f8dfddb33 100644 --- a/client/internal/profilemanager/config_mdm_test.go +++ b/client/internal/profilemanager/config_mdm_test.go @@ -130,6 +130,32 @@ func TestApply_MDMBoolKeysOverrideOnDiskValue(t *testing.T) { assert.True(t, cfg.Policy().HasKey(mdm.KeyRosenpassEnabled)) } +func TestApply_MDMLocalMetrics(t *testing.T) { + tmp := filepath.Join(t.TempDir(), "config.json") + + // Seed without MDM. + withMDMPolicy(t, mdm.NewPolicy(nil)) + _, err := UpdateOrCreateConfig(ConfigInput{ + ConfigPath: tmp, + LocalMetricsEnabled: boolPtr(false), + }) + require.NoError(t, err) + + withMDMPolicy(t, mdm.NewPolicy(map[string]any{ + mdm.KeyEnableLocalMetrics: true, + mdm.KeyLocalMetricsAddress: "127.0.0.1:9292", + })) + + cfg, err := UpdateOrCreateConfig(ConfigInput{ConfigPath: tmp}) + require.NoError(t, err) + require.NotNil(t, cfg) + + assert.True(t, cfg.LocalMetricsEnabled, "MDM override should flip on-disk false to true") + assert.Equal(t, "127.0.0.1:9292", cfg.LocalMetricsAddress) + assert.True(t, cfg.Policy().HasKey(mdm.KeyEnableLocalMetrics)) + assert.True(t, cfg.Policy().HasKey(mdm.KeyLocalMetricsAddress)) +} + func TestApply_MDMLazyConnection(t *testing.T) { cases := []struct { name string diff --git a/client/mdm/canonical_loaders.go b/client/mdm/canonical_loaders.go index 29288b511..eb9db07c4 100644 --- a/client/mdm/canonical_loaders.go +++ b/client/mdm/canonical_loaders.go @@ -27,6 +27,8 @@ var allKeys = []string{ KeyRosenpassEnabled, KeyRosenpassPermissive, KeyWireguardPort, + KeyEnableLocalMetrics, + KeyLocalMetricsAddress, KeySplitTunnelMode, KeySplitTunnelApps, KeyLazyConnection, diff --git a/client/mdm/canonical_loaders_test.go b/client/mdm/canonical_loaders_test.go new file mode 100644 index 000000000..330a15c47 --- /dev/null +++ b/client/mdm/canonical_loaders_test.go @@ -0,0 +1,52 @@ +//go:build windows || darwin + +package mdm + +import ( + "go/ast" + "go/parser" + "go/token" + "slices" + "strconv" + "testing" +) + +// TestAllKeysCoversEveryPolicyKey guards against the drift that adding a Key* +// constant without listing it in allKeys causes: the desktop loaders resolve +// value names through canonicalKey, so an unlisted key is silently discarded as +// unknown. policy.go is parsed rather than hand-mirrored so the test cannot go +// stale in the same way. +func TestAllKeysCoversEveryPolicyKey(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "policy.go", nil, 0) + if err != nil { + t.Fatalf("parse policy.go: %v", err) + } + + for _, decl := range file.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.CONST { + continue + } + for _, spec := range gen.Specs { + value, ok := spec.(*ast.ValueSpec) + if !ok || len(value.Names) != 1 || len(value.Values) != 1 { + continue + } + name := value.Names[0].Name + if len(name) < 4 || name[:3] != "Key" { + continue + } + lit, ok := value.Values[0].(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + continue + } + key, err := strconv.Unquote(lit.Value) + if err != nil { + t.Fatalf("unquote %s: %v", name, err) + } + if !slices.Contains(allKeys, key) { + t.Errorf("%s (%q) is missing from allKeys, so the desktop loaders discard it as unknown", name, key) + } + } + } +} diff --git a/client/mdm/policy.go b/client/mdm/policy.go index 1feff28f8..6c64acfc8 100644 --- a/client/mdm/policy.go +++ b/client/mdm/policy.go @@ -47,6 +47,8 @@ const ( KeyRosenpassEnabled = "rosenpassEnabled" KeyRosenpassPermissive = "rosenpassPermissive" KeyWireguardPort = "wireguardPort" + KeyEnableLocalMetrics = "enableLocalMetrics" + KeyLocalMetricsAddress = "localMetricsAddress" // Split tunnel is modeled as a single conceptual policy with two // registry/plist values. KeySplitTunnelMode is the discriminator diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index b438a310a..089f3b95b 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -343,6 +343,8 @@ type LoginRequest struct { DisableSSHAuth *bool `protobuf:"varint,38,opt,name=disableSSHAuth,proto3,oneof" json:"disableSSHAuth,omitempty"` SshJWTCacheTTL *int32 `protobuf:"varint,39,opt,name=sshJWTCacheTTL,proto3,oneof" json:"sshJWTCacheTTL,omitempty"` DisableIpv6 *bool `protobuf:"varint,40,opt,name=disable_ipv6,json=disableIpv6,proto3,oneof" json:"disable_ipv6,omitempty"` + EnableLocalMetrics *bool `protobuf:"varint,41,opt,name=enable_local_metrics,json=enableLocalMetrics,proto3,oneof" json:"enable_local_metrics,omitempty"` + LocalMetricsAddress *string `protobuf:"bytes,42,opt,name=local_metrics_address,json=localMetricsAddress,proto3,oneof" json:"local_metrics_address,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -658,6 +660,20 @@ func (x *LoginRequest) GetDisableIpv6() bool { return false } +func (x *LoginRequest) GetEnableLocalMetrics() bool { + if x != nil && x.EnableLocalMetrics != nil { + return *x.EnableLocalMetrics + } + return false +} + +func (x *LoginRequest) GetLocalMetricsAddress() string { + if x != nil && x.LocalMetricsAddress != nil { + return *x.LocalMetricsAddress + } + return "" +} + type LoginResponse struct { state protoimpl.MessageState `protogen:"open.v1"` NeedsSSOLogin bool `protobuf:"varint,1,opt,name=needsSSOLogin,proto3" json:"needsSSOLogin,omitempty"` @@ -4233,6 +4249,8 @@ type SetConfigRequest struct { DisableSSHAuth *bool `protobuf:"varint,33,opt,name=disableSSHAuth,proto3,oneof" json:"disableSSHAuth,omitempty"` SshJWTCacheTTL *int32 `protobuf:"varint,34,opt,name=sshJWTCacheTTL,proto3,oneof" json:"sshJWTCacheTTL,omitempty"` DisableIpv6 *bool `protobuf:"varint,35,opt,name=disable_ipv6,json=disableIpv6,proto3,oneof" json:"disable_ipv6,omitempty"` + EnableLocalMetrics *bool `protobuf:"varint,36,opt,name=enable_local_metrics,json=enableLocalMetrics,proto3,oneof" json:"enable_local_metrics,omitempty"` + LocalMetricsAddress *string `protobuf:"bytes,37,opt,name=local_metrics_address,json=localMetricsAddress,proto3,oneof" json:"local_metrics_address,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4512,6 +4530,20 @@ func (x *SetConfigRequest) GetDisableIpv6() bool { return false } +func (x *SetConfigRequest) GetEnableLocalMetrics() bool { + if x != nil && x.EnableLocalMetrics != nil { + return *x.EnableLocalMetrics + } + return false +} + +func (x *SetConfigRequest) GetLocalMetricsAddress() string { + if x != nil && x.LocalMetricsAddress != nil { + return *x.LocalMetricsAddress + } + return "" +} + type SetConfigResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -7032,7 +7064,7 @@ var File_daemon_proto protoreflect.FileDescriptor const file_daemon_proto_rawDesc = "" + "\n" + "\fdaemon.proto\x12\x06daemon\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\"\x0e\n" + - "\fEmptyRequest\"\xef\x12\n" + + "\fEmptyRequest\"\x92\x14\n" + "\fLoginRequest\x12\x1a\n" + "\bsetupKey\x18\x01 \x01(\tR\bsetupKey\x12&\n" + "\fpreSharedKey\x18\x02 \x01(\tB\x02\x18\x01R\fpreSharedKey\x12$\n" + @@ -7077,7 +7109,9 @@ const file_daemon_proto_rawDesc = "" + "\x1denableSSHRemotePortForwarding\x18% \x01(\bH\x18R\x1denableSSHRemotePortForwarding\x88\x01\x01\x12+\n" + "\x0edisableSSHAuth\x18& \x01(\bH\x19R\x0edisableSSHAuth\x88\x01\x01\x12+\n" + "\x0esshJWTCacheTTL\x18' \x01(\x05H\x1aR\x0esshJWTCacheTTL\x88\x01\x01\x12&\n" + - "\fdisable_ipv6\x18( \x01(\bH\x1bR\vdisableIpv6\x88\x01\x01B\x13\n" + + "\fdisable_ipv6\x18( \x01(\bH\x1bR\vdisableIpv6\x88\x01\x01\x125\n" + + "\x14enable_local_metrics\x18) \x01(\bH\x1cR\x12enableLocalMetrics\x88\x01\x01\x127\n" + + "\x15local_metrics_address\x18* \x01(\tH\x1dR\x13localMetricsAddress\x88\x01\x01B\x13\n" + "\x11_rosenpassEnabledB\x10\n" + "\x0e_interfaceNameB\x10\n" + "\x0e_wireguardPortB\x17\n" + @@ -7105,7 +7139,9 @@ const file_daemon_proto_rawDesc = "" + "\x1e_enableSSHRemotePortForwardingB\x11\n" + "\x0f_disableSSHAuthB\x11\n" + "\x0f_sshJWTCacheTTLB\x0f\n" + - "\r_disable_ipv6\"\xb5\x01\n" + + "\r_disable_ipv6B\x17\n" + + "\x15_enable_local_metricsB\x18\n" + + "\x16_local_metrics_address\"\xb5\x01\n" + "\rLoginResponse\x12$\n" + "\rneedsSSOLogin\x18\x01 \x01(\bR\rneedsSSOLogin\x12\x1a\n" + "\buserCode\x18\x02 \x01(\tR\buserCode\x12(\n" + @@ -7400,7 +7436,7 @@ const file_daemon_proto_rawDesc = "" + "\f_profileNameB\v\n" + "\t_username\"'\n" + "\x15SwitchProfileResponse\x12\x0e\n" + - "\x02id\x18\x01 \x01(\tR\x02id\"\x98\x11\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"\xbb\x12\n" + "\x10SetConfigRequest\x12\x1a\n" + "\busername\x18\x01 \x01(\tR\busername\x12 \n" + "\vprofileName\x18\x02 \x01(\tR\vprofileName\x12$\n" + @@ -7440,7 +7476,9 @@ const file_daemon_proto_rawDesc = "" + "\x1denableSSHRemotePortForwarding\x18 \x01(\bH\x15R\x1denableSSHRemotePortForwarding\x88\x01\x01\x12+\n" + "\x0edisableSSHAuth\x18! \x01(\bH\x16R\x0edisableSSHAuth\x88\x01\x01\x12+\n" + "\x0esshJWTCacheTTL\x18\" \x01(\x05H\x17R\x0esshJWTCacheTTL\x88\x01\x01\x12&\n" + - "\fdisable_ipv6\x18# \x01(\bH\x18R\vdisableIpv6\x88\x01\x01B\x13\n" + + "\fdisable_ipv6\x18# \x01(\bH\x18R\vdisableIpv6\x88\x01\x01\x125\n" + + "\x14enable_local_metrics\x18$ \x01(\bH\x19R\x12enableLocalMetrics\x88\x01\x01\x127\n" + + "\x15local_metrics_address\x18% \x01(\tH\x1aR\x13localMetricsAddress\x88\x01\x01B\x13\n" + "\x11_rosenpassEnabledB\x10\n" + "\x0e_interfaceNameB\x10\n" + "\x0e_wireguardPortB\x17\n" + @@ -7465,7 +7503,9 @@ const file_daemon_proto_rawDesc = "" + "\x1e_enableSSHRemotePortForwardingB\x11\n" + "\x0f_disableSSHAuthB\x11\n" + "\x0f_sshJWTCacheTTLB\x0f\n" + - "\r_disable_ipv6\"\x13\n" + + "\r_disable_ipv6B\x17\n" + + "\x15_enable_local_metricsB\x18\n" + + "\x16_local_metrics_address\"\x13\n" + "\x11SetConfigResponse\"Q\n" + "\x11AddProfileRequest\x12\x1a\n" + "\busername\x18\x01 \x01(\tR\busername\x12 \n" + diff --git a/client/proto/daemon.proto b/client/proto/daemon.proto index a3e3f4500..ad59a78f8 100644 --- a/client/proto/daemon.proto +++ b/client/proto/daemon.proto @@ -242,6 +242,9 @@ message LoginRequest { optional bool disableSSHAuth = 38; optional int32 sshJWTCacheTTL = 39; optional bool disable_ipv6 = 40; + + optional bool enable_local_metrics = 41; + optional string local_metrics_address = 42; } message LoginResponse { @@ -766,6 +769,9 @@ message SetConfigRequest { optional bool disableSSHAuth = 33; optional int32 sshJWTCacheTTL = 34; optional bool disable_ipv6 = 35; + + optional bool enable_local_metrics = 36; + optional string local_metrics_address = 37; } message SetConfigResponse{} diff --git a/client/server/mdm.go b/client/server/mdm.go index 9836c6bea..552fba94f 100644 --- a/client/server/mdm.go +++ b/client/server/mdm.go @@ -233,6 +233,24 @@ func conflictString(key, got string) conflictCheck { } } +// conflictStringPtr is conflictString for optional proto fields, where an +// explicit empty value is still a request to change the setting. If p is +// nil the field is treated as matching (no override requested); otherwise +// the check returns true only when the policy contains the key and its +// value equals *p. +func conflictStringPtr(key string, p *string) conflictCheck { + return conflictCheck{ + key: key, + check: func(pol *mdm.Policy) bool { + if p == nil { + return true + } + want, ok := pol.GetString(key) + return ok && want == *p + }, + } +} + // conflictInt64 builds a conflictCheck for an integer MDM key. If p is // nil the field is treated as matching; otherwise the check returns // true only when the policy contains the key and its int value equals *p. @@ -301,6 +319,8 @@ func mdmManagedFieldConflicts(msg *proto.SetConfigRequest, policy *mdm.Policy) [ conflictBool(mdm.KeyDisableServerRoutes, msg.DisableServerRoutes), conflictBool(mdm.KeyBlockInbound, msg.BlockInbound), conflictInt64(mdm.KeyWireguardPort, msg.WireguardPort), + conflictBool(mdm.KeyEnableLocalMetrics, msg.EnableLocalMetrics), + conflictStringPtr(mdm.KeyLocalMetricsAddress, msg.LocalMetricsAddress), }) } @@ -346,7 +366,9 @@ func setConfigRequestHasConfigOverrides(msg *proto.SetConfigRequest) bool { msg.EnableSSHLocalPortForwarding != nil || msg.EnableSSHRemotePortForwarding != nil || msg.DisableSSHAuth != nil || - msg.SshJWTCacheTTL != nil + msg.SshJWTCacheTTL != nil || + msg.EnableLocalMetrics != nil || + msg.LocalMetricsAddress != nil } // loginRequestHasConfigOverrides reports whether the LoginRequest @@ -381,7 +403,9 @@ func loginRequestHasConfigOverrides(msg *proto.LoginRequest) bool { msg.BlockLanAccess != nil || msg.DisableNotifications != nil || len(msg.DnsLabels) > 0 || msg.CleanDNSLabels || - msg.BlockInbound != nil + msg.BlockInbound != nil || + msg.EnableLocalMetrics != nil || + msg.LocalMetricsAddress != nil } // loginRequestMDMConflicts mirrors mdmManagedFieldConflicts but for the @@ -422,6 +446,8 @@ func loginRequestMDMConflicts(msg *proto.LoginRequest, policy *mdm.Policy) []str conflictBool(mdm.KeyDisableServerRoutes, msg.DisableServerRoutes), conflictBool(mdm.KeyBlockInbound, msg.BlockInbound), conflictInt64(mdm.KeyWireguardPort, msg.WireguardPort), + conflictBool(mdm.KeyEnableLocalMetrics, msg.EnableLocalMetrics), + conflictStringPtr(mdm.KeyLocalMetricsAddress, msg.LocalMetricsAddress), }) } diff --git a/client/server/server.go b/client/server/server.go index f33e19075..23dccc9b1 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -23,6 +23,9 @@ import ( "github.com/netbirdio/netbird/client/internal/auth" "github.com/netbirdio/netbird/client/internal/expose" + "github.com/prometheus/client_golang/prometheus" + + "github.com/netbirdio/netbird/client/internal/localmetrics" "github.com/netbirdio/netbird/client/internal/profilemanager" sleephandler "github.com/netbirdio/netbird/client/internal/sleep/handler" "github.com/netbirdio/netbird/client/mdm" @@ -108,6 +111,7 @@ type Server struct { statusRecorder *peer.Status sessionWatcher *internal.SessionWatcher + localMetrics *localmetrics.Manager probeThrottle *probeThrottle persistSyncResponse bool @@ -171,9 +175,28 @@ func New(ctx context.Context, logFile string, configFile string, profilesDisable s.sleepHandler = sleephandler.New(agent) s.startSleepDetector() + s.localMetrics = localmetrics.NewManager(ctx, s.statusRecorder, s.clientMetricsGatherer) + return s } +// clientMetricsGatherer returns the Prometheus gatherer of the running +// engine's client metrics, or nil when no engine is running. +func (s *Server) clientMetricsGatherer() prometheus.Gatherer { + s.mutex.Lock() + connectClient := s.connectClient + s.mutex.Unlock() + + if connectClient == nil { + return nil + } + engine := connectClient.Engine() + if engine == nil { + return nil + } + return engine.GetClientMetrics().PrometheusGatherer() +} + func (s *Server) Start() error { s.mutex.Lock() defer s.mutex.Unlock() @@ -254,6 +277,7 @@ func (s *Server) Start() error { s.statusRecorder.UpdateManagementAddress(config.ManagementURL.String()) s.statusRecorder.UpdateRosenpass(config.RosenpassEnabled, config.RosenpassPermissive) + s.localMetrics.Reconcile(config.LocalMetricsEnabled, config.LocalMetricsAddress) if s.sessionWatcher == nil { s.sessionWatcher = internal.NewSessionWatcher(s.rootCtx, s.statusRecorder) @@ -477,11 +501,18 @@ func (s *Server) SetConfig(callerCtx context.Context, msg *proto.SetConfigReques return nil, err } - if _, err := profilemanager.UpdateConfig(config); err != nil { + updatedConf, err := profilemanager.UpdateConfig(config) + if err != nil { log.Errorf("failed to update profile config: %v", err) return nil, fmt.Errorf("failed to update profile config: %w", err) } + if activeProf, err := s.profileManager.GetActiveProfileState(); err == nil { + if activePath, err := activeProf.FilePath(); err == nil && activePath == config.ConfigPath { + s.localMetrics.Reconcile(updatedConf.LocalMetricsEnabled, updatedConf.LocalMetricsAddress) + } + } + return &proto.SetConfigResponse{}, nil } @@ -551,6 +582,8 @@ func (s *Server) setConfigInputFromRequest(msg *proto.SetConfigRequest) (profile config.RosenpassEnabled = msg.RosenpassEnabled config.RosenpassPermissive = msg.RosenpassPermissive + config.LocalMetricsEnabled = msg.EnableLocalMetrics + config.LocalMetricsAddress = msg.LocalMetricsAddress config.DisableAutoConnect = msg.DisableAutoConnect config.ServerSSHAllowed = msg.ServerSSHAllowed config.NetworkMonitor = msg.NetworkMonitor @@ -657,6 +690,8 @@ func (s *Server) Login(callerCtx context.Context, msg *proto.LoginRequest) (*pro s.config = config s.mutex.Unlock() + s.localMetrics.Reconcile(config.LocalMetricsEnabled, config.LocalMetricsAddress) + // A probe that errors leaves the login undecided: Management unreachable, a // restart mid-request, an internal error. Those are returned for the caller // to retry, because turning them into an SSO prompt asks the user to solve @@ -1007,6 +1042,7 @@ func (s *Server) Up(callerCtx context.Context, msg *proto.UpRequest) (*proto.UpR s.statusRecorder.UpdateManagementAddress(s.config.ManagementURL.String()) s.statusRecorder.UpdateRosenpass(s.config.RosenpassEnabled, s.config.RosenpassPermissive) + s.localMetrics.Reconcile(s.config.LocalMetricsEnabled, s.config.LocalMetricsAddress) s.clientRunning = true s.clientRunningChan = make(chan struct{}) @@ -1184,6 +1220,7 @@ func (s *Server) SwitchProfile(callerCtx context.Context, msg *proto.SwitchProfi } s.config = config + s.localMetrics.Reconcile(config.LocalMetricsEnabled, config.LocalMetricsAddress) if msg != nil && msg.ProfileName != nil { s.publishProfileListChanged(*msg.ProfileName) diff --git a/client/server/setconfig_mdm_test.go b/client/server/setconfig_mdm_test.go index ae323ea8c..ad3b7ade7 100644 --- a/client/server/setconfig_mdm_test.go +++ b/client/server/setconfig_mdm_test.go @@ -136,6 +136,51 @@ func TestSetConfig_MDMReject_MultipleFields(t *testing.T) { }, v.GetFields()) } +func TestSetConfig_MDMReject_LocalMetrics(t *testing.T) { + withMDMPolicy(t, mdm.NewPolicy(map[string]any{ + mdm.KeyEnableLocalMetrics: true, + mdm.KeyLocalMetricsAddress: "127.0.0.1:9191", + })) + + s, ctx, profName, username, _ := setupServerWithProfile(t) + + enabled := false + addr := "0.0.0.0:9999" + _, err := s.SetConfig(ctx, &proto.SetConfigRequest{ + ProfileName: profName, + Username: username, + EnableLocalMetrics: &enabled, + LocalMetricsAddress: &addr, + }) + + v := extractViolation(t, err) + assert.ElementsMatch(t, []string{ + mdm.KeyEnableLocalMetrics, + mdm.KeyLocalMetricsAddress, + }, v.GetFields()) +} + +// An explicitly empty address still changes the effective listen address +// (the manager falls back to the default), so presence must be honored +// rather than collapsed to "field not set". +func TestSetConfig_MDMReject_LocalMetricsEmptyAddress(t *testing.T) { + withMDMPolicy(t, mdm.NewPolicy(map[string]any{ + mdm.KeyLocalMetricsAddress: "127.0.0.1:9999", + })) + + s, ctx, profName, username, _ := setupServerWithProfile(t) + + addr := "" + _, err := s.SetConfig(ctx, &proto.SetConfigRequest{ + ProfileName: profName, + Username: username, + LocalMetricsAddress: &addr, + }) + + v := extractViolation(t, err) + assert.ElementsMatch(t, []string{mdm.KeyLocalMetricsAddress}, v.GetFields()) +} + func TestSetConfig_MDMReject_AllOrNothing(t *testing.T) { // MDM enforces ManagementURL only; user request touches both the // enforced field AND a non-enforced field (RosenpassEnabled). diff --git a/client/server/setconfig_test.go b/client/server/setconfig_test.go index db7a26f03..d8309f519 100644 --- a/client/server/setconfig_test.go +++ b/client/server/setconfig_test.go @@ -76,6 +76,8 @@ func TestSetConfig_AllFieldsSaved(t *testing.T) { disableIPv6 := true mtu := int64(1280) sshJWTCacheTTL := int32(300) + enableLocalMetrics := true + localMetricsAddress := "127.0.0.1:9292" req := &proto.SetConfigRequest{ ProfileName: profName, @@ -107,6 +109,8 @@ func TestSetConfig_AllFieldsSaved(t *testing.T) { DnsRouteInterval: durationpb.New(2 * time.Minute), Mtu: &mtu, SshJWTCacheTTL: &sshJWTCacheTTL, + EnableLocalMetrics: &enableLocalMetrics, + LocalMetricsAddress: &localMetricsAddress, } _, err = s.SetConfig(ctx, req) @@ -153,6 +157,8 @@ func TestSetConfig_AllFieldsSaved(t *testing.T) { require.Equal(t, uint16(mtu), cfg.MTU) require.NotNil(t, cfg.SSHJWTCacheTTL) require.Equal(t, int(sshJWTCacheTTL), *cfg.SSHJWTCacheTTL) + require.Equal(t, enableLocalMetrics, cfg.LocalMetricsEnabled) + require.Equal(t, localMetricsAddress, cfg.LocalMetricsAddress) verifyAllFieldsCovered(t, req) } @@ -205,6 +211,8 @@ func verifyAllFieldsCovered(t *testing.T, req *proto.SetConfigRequest) { "EnableSSHRemotePortForwarding": true, "DisableSSHAuth": true, "SshJWTCacheTTL": true, + "EnableLocalMetrics": true, + "LocalMetricsAddress": true, } val := reflect.ValueOf(req).Elem() @@ -264,6 +272,8 @@ func TestCLIFlags_MappedToSetConfig(t *testing.T) { "enable-ssh-remote-port-forwarding": "EnableSSHRemotePortForwarding", "disable-ssh-auth": "DisableSSHAuth", "ssh-jwt-cache-ttl": "SshJWTCacheTTL", + "enable-local-metrics": "EnableLocalMetrics", + "local-metrics-address": "LocalMetricsAddress", } // SetConfigRequest fields that don't have CLI flags (settable only via UI or other means). diff --git a/client/server/ssh_gate.go b/client/server/ssh_gate.go index ca1b4c4ee..3b62f5e56 100644 --- a/client/server/ssh_gate.go +++ b/client/server/ssh_gate.go @@ -14,6 +14,7 @@ import ( "github.com/netbirdio/netbird/client/internal/daemonaddr" "github.com/netbirdio/netbird/client/internal/ipcauth" + "github.com/netbirdio/netbird/client/internal/localmetrics" "github.com/netbirdio/netbird/client/internal/profilemanager" "github.com/netbirdio/netbird/client/proto" "github.com/netbirdio/netbird/util" @@ -30,6 +31,8 @@ import ( // management identity hands SSH authorization decisions, including which // keys and users are accepted, to whoever controls that identity. Changing // the management URL and deregistering the peer are both ways to do that. +// - Binding the local metrics endpoint to a non-loopback address publishes +// peer names and connectivity state to the network without authentication. // // Everything else stays unauthenticated, so this is not an authorization model: // it only refuses the changes that would let a local user become root. A caller @@ -39,27 +42,33 @@ import ( // user-to-root boundary. Fields are nil or empty when the request leaves them // untouched. type privilegedConfigChange struct { - managementURL string - serverSSHAllowed *bool - enableSSHRoot *bool - disableSSHAuth *bool + managementURL string + serverSSHAllowed *bool + enableSSHRoot *bool + disableSSHAuth *bool + enableLocalMetrics *bool + localMetricsAddress *string } func privilegedChangeFromSetConfig(msg *proto.SetConfigRequest) privilegedConfigChange { return privilegedConfigChange{ - managementURL: msg.GetManagementUrl(), - serverSSHAllowed: msg.ServerSSHAllowed, - enableSSHRoot: msg.EnableSSHRoot, - disableSSHAuth: msg.DisableSSHAuth, + managementURL: msg.GetManagementUrl(), + serverSSHAllowed: msg.ServerSSHAllowed, + enableSSHRoot: msg.EnableSSHRoot, + disableSSHAuth: msg.DisableSSHAuth, + enableLocalMetrics: msg.EnableLocalMetrics, + localMetricsAddress: msg.LocalMetricsAddress, } } func privilegedChangeFromLogin(msg *proto.LoginRequest) privilegedConfigChange { return privilegedConfigChange{ - managementURL: msg.GetManagementUrl(), - serverSSHAllowed: msg.ServerSSHAllowed, - enableSSHRoot: msg.EnableSSHRoot, - disableSSHAuth: msg.DisableSSHAuth, + managementURL: msg.GetManagementUrl(), + serverSSHAllowed: msg.ServerSSHAllowed, + enableSSHRoot: msg.EnableSSHRoot, + disableSSHAuth: msg.DisableSSHAuth, + enableLocalMetrics: msg.EnableLocalMetrics, + localMetricsAddress: msg.LocalMetricsAddress, } } @@ -83,6 +92,12 @@ func requirePrivilegeForConfigChange(ctx context.Context, stored *profilemanager return denyPrivileged(ctx, "enabling the NetBird SSH server", ipcauth.UpCommand("--allow-server-ssh")) } + if addr, exposes := exposesLocalMetrics(stored, change); exposes { + return denyPrivileged(ctx, + "exposing the local metrics endpoint on a non-loopback address", + ipcauth.UpCommand("--enable-local-metrics --local-metrics-address "+addr)) + } + // Only guard the management binding while the SSH server is enabled: that is // when the management identity decides who may open a shell here. if !sshServerEnabled(stored) { @@ -245,6 +260,48 @@ func sshServerCurrentlyAllowed(cfg *profilemanager.Config) *bool { return &enabled } +// exposesLocalMetrics reports whether the change would leave the metrics +// endpoint enabled on an address that is not confirmed loopback, and returns +// that address. A request that restates the stored state is not a change, so a +// settings form resubmitted after an administrator opened the endpoint is not +// refused. +func exposesLocalMetrics(stored *profilemanager.Config, change privilegedConfigChange) (string, bool) { + storedEnabled, storedAddr := storedLocalMetrics(stored) + + enabled := storedEnabled + if change.enableLocalMetrics != nil { + enabled = *change.enableLocalMetrics + } + addr := storedAddr + if change.localMetricsAddress != nil { + addr = metricsAddrOrDefault(*change.localMetricsAddress) + } + + if !enabled || localmetrics.IsLoopback(addr) { + return "", false + } + if storedEnabled && storedAddr == addr { + return "", false + } + return addr, true +} + +// storedLocalMetrics reads the metrics settings from the stored config, +// tolerating a config that does not exist yet. +func storedLocalMetrics(cfg *profilemanager.Config) (bool, string) { + if cfg == nil { + return false, localmetrics.DefaultListenAddress + } + return cfg.LocalMetricsEnabled, metricsAddrOrDefault(cfg.LocalMetricsAddress) +} + +func metricsAddrOrDefault(addr string) string { + if addr == "" { + return localmetrics.DefaultListenAddress + } + return addr +} + // sameManagementURL reports whether requested addresses the same management // server as stored, comparing scheme, host and effective port so that an // equivalent spelling ("https://api.netbird.io" for a stored diff --git a/client/server/ssh_gate_test.go b/client/server/ssh_gate_test.go index cbd345f16..d71cd86ef 100644 --- a/client/server/ssh_gate_test.go +++ b/client/server/ssh_gate_test.go @@ -61,6 +61,8 @@ func noIdentityCtx() context.Context { return context.Background() } func boolPtr(v bool) *bool { return &v } +func strPtr(v string) *string { return &v } + func mustURL(t *testing.T, raw string) *url.URL { t.Helper() u, err := url.Parse(raw) @@ -194,6 +196,102 @@ func TestRequirePrivilegeForConfigChange_SSHFlags(t *testing.T) { } } +func TestRequirePrivilegeForConfigChange_LocalMetrics(t *testing.T) { + exposed := &profilemanager.Config{LocalMetricsEnabled: true, LocalMetricsAddress: "0.0.0.0:9191"} + + tests := []struct { + name string + stored *profilemanager.Config + change privilegedConfigChange + privileged bool + wantDeny bool + }{ + { + name: "binding a non-loopback address unprivileged is refused", + stored: &profilemanager.Config{}, + change: privilegedConfigChange{enableLocalMetrics: boolPtr(true), localMetricsAddress: strPtr("0.0.0.0:9191")}, + wantDeny: true, + }, + { + name: "binding a non-loopback address as root is allowed", + stored: &profilemanager.Config{}, + change: privilegedConfigChange{enableLocalMetrics: boolPtr(true), localMetricsAddress: strPtr("0.0.0.0:9191")}, + privileged: true, + }, + { + name: "enabling on the default loopback address is not guarded", + stored: &profilemanager.Config{}, + change: privilegedConfigChange{enableLocalMetrics: boolPtr(true)}, + }, + { + name: "enabling on an explicit loopback address is not guarded", + stored: &profilemanager.Config{}, + change: privilegedConfigChange{enableLocalMetrics: boolPtr(true), localMetricsAddress: strPtr("127.0.0.1:9999")}, + }, + { + name: "enabling on the IPv6 loopback address is not guarded", + stored: &profilemanager.Config{}, + change: privilegedConfigChange{enableLocalMetrics: boolPtr(true), localMetricsAddress: strPtr("[::1]:9191")}, + }, + { + // The address alone does nothing while the endpoint stays off. + name: "a non-loopback address without enabling is not guarded", + stored: &profilemanager.Config{}, + change: privilegedConfigChange{localMetricsAddress: strPtr("0.0.0.0:9191")}, + }, + { + name: "widening an already enabled loopback endpoint is refused", + stored: &profilemanager.Config{LocalMetricsEnabled: true, LocalMetricsAddress: "127.0.0.1:9191"}, + change: privilegedConfigChange{localMetricsAddress: strPtr("0.0.0.0:9191")}, + wantDeny: true, + }, + { + name: "restating an already exposed endpoint is not a change", + stored: exposed, + change: privilegedConfigChange{enableLocalMetrics: boolPtr(true), localMetricsAddress: strPtr("0.0.0.0:9191")}, + }, + { + name: "turning an exposed endpoint off is not guarded", + stored: exposed, + change: privilegedConfigChange{enableLocalMetrics: boolPtr(false)}, + }, + { + name: "re-enabling an exposed endpoint that was turned off is refused", + stored: &profilemanager.Config{LocalMetricsEnabled: false, LocalMetricsAddress: "0.0.0.0:9191"}, + change: privilegedConfigChange{enableLocalMetrics: boolPtr(true)}, + wantDeny: true, + }, + { + // Fail closed: an address that cannot be parsed is not confirmed loopback. + name: "an unparseable address is refused", + stored: &profilemanager.Config{}, + change: privilegedConfigChange{enableLocalMetrics: boolPtr(true), localMetricsAddress: strPtr("not-an-address")}, + wantDeny: true, + }, + { + name: "a profile with no config yet counts as off, so exposing is refused", + stored: nil, + change: privilegedConfigChange{enableLocalMetrics: boolPtr(true), localMetricsAddress: strPtr("0.0.0.0:9191")}, + wantDeny: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := userCtx() + if tt.privileged { + ctx = rootCtx() + } + err := requirePrivilegeForConfigChange(ctx, tt.stored, tt.change) + if tt.wantDeny { + assertDenied(t, err) + return + } + assertAllowed(t, err) + }) + } +} + func TestRequirePrivilegeForConfigChange_ManagementURL(t *testing.T) { sshOn := func(raw string) *profilemanager.Config { return &profilemanager.Config{ServerSSHAllowed: boolPtr(true), ManagementURL: mustURL(t, raw)} diff --git a/go.mod b/go.mod index 09c3df95b..cede9c22d 100644 --- a/go.mod +++ b/go.mod @@ -100,6 +100,7 @@ require ( github.com/pires/go-proxyproto v0.11.0 github.com/pkg/sftp v1.13.9 github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 github.com/quic-go/quic-go v0.59.1 github.com/redis/go-redis/v9 v9.7.3 github.com/rs/xid v1.3.0 @@ -250,6 +251,7 @@ require ( github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/koron/go-ssdp v0.0.4 // indirect github.com/kr/fs v0.1.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lib/pq v1.12.3 // indirect github.com/libdns/libdns v0.2.2 // indirect github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect @@ -290,7 +292,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/pquerna/otp v1.5.0 // indirect - github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.19.2 // indirect diff --git a/infrastructure_files/observability/grafana/dashboards/client.json b/infrastructure_files/observability/grafana/dashboards/client.json new file mode 100644 index 000000000..05306a972 --- /dev/null +++ b/infrastructure_files/observability/grafana/dashboards/client.json @@ -0,0 +1,1107 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.1.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Connection state", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "netbird_management_connected{job=~\"$job\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Management connected", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "netbird_signal_connected{job=~\"$job\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Signal connected", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "netbird_peers{job=~\"$job\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Known peers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(netbird_peers_connected{job=~\"$job\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Connected peers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "netbird_peers_connected{job=~\"$job\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{connection_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Connected peers by connection type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "netbird_peer_latency_seconds{job=~\"$job\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "Peer latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 8, + "panels": [], + "title": "Peer connection establishment", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum(increase(netbird_peer_connection_stage_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",stage=\"total\"}[$__rate_interval])) by (le,connection_type))", + "instant": false, + "legendFormat": "{{connection_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Connection establishment duration (p50)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum(increase(netbird_peer_connection_stage_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le,stage))", + "instant": false, + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Connection establishment stages (p50)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 11, + "panels": [], + "title": "Management interactions", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum(increase(netbird_sync_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le))", + "instant": false, + "legendFormat": "sync", + "range": true, + "refId": "A" + } + ], + "title": "Sync processing duration (p50)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum(increase(netbird_sync_phase_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le,phase))", + "instant": false, + "legendFormat": "{{phase}}", + "range": true, + "refId": "A" + } + ], + "title": "Sync phase duration (p50)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum(increase(netbird_login_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le,success))", + "instant": false, + "legendFormat": "success={{success}}", + "range": true, + "refId": "A" + } + ], + "title": "Login duration (p50)", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [ + "netbird", + "client" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(netbird_management_connected,job)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(netbird_management_connected,job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(netbird_management_connected{job=~\"$job\"},instance)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(netbird_management_connected{job=~\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Netbird / Client", + "uid": "netbird-client-v001", + "version": 1, + "weekStart": "" +}