diff --git a/client/cmd/root.go b/client/cmd/root.go index f1ef32717..7dae4147c 100644 --- a/client/cmd/root.go +++ b/client/cmd/root.go @@ -23,6 +23,7 @@ import ( "google.golang.org/grpc/credentials/insecure" daddr "github.com/netbirdio/netbird/client/internal/daemonaddr" + "github.com/netbirdio/netbird/client/internal/localmetrics" "github.com/netbirdio/netbird/client/internal/profilemanager" ) @@ -31,6 +32,8 @@ const ( dnsResolverAddress = "dns-resolver-address" enableRosenpassFlag = "enable-rosenpass" rosenpassPermissiveFlag = "rosenpass-permissive" + enableLocalMetricsFlag = "enable-local-metrics" + localMetricsAddressFlag = "local-metrics-address" preSharedKeyFlag = "preshared-key" interfaceNameFlag = "interface-name" wireguardPortFlag = "wireguard-port" @@ -79,6 +82,8 @@ var ( updateSettingsDisabled bool captureEnabled bool networksDisabled bool + localMetricsEnabled bool + localMetricsAddr string rootCmd = &cobra.Command{ Use: "netbird", @@ -212,6 +217,8 @@ func init() { upCmd.PersistentFlags().BoolVar(&rosenpassEnabled, enableRosenpassFlag, false, "[Experimental] Enable Rosenpass feature. If enabled, the connection will be post-quantum secured via Rosenpass.") upCmd.PersistentFlags().BoolVar(&rosenpassPermissive, rosenpassPermissiveFlag, false, "[Experimental] Enable Rosenpass in permissive mode to allow this peer to accept WireGuard connections without requiring Rosenpass functionality from peers that do not have Rosenpass enabled.") upCmd.PersistentFlags().BoolVar(&autoConnectDisabled, disableAutoConnectFlag, false, "Disables auto-connect feature. If enabled, then the client won't connect automatically when the service starts.") + upCmd.PersistentFlags().BoolVar(&localMetricsEnabled, enableLocalMetricsFlag, false, "Enables a local Prometheus /metrics endpoint exposing connection state (peers, latency, P2P vs relay).") + upCmd.PersistentFlags().StringVar(&localMetricsAddr, localMetricsAddressFlag, localmetrics.DefaultListenAddress, "Listen address of the local Prometheus /metrics endpoint.") upCmd.PersistentFlags().BoolVar(&lazyConnEnabled, enableLazyConnectionFlag, false, "Deprecated: no longer used. Lazy connections are controlled by the server and the NB_LAZY_CONN environment variable.") _ = upCmd.PersistentFlags().MarkDeprecated(enableLazyConnectionFlag, "no longer used; lazy connections are controlled by the server and the NB_LAZY_CONN environment variable") diff --git a/client/cmd/up.go b/client/cmd/up.go index 8b3de3c66..2f20407b3 100644 --- a/client/cmd/up.go +++ b/client/cmd/up.go @@ -479,6 +479,13 @@ func setupSetConfigReq(customDNSAddressConverted []byte, cmd *cobra.Command, pro req.DisableIpv6 = &disableIPv6 } + if cmd.Flag(enableLocalMetricsFlag).Changed { + req.EnableLocalMetrics = &localMetricsEnabled + } + if cmd.Flag(localMetricsAddressFlag).Changed { + req.LocalMetricsAddress = &localMetricsAddr + } + return &req } @@ -596,6 +603,14 @@ func setupConfig(customDNSAddressConverted []byte, cmd *cobra.Command, configFil ic.DisableIPv6 = &disableIPv6 } + if cmd.Flag(enableLocalMetricsFlag).Changed { + ic.LocalMetricsEnabled = &localMetricsEnabled + } + + if cmd.Flag(localMetricsAddressFlag).Changed { + ic.LocalMetricsAddress = &localMetricsAddr + } + return &ic, nil } @@ -658,6 +673,14 @@ func setupLoginRequest(providedSetupKey string, customDNSAddressConverted []byte loginRequest.DisableAutoConnect = &autoConnectDisabled } + if cmd.Flag(enableLocalMetricsFlag).Changed { + loginRequest.EnableLocalMetrics = &localMetricsEnabled + } + + if cmd.Flag(localMetricsAddressFlag).Changed { + loginRequest.LocalMetricsAddress = &localMetricsAddr + } + if cmd.Flag(interfaceNameFlag).Changed { if err := parseInterfaceName(interfaceName); err != nil { return nil, err diff --git a/client/internal/localmetrics/localmetrics.go b/client/internal/localmetrics/localmetrics.go new file mode 100644 index 000000000..d4a7a862b --- /dev/null +++ b/client/internal/localmetrics/localmetrics.go @@ -0,0 +1,210 @@ +// Package localmetrics exposes client connection state as a local +// Prometheus /metrics endpoint. +package localmetrics + +import ( + "context" + "errors" + "net/http" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + dto "github.com/prometheus/client_model/go" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/peer" +) + +// DefaultListenAddress is used when local metrics are enabled without an explicit address. +const DefaultListenAddress = "127.0.0.1:9191" + +const shutdownTimeout = 3 * time.Second + +// statusSource provides the connection state snapshots the collector reads on scrape. +type statusSource interface { + GetPeerStates() []peer.State + GetManagementState() peer.ManagementState + GetSignalState() peer.SignalState +} + +// GathererProvider returns the current client metrics gatherer, or nil when +// no engine is running. It is called on every scrape. +type GathererProvider func() prometheus.Gatherer + +// Manager runs the local /metrics HTTP endpoint according to the active +// client configuration. Reconcile is safe to call on every config change. +type Manager struct { + status statusSource + clientMetrics GathererProvider + + mu sync.Mutex + srv *http.Server + addr string +} + +// NewManager creates a manager that serves metrics from status and +// clientMetrics and shuts down when ctx is canceled. +func NewManager(ctx context.Context, status statusSource, clientMetrics GathererProvider) *Manager { + m := &Manager{status: status, clientMetrics: clientMetrics} + go func() { + <-ctx.Done() + m.Stop() + }() + return m +} + +// Reconcile starts, stops, or restarts the metrics endpoint to match the +// desired state. An empty addr falls back to DefaultListenAddress. +func (m *Manager) Reconcile(enabled bool, addr string) { + if addr == "" { + addr = DefaultListenAddress + } + + m.mu.Lock() + defer m.mu.Unlock() + + if !enabled { + m.stop() + return + } + if m.srv != nil && m.addr == addr { + return + } + m.stop() + + registry := prometheus.NewRegistry() + registry.MustRegister(newCollector(m.status)) + + gatherers := prometheus.Gatherers{registry, prometheus.GathererFunc(func() ([]*dto.MetricFamily, error) { + if m.clientMetrics == nil { + return nil, nil + } + g := m.clientMetrics() + if g == nil { + return nil, nil + } + return g.Gather() + })} + + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{})) + + srv := &http.Server{Addr: addr, Handler: mux} + m.srv = srv + m.addr = addr + + log.Infof("serving local metrics on http://%s/metrics", addr) + go func() { + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Errorf("failed to serve local metrics on %s: %v", addr, err) + } + }() +} + +// Stop shuts down the metrics endpoint if it is running. +func (m *Manager) Stop() { + m.mu.Lock() + defer m.mu.Unlock() + m.stop() +} + +// stop shuts down the running server. Callers must hold m.mu. +func (m *Manager) stop() { + if m.srv == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + if err := m.srv.Shutdown(ctx); err != nil { + log.Debugf("failed to shut down local metrics server: %v", err) + } + m.srv = nil + m.addr = "" +} + +// collector converts status recorder snapshots into Prometheus metrics at scrape time. +type collector struct { + status statusSource + + managementConnected *prometheus.Desc + signalConnected *prometheus.Desc + peersTotal *prometheus.Desc + peersConnected *prometheus.Desc + peerLatency *prometheus.Desc +} + +func newCollector(status statusSource) *collector { + return &collector{ + status: status, + managementConnected: prometheus.NewDesc( + "netbird_management_connected", + "Whether the client is connected to the management service (1 connected, 0 disconnected).", + nil, nil, + ), + signalConnected: prometheus.NewDesc( + "netbird_signal_connected", + "Whether the client is connected to the signal service (1 connected, 0 disconnected).", + nil, nil, + ), + peersTotal: prometheus.NewDesc( + "netbird_peers", + "Number of peers known to this client.", + nil, nil, + ), + peersConnected: prometheus.NewDesc( + "netbird_peers_connected", + "Number of connected peers by connection type.", + []string{"connection_type"}, nil, + ), + peerLatency: prometheus.NewDesc( + "netbird_peer_latency_seconds", + "Round-trip latency per directly connected peer; relayed connections have no latency measurement.", + []string{"peer"}, nil, + ), + } +} + +// Describe implements prometheus.Collector. +func (c *collector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.managementConnected + ch <- c.signalConnected + ch <- c.peersTotal + ch <- c.peersConnected + ch <- c.peerLatency +} + +// Collect implements prometheus.Collector. +func (c *collector) Collect(ch chan<- prometheus.Metric) { + ch <- prometheus.MustNewConstMetric(c.managementConnected, prometheus.GaugeValue, boolToFloat(c.status.GetManagementState().Connected)) + ch <- prometheus.MustNewConstMetric(c.signalConnected, prometheus.GaugeValue, boolToFloat(c.status.GetSignalState().Connected)) + + peers := c.status.GetPeerStates() + ch <- prometheus.MustNewConstMetric(c.peersTotal, prometheus.GaugeValue, float64(len(peers))) + + var p2p, relayed float64 + for _, p := range peers { + if p.ConnStatus != peer.StatusConnected { + continue + } + if p.Relayed { + relayed++ + continue + } + p2p++ + + if latency := p.Latency.Seconds(); latency > 0 { + ch <- prometheus.MustNewConstMetric(c.peerLatency, prometheus.GaugeValue, latency, p.FQDN) + } + } + ch <- prometheus.MustNewConstMetric(c.peersConnected, prometheus.GaugeValue, p2p, "p2p") + ch <- prometheus.MustNewConstMetric(c.peersConnected, prometheus.GaugeValue, relayed, "relay") +} + +func boolToFloat(b bool) float64 { + if b { + return 1 + } + return 0 +} diff --git a/client/internal/localmetrics/localmetrics_test.go b/client/internal/localmetrics/localmetrics_test.go new file mode 100644 index 000000000..77b522ecb --- /dev/null +++ b/client/internal/localmetrics/localmetrics_test.go @@ -0,0 +1,97 @@ +package localmetrics + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/client/internal/peer" +) + +type stubStatus struct { + peers []peer.State + management peer.ManagementState + signal peer.SignalState +} + +func (s *stubStatus) GetPeerStates() []peer.State { return s.peers } +func (s *stubStatus) GetManagementState() peer.ManagementState { return s.management } +func (s *stubStatus) GetSignalState() peer.SignalState { return s.signal } + +func testStatus() *stubStatus { + return &stubStatus{ + management: peer.ManagementState{Connected: true}, + signal: peer.SignalState{Connected: true}, + peers: []peer.State{ + {FQDN: "peer-a.netbird.cloud", IP: "100.90.0.1", ConnStatus: peer.StatusConnected, Relayed: false, Latency: 12 * time.Millisecond}, + {FQDN: "peer-b.netbird.cloud", IP: "100.90.0.2", ConnStatus: peer.StatusConnected, Relayed: false, Latency: 36 * time.Millisecond}, + {FQDN: "peer-c.netbird.cloud", IP: "100.90.0.3", ConnStatus: peer.StatusConnected, Relayed: true}, + {FQDN: "peer-d.netbird.cloud", IP: "100.90.0.4", ConnStatus: peer.StatusIdle}, + }, + } +} + +func TestCollector(t *testing.T) { + c := newCollector(testStatus()) + + expected := ` +# HELP netbird_management_connected Whether the client is connected to the management service (1 connected, 0 disconnected). +# TYPE netbird_management_connected gauge +netbird_management_connected 1 +# HELP netbird_peer_latency_seconds Round-trip latency per directly connected peer; relayed connections have no latency measurement. +# TYPE netbird_peer_latency_seconds gauge +netbird_peer_latency_seconds{peer="peer-a.netbird.cloud"} 0.012 +netbird_peer_latency_seconds{peer="peer-b.netbird.cloud"} 0.036 +# HELP netbird_peers Number of peers known to this client. +# TYPE netbird_peers gauge +netbird_peers 4 +# HELP netbird_peers_connected Number of connected peers by connection type. +# TYPE netbird_peers_connected gauge +netbird_peers_connected{connection_type="p2p"} 2 +netbird_peers_connected{connection_type="relay"} 1 +# HELP netbird_signal_connected Whether the client is connected to the signal service (1 connected, 0 disconnected). +# TYPE netbird_signal_connected gauge +netbird_signal_connected 1 +` + require.NoError(t, testutil.CollectAndCompare(c, strings.NewReader(expected))) +} + +func TestServe(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err, "must find a free port") + addr := ln.Addr().String() + require.NoError(t, ln.Close()) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + m := NewManager(ctx, testStatus(), nil) + m.Reconcile(true, addr) + + var body string + require.Eventually(t, func() bool { + resp, err := http.Get(fmt.Sprintf("http://%s/metrics", addr)) + if err != nil { + return false + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil || resp.StatusCode != http.StatusOK { + return false + } + body = string(data) + return true + }, 2*time.Second, 50*time.Millisecond, "metrics endpoint should come up") + + assert.Contains(t, body, "netbird_peers 4") + assert.Contains(t, body, `netbird_peers_connected{connection_type="relay"} 1`) + assert.Contains(t, body, `netbird_peer_latency_seconds{peer="peer-a.netbird.cloud"} 0.012`) +} diff --git a/client/internal/metrics/influxdb.go b/client/internal/metrics/influxdb.go index 4ba14bf44..717544f6a 100644 --- a/client/internal/metrics/influxdb.go +++ b/client/internal/metrics/influxdb.go @@ -45,30 +45,13 @@ func (m *influxDBMetrics) RecordConnectionStages( isReconnection bool, timestamps ConnectionStageTimestamps, ) { - var signalingReceivedToConnection, connectionToWgHandshake, totalDuration float64 - - if !timestamps.SignalingReceived.IsZero() && !timestamps.ConnectionReady.IsZero() { - signalingReceivedToConnection = timestamps.ConnectionReady.Sub(timestamps.SignalingReceived).Seconds() - } - - if !timestamps.ConnectionReady.IsZero() && !timestamps.WgHandshakeSuccess.IsZero() { - connectionToWgHandshake = timestamps.WgHandshakeSuccess.Sub(timestamps.ConnectionReady).Seconds() - } - - if !timestamps.SignalingReceived.IsZero() && !timestamps.WgHandshakeSuccess.IsZero() { - totalDuration = timestamps.WgHandshakeSuccess.Sub(timestamps.SignalingReceived).Seconds() - } - - attemptType := "initial" - if isReconnection { - attemptType = "reconnection" - } + signalingReceivedToConnection, connectionToWgHandshake, totalDuration := timestamps.Durations() connTypeStr := connectionType.String() tags := fmt.Sprintf("deployment_type=%s,connection_type=%s,attempt_type=%s,version=%s,os=%s,arch=%s,peer_id=%s,connection_pair_id=%s", agentInfo.DeploymentType.String(), connTypeStr, - attemptType, + attemptType(isReconnection), agentInfo.Version, agentInfo.OS, agentInfo.Arch, @@ -94,7 +77,7 @@ func (m *influxDBMetrics) RecordConnectionStages( m.trimLocked() log.Tracef("peer connection metrics [%s, %s, %s]: signalingReceived→connection: %.3fs, connection→wg_handshake: %.3fs, total: %.3fs", - agentInfo.DeploymentType.String(), connTypeStr, attemptType, signalingReceivedToConnection, connectionToWgHandshake, totalDuration) + agentInfo.DeploymentType.String(), connTypeStr, attemptType(isReconnection), signalingReceivedToConnection, connectionToWgHandshake, totalDuration) } func (m *influxDBMetrics) RecordSyncDuration(_ context.Context, agentInfo AgentInfo, duration time.Duration) { diff --git a/client/internal/metrics/metrics.go b/client/internal/metrics/metrics.go index cfe477107..5edf1d9c7 100644 --- a/client/internal/metrics/metrics.go +++ b/client/internal/metrics/metrics.go @@ -89,6 +89,21 @@ type ConnectionStageTimestamps struct { WgHandshakeSuccess time.Time } +// Durations returns the stage durations in seconds. A duration is zero when +// either of its timestamps is missing. +func (c ConnectionStageTimestamps) Durations() (signalingToConnection, connectionToWgHandshake, total float64) { + if !c.SignalingReceived.IsZero() && !c.ConnectionReady.IsZero() { + signalingToConnection = c.ConnectionReady.Sub(c.SignalingReceived).Seconds() + } + if !c.ConnectionReady.IsZero() && !c.WgHandshakeSuccess.IsZero() { + connectionToWgHandshake = c.WgHandshakeSuccess.Sub(c.ConnectionReady).Seconds() + } + if !c.SignalingReceived.IsZero() && !c.WgHandshakeSuccess.IsZero() { + total = c.WgHandshakeSuccess.Sub(c.SignalingReceived).Seconds() + } + return signalingToConnection, connectionToWgHandshake, total +} + // String returns a human-readable representation of the connection stage timestamps func (c ConnectionStageTimestamps) String() string { return fmt.Sprintf("ConnectionStageTimestamps{SignalingReceived=%v, ConnectionReady=%v, WgHandshakeSuccess=%v}", @@ -279,3 +294,11 @@ func (c *ClientMetrics) stopPushLocked() { c.wg.Wait() c.push.Store(nil) } + +// attemptType returns the metric label for an initial vs reconnection attempt. +func attemptType(isReconnection bool) string { + if isReconnection { + return "reconnection" + } + return "initial" +} diff --git a/client/internal/metrics/metrics_default.go b/client/internal/metrics/metrics_default.go index 927ab51d1..3798adab6 100644 --- a/client/internal/metrics/metrics_default.go +++ b/client/internal/metrics/metrics_default.go @@ -2,10 +2,24 @@ package metrics +import "github.com/prometheus/client_golang/prometheus" + // NewClientMetrics creates a new ClientMetrics instance func NewClientMetrics(agentInfo AgentInfo) *ClientMetrics { return &ClientMetrics{ - impl: newInfluxDBMetrics(), + impl: newPrometheusMetrics(newInfluxDBMetrics()), agentInfo: agentInfo, } } + +// PrometheusGatherer returns the registry with the mirrored Prometheus +// metrics, or nil when unavailable. +func (c *ClientMetrics) PrometheusGatherer() prometheus.Gatherer { + if c == nil { + return nil + } + if pm, ok := c.impl.(*prometheusMetrics); ok { + return pm.Gatherer() + } + return nil +} diff --git a/client/internal/metrics/prometheus.go b/client/internal/metrics/prometheus.go new file mode 100644 index 000000000..7f5020ea9 --- /dev/null +++ b/client/internal/metrics/prometheus.go @@ -0,0 +1,119 @@ +//go:build !js + +package metrics + +import ( + "context" + "io" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// prometheusMetrics mirrors recorded client metrics into a Prometheus +// registry for the local /metrics endpoint, then delegates to the wrapped +// implementation. Export and Reset pass through untouched: Prometheus +// metrics are cumulative and pull-based. +type prometheusMetrics struct { + next metricsImplementation + registry *prometheus.Registry + + connectionStages *prometheus.HistogramVec + syncDuration prometheus.Histogram + syncPhaseDuration *prometheus.HistogramVec + loginDuration *prometheus.HistogramVec +} + +func newPrometheusMetrics(next metricsImplementation) *prometheusMetrics { + connectionBuckets := []float64{.05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60} + + m := &prometheusMetrics{ + next: next, + registry: prometheus.NewRegistry(), + connectionStages: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "netbird_peer_connection_stage_duration_seconds", + Help: "Duration of peer connection establishment stages.", + Buckets: connectionBuckets, + }, []string{"stage", "connection_type", "attempt_type"}), + syncDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "netbird_sync_duration_seconds", + Help: "Duration of management sync message processing.", + Buckets: prometheus.DefBuckets, + }), + syncPhaseDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "netbird_sync_phase_duration_seconds", + Help: "Duration of individual sync processing phases.", + Buckets: prometheus.DefBuckets, + }, []string{"phase"}), + loginDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "netbird_login_duration_seconds", + Help: "Duration of logins to the management service.", + Buckets: prometheus.DefBuckets, + }, []string{"success"}), + } + + m.registry.MustRegister(m.connectionStages, m.syncDuration, m.syncPhaseDuration, m.loginDuration) + return m +} + +// Gatherer returns the registry holding the mirrored metrics. +func (m *prometheusMetrics) Gatherer() prometheus.Gatherer { + return m.registry +} + +// RecordConnectionStages implements metricsImplementation. +func (m *prometheusMetrics) RecordConnectionStages( + ctx context.Context, + agentInfo AgentInfo, + connectionPairID string, + connectionType ConnectionType, + isReconnection bool, + timestamps ConnectionStageTimestamps, +) { + attempt := attemptType(isReconnection) + connType := connectionType.String() + + signalingToConnection, connectionToWgHandshake, total := timestamps.Durations() + if signalingToConnection > 0 { + m.connectionStages.WithLabelValues("signaling_to_connection", connType, attempt).Observe(signalingToConnection) + } + if connectionToWgHandshake > 0 { + m.connectionStages.WithLabelValues("connection_to_wg_handshake", connType, attempt).Observe(connectionToWgHandshake) + } + if total > 0 { + m.connectionStages.WithLabelValues("total", connType, attempt).Observe(total) + } + + m.next.RecordConnectionStages(ctx, agentInfo, connectionPairID, connectionType, isReconnection, timestamps) +} + +// RecordSyncDuration implements metricsImplementation. +func (m *prometheusMetrics) RecordSyncDuration(ctx context.Context, agentInfo AgentInfo, duration time.Duration) { + m.syncDuration.Observe(duration.Seconds()) + m.next.RecordSyncDuration(ctx, agentInfo, duration) +} + +// RecordSyncPhase implements metricsImplementation. +func (m *prometheusMetrics) RecordSyncPhase(ctx context.Context, agentInfo AgentInfo, phase string, duration time.Duration) { + m.syncPhaseDuration.WithLabelValues(phase).Observe(duration.Seconds()) + m.next.RecordSyncPhase(ctx, agentInfo, phase, duration) +} + +// RecordLoginDuration implements metricsImplementation. +func (m *prometheusMetrics) RecordLoginDuration(ctx context.Context, agentInfo AgentInfo, duration time.Duration, success bool) { + m.loginDuration.WithLabelValues(strconv.FormatBool(success)).Observe(duration.Seconds()) + m.next.RecordLoginDuration(ctx, agentInfo, duration, success) +} + +// Export implements metricsImplementation by delegating to the wrapped +// implementation; Prometheus metrics are pulled via the registry instead. +func (m *prometheusMetrics) Export(w io.Writer) error { + return m.next.Export(w) +} + +// Reset implements metricsImplementation by delegating to the wrapped +// implementation; Prometheus metrics must not be cleared on push. +func (m *prometheusMetrics) Reset() { + m.next.Reset() +} diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index a987482fe..acea36d3e 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -1172,6 +1172,18 @@ func (d *Status) GetResolvedDomainsStates() map[domain.Domain]ResolvedDomainInfo return maps.Clone(d.resolvedDomainsStates) } +// GetPeerStates returns a snapshot of all known peer states. +func (d *Status) GetPeerStates() []State { + d.mux.RLock() + defer d.mux.RUnlock() + + states := make([]State, 0, len(d.peers)) + for _, state := range d.peers { + states = append(states, state) + } + return states +} + // GetFullStatus gets full status func (d *Status) GetFullStatus() FullStatus { fullStatus := FullStatus{ diff --git a/client/internal/profilemanager/config.go b/client/internal/profilemanager/config.go index ed2f21999..8973f1a64 100644 --- a/client/internal/profilemanager/config.go +++ b/client/internal/profilemanager/config.go @@ -102,6 +102,9 @@ type ConfigInput struct { DNSLabels domain.List MTU *uint16 + + LocalMetricsEnabled *bool + LocalMetricsAddress *string } // Config Configuration type @@ -142,6 +145,11 @@ type Config struct { DNSLabels domain.List + // LocalMetricsEnabled enables the local Prometheus /metrics endpoint. + LocalMetricsEnabled bool + // LocalMetricsAddress is the listen address of the local /metrics endpoint. + LocalMetricsAddress string + // SSHKey is a private SSH key in a PEM format SSHKey string @@ -386,6 +394,18 @@ func (config *Config) apply(input ConfigInput) (updated bool, err error) { updated = true } + if input.LocalMetricsEnabled != nil && *input.LocalMetricsEnabled != config.LocalMetricsEnabled { + log.Infof("switching local metrics to %t", *input.LocalMetricsEnabled) + config.LocalMetricsEnabled = *input.LocalMetricsEnabled + updated = true + } + + if input.LocalMetricsAddress != nil && *input.LocalMetricsAddress != config.LocalMetricsAddress { + log.Infof("switching local metrics address to %s", *input.LocalMetricsAddress) + config.LocalMetricsAddress = *input.LocalMetricsAddress + updated = true + } + if input.NetworkMonitor != nil && (config.NetworkMonitor == nil || *input.NetworkMonitor != *config.NetworkMonitor) { log.Infof("switching Network Monitor to %t", *input.NetworkMonitor) config.NetworkMonitor = input.NetworkMonitor diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index 6fbb09958..c78f1557f 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -343,6 +343,8 @@ type LoginRequest struct { DisableSSHAuth *bool `protobuf:"varint,38,opt,name=disableSSHAuth,proto3,oneof" json:"disableSSHAuth,omitempty"` SshJWTCacheTTL *int32 `protobuf:"varint,39,opt,name=sshJWTCacheTTL,proto3,oneof" json:"sshJWTCacheTTL,omitempty"` DisableIpv6 *bool `protobuf:"varint,40,opt,name=disable_ipv6,json=disableIpv6,proto3,oneof" json:"disable_ipv6,omitempty"` + EnableLocalMetrics *bool `protobuf:"varint,41,opt,name=enable_local_metrics,json=enableLocalMetrics,proto3,oneof" json:"enable_local_metrics,omitempty"` + LocalMetricsAddress *string `protobuf:"bytes,42,opt,name=local_metrics_address,json=localMetricsAddress,proto3,oneof" json:"local_metrics_address,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -658,6 +660,20 @@ func (x *LoginRequest) GetDisableIpv6() bool { return false } +func (x *LoginRequest) GetEnableLocalMetrics() bool { + if x != nil && x.EnableLocalMetrics != nil { + return *x.EnableLocalMetrics + } + return false +} + +func (x *LoginRequest) GetLocalMetricsAddress() string { + if x != nil && x.LocalMetricsAddress != nil { + return *x.LocalMetricsAddress + } + return "" +} + type LoginResponse struct { state protoimpl.MessageState `protogen:"open.v1"` NeedsSSOLogin bool `protobuf:"varint,1,opt,name=needsSSOLogin,proto3" json:"needsSSOLogin,omitempty"` @@ -4210,6 +4226,8 @@ type SetConfigRequest struct { DisableSSHAuth *bool `protobuf:"varint,33,opt,name=disableSSHAuth,proto3,oneof" json:"disableSSHAuth,omitempty"` SshJWTCacheTTL *int32 `protobuf:"varint,34,opt,name=sshJWTCacheTTL,proto3,oneof" json:"sshJWTCacheTTL,omitempty"` DisableIpv6 *bool `protobuf:"varint,35,opt,name=disable_ipv6,json=disableIpv6,proto3,oneof" json:"disable_ipv6,omitempty"` + EnableLocalMetrics *bool `protobuf:"varint,36,opt,name=enable_local_metrics,json=enableLocalMetrics,proto3,oneof" json:"enable_local_metrics,omitempty"` + LocalMetricsAddress *string `protobuf:"bytes,37,opt,name=local_metrics_address,json=localMetricsAddress,proto3,oneof" json:"local_metrics_address,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4489,6 +4507,20 @@ func (x *SetConfigRequest) GetDisableIpv6() bool { return false } +func (x *SetConfigRequest) GetEnableLocalMetrics() bool { + if x != nil && x.EnableLocalMetrics != nil { + return *x.EnableLocalMetrics + } + return false +} + +func (x *SetConfigRequest) GetLocalMetricsAddress() string { + if x != nil && x.LocalMetricsAddress != nil { + return *x.LocalMetricsAddress + } + return "" +} + type SetConfigResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -6987,7 +7019,7 @@ var File_daemon_proto protoreflect.FileDescriptor const file_daemon_proto_rawDesc = "" + "\n" + "\fdaemon.proto\x12\x06daemon\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\"\x0e\n" + - "\fEmptyRequest\"\xef\x12\n" + + "\fEmptyRequest\"\x92\x14\n" + "\fLoginRequest\x12\x1a\n" + "\bsetupKey\x18\x01 \x01(\tR\bsetupKey\x12&\n" + "\fpreSharedKey\x18\x02 \x01(\tB\x02\x18\x01R\fpreSharedKey\x12$\n" + @@ -7032,7 +7064,9 @@ const file_daemon_proto_rawDesc = "" + "\x1denableSSHRemotePortForwarding\x18% \x01(\bH\x18R\x1denableSSHRemotePortForwarding\x88\x01\x01\x12+\n" + "\x0edisableSSHAuth\x18& \x01(\bH\x19R\x0edisableSSHAuth\x88\x01\x01\x12+\n" + "\x0esshJWTCacheTTL\x18' \x01(\x05H\x1aR\x0esshJWTCacheTTL\x88\x01\x01\x12&\n" + - "\fdisable_ipv6\x18( \x01(\bH\x1bR\vdisableIpv6\x88\x01\x01B\x13\n" + + "\fdisable_ipv6\x18( \x01(\bH\x1bR\vdisableIpv6\x88\x01\x01\x125\n" + + "\x14enable_local_metrics\x18) \x01(\bH\x1cR\x12enableLocalMetrics\x88\x01\x01\x127\n" + + "\x15local_metrics_address\x18* \x01(\tH\x1dR\x13localMetricsAddress\x88\x01\x01B\x13\n" + "\x11_rosenpassEnabledB\x10\n" + "\x0e_interfaceNameB\x10\n" + "\x0e_wireguardPortB\x17\n" + @@ -7060,7 +7094,9 @@ const file_daemon_proto_rawDesc = "" + "\x1e_enableSSHRemotePortForwardingB\x11\n" + "\x0f_disableSSHAuthB\x11\n" + "\x0f_sshJWTCacheTTLB\x0f\n" + - "\r_disable_ipv6\"\xb5\x01\n" + + "\r_disable_ipv6B\x17\n" + + "\x15_enable_local_metricsB\x18\n" + + "\x16_local_metrics_address\"\xb5\x01\n" + "\rLoginResponse\x12$\n" + "\rneedsSSOLogin\x18\x01 \x01(\bR\rneedsSSOLogin\x12\x1a\n" + "\buserCode\x18\x02 \x01(\tR\buserCode\x12(\n" + @@ -7353,7 +7389,7 @@ const file_daemon_proto_rawDesc = "" + "\f_profileNameB\v\n" + "\t_username\"'\n" + "\x15SwitchProfileResponse\x12\x0e\n" + - "\x02id\x18\x01 \x01(\tR\x02id\"\x98\x11\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"\xbb\x12\n" + "\x10SetConfigRequest\x12\x1a\n" + "\busername\x18\x01 \x01(\tR\busername\x12 \n" + "\vprofileName\x18\x02 \x01(\tR\vprofileName\x12$\n" + @@ -7393,7 +7429,9 @@ const file_daemon_proto_rawDesc = "" + "\x1denableSSHRemotePortForwarding\x18 \x01(\bH\x15R\x1denableSSHRemotePortForwarding\x88\x01\x01\x12+\n" + "\x0edisableSSHAuth\x18! \x01(\bH\x16R\x0edisableSSHAuth\x88\x01\x01\x12+\n" + "\x0esshJWTCacheTTL\x18\" \x01(\x05H\x17R\x0esshJWTCacheTTL\x88\x01\x01\x12&\n" + - "\fdisable_ipv6\x18# \x01(\bH\x18R\vdisableIpv6\x88\x01\x01B\x13\n" + + "\fdisable_ipv6\x18# \x01(\bH\x18R\vdisableIpv6\x88\x01\x01\x125\n" + + "\x14enable_local_metrics\x18$ \x01(\bH\x19R\x12enableLocalMetrics\x88\x01\x01\x127\n" + + "\x15local_metrics_address\x18% \x01(\tH\x1aR\x13localMetricsAddress\x88\x01\x01B\x13\n" + "\x11_rosenpassEnabledB\x10\n" + "\x0e_interfaceNameB\x10\n" + "\x0e_wireguardPortB\x17\n" + @@ -7418,7 +7456,9 @@ const file_daemon_proto_rawDesc = "" + "\x1e_enableSSHRemotePortForwardingB\x11\n" + "\x0f_disableSSHAuthB\x11\n" + "\x0f_sshJWTCacheTTLB\x0f\n" + - "\r_disable_ipv6\"\x13\n" + + "\r_disable_ipv6B\x17\n" + + "\x15_enable_local_metricsB\x18\n" + + "\x16_local_metrics_address\"\x13\n" + "\x11SetConfigResponse\"Q\n" + "\x11AddProfileRequest\x12\x1a\n" + "\busername\x18\x01 \x01(\tR\busername\x12 \n" + diff --git a/client/proto/daemon.proto b/client/proto/daemon.proto index 8d5294eb7..fd1ae8213 100644 --- a/client/proto/daemon.proto +++ b/client/proto/daemon.proto @@ -242,6 +242,9 @@ message LoginRequest { optional bool disableSSHAuth = 38; optional int32 sshJWTCacheTTL = 39; optional bool disable_ipv6 = 40; + + optional bool enable_local_metrics = 41; + optional string local_metrics_address = 42; } message LoginResponse { @@ -757,6 +760,9 @@ message SetConfigRequest { optional bool disableSSHAuth = 33; optional int32 sshJWTCacheTTL = 34; optional bool disable_ipv6 = 35; + + optional bool enable_local_metrics = 36; + optional string local_metrics_address = 37; } message SetConfigResponse{} diff --git a/client/server/server.go b/client/server/server.go index 363f716a9..2569027ca 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -23,6 +23,9 @@ import ( "github.com/netbirdio/netbird/client/internal/auth" "github.com/netbirdio/netbird/client/internal/expose" + "github.com/prometheus/client_golang/prometheus" + + "github.com/netbirdio/netbird/client/internal/localmetrics" "github.com/netbirdio/netbird/client/internal/profilemanager" sleephandler "github.com/netbirdio/netbird/client/internal/sleep/handler" "github.com/netbirdio/netbird/client/mdm" @@ -99,6 +102,7 @@ type Server struct { statusRecorder *peer.Status sessionWatcher *internal.SessionWatcher + localMetrics *localmetrics.Manager probeThrottle *probeThrottle persistSyncResponse bool @@ -155,9 +159,28 @@ func New(ctx context.Context, logFile string, configFile string, profilesDisable s.sleepHandler = sleephandler.New(agent) s.startSleepDetector() + s.localMetrics = localmetrics.NewManager(ctx, s.statusRecorder, s.clientMetricsGatherer) + return s } +// clientMetricsGatherer returns the Prometheus gatherer of the running +// engine's client metrics, or nil when no engine is running. +func (s *Server) clientMetricsGatherer() prometheus.Gatherer { + s.mutex.Lock() + connectClient := s.connectClient + s.mutex.Unlock() + + if connectClient == nil { + return nil + } + engine := connectClient.Engine() + if engine == nil { + return nil + } + return engine.GetClientMetrics().PrometheusGatherer() +} + func (s *Server) Start() error { s.mutex.Lock() defer s.mutex.Unlock() @@ -238,6 +261,7 @@ func (s *Server) Start() error { s.statusRecorder.UpdateManagementAddress(config.ManagementURL.String()) s.statusRecorder.UpdateRosenpass(config.RosenpassEnabled, config.RosenpassPermissive) + s.localMetrics.Reconcile(config.LocalMetricsEnabled, config.LocalMetricsAddress) if s.sessionWatcher == nil { s.sessionWatcher = internal.NewSessionWatcher(s.rootCtx, s.statusRecorder) @@ -416,11 +440,18 @@ func (s *Server) SetConfig(callerCtx context.Context, msg *proto.SetConfigReques return nil, err } - if _, err := profilemanager.UpdateConfig(config); err != nil { + updatedConf, err := profilemanager.UpdateConfig(config) + if err != nil { log.Errorf("failed to update profile config: %v", err) return nil, fmt.Errorf("failed to update profile config: %w", err) } + if activeProf, err := s.profileManager.GetActiveProfileState(); err == nil { + if activePath, err := activeProf.FilePath(); err == nil && activePath == config.ConfigPath { + s.localMetrics.Reconcile(updatedConf.LocalMetricsEnabled, updatedConf.LocalMetricsAddress) + } + } + return &proto.SetConfigResponse{}, nil } @@ -490,6 +521,8 @@ func (s *Server) setConfigInputFromRequest(msg *proto.SetConfigRequest) (profile config.RosenpassEnabled = msg.RosenpassEnabled config.RosenpassPermissive = msg.RosenpassPermissive + config.LocalMetricsEnabled = msg.EnableLocalMetrics + config.LocalMetricsAddress = msg.LocalMetricsAddress config.DisableAutoConnect = msg.DisableAutoConnect config.ServerSSHAllowed = msg.ServerSSHAllowed config.NetworkMonitor = msg.NetworkMonitor @@ -943,6 +976,7 @@ func (s *Server) Up(callerCtx context.Context, msg *proto.UpRequest) (*proto.UpR s.statusRecorder.UpdateManagementAddress(s.config.ManagementURL.String()) s.statusRecorder.UpdateRosenpass(s.config.RosenpassEnabled, s.config.RosenpassPermissive) + s.localMetrics.Reconcile(s.config.LocalMetricsEnabled, s.config.LocalMetricsAddress) s.clientRunning = true s.clientRunningChan = make(chan struct{}) diff --git a/client/server/setconfig_test.go b/client/server/setconfig_test.go index 0e55257a9..608720344 100644 --- a/client/server/setconfig_test.go +++ b/client/server/setconfig_test.go @@ -73,6 +73,8 @@ func TestSetConfig_AllFieldsSaved(t *testing.T) { disableIPv6 := true mtu := int64(1280) sshJWTCacheTTL := int32(300) + enableLocalMetrics := true + localMetricsAddress := "127.0.0.1:9292" req := &proto.SetConfigRequest{ ProfileName: profName, @@ -104,6 +106,8 @@ func TestSetConfig_AllFieldsSaved(t *testing.T) { DnsRouteInterval: durationpb.New(2 * time.Minute), Mtu: &mtu, SshJWTCacheTTL: &sshJWTCacheTTL, + EnableLocalMetrics: &enableLocalMetrics, + LocalMetricsAddress: &localMetricsAddress, } _, err = s.SetConfig(ctx, req) @@ -150,6 +154,8 @@ func TestSetConfig_AllFieldsSaved(t *testing.T) { require.Equal(t, uint16(mtu), cfg.MTU) require.NotNil(t, cfg.SSHJWTCacheTTL) require.Equal(t, int(sshJWTCacheTTL), *cfg.SSHJWTCacheTTL) + require.Equal(t, enableLocalMetrics, cfg.LocalMetricsEnabled) + require.Equal(t, localMetricsAddress, cfg.LocalMetricsAddress) verifyAllFieldsCovered(t, req) } @@ -202,6 +208,8 @@ func verifyAllFieldsCovered(t *testing.T, req *proto.SetConfigRequest) { "EnableSSHRemotePortForwarding": true, "DisableSSHAuth": true, "SshJWTCacheTTL": true, + "EnableLocalMetrics": true, + "LocalMetricsAddress": true, } val := reflect.ValueOf(req).Elem() @@ -261,6 +269,8 @@ func TestCLIFlags_MappedToSetConfig(t *testing.T) { "enable-ssh-remote-port-forwarding": "EnableSSHRemotePortForwarding", "disable-ssh-auth": "DisableSSHAuth", "ssh-jwt-cache-ttl": "SshJWTCacheTTL", + "enable-local-metrics": "EnableLocalMetrics", + "local-metrics-address": "LocalMetricsAddress", } // SetConfigRequest fields that don't have CLI flags (settable only via UI or other means). diff --git a/go.mod b/go.mod index 3f1f8e414..7efbebf3a 100644 --- a/go.mod +++ b/go.mod @@ -98,6 +98,7 @@ require ( github.com/pires/go-proxyproto v0.11.0 github.com/pkg/sftp v1.13.9 github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 github.com/quic-go/quic-go v0.55.0 github.com/redis/go-redis/v9 v9.7.3 github.com/rs/xid v1.3.0 @@ -249,6 +250,7 @@ require ( github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/koron/go-ssdp v0.0.4 // indirect github.com/kr/fs v0.1.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lib/pq v1.12.3 // indirect github.com/libdns/libdns v0.2.2 // indirect github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect @@ -289,7 +291,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/pquerna/otp v1.5.0 // indirect - github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.19.2 // indirect diff --git a/infrastructure_files/observability/grafana/dashboards/client.json b/infrastructure_files/observability/grafana/dashboards/client.json new file mode 100644 index 000000000..05306a972 --- /dev/null +++ b/infrastructure_files/observability/grafana/dashboards/client.json @@ -0,0 +1,1107 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.1.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Connection state", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "netbird_management_connected{job=~\"$job\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Management connected", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "netbird_signal_connected{job=~\"$job\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Signal connected", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "netbird_peers{job=~\"$job\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Known peers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(netbird_peers_connected{job=~\"$job\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Connected peers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "netbird_peers_connected{job=~\"$job\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{connection_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Connected peers by connection type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "netbird_peer_latency_seconds{job=~\"$job\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "Peer latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 8, + "panels": [], + "title": "Peer connection establishment", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum(increase(netbird_peer_connection_stage_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",stage=\"total\"}[$__rate_interval])) by (le,connection_type))", + "instant": false, + "legendFormat": "{{connection_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Connection establishment duration (p50)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum(increase(netbird_peer_connection_stage_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le,stage))", + "instant": false, + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Connection establishment stages (p50)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 11, + "panels": [], + "title": "Management interactions", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum(increase(netbird_sync_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le))", + "instant": false, + "legendFormat": "sync", + "range": true, + "refId": "A" + } + ], + "title": "Sync processing duration (p50)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum(increase(netbird_sync_phase_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le,phase))", + "instant": false, + "legendFormat": "{{phase}}", + "range": true, + "refId": "A" + } + ], + "title": "Sync phase duration (p50)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum(increase(netbird_login_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le,success))", + "instant": false, + "legendFormat": "success={{success}}", + "range": true, + "refId": "A" + } + ], + "title": "Login duration (p50)", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [ + "netbird", + "client" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(netbird_management_connected,job)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(netbird_management_connected,job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(netbird_management_connected{job=~\"$job\"},instance)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(netbird_management_connected{job=~\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Netbird / Client", + "uid": "netbird-client-v001", + "version": 1, + "weekStart": "" +}