diff --git a/src/components/NavigationDocs.jsx b/src/components/NavigationDocs.jsx index c8593e33..8c4e016f 100644 --- a/src/components/NavigationDocs.jsx +++ b/src/components/NavigationDocs.jsx @@ -578,6 +578,19 @@ export const docsNavigation = [ }, ], }, + { + title: 'Observability', + isOpen: false, + links: [ + { title: 'Overview', href: '/selfhosted/observability' }, + { title: 'Management', href: '/selfhosted/observability/management' }, + { title: 'Signal', href: '/selfhosted/observability/signal' }, + { title: 'Relay', href: '/selfhosted/observability/relay' }, + { title: 'Proxy', href: '/selfhosted/observability/proxy' }, + { title: 'Combined', href: '/selfhosted/observability/combined' }, + { title: 'Dashboards', href: '/selfhosted/observability/dashboards' }, + ], + }, { title: 'Authentication', isOpen: false, diff --git a/src/pages/selfhosted/observability/combined.mdx b/src/pages/selfhosted/observability/combined.mdx new file mode 100644 index 00000000..425fd5d4 --- /dev/null +++ b/src/pages/selfhosted/observability/combined.mdx @@ -0,0 +1,55 @@ +export const description = 'Metrics configuration for the NetBird combined container.' + +# Combined deployment metrics + +The combined container (the default for new installations via `getting-started.sh`) runs Management, Signal, and Relay in a single process. All three services share **one** OpenTelemetry meter provider and emit to a **single** `/metrics` endpoint. + +## Endpoint + +| Setting | Default | +| --- | --- | +| Address | `0.0.0.0` | +| Port | `9090` | +| Path | `/metrics` | +| Config key | `server.metricsPort` | +| Config file | `config.yaml` | + +Set it in `config.yaml`: + +```yaml +server: + metricsPort: 9090 +``` + +Then restart the container: + +```bash +docker compose restart netbird-server +``` + +A single Prometheus job scrapes all three services: + +```yaml +scrape_configs: + - job_name: 'netbird' + static_configs: + - targets: ['netbird.example.com:9090'] +``` + + +For the full set of combined-container settings, see the [config.yaml.example](https://github.com/netbirdio/netbird/blob/main/combined/config.yaml.example) reference file. + + +## What's emitted + +The combined `/metrics` endpoint returns the union of: + +- All [Management metrics](/selfhosted/observability/management) — `management_*` series. +- All [Signal metrics](/selfhosted/observability/signal), each one rewritten with a `signal_` prefix so it doesn't collide with the rest of the endpoint. For example, standalone Signal exposes `active_peers` and `messages_forwarded_total`; in the combined container these become `signal_active_peers` and `signal_messages_forwarded_total`. +- All [Relay metrics](/selfhosted/observability/relay) — `relay_*` series. Emitted only when the embedded relay is enabled (no `relays` override set in `config.yaml`). + +If you point the combined server at an external Signal (`server.signalUri`) or external Relay (`server.relays`), the corresponding local service is disabled and its metrics will not appear on this endpoint — scrape the external service directly instead. + +## Health endpoint + +The combined container also exposes a separate healthcheck server on `:9000` by default (`server.healthcheckAddress`). The healthcheck endpoint does not expose metrics. diff --git a/src/pages/selfhosted/observability/dashboards.mdx b/src/pages/selfhosted/observability/dashboards.mdx new file mode 100644 index 00000000..3309f987 --- /dev/null +++ b/src/pages/selfhosted/observability/dashboards.mdx @@ -0,0 +1,53 @@ +export const description = 'Ready-made Grafana dashboards for NetBird Management, Signal, and Relay.' + +# Grafana dashboards + +NetBird ships ready-to-use Grafana dashboards for the Management, Signal, and Relay services. They are maintained in the `netbirdio/netbird` repository under [`infrastructure_files/observability/grafana/dashboards`](https://github.com/netbirdio/netbird/tree/main/infrastructure_files/observability/grafana/dashboards) and import directly into Grafana. + +## Available dashboards + +| Service | Dashboard JSON | +| --- | --- | +| Management | [`management.json`](https://github.com/netbirdio/netbird/blob/main/infrastructure_files/observability/grafana/dashboards/management.json) | +| Signal | [`signal.json`](https://github.com/netbirdio/netbird/blob/main/infrastructure_files/observability/grafana/dashboards/signal.json) | +| Relay | [`relay.json`](https://github.com/netbirdio/netbird/blob/main/infrastructure_files/observability/grafana/dashboards/relay.json) | + +### Management + +Covers peer-update fan-out, store latency, gRPC `Sync` / `Login` / `GetServerKey` rates and latencies, HTTP API request rates and latencies by endpoint and method, IdP request rates, network-map object counts, and update-channel queue length and operation durations. + +### Signal + +Covers active peers, peer connection durations, message forwarding throughput and latency, message-forward failures, registration / deregistration rates and failure rates, and gRPC RPC rates and latencies for the Signal service. + +### Relay + +Covers connected peers (total / active / idle), peer authentication latency, peer store latency, and inbound/outbound relay traffic bandwidth. + +## Importing a dashboard + +1. In Grafana, go to **Dashboards → New → Import**. +2. Upload the JSON file (or paste its contents) and click **Load**. +3. Select your Prometheus datasource and complete the import. + +## Dashboard variables + +The dashboards expose these template variables: + +| Variable | Purpose | +| --- | --- | +| `datasource` | Selects the Prometheus datasource. | +| `cluster` | Filters NetBird instances by cluster. | +| `environment` | Filters by environment (`dev`, `staging`, `uat`, `prod`). | +| `job` | Selects a specific NetBird instance when multiple are running. | +| `host` | Filters metrics by host. | + +Your deployment may use only a subset of these variables; unused ones can be left at the default `All`. + + +The Management dashboard expects HTTP request metrics to carry an `exported_endpoint` label rather than `endpoint`. If your Prometheus relabeling drops or renames this label, edit the dashboard panel queries accordingly. + + +## Scraping for the dashboards + +The dashboards assume a Prometheus scrape configuration that reaches the `/metrics` endpoint of each NetBird service. See [Service endpoints](/selfhosted/observability#service-endpoints) for defaults and per-service overrides. diff --git a/src/pages/selfhosted/observability/index.mdx b/src/pages/selfhosted/observability/index.mdx new file mode 100644 index 00000000..044e27bb --- /dev/null +++ b/src/pages/selfhosted/observability/index.mdx @@ -0,0 +1,89 @@ +export const description = 'Overview of OpenTelemetry metrics exposed by NetBird Management, Signal, Relay, and Proxy.' + +# Observability + +NetBird services emit runtime metrics via [OpenTelemetry](https://opentelemetry.io/) and expose them in [Prometheus](https://prometheus.io/) exposition format (with OpenMetrics 1.0.0 support). You can scrape these endpoints to monitor traffic, latency, peer activity, and the internal health of each service, and to render the [Grafana dashboards](/selfhosted/observability/dashboards) NetBird ships. + +This section documents what each service records, the default listening endpoint, and how to change it. + +## Service endpoints + +Every server-side component exposes a `/metrics` HTTP endpoint. By default, Management, Signal, Relay, and Combined listen on port `9090`; Proxy listens on port `8080`. + +| Service | Default endpoint | How to change | +| --- | --- | --- | +| [Management](/selfhosted/observability/management) | `http://0.0.0.0:9090/metrics` | `--metrics-port` flag | +| [Signal](/selfhosted/observability/signal) | `http://0.0.0.0:9090/metrics` | `--metrics-port` flag | +| [Relay](/selfhosted/observability/relay) | `http://0.0.0.0:9090/metrics` | `--metrics-port` flag | +| [Proxy](/selfhosted/observability/proxy) | `http://localhost:8080/metrics` | `--health-addr` flag / `NB_PROXY_HEALTH_ADDRESS` | +| [Combined](/selfhosted/observability/combined) | `http://0.0.0.0:9090/metrics` | `server.metricsPort` in `config.yaml` | + + +The proxy exposes `/metrics` on the same HTTP server as its health probes; there is no separate metrics port. + + +## Combined deployments + +In the combined container (the default for new installations via `getting-started.sh`), Management, Signal, and Relay all share **one** metrics server and **one** `/metrics` endpoint. The port is set with `server.metricsPort` in `config.yaml` (default `9090`) — see [Combined](/selfhosted/observability/combined). + +## Multi-container deployments + +In the older multi-container setup, each service runs its own metrics server. To avoid port collisions when colocating services on a single host, override `--metrics-port` per container, for example: + +```bash +netbird-mgmt --metrics-port 9090 +netbird-signal --metrics-port 9091 +netbird-relay --metrics-port 9092 +``` + +## Exposition format + +All endpoints return Prometheus text format with OpenMetrics support enabled: + +```bash +curl http://localhost:9090/metrics +``` + +## Naming conventions + +NetBird services register metrics with OpenTelemetry instrument names (using dots as separators, e.g. `management.grpc.sync.request.counter`). The Prometheus exporter rewrites them on the way out: + +- Dots become underscores: `management.grpc.sync.request.counter` → `management_grpc_sync_request_counter`. +- **Counters** get a `_total` suffix unless the instrument name already ends in `_total`. So `management.grpc.sync.request.counter` → `management_grpc_sync_request_counter_total`, while `registrations_total` stays `registrations_total`. +- **UpDownCounters** and **ObservableGauges** are emitted as Prometheus `gauge` types. The name is unchanged (e.g. `proxy_domains_count`, `relay_peers_active`). +- **Histograms** expand to three series — `_bucket`, `_sum`, `_count`. When the instrument was registered with a unit (`milliseconds`, `microseconds`, `bytes`, …), that unit is appended to the base name unless the name already ends in it. So `proxy.peer.add.duration.ms` with unit `milliseconds` becomes `proxy_peer_add_duration_ms_milliseconds_bucket`, but `relay_peer_authentication_time_milliseconds` (no unit registered) stays `relay_peer_authentication_time_milliseconds_bucket`. + +Service prefixes are baked into the OTel instrument names themselves: + +- **Management** metrics start with `management_*`. +- **Relay** metrics start with `relay_*`. +- **Proxy** metrics start with `proxy_*`. +- **Signal** metrics are **unprefixed** when Signal runs standalone (`active_peers`, `messages_forwarded_total`, …). In the combined container they are emitted with a `signal_` prefix (`signal_active_peers`, `signal_messages_forwarded_total`, …) so they don't collide with the other services on the shared `/metrics` endpoint. + +The metric tables on the per-service pages show the on-the-wire Prometheus names, so you can copy them straight into a PromQL query. + +You can scrape them with a standard Prometheus `scrape_config`: + +```yaml +scrape_configs: + - job_name: 'netbird-management' + static_configs: + - targets: ['management.example.com:9090'] + - job_name: 'netbird-signal' + static_configs: + - targets: ['signal.example.com:9090'] + - job_name: 'netbird-relay' + static_configs: + - targets: ['relay.example.com:9090'] + - job_name: 'netbird-proxy' + static_configs: + - targets: ['proxy.example.com:8080'] +``` + +## Network exposure + +Most metrics endpoints listen on all interfaces (0.0.0.0) and have no authentication; the Proxy endpoint listens on localhost by default. Restrict access at the firewall, reverse proxy, or service-mesh layer so that only your Prometheus scraper and operators can reach them. + +## Grafana dashboards + +NetBird ships ready-to-use Grafana dashboards for Management, Signal, and Relay. See [Dashboards](/selfhosted/observability/dashboards) for download links and required dashboard variables. diff --git a/src/pages/selfhosted/observability/management.mdx b/src/pages/selfhosted/observability/management.mdx new file mode 100644 index 00000000..83649d69 --- /dev/null +++ b/src/pages/selfhosted/observability/management.mdx @@ -0,0 +1,135 @@ +export const description = 'Metrics recorded by the NetBird Management service.' + +# Management metrics + +The Management service exposes Prometheus metrics describing the HTTP API, gRPC API, identity-provider integration, store/database, the peer update channel, the account manager, and ephemeral peer cleanup. All metric names are prefixed with `management_`. + +## Endpoint + +| Setting | Default | +| --- | --- | +| Address | `0.0.0.0` | +| Port | `9090` | +| Path | `/metrics` | +| Flag | `--metrics-port` | + +```bash +netbird-mgmt --metrics-port 9090 +``` + +In the combined container, the metrics port is set with `server.metricsPort` in `config.yaml` and is shared with Signal and Relay. See [Combined](/selfhosted/observability/combined). + + +The metric tables below show the on-the-wire Prometheus names. Counter names end in `_total`; histograms expand to `_bucket` / `_sum` / `_count`. When the instrument was registered with a unit, that unit is appended to the base histogram name (`_milliseconds`, `_microseconds`). See [Naming conventions](/selfhosted/observability#naming-conventions) for the full transformation rules. + + +## HTTP API + +Traffic and latency for the dashboard / public API HTTP server. + +| Metric | Type | Description | +| --- | --- | --- | +| `management_http_request_counter_total` | Counter | Requests received, labeled by endpoint and method. | +| `management_http_response_counter_total` | Counter | Responses returned, labeled by endpoint and status. | +| `management_http_request_duration_ms_milliseconds` | Histogram | Per-endpoint request duration. | +| `management_http_request_duration_ms_total_milliseconds` | Histogram | Aggregate request duration across all endpoints. | + + +The service also records aggregate counters `management.http.request.counter.total`, `management.http.response.counter.total`, and `management.http.response.code.total`. Because counter names already ending in `_total` are not re-suffixed by the Prometheus exporter, these aggregate counters appear under the same `management_http_request_counter_total` / `management_http_response_counter_total` names as the per-endpoint counters — they are distinguishable only by their (absent) endpoint label. + + +## gRPC API + +The management gRPC service used by peers (`Sync`, `Login`, `GetServerKey`). + +| Metric | Type | Description | +| --- | --- | --- | +| `management_grpc_sync_request_counter_total` | Counter | `Sync` requests received. | +| `management_grpc_sync_request_blocked_counter_total` | Counter | `Sync` requests rejected by rate-limiting. | +| `management_grpc_login_request_counter_total` | Counter | `Login` requests received. | +| `management_grpc_login_request_blocked_counter_total` | Counter | `Login` requests rejected by rate-limiting. | +| `management_grpc_login_request_high_latency_counter_total` | Counter | `Login` requests above the high-latency threshold. | +| `management_grpc_key_request_counter_total` | Counter | `GetServerKey` requests received. | +| `management_grpc_connected_streams` | Gauge | Currently open `Sync` streams. | +| `management_grpc_sync_request_duration_ms_milliseconds` | Histogram | `Sync` request duration. | +| `management_grpc_login_request_duration_ms_milliseconds` | Histogram | `Login` request duration. | +| `management_grpc_sync_request_duration_p95_by_account_ms_milliseconds` | Histogram | p95 of `Sync` request duration, bucketed by account. | +| `management_grpc_login_request_duration_p95_by_account_ms_milliseconds` | Histogram | p95 of `Login` request duration, bucketed by account. | + +## Identity provider (IdP) + +Calls made to the configured IdP (Auth0, Okta, Entra ID, Keycloak, etc.). + +| Metric | Type | Description | +| --- | --- | --- | +| `management_idp_authenticate_request_counter_total` | Counter | Authentication requests against the IdP. | +| `management_idp_get_account_counter_total` | Counter | `GetAccount` lookups. | +| `management_idp_get_accounts_counter_total` | Counter | `GetAllAccounts` lookups. | +| `management_idp_get_user_by_id_counter_total` | Counter | `GetUserByID` lookups. | +| `management_idp_get_user_by_email_counter_total` | Counter | `GetUserByEmail` lookups. | +| `management_idp_create_user_counter_total` | Counter | Users created in the IdP. | +| `management_idp_delete_user_counter_total` | Counter | Users deleted in the IdP. | +| `management_idp_update_user_meta_counter_total` | Counter | User metadata updates. | +| `management_idp_request_error_counter_total` | Counter | Failed IdP HTTP calls. | +| `management_idp_request_status_error_counter_total` | Counter | IdP calls that returned a non-2xx status. | + +## Store (database) + +Latency and contention on the management store (SQLite or PostgreSQL). + +| Metric | Type | Description | +| --- | --- | --- | +| `management_store_global_lock_acquisition_duration_micro_microseconds` | Histogram | Global lock acquisition time. | +| `management_store_global_lock_acquisition_duration_ms_milliseconds` | Histogram | Global lock acquisition time. | +| `management_store_persistence_duration_micro_microseconds` | Histogram | Persistence operation duration. | +| `management_store_persistence_duration_ms_milliseconds` | Histogram | Persistence operation duration. | +| `management_store_transaction_duration_ms_milliseconds` | Histogram | Store transaction duration. | + +## Update channel + +The fan-out path that pushes network-map updates to connected peers. + +| Metric | Type | Description | +| --- | --- | --- | +| `management_updatechannel_create_duration_micro_microseconds` | Histogram | Channel creation duration. | +| `management_updatechannel_close_one_duration_micro_microseconds` | Histogram | Close-single-channel duration. | +| `management_updatechannel_close_multiple_duration_micro_microseconds` | Histogram | Close-multiple-channels duration. | +| `management_updatechannel_close_multiple_channels` | Histogram | Number of channels closed per close-multiple call. | +| `management_updatechannel_send_duration_micro_microseconds` | Histogram | Time to send an update on a channel. | +| `management_updatechannel_get_all_duration_micro_microseconds` | Histogram | Time to enumerate all peer channels. | +| `management_updatechannel_get_all_peers` | Histogram | Number of peers returned by enumeration. | +| `management_updatechannel_haschannel_duration_micro_microseconds` | Histogram | `HasChannel` lookup duration. | +| `management_updatechannel_calc_posturechecks_duration_micro_microseconds` | Histogram | Posture-check computation duration. | +| `management_updatechannel_calc_networkmap_duration_ms_milliseconds` | Histogram | Per-peer network-map computation duration. | +| `management_updatechannel_merge_networkmap_duration_micro_microseconds` | Histogram | Network-map merge duration. | +| `management_updatechannel_tosyncresponse_duration_micro_microseconds` | Histogram | Time to convert a network map to a `SyncResponse`. | + +## Account manager + +Account-wide operations: peer status updates, metadata, network-map computation. + +| Metric | Type | Description | +| --- | --- | --- | +| `management_account_update_account_peers_duration_ms_milliseconds` | Histogram | Duration of an account-wide peer update fan-out. | +| `management_account_update_account_peers_counter_total` | Counter | Account-wide peer updates triggered. | +| `management_network_map_counter_total` | Counter | Network-map computations performed. | +| `management_account_get_peer_network_map_duration_ms_milliseconds` | Histogram | Per-peer `GetPeerNetworkMap` duration. | +| `management_account_network_map_object_count` | Histogram | Number of objects in computed network maps. | +| `management_account_peer_meta_update_counter_total` | Counter | Peer metadata updates. | +| `management_account_peer_status_update_counter_total` | Counter | Peer status updates. | +| `management_account_peer_status_update_duration_ms_milliseconds` | Histogram | Peer status update duration. | + +## Ephemeral peers + +Background cleanup of ephemeral peers (peers registered via ephemeral setup keys). + +| Metric | Type | Description | +| --- | --- | --- | +| `management_ephemeral_peers_pending` | Gauge | Ephemeral peers waiting for cleanup. | +| `management_ephemeral_cleanup_runs_counter_total` | Counter | Number of cleanup runs executed. | +| `management_ephemeral_peers_cleaned_counter_total` | Counter | Ephemeral peers removed by cleanup. | +| `management_ephemeral_cleanup_errors_counter_total` | Counter | Errors encountered during cleanup. | + +## Grafana dashboard + +A ready-made Management dashboard is available under [Dashboards](/selfhosted/observability/dashboards). diff --git a/src/pages/selfhosted/observability/proxy.mdx b/src/pages/selfhosted/observability/proxy.mdx new file mode 100644 index 00000000..e43744eb --- /dev/null +++ b/src/pages/selfhosted/observability/proxy.mdx @@ -0,0 +1,70 @@ +export const description = 'Metrics recorded by the NetBird Proxy service.' + +# Proxy metrics + +The NetBird Proxy serves the [Reverse Proxy](/manage/reverse-proxy) feature: HTTP / TLS / TCP / UDP traffic forwarded to peers without requiring those peers to be reachable directly. Its metrics describe the inbound HTTP layer, the L4 (TCP/UDP) listeners, ACME certificate issuance, and the sync loop with Management. All metric names are prefixed with `proxy_`. + +## Endpoint + +Unlike the other services, the proxy does not run a dedicated metrics server. Instead it mounts `/metrics` on the same HTTP server as its health probes. + +| Setting | Default | +| --- | --- | +| Address | `localhost:8080` | +| Path | `/metrics` | +| Flag | `--health-addr` | +| Env var | `NB_PROXY_HEALTH_ADDRESS` | + +```bash +netbird-proxy --health-addr 0.0.0.0:8080 +``` + +To scrape the proxy from another host, bind the health server to a routable interface (`0.0.0.0:8080` or a specific address) instead of `localhost`. + + +The same endpoint serves liveness, readiness, and startup probes. If you expose it publicly, restrict access to your Prometheus scraper and your orchestrator's health-check source. + + + +The metric tables below show the on-the-wire Prometheus names. Counter names end in `_total`; histograms expand to `_bucket` / `_sum` / `_count`. See [Naming conventions](/selfhosted/observability#naming-conventions) for the full transformation rules. + + +## HTTP + +Inbound HTTP/HTTPS traffic terminated by the proxy and forwarded to peers. + +| Metric | Type | Description | +| --- | --- | --- | +| `proxy_http_request_counter_total` | Counter | Total HTTP requests handled. | +| `proxy_http_active_requests` | Gauge | In-flight HTTP requests. | +| `proxy_http_request_duration_ms_milliseconds` | Histogram | End-to-end request duration. | +| `proxy_backend_duration_ms_milliseconds` | Histogram | Round-trip time to the upstream peer. | +| `proxy_domains_count` | Gauge | Domains currently configured. | +| `proxy_paths_count` | Gauge | Path routes currently configured. | +| `proxy_certificate_issue_duration_ms_milliseconds` | Histogram | ACME certificate issuance duration. | + +## L4 (TCP/UDP) + +Layer-4 passthrough services configured on the proxy. + +| Metric | Type | Description | +| --- | --- | --- | +| `proxy_l4_services_count` | Gauge | L4 services currently configured. | +| `proxy_tcp_active_connections` | Gauge | Active TCP/TLS connections. | +| `proxy_tcp_connections_total` | Counter | Total TCP/TLS connections accepted. | +| `proxy_tcp_connection_duration_ms_milliseconds` | Histogram | TCP/TLS connection duration. | +| `proxy_tcp_bytes_total` | Counter | Bytes transferred over TCP/TLS. | +| `proxy_udp_active_sessions` | Gauge | Active UDP relay sessions. | +| `proxy_udp_sessions_total` | Counter | Total UDP sessions opened. | +| `proxy_udp_packets_total` | Counter | UDP packets relayed. | +| `proxy_udp_bytes_total` | Counter | Bytes transferred over UDP. | + +## Sync with Management + +The proxy receives configuration (domains, paths, L4 services) from Management over a gRPC stream. + +| Metric | Type | Description | +| --- | --- | --- | +| `proxy_sync_snapshot_duration_ms_milliseconds` | Histogram | Time to apply the initial snapshot from Management. | +| `proxy_sync_batch_duration_ms_milliseconds` | Histogram | Time to apply a mapping batch. | +| `proxy_peer_add_duration_ms_milliseconds` | Histogram | Time to add a peer to the embedded client (key generation + gRPC + embed). | diff --git a/src/pages/selfhosted/observability/relay.mdx b/src/pages/selfhosted/observability/relay.mdx new file mode 100644 index 00000000..10a634cc --- /dev/null +++ b/src/pages/selfhosted/observability/relay.mdx @@ -0,0 +1,46 @@ +export const description = 'Metrics recorded by the NetBird Relay service.' + +# Relay metrics + +The Relay service forwards encrypted WireGuard traffic between peers that cannot establish a direct P2P connection. Its metrics cover connected peers, throughput, and the per-peer authentication and bookkeeping paths. All metric names are prefixed with `relay_`. + +## Endpoint + +| Setting | Default | +| --- | --- | +| Address | `0.0.0.0` | +| Port | `9090` | +| Path | `/metrics` | +| Flag | `--metrics-port` | + +```bash +netbird-relay --metrics-port 9090 +``` + +In the combined container, the metrics port is set with `server.metricsPort` in `config.yaml` and is shared with Management and Signal. See [Combined](/selfhosted/observability/combined). + + +The relay also runs a separate healthcheck server on `:9000` (configurable with `--health-listen-address` / `-H`). The healthcheck server does **not** expose metrics. + + +## Peers + +| Metric | Type | Description | +| --- | --- | --- | +| `relay_peers` | Gauge | Currently connected peers. | +| `relay_peers_active` | Gauge | Peers that exchanged traffic within the last 30 seconds. | +| `relay_peers_idle` | Gauge | Connected peers that have been idle for at least 30 seconds. | +| `relay_peer_authentication_time_milliseconds` | Histogram | Time to authenticate a peer. Unit: milliseconds. | +| `relay_peer_store_time_milliseconds` | Histogram | Time to register a peer in the in-memory store. Unit: milliseconds. | +| `relay_peer_reconnections_total` | Counter | Peer reconnections (existing peer ID re-registering). | + +## Traffic + +| Metric | Type | Description | +| --- | --- | --- | +| `relay_transfer_sent_bytes_total` | Counter | Bytes sent to peers. | +| `relay_transfer_received_bytes_total` | Counter | Bytes received from peers. | + +## Grafana dashboard + +A ready-made Relay dashboard is available under [Dashboards](/selfhosted/observability/dashboards). diff --git a/src/pages/selfhosted/observability/signal.mdx b/src/pages/selfhosted/observability/signal.mdx new file mode 100644 index 00000000..7f005bb8 --- /dev/null +++ b/src/pages/selfhosted/observability/signal.mdx @@ -0,0 +1,53 @@ +export const description = 'Metrics recorded by the NetBird Signal service.' + +# Signal metrics + +The Signal service brokers WireGuard handshake messages between peers. Its metrics describe peer connectivity, message forwarding, and the gRPC server itself. + +## Endpoint + +| Setting | Default | +| --- | --- | +| Address | `0.0.0.0` | +| Port | `9090` | +| Path | `/metrics` | +| Flag | `--metrics-port` | + +```bash +netbird-signal run --metrics-port 9090 +``` + +In the combined container, the metrics port is set with `server.metricsPort` in `config.yaml` and is shared with Management and Relay. See [Combined](/selfhosted/observability/combined). + + +Signal metric names are **unprefixed** when Signal runs as a standalone service. The combined container injects a `signal_` prefix on every metric name to avoid collisions on the shared `/metrics` endpoint — for example, `active_peers` becomes `signal_active_peers`. The tables below show the standalone names. + + + +The Signal gRPC server is also instrumented with the OpenTelemetry gRPC stats handler. Standard gRPC instrumentation series — including `rpc_server_call_duration_seconds_bucket`, `rpc_server_request_size_bytes_bucket`, and `rpc_server_response_size_bytes_bucket` — appear alongside the application metrics below. + + +## Peer connectivity + +| Metric | Type | Description | +| --- | --- | --- | +| `active_peers` | Gauge | Currently connected peers. | +| `peer_connection_duration_seconds` | Histogram | Duration of a peer's stream connection. Unit: seconds. | +| `registrations_total` | Counter | Peer registrations on the Signal stream. | +| `deregistrations_total` | Counter | Peer deregistrations. | +| `registration_failures_total` | Counter | Peer registration failures. | +| `registration_delay_milliseconds` | Histogram | Delay between stream open and successful registration. Unit: milliseconds. | +| `get_registration_delay_milliseconds` | Histogram | Lookup time for a registered peer. Unit: milliseconds. | + +## Message forwarding + +| Metric | Type | Description | +| --- | --- | --- | +| `messages_forwarded_total` | Counter | Messages successfully relayed between peers. | +| `message_forward_failures_total` | Counter | Forward attempts that failed (peer offline, send error, etc.). | +| `message_forward_latency_milliseconds` | Histogram | End-to-end forward latency. Unit: milliseconds. | +| `message_size_bytes` | Histogram | Size of forwarded messages. Unit: bytes. | + +## Grafana dashboard + +A ready-made Signal dashboard is available under [Dashboards](/selfhosted/observability/dashboards).