From d10875c43dfb392d1f970376a07e6151e04ca489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Otto=20Kr=C3=B6pke?= Date: Thu, 15 Oct 2026 19:33:06 +0200 Subject: [PATCH] remote_fx: fix swapped skipped-frame counters and sample fraction rates (#2716) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Otto Kröpke --- UPGRADE.md | 14 +- docs/collector.remote_fx.md | 51 ++- docs/collector.terminal_services.md | 2 +- internal/collector/remote_fx/remote_fx.go | 344 ++++++++++-------- .../remote_fx/remote_fx_internal_test.go | 221 +++++++++++ internal/collector/remote_fx/types.go | 16 +- 6 files changed, 471 insertions(+), 177 deletions(-) create mode 100644 internal/collector/remote_fx/remote_fx_internal_test.go diff --git a/UPGRADE.md b/UPGRADE.md index cbf739e1..4f693dbf 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -17,9 +17,9 @@ Most installations need no changes. Go through the checklist below and read the | You use the `container` collector | Replace `containerd-state-dir` with `cri-endpoint` | [container collector](#container-collector) | | You query `windows_gpu_info` | Replace the `phys` label with `device_number` | [GPU collector](#gpu-collector) | | You alert on BitLocker status | Replace `status="disabled"` with `status="off"` | [BitLocker status](#bitlocker-status) | -| You query Hyper-V, logical disk queue, cache, IIS output cache, memory, scheduled task, SMB server share or terminal services metrics | Update queries | [Renamed, deprecated and retyped metrics](#renamed-deprecated-and-retyped-metrics) | +| You query Hyper-V, logical disk queue, cache, IIS output cache, memory, RemoteFX, scheduled task, SMB server share or terminal services metrics | Update queries | [Renamed, deprecated and retyped metrics](#renamed-deprecated-and-retyped-metrics) | | You enable the `scheduled_task` collector and use tasks below `\Microsoft\` or missed runs | Set `--collector.scheduled_task.exclude` or `missed-runs` | [scheduled_task collector](#scheduled_task-collector) | -| You chart SQL Server latencies, SQL Server lock counts or DHCP denials | Expect corrected values | [Corrected values](#corrected-values) | +| You chart SQL Server latencies, SQL Server lock counts, DHCP denials or RemoteFX skipped frames | Expect corrected values | [Corrected values](#corrected-values) | | You enable the `hyperv` or `mscluster` collector | Check the new default sub-collectors | [New default sub-collectors](#new-default-sub-collectors) | | You set scrape timeouts above 5 minutes or below the timeout margin | Review the new limits | [Scrape timeouts](#scrape-timeouts) | | You use the sample Grafana dashboard | Use Grafana 13 or later and import the new dashboard | [Dashboard and alerts](#dashboard-and-alerts) | @@ -219,6 +219,15 @@ Renamed: | `windows_smb_server_shares_write_requests_count_total` | `windows_smb_server_shares_write_requests_total` | Same value | | `windows_smb_server_shares_metadata_requests_count_total` | `windows_smb_server_shares_metadata_requests_total` | Same value | | `windows_smb_server_shares_files_opened_count_total` | `windows_smb_server_shares_files_opened_total` | Same value | +| `windows_remote_fx_net_current_tcp_bandwidth` | `windows_remote_fx_net_current_tcp_bandwidth_bytes_per_second` | Unit suffix added, same value | +| `windows_remote_fx_net_current_udp_bandwidth` | `windows_remote_fx_net_current_udp_bandwidth_bytes_per_second` | Unit suffix added, same value | +| `windows_remote_fx_gfx_frame_quality` | `windows_remote_fx_gfx_frame_quality_ratio` | Ratio from 0 to 1 instead of percent | +| `windows_remote_fx_net_loss_rate` | `windows_remote_fx_net_lost_packets_total` and `windows_remote_fx_net_loss_sampled_packets_total` | Counters, see below | +| `windows_remote_fx_net_fec_rate` | `windows_remote_fx_net_fec_packets_total` and `windows_remote_fx_net_fec_sampled_packets_total` | Counters, see below | +| `windows_remote_fx_net_retransmission_rate` | `windows_remote_fx_net_retransmitted_packets_total` and `windows_remote_fx_net_retransmission_sampled_packets_total` | Counters, see below | + +The RemoteFX loss, FEC and retransmission gauges were not percentages. They exported the raw numerator of a Windows sample fraction, a running count that only grows. +The numerator and its base are now two counters. Get the ratio with `rate(windows_remote_fx_net_lost_packets_total[5m]) / rate(windows_remote_fx_net_loss_sampled_packets_total[5m])`, and likewise with `windows_remote_fx_net_fec_packets_total` / `windows_remote_fx_net_fec_sampled_packets_total` and `windows_remote_fx_net_retransmitted_packets_total` / `windows_remote_fx_net_retransmission_sampled_packets_total`. Removed: @@ -277,6 +286,7 @@ These metrics keep their names, but 0.31 reported wrong values. Graphs show a st - `windows_mssql_databases_xtp_controller_dlc_peak_latency_seconds`, `windows_mssql_dbreplica_database_flow_control_wait_seconds` and `windows_mssql_dbreplica_group_commit_stall_seconds` are now in seconds. SQL Server reports microseconds, which 0.31 did not convert correctly. - `windows_mssql_locks_count` is no longer divided by 1000. - `windows_dhcp_denied_due_to_nonmatch_total` now counts denials due to a non-match. In 0.31 it repeated the match denial count. +- `windows_remote_fx_gfx_frames_skipped_insufficient_resource_total` with `resource="client"` and `resource="server"` now count the matching frames. In 0.31 the two label values were swapped. - `windows_netframework_clrmemory_gc_time_percent` no longer overflows, and it is left out when Windows reports no base value. The `netframework` collector now reads the CLR counters through PDH instead of WMI; names, labels and types are unchanged. - The IIS URI cache flush and kernel cache item metrics read the right Windows counters, and the metadata cache hit metrics are no longer always `0`. - `windows_logical_disk_readonly`, the FSRM quota template information, the SMTP general failure totals and the Hyper-V root partition interrupt mappings were documented but never exported. They are exported now. diff --git a/docs/collector.remote_fx.md b/docs/collector.remote_fx.md index 0f8113cb..2fb30ce7 100644 --- a/docs/collector.remote_fx.md +++ b/docs/collector.remote_fx.md @@ -20,37 +20,56 @@ Name | Description | Type | Labels -----|-------------|------|------- `windows_remote_fx_net_base_udp_rtt_seconds` | Base UDP round-trip time (RTT) detected in seconds. | gauge | `session_name` `windows_remote_fx_net_base_tcp_rtt_seconds` | Base TCP round-trip time (RTT) detected in seconds. | gauge | `session_name` -`windows_remote_fx_net_current_tcp_bandwidth` | TCP Bandwidth detected in bytes per second. | gauge | `session_name` +`windows_remote_fx_net_current_tcp_bandwidth_bytes_per_second` | TCP bandwidth detected in bytes per second. | gauge | `session_name` `windows_remote_fx_net_current_tcp_rtt_seconds` | Average TCP round-trip time (RTT) detected in seconds. | gauge | `session_name` -`windows_remote_fx_net_current_udp_bandwidth` | UDP Bandwidth detected in bytes per second. | gauge | `session_name` +`windows_remote_fx_net_current_udp_bandwidth_bytes_per_second` | UDP bandwidth detected in bytes per second. | gauge | `session_name` `windows_remote_fx_net_current_udp_rtt_seconds` | Average UDP round-trip time (RTT) detected in seconds. | gauge | `session_name` -`windows_remote_fx_net_received_bytes_total` | Total bytes received over the network session. | counter | `session_name` -`windows_remote_fx_net_sent_bytes_total` | Total bytes sent over the network session. | counter | `session_name` -`windows_remote_fx_net_udp_packets_received_total` | Rate in packets per second at which packets are received over UDP. | counter | `session_name` -`windows_remote_fx_net_udp_packets_sent_total` | Rate in packets per second at which packets are sent over UDP. | counter | `session_name` -`windows_remote_fx_net_loss_rate` | Network packet loss rate detected over the RemoteFX session, expressed as a percentage. | gauge | `session_name` -`windows_remote_fx_net_fec_rate` | Forward Error Correction (FEC) rate applied to packets sent over the RemoteFX session, expressed as a percentage. | gauge | `session_name` -`windows_remote_fx_net_retransmission_rate` | Rate of packets retransmitted over the RemoteFX session, expressed as a percentage. | gauge | `session_name` +`windows_remote_fx_net_received_bytes_total` | Total number of bytes received since the connection started. | counter | `session_name` +`windows_remote_fx_net_sent_bytes_total` | Total number of bytes sent since the connection started. | counter | `session_name` +`windows_remote_fx_net_udp_packets_received_total` | Total number of packets received over UDP. | counter | `session_name` +`windows_remote_fx_net_udp_packets_sent_total` | Total number of packets sent over UDP. | counter | `session_name` +`windows_remote_fx_net_lost_packets_total` | Number of lost packets. Numerator of the loss rate counter. | counter | `session_name` +`windows_remote_fx_net_loss_sampled_packets_total` | Number of packets sampled for the loss rate. Denominator (base) of the loss rate counter. | counter | `session_name` +`windows_remote_fx_net_fec_packets_total` | Number of Forward Error Correction (FEC) packets. Numerator of the FEC rate counter. | counter | `session_name` +`windows_remote_fx_net_fec_sampled_packets_total` | Number of packets sampled for the FEC rate. Denominator (base) of the FEC rate counter. | counter | `session_name` +`windows_remote_fx_net_retransmitted_packets_total` | Number of retransmitted packets. Numerator of the retransmission rate counter. | counter | `session_name` +`windows_remote_fx_net_retransmission_sampled_packets_total` | Number of packets sampled for the retransmission rate. Denominator (base) of the retransmission rate counter. | counter | `session_name` +Windows reports the loss, FEC and retransmission rates as sample fractions: two cumulative values whose +deltas give the ratio. Divide the rate of the numerator by the rate of its base, see [Useful queries](#useful-queries). +The RTT counters appear to be measured only with UDP transport; they stay `0` for sessions over TCP. +Windows keeps `Total Sent Bytes` and `Total Received Bytes` in 32 bits, so the byte counters wrap after 4 GiB, +which looks like a counter reset. ## Metrics (Graphics) Name | Description | Type | Labels -----|-------------|------|------- -`windows_remote_fx_gfx_average_encoding_time_seconds` | Average frame encoding time. | gauge | `session_name` -`windows_remote_fx_gfx_frame_quality` | Quality of the output frame expressed as a percentage of the quality of the source frame. | gauge | `session_name` -`windows_remote_fx_gfx_frames_skipped_insufficient_resource_total` | Number of frames skipped per second due to insufficient resources. resources are client, server or network. | counter | `session_name`, `resource` +`windows_remote_fx_gfx_average_encoding_time_seconds` | Average frame encoding time in seconds. | gauge | `session_name` +`windows_remote_fx_gfx_frame_quality_ratio` | Quality of the output frame relative to the quality of the source frame (0-1). | gauge | `session_name` +`windows_remote_fx_gfx_frames_skipped_insufficient_resource_total` | Total number of frames skipped due to insufficient resources. `resource` is `client`, `network` or `server`. | counter | `session_name`, `resource` `windows_remote_fx_gfx_graphics_compression_ratio` | Ratio of the number of bytes encoded to the number of bytes input. | gauge | `session_name` -`windows_remote_fx_gfx_input_frames_total` | Number of sources frames provided as input to RemoteFX graphics per second. | counter | `session_name` -`windows_remote_fx_gfx_output_frames_total` | Number of frames sent to the client per second. | counter | `session_name` -`windows_remote_fx_gfx_source_frames_total` | Number of frames composed by the source (DWM) per second. | counter | `session_name` +`windows_remote_fx_gfx_input_frames_total` | Total number of source frames provided as input to RemoteFX graphics. | counter | `session_name` +`windows_remote_fx_gfx_output_frames_total` | Total number of frames sent to the client. | counter | `session_name` +`windows_remote_fx_gfx_source_frames_total` | Total number of frames composed by the source (DWM). | counter | `session_name` ### Example metric _This collector does not yet have explained examples, we would appreciate your help adding them!_ ## Useful queries -_This collector does not yet have any useful queries added, we would appreciate your help adding them!_ + +Packet loss ratio of each RDP session over the last 5 minutes: + +``` +rate(windows_remote_fx_net_lost_packets_total[5m]) / rate(windows_remote_fx_net_loss_sampled_packets_total[5m]) +``` + +Frames skipped per second, by cause: + +``` +sum by (session_name, resource) (rate(windows_remote_fx_gfx_frames_skipped_insufficient_resource_total[5m])) +``` ## Alerting examples _This collector does not yet have alerting examples, we would appreciate your help adding them!_ diff --git a/docs/collector.terminal_services.md b/docs/collector.terminal_services.md index 45cca9b9..43e7a7f6 100644 --- a/docs/collector.terminal_services.md +++ b/docs/collector.terminal_services.md @@ -124,7 +124,7 @@ windows_terminal_services_working_set_bytes_peak{session_name="RDP-Tcp 0"} 6.748 Use metrics can be combined with other metrics to create useful queries. For example, with remote_fx metrics: ``` -windows_remote_fx_net_loss_rate * on(session_name) group_left(user) (windows_terminal_services_session_info == 1) +(rate(windows_remote_fx_net_lost_packets_total[5m]) / rate(windows_remote_fx_net_loss_sampled_packets_total[5m])) * on(session_name) group_left(user) (windows_terminal_services_session_info == 1) ``` ## Alerting examples diff --git a/internal/collector/remote_fx/remote_fx.go b/internal/collector/remote_fx/remote_fx.go index 10042ee3..ac0b3a95 100644 --- a/internal/collector/remote_fx/remote_fx.go +++ b/internal/collector/remote_fx/remote_fx.go @@ -59,9 +59,12 @@ type Collector struct { currentTCPRTT *prometheus.Desc currentUDPBandwidth *prometheus.Desc currentUDPRTT *prometheus.Desc - fecRate *prometheus.Desc - lossRate *prometheus.Desc - retransmissionRate *prometheus.Desc + fec *prometheus.Desc + fecBase *prometheus.Desc + loss *prometheus.Desc + lossBase *prometheus.Desc + retransmission *prometheus.Desc + retransmissionBase *prometheus.Desc totalReceivedBytes *prometheus.Desc totalSentBytes *prometheus.Desc udpPacketsReceivedPerSec *prometheus.Desc @@ -105,129 +108,7 @@ func (c *Collector) Close() error { } func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error { - // net - c.baseTCPRTT = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_base_tcp_rtt_seconds"), - "Base TCP round-trip time (RTT) detected in seconds", - []string{"session_name"}, - nil, - ) - c.baseUDPRTT = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_base_udp_rtt_seconds"), - "Base UDP round-trip time (RTT) detected in seconds.", - []string{"session_name"}, - nil, - ) - c.currentTCPBandwidth = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_current_tcp_bandwidth"), - "TCP Bandwidth detected in bytes per second.", - []string{"session_name"}, - nil, - ) - c.currentTCPRTT = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_current_tcp_rtt_seconds"), - "Average TCP round-trip time (RTT) detected in seconds.", - []string{"session_name"}, - nil, - ) - c.currentUDPBandwidth = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_current_udp_bandwidth"), - "UDP Bandwidth detected in bytes per second.", - []string{"session_name"}, - nil, - ) - c.currentUDPRTT = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_current_udp_rtt_seconds"), - "Average UDP round-trip time (RTT) detected in seconds.", - []string{"session_name"}, - nil, - ) - c.totalReceivedBytes = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_received_bytes_total"), - "(TotalReceivedBytes)", - []string{"session_name"}, - nil, - ) - c.totalSentBytes = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_sent_bytes_total"), - "(TotalSentBytes)", - []string{"session_name"}, - nil, - ) - c.udpPacketsReceivedPerSec = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_udp_packets_received_total"), - "Rate in packets per second at which packets are received over UDP.", - []string{"session_name"}, - nil, - ) - c.udpPacketsSentPerSec = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_udp_packets_sent_total"), - "Rate in packets per second at which packets are sent over UDP.", - []string{"session_name"}, - nil, - ) - c.fecRate = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_fec_rate"), - "Forward Error Correction (FEC) percentage", - []string{"session_name"}, - nil, - ) - c.lossRate = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_loss_rate"), - "Loss percentage", - []string{"session_name"}, - nil, - ) - c.retransmissionRate = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "net_retransmission_rate"), - "Percentage of packets that have been retransmitted", - []string{"session_name"}, - nil, - ) - - // gfx - c.averageEncodingTime = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "gfx_average_encoding_time_seconds"), - "Average frame encoding time in seconds", - []string{"session_name"}, - nil, - ) - c.frameQuality = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "gfx_frame_quality"), - "Quality of the output frame expressed as a percentage of the quality of the source frame.", - []string{"session_name"}, - nil, - ) - c.framesSkippedPerSecondInsufficientResources = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "gfx_frames_skipped_insufficient_resource_total"), - "Number of frames skipped per second due to insufficient client resources.", - []string{"session_name", "resource"}, - nil, - ) - c.graphicsCompressionRatio = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "gfx_graphics_compression_ratio"), - "Ratio of the number of bytes encoded to the number of bytes input.", - []string{"session_name"}, - nil, - ) - c.inputFramesPerSecond = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "gfx_input_frames_total"), - "Number of sources frames provided as input to RemoteFX graphics per second.", - []string{"session_name"}, - nil, - ) - c.outputFramesPerSecond = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "gfx_output_frames_total"), - "Number of frames sent to the client per second.", - []string{"session_name"}, - nil, - ) - c.sourceFramesPerSecond = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "gfx_source_frames_total"), - "Number of frames composed by the source (DWM) per second.", - []string{"session_name"}, - nil, - ) + c.buildDescriptors() var err error @@ -246,29 +127,172 @@ func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error { return errors.Join(errs...) } +func (c *Collector) buildDescriptors() { + // net + c.baseTCPRTT = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_base_tcp_rtt_seconds"), + "Base TCP round-trip time (RTT) detected in seconds.", + []string{"session_name"}, + nil, + ) + c.baseUDPRTT = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_base_udp_rtt_seconds"), + "Base UDP round-trip time (RTT) detected in seconds.", + []string{"session_name"}, + nil, + ) + c.currentTCPBandwidth = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_current_tcp_bandwidth_bytes_per_second"), + "TCP bandwidth detected in bytes per second.", + []string{"session_name"}, + nil, + ) + c.currentTCPRTT = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_current_tcp_rtt_seconds"), + "Average TCP round-trip time (RTT) detected in seconds.", + []string{"session_name"}, + nil, + ) + c.currentUDPBandwidth = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_current_udp_bandwidth_bytes_per_second"), + "UDP bandwidth detected in bytes per second.", + []string{"session_name"}, + nil, + ) + c.currentUDPRTT = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_current_udp_rtt_seconds"), + "Average UDP round-trip time (RTT) detected in seconds.", + []string{"session_name"}, + nil, + ) + c.totalReceivedBytes = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_received_bytes_total"), + "Total number of bytes received since the connection started.", + []string{"session_name"}, + nil, + ) + c.totalSentBytes = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_sent_bytes_total"), + "Total number of bytes sent since the connection started.", + []string{"session_name"}, + nil, + ) + c.udpPacketsReceivedPerSec = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_udp_packets_received_total"), + "Total number of packets received over UDP.", + []string{"session_name"}, + nil, + ) + c.udpPacketsSentPerSec = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_udp_packets_sent_total"), + "Total number of packets sent over UDP.", + []string{"session_name"}, + nil, + ) + c.fec = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_fec_packets_total"), + "Number of Forward Error Correction (FEC) packets. Numerator of the FEC rate counter; divide its rate by the rate of windows_remote_fx_net_fec_sampled_packets_total to get the FEC ratio.", + []string{"session_name"}, + nil, + ) + c.fecBase = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_fec_sampled_packets_total"), + "Number of packets sampled for the FEC rate. Denominator (base) of the FEC rate counter.", + []string{"session_name"}, + nil, + ) + c.loss = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_lost_packets_total"), + "Number of lost packets. Numerator of the loss rate counter; divide its rate by the rate of windows_remote_fx_net_loss_sampled_packets_total to get the loss ratio.", + []string{"session_name"}, + nil, + ) + c.lossBase = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_loss_sampled_packets_total"), + "Number of packets sampled for the loss rate. Denominator (base) of the loss rate counter.", + []string{"session_name"}, + nil, + ) + c.retransmission = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_retransmitted_packets_total"), + "Number of retransmitted packets. Numerator of the retransmission rate counter; divide its rate by the rate of windows_remote_fx_net_retransmission_sampled_packets_total to get the retransmission ratio.", + []string{"session_name"}, + nil, + ) + c.retransmissionBase = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "net_retransmission_sampled_packets_total"), + "Number of packets sampled for the retransmission rate. Denominator (base) of the retransmission rate counter.", + []string{"session_name"}, + nil, + ) + + // gfx + c.averageEncodingTime = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "gfx_average_encoding_time_seconds"), + "Average frame encoding time in seconds.", + []string{"session_name"}, + nil, + ) + c.frameQuality = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "gfx_frame_quality_ratio"), + "Quality of the output frame relative to the quality of the source frame (0-1).", + []string{"session_name"}, + nil, + ) + c.framesSkippedPerSecondInsufficientResources = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "gfx_frames_skipped_insufficient_resource_total"), + "Total number of frames skipped due to insufficient client, network or server resources.", + []string{"session_name", "resource"}, + nil, + ) + c.graphicsCompressionRatio = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "gfx_graphics_compression_ratio"), + "Ratio of the number of bytes encoded to the number of bytes input.", + []string{"session_name"}, + nil, + ) + c.inputFramesPerSecond = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "gfx_input_frames_total"), + "Total number of source frames provided as input to RemoteFX graphics.", + []string{"session_name"}, + nil, + ) + c.outputFramesPerSecond = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "gfx_output_frames_total"), + "Total number of frames sent to the client.", + []string{"session_name"}, + nil, + ) + c.sourceFramesPerSecond = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "gfx_source_frames_total"), + "Total number of frames composed by the source (DWM).", + []string{"session_name"}, + nil, + ) +} + // Collect sends the metric values for each metric // to the provided prometheus Metric channel. func (c *Collector) Collect(ch chan<- prometheus.Metric, _ time.Duration) error { errs := make([]error, 0) - if err := c.collectRemoteFXNetworkCount(ch); err != nil { + if err := c.perfDataCollectorNetwork.Collect(&c.perfDataObjectNetwork); err != nil { errs = append(errs, fmt.Errorf("failed collecting RemoteFX Network metrics: %w", err)) + } else { + c.collectNetworkMetrics(ch, c.perfDataObjectNetwork) } - if err := c.collectRemoteFXGraphicsCounters(ch); err != nil { + if err := c.perfDataCollectorGraphics.Collect(&c.perfDataObjectGraphics); err != nil { errs = append(errs, fmt.Errorf("failed collecting RemoteFX Graphics metrics: %w", err)) + } else { + c.collectGraphicsMetrics(ch, c.perfDataObjectGraphics) } return errors.Join(errs...) } -func (c *Collector) collectRemoteFXNetworkCount(ch chan<- prometheus.Metric) error { - err := c.perfDataCollectorNetwork.Collect(&c.perfDataObjectNetwork) - if err != nil { - return fmt.Errorf("failed to collect RemoteFX Network metrics: %w", err) - } - - for _, data := range c.perfDataObjectNetwork { +func (c *Collector) collectNetworkMetrics(ch chan<- prometheus.Metric, rows []perfDataCounterValuesNetwork) { + for _, data := range rows { // only connect metrics for remote named sessions sessionName := normalizeSessionName(data.Name) if n := strings.ToLower(sessionName); n == "" || n == "services" || n == "console" { @@ -289,6 +313,7 @@ func (c *Collector) collectRemoteFXNetworkCount(ch chan<- prometheus.Metric) err sessionName, ) + // Current TCP/UDP Bandwidth is in kbit/s (1000 bit/s). ch <- prometheus.MustNewConstMetric( c.currentTCPBandwidth, prometheus.GaugeValue, @@ -346,37 +371,51 @@ func (c *Collector) collectRemoteFXNetworkCount(ch chan<- prometheus.Metric) err ) ch <- prometheus.MustNewConstMetric( - c.fecRate, - prometheus.GaugeValue, + c.fec, + prometheus.CounterValue, data.FECRate, sessionName, ) ch <- prometheus.MustNewConstMetric( - c.lossRate, - prometheus.GaugeValue, + c.fecBase, + prometheus.CounterValue, + data.FECRateBase, + sessionName, + ) + + ch <- prometheus.MustNewConstMetric( + c.loss, + prometheus.CounterValue, data.LossRate, sessionName, ) ch <- prometheus.MustNewConstMetric( - c.retransmissionRate, - prometheus.GaugeValue, + c.lossBase, + prometheus.CounterValue, + data.LossRateBase, + sessionName, + ) + + ch <- prometheus.MustNewConstMetric( + c.retransmission, + prometheus.CounterValue, data.RetransmissionRate, sessionName, ) - } - return nil + ch <- prometheus.MustNewConstMetric( + c.retransmissionBase, + prometheus.CounterValue, + data.RetransmissionRateBase, + sessionName, + ) + } } -func (c *Collector) collectRemoteFXGraphicsCounters(ch chan<- prometheus.Metric) error { - err := c.perfDataCollectorGraphics.Collect(&c.perfDataObjectGraphics) - if err != nil { - return fmt.Errorf("failed to collect RemoteFX Graphics metrics: %w", err) - } - - for _, data := range c.perfDataObjectGraphics { +func (c *Collector) collectGraphicsMetrics(ch chan<- prometheus.Metric, rows []perfDataCounterValuesGraphics) { + for _, data := range rows { // only connect metrics for remote named sessions sessionName := normalizeSessionName(data.Name) if n := strings.ToLower(sessionName); n == "" || n == "services" || n == "console" { @@ -390,10 +429,11 @@ func (c *Collector) collectRemoteFXGraphicsCounters(ch chan<- prometheus.Metric) sessionName, ) + // Frame Quality is a percentage. ch <- prometheus.MustNewConstMetric( c.frameQuality, prometheus.GaugeValue, - data.FrameQuality, + data.FrameQuality/100, sessionName, ) @@ -449,8 +489,6 @@ func (c *Collector) collectRemoteFXGraphicsCounters(ch chan<- prometheus.Metric) sessionName, ) } - - return nil } // normalizeSessionName ensure that the session is the same between WTS API and performance counters. diff --git a/internal/collector/remote_fx/remote_fx_internal_test.go b/internal/collector/remote_fx/remote_fx_internal_test.go new file mode 100644 index 00000000..93063610 --- /dev/null +++ b/internal/collector/remote_fx/remote_fx_internal_test.go @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package remote_fx + +import ( + "reflect" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" +) + +type expectedSample struct { + metricType dto.MetricType + value float64 +} + +func TestCollectNetworkMetrics(t *testing.T) { + t.Parallel() + + rows := []perfDataCounterValuesNetwork{ + {Name: "Services", LossRate: 99}, + { + Name: "RDP-tcp 0", + BaseTCPRTT: 10, + BaseUDPRTT: 20, + CurrentTCPBandwidth: 151159, + CurrentTCPRTT: 30, + CurrentUDPBandwidth: 8, + CurrentUDPRTT: 40, + TotalReceivedBytes: 3920173, + TotalSentBytes: 29262741, + UDPPacketsReceivedPersec: 5, + UDPPacketsSentPersec: 6, + FECRate: 7, + FECRateBase: 70, + LossRate: 2, + LossRateBase: 200, + RetransmissionRate: 3, + RetransmissionRateBase: 300, + }, + } + + expected := map[string]expectedSample{ + "windows_remote_fx_net_base_tcp_rtt_seconds": {dto.MetricType_GAUGE, 0.01}, + "windows_remote_fx_net_base_udp_rtt_seconds": {dto.MetricType_GAUGE, 0.02}, + "windows_remote_fx_net_current_tcp_bandwidth_bytes_per_second": {dto.MetricType_GAUGE, 18894875}, + "windows_remote_fx_net_current_tcp_rtt_seconds": {dto.MetricType_GAUGE, 0.03}, + "windows_remote_fx_net_current_udp_bandwidth_bytes_per_second": {dto.MetricType_GAUGE, 1000}, + "windows_remote_fx_net_current_udp_rtt_seconds": {dto.MetricType_GAUGE, 0.04}, + "windows_remote_fx_net_received_bytes_total": {dto.MetricType_COUNTER, 3920173}, + "windows_remote_fx_net_sent_bytes_total": {dto.MetricType_COUNTER, 29262741}, + "windows_remote_fx_net_udp_packets_received_total": {dto.MetricType_COUNTER, 5}, + "windows_remote_fx_net_udp_packets_sent_total": {dto.MetricType_COUNTER, 6}, + "windows_remote_fx_net_fec_packets_total": {dto.MetricType_COUNTER, 7}, + "windows_remote_fx_net_fec_sampled_packets_total": {dto.MetricType_COUNTER, 70}, + "windows_remote_fx_net_lost_packets_total": {dto.MetricType_COUNTER, 2}, + "windows_remote_fx_net_loss_sampled_packets_total": {dto.MetricType_COUNTER, 200}, + "windows_remote_fx_net_retransmitted_packets_total": {dto.MetricType_COUNTER, 3}, + "windows_remote_fx_net_retransmission_sampled_packets_total": {dto.MetricType_COUNTER, 300}, + } + + c := New(nil) + c.buildDescriptors() + + families := gather(t, func(ch chan<- prometheus.Metric) { c.collectNetworkMetrics(ch, rows) }) + require.Len(t, families, len(expected)) + + for _, family := range families { + want, ok := expected[family.GetName()] + require.True(t, ok, "unexpected metric %s", family.GetName()) + require.Equal(t, want.metricType, family.GetType(), family.GetName()) + require.Len(t, family.GetMetric(), 1, family.GetName()) + + metric := family.GetMetric()[0] + require.Equal(t, map[string]string{"session_name": "RDP-Tcp 0"}, labels(metric)) + require.InDelta(t, want.value, value(family.GetType(), metric), 1e-9, family.GetName()) + } +} + +func TestCollectGraphicsMetrics(t *testing.T) { + t.Parallel() + + rows := []perfDataCounterValuesGraphics{ + {Name: "Console", FrameQuality: 100}, + { + Name: "RDP-Tcp 0", + AverageEncodingTime: 4, + FrameQuality: 85, + FramesSkippedPerSecondInsufficientClientResources: 11, + FramesSkippedPerSecondInsufficientNetworkResources: 12, + FramesSkippedPerSecondInsufficientServerResources: 13, + GraphicsCompressionratio: 2, + InputFramesPerSecond: 27086, + OutputFramesPerSecond: 26142, + SourceFramesPerSecond: 9, + }, + } + + expected := map[string]expectedSample{ + "windows_remote_fx_gfx_average_encoding_time_seconds": {dto.MetricType_GAUGE, 0.004}, + "windows_remote_fx_gfx_frame_quality_ratio": {dto.MetricType_GAUGE, 0.85}, + "windows_remote_fx_gfx_graphics_compression_ratio": {dto.MetricType_GAUGE, 2}, + "windows_remote_fx_gfx_input_frames_total": {dto.MetricType_COUNTER, 27086}, + "windows_remote_fx_gfx_output_frames_total": {dto.MetricType_COUNTER, 26142}, + "windows_remote_fx_gfx_source_frames_total": {dto.MetricType_COUNTER, 9}, + } + skipped := map[string]float64{"client": 11, "network": 12, "server": 13} + + c := New(nil) + c.buildDescriptors() + + families := gather(t, func(ch chan<- prometheus.Metric) { c.collectGraphicsMetrics(ch, rows) }) + require.Len(t, families, len(expected)+1) + + for _, family := range families { + if family.GetName() == "windows_remote_fx_gfx_frames_skipped_insufficient_resource_total" { + require.Equal(t, dto.MetricType_COUNTER, family.GetType()) + require.Len(t, family.GetMetric(), len(skipped)) + + for _, metric := range family.GetMetric() { + l := labels(metric) + require.Equal(t, "RDP-Tcp 0", l["session_name"]) + require.InDelta(t, skipped[l["resource"]], metric.GetCounter().GetValue(), 0, l["resource"]) + } + + continue + } + + want, ok := expected[family.GetName()] + require.True(t, ok, "unexpected metric %s", family.GetName()) + require.Equal(t, want.metricType, family.GetType(), family.GetName()) + require.Len(t, family.GetMetric(), 1, family.GetName()) + + metric := family.GetMetric()[0] + require.Equal(t, map[string]string{"session_name": "RDP-Tcp 0"}, labels(metric)) + require.InDelta(t, want.value, value(family.GetType(), metric), 1e-9, family.GetName()) + } +} + +// TestFramesSkippedCounterNames guards the mapping from struct field to +// Windows counter; the client and server counters were swapped before. +func TestFramesSkippedCounterNames(t *testing.T) { + t.Parallel() + + typ := reflect.TypeFor[perfDataCounterValuesGraphics]() + + for _, resource := range []string{"Client", "Network", "Server"} { + field, ok := typ.FieldByName("FramesSkippedPerSecondInsufficient" + resource + "Resources") + require.True(t, ok, resource) + require.Equal(t, "Frames Skipped/Second - Insufficient "+resource+" Resources", field.Tag.Get("perfdata")) + } + + for _, counter := range []string{"FEC rate", "Loss rate", "Retransmission rate"} { + found := 0 + + for field := range reflect.TypeFor[perfDataCounterValuesNetwork]().Fields() { + if strings.HasPrefix(field.Tag.Get("perfdata"), counter) { + found++ + } + } + + require.Equal(t, 2, found, "%s needs numerator and base", counter) + } +} + +func gather(t *testing.T, collect func(ch chan<- prometheus.Metric)) []*dto.MetricFamily { + t.Helper() + + registry := prometheus.NewPedanticRegistry() + require.NoError(t, registry.Register(metricCollector(collect))) + + families, err := registry.Gather() + require.NoError(t, err) + + return families +} + +type metricCollector func(ch chan<- prometheus.Metric) + +func (c metricCollector) Describe(ch chan<- *prometheus.Desc) { + prometheus.DescribeByCollect(c, ch) +} + +func (c metricCollector) Collect(ch chan<- prometheus.Metric) { + c(ch) +} + +func labels(metric *dto.Metric) map[string]string { + result := make(map[string]string, len(metric.GetLabel())) + for _, label := range metric.GetLabel() { + result[label.GetName()] = label.GetValue() + } + + return result +} + +func value(metricType dto.MetricType, metric *dto.Metric) float64 { + if metricType == dto.MetricType_COUNTER { + return metric.GetCounter().GetValue() + } + + return metric.GetGauge().GetValue() +} diff --git a/internal/collector/remote_fx/types.go b/internal/collector/remote_fx/types.go index 7dcd3cdd..c426fe5e 100644 --- a/internal/collector/remote_fx/types.go +++ b/internal/collector/remote_fx/types.go @@ -30,9 +30,15 @@ type perfDataCounterValuesNetwork struct { TotalSentBytes float64 `perfdata:"Total Sent Bytes"` UDPPacketsReceivedPersec float64 `perfdata:"UDP Packets Received/sec"` UDPPacketsSentPersec float64 `perfdata:"UDP Packets Sent/sec"` - FECRate float64 `perfdata:"FEC rate"` - LossRate float64 `perfdata:"Loss rate"` - RetransmissionRate float64 `perfdata:"Retransmission rate"` + // FEC Rate, Loss Rate and Retransmission Rate are PERF_SAMPLE_FRACTION + // counters: the first value is a cumulative numerator, the second value + // the cumulative base. Neither is a percentage on its own. + FECRate float64 `perfdata:"FEC rate"` + FECRateBase float64 `perfdata:"FEC rate,secondvalue"` + LossRate float64 `perfdata:"Loss rate"` + LossRateBase float64 `perfdata:"Loss rate,secondvalue"` + RetransmissionRate float64 `perfdata:"Retransmission rate"` + RetransmissionRateBase float64 `perfdata:"Retransmission rate,secondvalue"` } type perfDataCounterValuesGraphics struct { @@ -40,9 +46,9 @@ type perfDataCounterValuesGraphics struct { AverageEncodingTime float64 `perfdata:"Average Encoding Time"` FrameQuality float64 `perfdata:"Frame Quality"` - FramesSkippedPerSecondInsufficientClientResources float64 `perfdata:"Frames Skipped/Second - Insufficient Server Resources"` + FramesSkippedPerSecondInsufficientClientResources float64 `perfdata:"Frames Skipped/Second - Insufficient Client Resources"` FramesSkippedPerSecondInsufficientNetworkResources float64 `perfdata:"Frames Skipped/Second - Insufficient Network Resources"` - FramesSkippedPerSecondInsufficientServerResources float64 `perfdata:"Frames Skipped/Second - Insufficient Client Resources"` + FramesSkippedPerSecondInsufficientServerResources float64 `perfdata:"Frames Skipped/Second - Insufficient Server Resources"` GraphicsCompressionratio float64 `perfdata:"Graphics Compression ratio"` InputFramesPerSecond float64 `perfdata:"Input Frames/Second"` OutputFramesPerSecond float64 `perfdata:"Output Frames/Second"`