From 8138b2e0738ae26ffe552c1e33f569940396b566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Otto=20Kr=C3=B6pke?= Date: Thu, 15 Oct 2026 20:34:37 +0200 Subject: [PATCH] iis: fix types, units and names of app pool, queue and worker metrics (#2715) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Otto Kröpke --- UPGRADE.md | 13 +- docs/collector.iis.md | 28 +-- internal/collector/iis/AGENTS.md | 12 ++ internal/collector/iis/iis.go | 14 +- internal/collector/iis/iis_app_pool_was.go | 26 ++- .../iis/iis_http_service_request_queues.go | 74 ++++--- internal/collector/iis/iis_w3svc_w3wp.go | 16 +- internal/collector/iis/metric_types_test.go | 199 ++++++++++++++++++ 8 files changed, 315 insertions(+), 67 deletions(-) create mode 100644 internal/collector/iis/metric_types_test.go diff --git a/UPGRADE.md b/UPGRADE.md index 0bff8ecb..24cd63d6 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -17,7 +17,7 @@ Most installations need no changes. Go through the checklist below and read the | You use the `container` collector | Replace `containerd-state-dir` with `cri-endpoint` | [container collector](#container-collector) | | You query `windows_gpu_info` | Replace the `phys` label with `device_number` | [GPU collector](#gpu-collector) | | You alert on BitLocker status | Replace `status="disabled"` with `status="off"` | [BitLocker status](#bitlocker-status) | -| You query Active Directory latency or security descriptor propagation, DFSR, Hyper-V, logical disk queue, cache, IIS output cache, memory, RemoteFX, scheduled task, SMB server share or terminal services metrics | Update queries | [Renamed, deprecated and retyped metrics](#renamed-deprecated-and-retyped-metrics) | +| You query Active Directory latency or security descriptor propagation, DFSR, Hyper-V, logical disk queue, cache, IIS, memory, RemoteFX, scheduled task, SMB server share or terminal services metrics | Update queries | [Renamed, deprecated and retyped metrics](#renamed-deprecated-and-retyped-metrics) | | You enable the `scheduled_task` collector and use tasks below `\Microsoft\` or missed runs | Set `--collector.scheduled_task.exclude` or `missed-runs` | [scheduled_task collector](#scheduled_task-collector) | | You chart SQL Server latencies, SQL Server lock counts, DHCP denials, AD LSA writes, AD computer creations, AD FS database query times or RemoteFX skipped frames | Expect corrected values | [Corrected values](#corrected-values) | | You enable the `hyperv` or `mscluster` collector | Check the new default sub-collectors | [New default sub-collectors](#new-default-sub-collectors) | @@ -246,6 +246,12 @@ Renamed: | `windows_dfsr_connection_bandwidth_savings_using_dfs_replication_bytes_total` | `windows_dfsr_connection_bandwidth_savings_using_dfs_replication_ratio` | Now a gauge, 0-1 instead of percent | | `windows_dfsr_folder_bandwidth_savings_using_dfs_replication_bytes_total` | `windows_dfsr_folder_bandwidth_savings_using_dfs_replication_ratio` | Now a gauge, 0-1 instead of percent | | `windows_dfsr_volume_usn_journal_unread_percentage` | `windows_dfsr_volume_usn_journal_unread_ratio` | 0-1 instead of percent | +| `windows_iis_current_application_pool_start_time` | `windows_iis_current_application_pool_uptime_seconds` | Same value: it always was the uptime in seconds, not a timestamp | +| `windows_iis_total_application_pool_start_time` | `windows_iis_total_application_pool_uptime_seconds` | Same value, now a gauge | +| `windows_iis_time_since_last_worker_process_failure` | `windows_iis_time_since_last_worker_process_failure_seconds` | Unit suffix added, same value | +| `windows_iis_http_request_total_rejected_request` | `windows_iis_http_requests_rejected_total` | Now a counter, use `rate()`; label `app` | +| `windows_iis_http_requests_arrival_rate` | `windows_iis_http_requests_arrived_total` | Was a running count, now a counter, use `rate()`; label `app` | +| `windows_iis_http_requests_max_queue_item_age` | `windows_iis_http_requests_max_queue_item_age_seconds` | Left out while the queue is empty; label `app` | The RemoteFX loss, FEC and retransmission gauges were not percentages. They exported the raw numerator of a Windows sample fraction, a running count that only grows. The numerator and its base are now two counters. Get the ratio with `rate(windows_remote_fx_net_lost_packets_total[5m]) / rate(windows_remote_fx_net_loss_sampled_packets_total[5m])`, and likewise with `windows_remote_fx_net_fec_packets_total` / `windows_remote_fx_net_fec_sampled_packets_total` and `windows_remote_fx_net_retransmitted_packets_total` / `windows_remote_fx_net_retransmission_sampled_packets_total`. @@ -285,17 +291,20 @@ Changed type, same name: | `windows_iis_server_output_cache_memory_bytes`, `windows_iis_worker_output_cache_memory_bytes` | counter | gauge | | `windows_iis_server_output_cache_active_flushed_items`, `windows_iis_worker_output_cache_active_flushed_items` | counter | gauge | | `windows_smb_server_shares_current_open_file_count`, `windows_smb_server_shares_tree_connect_count` | counter | gauge | +| `windows_iis_worker_current_requests`, `windows_iis_worker_current_websocket_requests` | counter | gauge | +| `windows_iis_worker_max_threads` | counter | gauge | | `windows_cache_copy_read_hits_total` | gauge | counter | | `windows_udp_datagram_received_total` | gauge | counter | | `windows_hyperv_legacy_network_adapter_bytes_dropped_total` | gauge | counter | -The IIS and SMB values are current occupancy, so query them directly instead of with `rate()`. +The IIS and SMB values are current occupancy or, for `worker_max_threads`, the configured thread pool limit, so query them directly instead of with `rate()`. If Prometheus scrapes with OpenMetrics, the IIS and SMB metrics were stored with a `_total` suffix, which counters get in that format. They now arrive without it. The cache, UDP and Hyper-V metrics already end in `_total`, so only their type metadata changes; `rate()` queries keep working. Label changes on existing metrics: - `windows_os_info` has an `installation_type` label, for example `Server` or `Client`. +- The IIS HTTP request queue metrics (`windows_iis_http_requests_*`) have the label `app` instead of `site`. The value is unchanged: it always was the application pool that owns the HTTP.sys request queue, not a site. They are now filtered with `--collector.iis.app-include` and `--collector.iis.app-exclude` instead of the site filters. - The `windows_netframework_clrmemory_*` metrics have a `process_id` label. - In the `windows_hyperv_dynamic_memory_vm_*` metrics, unnamed VM instances have `vm="(unknown)"` (or `(unknown)#N`) instead of `vm="------"`. - `windows_terminal_services_connection_broker_performance_total` no longer has a `connection="Pending"` series. It was the current number of pending connections, published as a counter. Use the gauge `windows_terminal_services_connection_broker_pending_connections`. diff --git a/docs/collector.iis.md b/docs/collector.iis.md index c5cc0db2..9ec47343 100644 --- a/docs/collector.iis.md +++ b/docs/collector.iis.md @@ -26,6 +26,8 @@ If given, an application needs to match the include regular expression in order If given, an application needs to *not* match the exclude regular expression in order for the corresponding metrics to be reported. +The application filters apply to the application pool, worker process and HTTP request queue metrics (`app` label). The site filters apply to the `Web Service` metrics (`site` label). + ## Collection duration `windows_exporter_collector_duration_seconds{collector="iis"}` can jump to about 250 ms on some scrapes, while it normally stays in the low milliseconds. At the same moments, some other collectors that read performance counters, such as `system`, `logical_disk` or `net`, can show the same duration. @@ -70,13 +72,13 @@ See [#2698](https://github.com/prometheus-community/windows_exporter/issues/2698 | `windows_iis_not_found_errors_total` | The number of requests that have been made since the service started that were not satisfied by the server because the requested document was not found. Usually reported as HTTP error 404 | counter | `site` | | `windows_iis_rejected_async_io_requests_total` | The total number of rejected asynchronous I/O requests since the start of the IIS service. | counter | `site` | | `windows_iis_current_application_pool_state` | The current status of the application pool (1 - Uninitialized, 2 - Initialized, 3 - Running, 4 - Disabling, 5 - Disabled, 6 - Shutdown Pending, 7 - Delete Pending) | gauge | `app`, `state` | -| `windows_iis_current_application_pool_start_time` | The Unix timestamp for the application pool start time | gauge | `app` | +| `windows_iis_current_application_pool_uptime_seconds` | The time, in seconds, since the application pool was last started or recycled | gauge | `app` | | `windows_iis_current_worker_processes` | The current number of worker processes that are running in the application pool | gauge | `app` | | `windows_iis_maximum_worker_processes` | The maximum number of worker processes that have been created for the application pool since Windows Process Activation Service (WAS) started | gauge | `app` | | `windows_iis_recent_worker_process_failures` | The number of times that worker processes for the application pool failed during the rapid-fail protection interval | gauge | `app` | -| `windows_iis_time_since_last_worker_process_failure` | The length of time, in seconds, since the last worker process failure occurred for the application pool | gauge | `app` | +| `windows_iis_time_since_last_worker_process_failure_seconds` | The length of time, in seconds, since the last worker process failure occurred for the application pool. Equals the pool uptime if no worker process failed | gauge | `app` | | `windows_iis_total_application_pool_recycles` | The number of times that the application pool has been recycled since Windows Process Activation Service (WAS) started | counter | `app` | -| `windows_iis_total_application_pool_start_time` | The Unix timestamp for the application pool of when the Windows Process Activation Service (WAS) started | counter | `app` | +| `windows_iis_total_application_pool_uptime_seconds` | The time, in seconds, since the application pool was first started after the Windows Process Activation Service (WAS) started | gauge | `app` | | `windows_iis_total_worker_processes_created` | The number of worker processes created for the application pool since Windows Process Activation Service (WAS) started | counter | `app` | | `windows_iis_total_worker_process_failures` | The number of times that worker processes have crashed since the application pool was started | counter | `app` | | `windows_iis_total_worker_process_ping_failures` | The number of times that Windows Process Activation Service (WAS) did not receive a response to ping messages sent to a worker process | counter | `app` | @@ -111,14 +113,14 @@ See [#2698](https://github.com/prometheus-community/windows_exporter/issues/2698 | `windows_iis_worker_output_cache_items_flushed_total` | Total number of items flushed from output cache (since service startup) | counter | `app`, `pid` | | `windows_iis_worker_output_cache_flushes_total` | Total number of flushes of output cache (since service startup) | counter | `app`, `pid` | | `windows_iis_worker_threads` | Number of threads actively processing requests in the worker process | gauge | `app`, `pid`, `state` | -| `windows_iis_worker_max_threads` | Maximum number of threads to which the thread pool can grow as needed | counter | `app`, `pid` | +| `windows_iis_worker_max_threads` | Maximum number of threads to which the thread pool can grow as needed | gauge | `app`, `pid` | | `windows_iis_worker_requests_total` | Total number of HTTP requests served by the worker process | counter | `app`, `pid` | -| `windows_iis_worker_current_requests` | Current number of requests being processed by the worker process | counter | `app`, `pid` | +| `windows_iis_worker_current_requests` | Current number of requests being processed by the worker process | gauge | `app`, `pid` | | `windows_iis_worker_request_errors_total` | Total number of requests that returned an error | counter | `app`, `pid`, `status_code` | -| `windows_iis_worker_current_websocket_requests` | the current number of WebSocket connections being processed by the IIS worker process. | counter | `app`, `pid` | -| `windows_iis_worker_websocket_connection_attempts_total` | the total number of attempted WebSocket connections since the start of the IIS worker process | counter | `app`, `pid` | -| `windows_iis_worker_websocket_connection_accepted_total` | the total number of WebSocket connections that have been successfully established since the start of the IIS worker process | counter | `app`, `pid` | -| `windows_iis_worker_websocket_connection_rejected_total` | the total number of WebSocket connections that have been rejected by the server since the start of the IIS worker process. Connections can be rejected for various reasons, such as capacity limitations, authentication failures, or configuration issues | counter | `app`, `pid` | +| `windows_iis_worker_current_websocket_requests` | Current number of WebSocket requests being processed by the worker process | gauge | `app`, `pid` | +| `windows_iis_worker_websocket_connection_attempts_total` | Total number of WebSocket connection attempts since the worker process started | counter | `app`, `pid` | +| `windows_iis_worker_websocket_connection_accepted_total` | Total number of WebSocket connections accepted since the worker process started | counter | `app`, `pid` | +| `windows_iis_worker_websocket_connection_rejected_total` | Total number of WebSocket connections rejected since the worker process started | counter | `app`, `pid` | | `windows_iis_server_cache_active_flushed_entries` | Number of file handles cached that will be closed when all current transfers complete | gauge | None | | `windows_iis_server_file_cache_memory_bytes` | Current number of bytes used by file cache | gauge | None | | `windows_iis_server_file_cache_max_memory_bytes` | Maximum number of bytes used by file cache | counter | None | @@ -147,10 +149,10 @@ See [#2698](https://github.com/prometheus-community/windows_exporter/issues/2698 | `windows_iis_server_output_cache_hits_total` | Total number of successful lookups in output cache (since service startup) | counter | None | | `windows_iis_server_output_cache_items_flushed_total` | Total number of items flushed from output cache (since service startup) | counter | None | | `windows_iis_server_output_cache_flushes_total` | Total number of flushes of output cache (since service startup) | counter | None | -| `windows_iis_http_requests_current_queue_size` | Http Request Current queue size | gauge | `site` | -| `windows_iis_http_request_total_rejected_request` | Http Request total rejected request | gauge | `site` | -| `windows_iis_http_requests_max_queue_item_age` | Http Request Max queue Item age | gauge | `site` | -| `windows_iis_http_requests_arrival_rate` | Http requests Arrival Rate | gauge | `site` | +| `windows_iis_http_requests_current_queue_size` | Number of requests in the HTTP.sys request queue of the application pool | gauge | `app` | +| `windows_iis_http_requests_rejected_total` | Total number of requests rejected by the HTTP.sys request queue of the application pool | counter | `app` | +| `windows_iis_http_requests_max_queue_item_age_seconds` | Age, in seconds, of the oldest request in the HTTP.sys request queue of the application pool. Not reported while the queue is empty | gauge | `app` | +| `windows_iis_http_requests_arrived_total` | Total number of requests that arrived in the HTTP.sys request queue of the application pool | counter | `app` | ### Example metric _This collector does not yet have explained examples, we would appreciate your help adding them!_ diff --git a/internal/collector/iis/AGENTS.md b/internal/collector/iis/AGENTS.md index b663bdf1..6705c3d6 100644 --- a/internal/collector/iis/AGENTS.md +++ b/internal/collector/iis/AGENTS.md @@ -276,3 +276,15 @@ Not read by the collector: - `PERF_COUNTER_COUNTER`: `Kernel: Uri Cache Hits/sec` - `PERF_RAW_FRACTION`: `File Cache Hits %`, `Kernel: URI Cache Hits %`, `Metadata Cache Hits %`, `URI Cache Hits %` - `PERF_SAMPLE_FRACTION`: `Output Cache Current Hits %` + +## Published metrics + +- Publish the `APP_POOL_WAS` elapsed-time counters as uptime gauges in + seconds (`*_uptime_seconds`), not as timestamps. +- `MaxQueueItemAge` of an empty queue becomes a large negative number + (now minus `math.MaxInt64` ticks). Leave the series out then; never publish + it as `0`. +- `HTTP Service Request Queues` instances are application pools: they use the + label `app` and the app include/exclude filters. +- Check published names, types and labels through registry gathering with the + `cacheFixture` collector fake, as in `metric_types_test.go`. diff --git a/internal/collector/iis/iis.go b/internal/collector/iis/iis.go index 5e01676b..f610323a 100644 --- a/internal/collector/iis/iis.go +++ b/internal/collector/iis/iis.go @@ -156,14 +156,22 @@ func (c *Collector) GetName() string { func (c *Collector) Close() error { c.perfDataCollectorWebService.Close() - c.perfDataCollectorHttpServiceRequestQueues.Close() - c.perfDataCollectorAppPoolWAS.Close() + + if c.perfDataCollectorHttpServiceRequestQueues != nil { + c.perfDataCollectorHttpServiceRequestQueues.Close() + } + + if c.perfDataCollectorAppPoolWAS != nil { + c.perfDataCollectorAppPoolWAS.Close() + } if c.w3SVCW3WPPerfDataCollector != nil { c.w3SVCW3WPPerfDataCollector.Close() } - c.w3SVCW3WPPerfDataCollectorV8.Close() + if c.w3SVCW3WPPerfDataCollectorV8 != nil { + c.w3SVCW3WPPerfDataCollectorV8.Close() + } if c.serviceCachePerfDataCollector != nil { c.serviceCachePerfDataCollector.Close() diff --git a/internal/collector/iis/iis_app_pool_was.go b/internal/collector/iis/iis_app_pool_was.go index 2971ceb6..79817d4f 100644 --- a/internal/collector/iis/iis_app_pool_was.go +++ b/internal/collector/iis/iis_app_pool_was.go @@ -21,12 +21,13 @@ import ( "fmt" "github.com/prometheus-community/windows_exporter/internal/pdh" + pdhtypes "github.com/prometheus-community/windows_exporter/internal/pdh/types" "github.com/prometheus-community/windows_exporter/internal/types" "github.com/prometheus/client_golang/prometheus" ) type collectorAppPoolWAS struct { - perfDataCollectorAppPoolWAS *pdh.Collector[perfDataCounterValuesAppPoolWAS] + perfDataCollectorAppPoolWAS pdhtypes.Collector[perfDataCounterValuesAppPoolWAS] perfDataObjectAppPoolWAS []perfDataCounterValuesAppPoolWAS currentApplicationPoolState *prometheus.Desc @@ -84,7 +85,14 @@ func (c *Collector) buildAppPoolWAS() error { return fmt.Errorf("failed to create APP_POOL_WAS collector: %w", err) } - // APP_POOL_WAS + c.buildAppPoolWASDescriptors() + + return nil +} + +func (c *Collector) buildAppPoolWASDescriptors() { + // The uptime counters are PERF_ELAPSED_TIME. Their raw value is the start + // time, which internal/pdh turns into the elapsed time in seconds. c.currentApplicationPoolState = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "current_application_pool_state"), "The current status of the application pool (1 - Uninitialized, 2 - Initialized, 3 - Running, 4 - Disabling, 5 - Disabled, 6 - Shutdown Pending, 7 - Delete Pending) (CurrentApplicationPoolState)", @@ -92,8 +100,8 @@ func (c *Collector) buildAppPoolWAS() error { nil, ) c.currentApplicationPoolUptime = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "current_application_pool_start_time"), - "The unix timestamp for the application pool start time (CurrentApplicationPoolUptime)", + prometheus.BuildFQName(types.Namespace, Name, "current_application_pool_uptime_seconds"), + "The time, in seconds, since the application pool was last started or recycled (CurrentApplicationPoolUptime)", []string{"app"}, nil, ) @@ -116,7 +124,7 @@ func (c *Collector) buildAppPoolWAS() error { nil, ) c.timeSinceLastWorkerProcessFailure = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "time_since_last_worker_process_failure"), + prometheus.BuildFQName(types.Namespace, Name, "time_since_last_worker_process_failure_seconds"), "The length of time, in seconds, since the last worker process failure occurred for the application pool (TimeSinceLastWorkerProcessFailure)", []string{"app"}, nil, @@ -128,8 +136,8 @@ func (c *Collector) buildAppPoolWAS() error { nil, ) c.totalApplicationPoolUptime = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "total_application_pool_start_time"), - "The unix timestamp for the application pool of when the Windows Process Activation Service (WAS) started (TotalApplicationPoolUptime)", + prometheus.BuildFQName(types.Namespace, Name, "total_application_pool_uptime_seconds"), + "The time, in seconds, since the application pool was first started after the Windows Process Activation Service (WAS) started (TotalApplicationPoolUptime)", []string{"app"}, nil, ) @@ -163,8 +171,6 @@ func (c *Collector) buildAppPoolWAS() error { []string{"app"}, nil, ) - - return nil } func (c *Collector) collectAppPoolWAS(ch chan<- prometheus.Metric) error { @@ -242,7 +248,7 @@ func (c *Collector) collectAppPoolWAS(ch chan<- prometheus.Metric) error { ch <- prometheus.MustNewConstMetric( c.totalApplicationPoolUptime, - prometheus.CounterValue, + prometheus.GaugeValue, data.TotalApplicationPoolUptime, data.Name, ) diff --git a/internal/collector/iis/iis_http_service_request_queues.go b/internal/collector/iis/iis_http_service_request_queues.go index ab680c32..ed02bf57 100644 --- a/internal/collector/iis/iis_http_service_request_queues.go +++ b/internal/collector/iis/iis_http_service_request_queues.go @@ -22,18 +22,19 @@ import ( "strings" "github.com/prometheus-community/windows_exporter/internal/pdh" + pdhtypes "github.com/prometheus-community/windows_exporter/internal/pdh/types" "github.com/prometheus-community/windows_exporter/internal/types" "github.com/prometheus/client_golang/prometheus" ) type collectorHttpServiceRequestQueues struct { - perfDataCollectorHttpServiceRequestQueues *pdh.Collector[perfDataCounterValuesHttpServiceRequestQueues] + perfDataCollectorHttpServiceRequestQueues pdhtypes.Collector[perfDataCounterValuesHttpServiceRequestQueues] perfDataObjectHttpServiceRequestQueues []perfDataCounterValuesHttpServiceRequestQueues - httpRequestQueuesCurrentQueueSize *prometheus.Desc - httpRequestQueuesTotalRejectedRequest *prometheus.Desc - httpRequestQueuesMaxQueueItemAge *prometheus.Desc - httpRequestQueuesArrivalRate *prometheus.Desc + httpRequestQueuesCurrentQueueSize *prometheus.Desc + httpRequestQueuesRejectedRequests *prometheus.Desc + httpRequestQueuesMaxQueueItemAge *prometheus.Desc + httpRequestQueuesArrivedRequests *prometheus.Desc } type perfDataCounterValuesHttpServiceRequestQueues struct { @@ -59,32 +60,38 @@ func (c *Collector) buildHttpServiceRequestQueues() error { return fmt.Errorf("failed to create Http Service collector: %w", err) } + c.buildHttpServiceRequestQueuesDescriptors() + + return nil +} + +func (c *Collector) buildHttpServiceRequestQueuesDescriptors() { + // The named instances of HTTP Service Request Queues are the HTTP.sys + // request queues of the IIS application pools, so the label is "app". c.httpRequestQueuesCurrentQueueSize = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "http_requests_current_queue_size"), - "Http Request Current Queue Size", - []string{"site"}, + "Number of requests in the HTTP.sys request queue of the application pool (CurrentQueueSize)", + []string{"app"}, nil, ) - c.httpRequestQueuesTotalRejectedRequest = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "http_request_total_rejected_request"), - "Http Request Total Rejected Request", - []string{"site"}, + c.httpRequestQueuesRejectedRequests = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "http_requests_rejected_total"), + "Total number of requests rejected by the HTTP.sys request queue of the application pool (RejectedRequests)", + []string{"app"}, nil, ) c.httpRequestQueuesMaxQueueItemAge = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "http_requests_max_queue_item_age"), - "Http Request Max Queue Item Age. The values might be bogus if the queue is empty.", - []string{"site"}, + prometheus.BuildFQName(types.Namespace, Name, "http_requests_max_queue_item_age_seconds"), + "Age, in seconds, of the oldest request in the HTTP.sys request queue of the application pool. Not reported while the queue is empty (MaxQueueItemAge)", + []string{"app"}, nil, ) - c.httpRequestQueuesArrivalRate = prometheus.NewDesc( - prometheus.BuildFQName(types.Namespace, Name, "http_requests_arrival_rate"), - "Http Request Arrival Rate", - []string{"site"}, + c.httpRequestQueuesArrivedRequests = prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, Name, "http_requests_arrived_total"), + "Total number of requests that arrived in the HTTP.sys request queue of the application pool (ArrivalRate)", + []string{"app"}, nil, ) - - return nil } func (c *Collector) collectHttpServiceRequestQueues(ch chan<- prometheus.Metric) error { @@ -103,7 +110,7 @@ func (c *Collector) collectHttpServiceRequestQueues(ch chan<- prometheus.Metric) continue } - if c.config.SiteExclude.MatchString(data.Name) || !c.config.SiteInclude.MatchString(data.Name) { + if c.config.AppExclude.MatchString(data.Name) || !c.config.AppInclude.MatchString(data.Name) { continue } @@ -115,22 +122,27 @@ func (c *Collector) collectHttpServiceRequestQueues(ch chan<- prometheus.Metric) ) ch <- prometheus.MustNewConstMetric( - c.httpRequestQueuesTotalRejectedRequest, - prometheus.GaugeValue, + c.httpRequestQueuesRejectedRequests, + prometheus.CounterValue, data.HttpRequestQueuesTotalRejectedRequests, data.Name, ) - ch <- prometheus.MustNewConstMetric( - c.httpRequestQueuesMaxQueueItemAge, - prometheus.GaugeValue, - data.HttpRequestQueuesMaxQueueItemAge, - data.Name, - ) + // MaxQueueItemAge is PERF_ELAPSED_TIME. On an empty queue its raw + // value is math.MaxInt64 instead of a start time, which gives a large + // negative elapsed time. That is no sample, so leave the series out. + if data.HttpRequestQueuesMaxQueueItemAge >= 0 { + ch <- prometheus.MustNewConstMetric( + c.httpRequestQueuesMaxQueueItemAge, + prometheus.GaugeValue, + data.HttpRequestQueuesMaxQueueItemAge, + data.Name, + ) + } ch <- prometheus.MustNewConstMetric( - c.httpRequestQueuesArrivalRate, - prometheus.GaugeValue, + c.httpRequestQueuesArrivedRequests, + prometheus.CounterValue, data.HttpRequestQueuesArrivalRate, data.Name, ) diff --git a/internal/collector/iis/iis_w3svc_w3wp.go b/internal/collector/iis/iis_w3svc_w3wp.go index 0daea12e..2408f3cf 100644 --- a/internal/collector/iis/iis_w3svc_w3wp.go +++ b/internal/collector/iis/iis_w3svc_w3wp.go @@ -29,7 +29,7 @@ import ( type collectorW3SVCW3WP struct { w3SVCW3WPPerfDataCollector pdhtypes.Collector[perfDataCounterValuesW3SVCW3WP] - w3SVCW3WPPerfDataCollectorV8 *pdh.Collector[perfDataCounterValuesW3SVCW3WPV8] + w3SVCW3WPPerfDataCollectorV8 pdhtypes.Collector[perfDataCounterValuesW3SVCW3WPV8] perfDataObjectW3SVCW3WP []perfDataCounterValuesW3SVCW3WP perfDataObjectW3SVCW3WPV8 []perfDataCounterValuesW3SVCW3WPV8 @@ -372,25 +372,25 @@ func (c *Collector) buildW3SVCW3WPDescriptors() { ) c.w3SVCW3WPWebSocketRequestsActive = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "worker_current_websocket_requests"), - "", + "Current number of WebSocket requests being processed by the worker process", []string{"app", "pid"}, nil, ) c.w3SVCW3WPWebSocketConnectionAttempts = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "worker_websocket_connection_attempts_total"), - "", + "Total number of WebSocket connection attempts since the worker process started", []string{"app", "pid"}, nil, ) c.w3SVCW3WPWebSocketConnectionsAccepted = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "worker_websocket_connection_accepted_total"), - "", + "Total number of WebSocket connections accepted since the worker process started", []string{"app", "pid"}, nil, ) c.w3SVCW3WPWebSocketConnectionsRejected = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "worker_websocket_connection_rejected_total"), - "", + "Total number of WebSocket connections rejected since the worker process started", []string{"app", "pid"}, nil, ) @@ -470,7 +470,7 @@ func (c *Collector) collectW3SVCW3WPv8(ch chan<- prometheus.Metric) error { ch <- prometheus.MustNewConstMetric( c.w3SVCW3WPWebSocketRequestsActive, - prometheus.CounterValue, + prometheus.GaugeValue, data.W3SVCW3WPWebSocketRequestsActive, name, pid, @@ -533,7 +533,7 @@ func (c *Collector) collectW3SVCW3WPv7(ch chan<- prometheus.Metric) error { ch <- prometheus.MustNewConstMetric( c.w3SVCW3WPMaximumThreads, - prometheus.CounterValue, + prometheus.GaugeValue, data.W3SVCW3WPMaximumThreads, name, pid, @@ -549,7 +549,7 @@ func (c *Collector) collectW3SVCW3WPv7(ch chan<- prometheus.Metric) error { ch <- prometheus.MustNewConstMetric( c.w3SVCW3WPRequestsActive, - prometheus.CounterValue, + prometheus.GaugeValue, data.W3SVCW3WPRequestsActive, name, pid, diff --git a/internal/collector/iis/metric_types_test.go b/internal/collector/iis/metric_types_test.go new file mode 100644 index 00000000..1ad1415b --- /dev/null +++ b/internal/collector/iis/metric_types_test.go @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package iis + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" +) + +type gatheredMetric struct { + kind dto.MetricType + value float64 + labels map[string]string +} + +// gatherCollect registers collect with a registry and returns the gathered +// metrics by name, so the test checks names, types and labels as published. +func gatherCollect(t *testing.T, collect func(ch chan<- prometheus.Metric) error) map[string][]gatheredMetric { + t.Helper() + + ch := make(chan prometheus.Metric, 1000) + require.NoError(t, collect(ch)) + close(ch) + + collected := make([]prometheus.Metric, 0, len(ch)) + for metric := range ch { + collected = append(collected, metric) + } + + registry := prometheus.NewPedanticRegistry() + require.NoError(t, registry.Register(collectFunc(func(ch chan<- prometheus.Metric) { + for _, metric := range collected { + ch <- metric + } + }))) + + families, err := registry.Gather() + require.NoError(t, err) + + result := make(map[string][]gatheredMetric) + + for _, family := range families { + for _, metric := range family.GetMetric() { + labels := make(map[string]string) + for _, label := range metric.GetLabel() { + labels[label.GetName()] = label.GetValue() + } + + var value float64 + + switch family.GetType() { + case dto.MetricType_COUNTER: + value = metric.GetCounter().GetValue() + case dto.MetricType_GAUGE: + value = metric.GetGauge().GetValue() + default: + t.Fatalf("unexpected type %s for %s", family.GetType(), family.GetName()) + } + + result[family.GetName()] = append(result[family.GetName()], gatheredMetric{kind: family.GetType(), value: value, labels: labels}) + } + } + + return result +} + +// collectFunc is an unchecked collector: it describes no metrics, so the +// registry validates the metrics it gathers instead of the descriptions. +type collectFunc func(ch chan<- prometheus.Metric) + +func (collectFunc) Describe(chan<- *prometheus.Desc) {} + +func (f collectFunc) Collect(ch chan<- prometheus.Metric) { f(ch) } + +func requireMetric(t *testing.T, metrics map[string][]gatheredMetric, name string, kind dto.MetricType, value float64, labels map[string]string) { + t.Helper() + + require.Len(t, metrics[name], 1, name) + require.Equal(t, kind, metrics[name][0].kind, name) + require.InDelta(t, value, metrics[name][0].value, 1e-9, name) + + for key, expected := range labels { + require.Equal(t, expected, metrics[name][0].labels[key], "%s label %s", name, key) + } +} + +func TestAppPoolWASUptimeMetrics(t *testing.T) { + t.Parallel() + + c := New(nil) + c.buildAppPoolWASDescriptors() + c.perfDataCollectorAppPoolWAS = cacheFixture[perfDataCounterValuesAppPoolWAS]{sources: map[string]float64{ + // internal/pdh turns the PERF_ELAPSED_TIME start times into elapsed seconds. + "Current Application Pool Uptime": 298, + "Total Application Pool Uptime": 600, + "Time Since Last Worker Process Failure": 298, + "Total Application Pool Recycles": 2, + }} + + metrics := gatherCollect(t, c.collectAppPoolWAS) + app := map[string]string{"app": "123_Example"} + + requireMetric(t, metrics, "windows_iis_current_application_pool_uptime_seconds", dto.MetricType_GAUGE, 298, app) + requireMetric(t, metrics, "windows_iis_total_application_pool_uptime_seconds", dto.MetricType_GAUGE, 600, app) + requireMetric(t, metrics, "windows_iis_time_since_last_worker_process_failure_seconds", dto.MetricType_GAUGE, 298, app) + requireMetric(t, metrics, "windows_iis_total_application_pool_recycles", dto.MetricType_COUNTER, 2, app) + require.NotContains(t, metrics, "windows_iis_current_application_pool_start_time") + require.NotContains(t, metrics, "windows_iis_total_application_pool_start_time") +} + +func TestHttpServiceRequestQueuesMetrics(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + maxAge float64 + wantMaxAge bool + }{ + // An empty queue reports math.MaxInt64 as start time, which internal/pdh + // turns into a large negative number of seconds. + {name: "empty queue", maxAge: -922337203685.4775, wantMaxAge: false}, + {name: "waiting request", maxAge: 1.5, wantMaxAge: true}, + {name: "request just queued", maxAge: 0, wantMaxAge: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + c := New(nil) + c.buildHttpServiceRequestQueuesDescriptors() + c.perfDataCollectorHttpServiceRequestQueues = cacheFixture[perfDataCounterValuesHttpServiceRequestQueues]{sources: map[string]float64{ + "CurrentQueueSize": 1, + "RejectedRequests": 3, + "MaxQueueItemAge": tc.maxAge, + "ArrivalRate": 67, + }} + + metrics := gatherCollect(t, c.collectHttpServiceRequestQueues) + app := map[string]string{"app": "123_Example"} + + requireMetric(t, metrics, "windows_iis_http_requests_current_queue_size", dto.MetricType_GAUGE, 1, app) + requireMetric(t, metrics, "windows_iis_http_requests_rejected_total", dto.MetricType_COUNTER, 3, app) + requireMetric(t, metrics, "windows_iis_http_requests_arrived_total", dto.MetricType_COUNTER, 67, app) + + if tc.wantMaxAge { + requireMetric(t, metrics, "windows_iis_http_requests_max_queue_item_age_seconds", dto.MetricType_GAUGE, tc.maxAge, app) + } else { + require.NotContains(t, metrics, "windows_iis_http_requests_max_queue_item_age_seconds") + } + }) + } +} + +func TestW3SVCW3WPCurrentValuesAreGauges(t *testing.T) { + t.Parallel() + + c := New(nil) + c.buildW3SVCW3WPDescriptors() + + sources := map[string]float64{ + "Maximum Threads Count": 256, + "Active Requests": 4, + "WebSocket Active Requests": 2, + "Total HTTP Requests Served": 10, + } + c.w3SVCW3WPPerfDataCollector = cacheFixture[perfDataCounterValuesW3SVCW3WP]{sources: sources} + c.w3SVCW3WPPerfDataCollectorV8 = cacheFixture[perfDataCounterValuesW3SVCW3WPV8]{sources: sources} + + metrics := gatherCollect(t, func(ch chan<- prometheus.Metric) error { + if err := c.collectW3SVCW3WPv7(ch); err != nil { + return err + } + + return c.collectW3SVCW3WPv8(ch) + }) + labels := map[string]string{"app": "Example", "pid": "123"} + + requireMetric(t, metrics, "windows_iis_worker_max_threads", dto.MetricType_GAUGE, 256, labels) + requireMetric(t, metrics, "windows_iis_worker_current_requests", dto.MetricType_GAUGE, 4, labels) + requireMetric(t, metrics, "windows_iis_worker_current_websocket_requests", dto.MetricType_GAUGE, 2, labels) + requireMetric(t, metrics, "windows_iis_worker_requests_total", dto.MetricType_COUNTER, 10, labels) +}