iis: add worker_responses_total as denominator for worker_request_errors_total (#2764)

Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>

docs/collector.iis.md

@@ -116,7 +116,8 @@ See [#2698](https://github.com/prometheus-community/windows_exporter/issues/2698
 | `windows_iis_worker_max_threads`                         | Maximum number of threads to which the thread pool can grow as needed                                                                                                                                                                                                                       | gauge   | `app`, `pid`                |
 | `windows_iis_worker_requests_total`                      | Total number of HTTP requests served by the worker process                                                                                                                                                                                                                                  | counter | `app`, `pid`                |
 | `windows_iis_worker_current_requests`                    | Current number of requests being processed by the worker process                                                                                                                                                                                                                            | gauge   | `app`, `pid`                |
-| `windows_iis_worker_request_errors_total`                | Total number of requests that returned an error                                                                                                                                                                                                                                             | counter | `app`, `pid`, `status_code` |
+| `windows_iis_worker_request_errors_total`                | Total number of responses with the status code `401`, `403`, `404` or `500`. Divide by `windows_iis_worker_responses_total`                                                                                                                                                                 | counter | `app`, `pid`, `status_code` |
+| `windows_iis_worker_responses_total`                     | Total number of responses sent by the worker process. Denominator for `windows_iis_worker_request_errors_total`, read in the same sample                                                                                                                                                    | counter | `app`, `pid`                |
 | `windows_iis_worker_current_websocket_requests`          | Current number of WebSocket requests being processed by the worker process                                                                                                                                                                                                                  | gauge   | `app`, `pid`                |
 | `windows_iis_worker_websocket_connection_attempts_total` | Total number of WebSocket connection attempts since the worker process started                                                                                                                                                                                                              | counter | `app`, `pid`                |
 | `windows_iis_worker_websocket_connection_accepted_total` | Total number of WebSocket connections accepted since the worker process started                                                                                                                                                                                                             | counter | `app`, `pid`                |
@@ -158,7 +159,21 @@ See [#2698](https://github.com/prometheus-community/windows_exporter/issues/2698
 _This collector does not yet have explained examples, we would appreciate your help adding them!_
 
 ## Useful queries
-_This collector does not yet have any useful queries added, we would appreciate your help adding them!_
+
+Share of HTTP 500 responses of each application pool over the last 5 minutes:
+
+```promql
+sum by (app) (rate(windows_iis_worker_request_errors_total{status_code="500"}[5m]))
+/
+sum by (app) (rate(windows_iis_worker_responses_total[5m]))
+```
+
+`windows_iis_worker_request_errors_total` and `windows_iis_worker_responses_total`
+are the value and the base of the `% <status> HTTP Response Sent` counters and
+come from the same sample. Don't divide the errors by
+`windows_iis_worker_requests_total`: that counter
+(`Total HTTP Requests Served`) is read in another query at a slightly different
+moment, so the ratio can exceed 1.
 
 ## Alerting examples
 _This collector does not yet have alerting examples, we would appreciate your help adding them!_

internal/collector/iis/AGENTS.md

@@ -56,7 +56,13 @@ notes were recorded on Windows Server 2025 (build 26100) with only
 - `% 401/403/404/500 HTTP Response Sent` of `W3SVC_W3WP` are
   `PERF_SAMPLE_FRACTION`. Raw is the cumulative count of that status, base is
   the cumulative request count (after one 200 and one 404 request: 404 raw 1,
-  base 2).
+  base 2). The collector publishes the raw value as
+  `worker_request_errors_total` and the base of `% 500 HTTP Response Sent`
+  (`secondvalue`) as `worker_responses_total`, from the same V8 query, so both
+  come from the same sample. Both are 32-bit and wrap at 2^32. It assumes that the four counters share one base.
+  `worker_requests_total` (`Total HTTP Requests Served`) is read in the other
+  `W3SVC_W3WP` query at a different moment and is not a valid denominator
+  ([#1538](https://github.com/prometheus-community/windows_exporter/issues/1538)).
 - `APP_POOL_WAS` uptimes and `Time Since Last Worker Process Failure` are
   `PERF_ELAPSED_TIME`. Raw is the start time as a FILETIME (100 ns since 1601),
   the second value is the current time, and `internal/pdh` returns the elapsed

internal/collector/iis/iis_w3svc_w3wp.go

@@ -75,6 +75,7 @@ type collectorW3SVCW3WP struct {
 
 	// IIS 8+ Only
 	w3SVCW3WPRequestErrorsTotal           *prometheus.Desc
+	w3SVCW3WPResponsesTotal               *prometheus.Desc
 	w3SVCW3WPWebSocketRequestsActive      *prometheus.Desc
 	w3SVCW3WPWebSocketConnectionAttempts  *prometheus.Desc
 	w3SVCW3WPWebSocketConnectionsAccepted *prometheus.Desc
@@ -139,6 +140,12 @@ type perfDataCounterValuesW3SVCW3WPV8 struct {
 	W3SVCW3WPRequestErrors403 float64 `perfdata:"% 403 HTTP Response Sent"`
 	W3SVCW3WPRequestErrors401 float64 `perfdata:"% 401 HTTP Response Sent"`
 
+	// The % ... HTTP Response Sent counters are PERF_SAMPLE_FRACTION. The second
+	// value is the base of the fraction, the cumulative number of responses the
+	// worker process sent. The four counters are assumed to share it, so one base
+	// is read.
+	W3SVCW3WPResponsesTotal float64 `perfdata:"% 500 HTTP Response Sent,secondvalue"`
+
 	W3SVCW3WPWebSocketRequestsActive      float64 `perfdata:"WebSocket Active Requests"`
 	W3SVCW3WPWebSocketConnectionAttempts  float64 `perfdata:"WebSocket Connection Attempts / Sec"`
 	W3SVCW3WPWebSocketConnectionsAccepted float64 `perfdata:"WebSocket Connections Accepted / Sec"`
@@ -366,10 +373,16 @@ func (c *Collector) buildW3SVCW3WPDescriptors() {
 	// W3SVC_W3WP_IIS8
 	c.w3SVCW3WPRequestErrorsTotal = prometheus.NewDesc(
 		prometheus.BuildFQName(types.Namespace, Name, "worker_request_errors_total"),
-		"Total number of requests that returned an error",
+		"Total number of responses with the status code 401, 403, 404 or 500. Divide by windows_iis_worker_responses_total",
 		[]string{"app", "pid", "status_code"},
 		nil,
 	)
+	c.w3SVCW3WPResponsesTotal = prometheus.NewDesc(
+		prometheus.BuildFQName(types.Namespace, Name, "worker_responses_total"),
+		"Total number of responses sent by the worker process. Denominator for windows_iis_worker_request_errors_total, read in the same sample",
+		[]string{"app", "pid"},
+		nil,
+	)
 	c.w3SVCW3WPWebSocketRequestsActive = prometheus.NewDesc(
 		prometheus.BuildFQName(types.Namespace, Name, "worker_current_websocket_requests"),
 		"Current number of WebSocket requests being processed by the worker process",
@@ -468,6 +481,14 @@ func (c *Collector) collectW3SVCW3WPv8(ch chan<- prometheus.Metric) error {
 			"500",
 		)
 
+		ch <- prometheus.MustNewConstMetric(
+			c.w3SVCW3WPResponsesTotal,
+			prometheus.CounterValue,
+			data.W3SVCW3WPResponsesTotal,
+			name,
+			pid,
+		)
+
 		ch <- prometheus.MustNewConstMetric(
 			c.w3SVCW3WPWebSocketRequestsActive,
 			prometheus.GaugeValue,

internal/collector/iis/metric_types_test.go

@@ -197,3 +197,38 @@ func TestW3SVCW3WPCurrentValuesAreGauges(t *testing.T) {
 	requireMetric(t, metrics, "windows_iis_worker_current_websocket_requests", dto.MetricType_GAUGE, 2, labels)
 	requireMetric(t, metrics, "windows_iis_worker_requests_total", dto.MetricType_COUNTER, 10, labels)
 }
+
+func TestW3SVCW3WPResponsesTotalIsErrorDenominator(t *testing.T) {
+	t.Parallel()
+
+	c := New(nil)
+	c.buildW3SVCW3WPDescriptors()
+
+	// PERF_SAMPLE_FRACTION: raw is the count per status code, the second value
+	// is the cumulative response count of the same sample.
+	c.w3SVCW3WPPerfDataCollectorV8 = cacheFixture[perfDataCounterValuesW3SVCW3WPV8]{sources: map[string]float64{
+		"% 401 HTTP Response Sent":             1,
+		"% 403 HTTP Response Sent":             2,
+		"% 404 HTTP Response Sent":             3,
+		"% 500 HTTP Response Sent":             4,
+		"% 500 HTTP Response Sent,secondvalue": 20,
+	}}
+
+	metrics := gatherCollect(t, c.collectW3SVCW3WPv8)
+	labels := map[string]string{"app": "Example", "pid": "123"}
+
+	requireMetric(t, metrics, "windows_iis_worker_responses_total", dto.MetricType_COUNTER, 20, labels)
+	require.NotContains(t, metrics["windows_iis_worker_responses_total"][0].labels, "status_code")
+
+	errorMetrics := metrics["windows_iis_worker_request_errors_total"]
+	require.Len(t, errorMetrics, 4)
+
+	want := map[string]float64{"401": 1, "403": 2, "404": 3, "500": 4}
+
+	for _, metric := range errorMetrics {
+		require.Equal(t, dto.MetricType_COUNTER, metric.kind)
+		require.Equal(t, labels["app"], metric.labels["app"])
+		require.Equal(t, labels["pid"], metric.labels["pid"])
+		require.InDelta(t, want[metric.labels["status_code"]], metric.value, 1e-9, metric.labels["status_code"])
+	}
+}