iis: fix types, units and names of app pool, queue and worker metrics (#2715)

Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>

internal/collector/iis/AGENTS.md

@@ -276,3 +276,15 @@ Not read by the collector:
 - `PERF_COUNTER_COUNTER`: `Kernel: Uri Cache Hits/sec`
 - `PERF_RAW_FRACTION`: `File Cache Hits %`, `Kernel: URI Cache Hits %`, `Metadata Cache Hits %`, `URI Cache Hits %`
 - `PERF_SAMPLE_FRACTION`: `Output Cache Current Hits %`
+
+## Published metrics
+
+- Publish the `APP_POOL_WAS` elapsed-time counters as uptime gauges in
+  seconds (`*_uptime_seconds`), not as timestamps.
+- `MaxQueueItemAge` of an empty queue becomes a large negative number
+  (now minus `math.MaxInt64` ticks). Leave the series out then; never publish
+  it as `0`.
+- `HTTP Service Request Queues` instances are application pools: they use the
+  label `app` and the app include/exclude filters.
+- Check published names, types and labels through registry gathering with the
+  `cacheFixture` collector fake, as in `metric_types_test.go`.

internal/collector/iis/iis.go

@@ -156,14 +156,22 @@ func (c *Collector) GetName() string {
 
 func (c *Collector) Close() error {
 	c.perfDataCollectorWebService.Close()
-	c.perfDataCollectorHttpServiceRequestQueues.Close()
-	c.perfDataCollectorAppPoolWAS.Close()
+
+	if c.perfDataCollectorHttpServiceRequestQueues != nil {
+		c.perfDataCollectorHttpServiceRequestQueues.Close()
+	}
+
+	if c.perfDataCollectorAppPoolWAS != nil {
+		c.perfDataCollectorAppPoolWAS.Close()
+	}
 
 	if c.w3SVCW3WPPerfDataCollector != nil {
 		c.w3SVCW3WPPerfDataCollector.Close()
 	}
 
-	c.w3SVCW3WPPerfDataCollectorV8.Close()
+	if c.w3SVCW3WPPerfDataCollectorV8 != nil {
+		c.w3SVCW3WPPerfDataCollectorV8.Close()
+	}
 
 	if c.serviceCachePerfDataCollector != nil {
 		c.serviceCachePerfDataCollector.Close()

internal/collector/iis/iis_app_pool_was.go

@@ -21,12 +21,13 @@ import (
 	"fmt"
 
 	"github.com/prometheus-community/windows_exporter/internal/pdh"
+	pdhtypes "github.com/prometheus-community/windows_exporter/internal/pdh/types"
 	"github.com/prometheus-community/windows_exporter/internal/types"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
 type collectorAppPoolWAS struct {
-	perfDataCollectorAppPoolWAS *pdh.Collector[perfDataCounterValuesAppPoolWAS]
+	perfDataCollectorAppPoolWAS pdhtypes.Collector[perfDataCounterValuesAppPoolWAS]
 	perfDataObjectAppPoolWAS    []perfDataCounterValuesAppPoolWAS
 
 	currentApplicationPoolState        *prometheus.Desc
@@ -84,7 +85,14 @@ func (c *Collector) buildAppPoolWAS() error {
 		return fmt.Errorf("failed to create APP_POOL_WAS collector: %w", err)
 	}
 
-	// APP_POOL_WAS
+	c.buildAppPoolWASDescriptors()
+
+	return nil
+}
+
+func (c *Collector) buildAppPoolWASDescriptors() {
+	// The uptime counters are PERF_ELAPSED_TIME. Their raw value is the start
+	// time, which internal/pdh turns into the elapsed time in seconds.
 	c.currentApplicationPoolState = prometheus.NewDesc(
 		prometheus.BuildFQName(types.Namespace, Name, "current_application_pool_state"),
 		"The current status of the application pool (1 - Uninitialized, 2 - Initialized, 3 - Running, 4 - Disabling, 5 - Disabled, 6 - Shutdown Pending, 7 - Delete Pending) (CurrentApplicationPoolState)",
@@ -92,8 +100,8 @@ func (c *Collector) buildAppPoolWAS() error {
 		nil,
 	)
 	c.currentApplicationPoolUptime = prometheus.NewDesc(
-		prometheus.BuildFQName(types.Namespace, Name, "current_application_pool_start_time"),
-		"The unix timestamp for the application pool start time (CurrentApplicationPoolUptime)",
+		prometheus.BuildFQName(types.Namespace, Name, "current_application_pool_uptime_seconds"),
+		"The time, in seconds, since the application pool was last started or recycled (CurrentApplicationPoolUptime)",
 		[]string{"app"},
 		nil,
 	)
@@ -116,7 +124,7 @@ func (c *Collector) buildAppPoolWAS() error {
 		nil,
 	)
 	c.timeSinceLastWorkerProcessFailure = prometheus.NewDesc(
-		prometheus.BuildFQName(types.Namespace, Name, "time_since_last_worker_process_failure"),
+		prometheus.BuildFQName(types.Namespace, Name, "time_since_last_worker_process_failure_seconds"),
 		"The length of time, in seconds, since the last worker process failure occurred for the application pool (TimeSinceLastWorkerProcessFailure)",
 		[]string{"app"},
 		nil,
@@ -128,8 +136,8 @@ func (c *Collector) buildAppPoolWAS() error {
 		nil,
 	)
 	c.totalApplicationPoolUptime = prometheus.NewDesc(
-		prometheus.BuildFQName(types.Namespace, Name, "total_application_pool_start_time"),
-		"The unix timestamp for the application pool of when the Windows Process Activation Service (WAS) started (TotalApplicationPoolUptime)",
+		prometheus.BuildFQName(types.Namespace, Name, "total_application_pool_uptime_seconds"),
+		"The time, in seconds, since the application pool was first started after the Windows Process Activation Service (WAS) started (TotalApplicationPoolUptime)",
 		[]string{"app"},
 		nil,
 	)
@@ -163,8 +171,6 @@ func (c *Collector) buildAppPoolWAS() error {
 		[]string{"app"},
 		nil,
 	)
-
-	return nil
 }
 
 func (c *Collector) collectAppPoolWAS(ch chan<- prometheus.Metric) error {
@@ -242,7 +248,7 @@ func (c *Collector) collectAppPoolWAS(ch chan<- prometheus.Metric) error {
 
 		ch <- prometheus.MustNewConstMetric(
 			c.totalApplicationPoolUptime,
-			prometheus.CounterValue,
+			prometheus.GaugeValue,
 			data.TotalApplicationPoolUptime,
 			data.Name,
 		)

internal/collector/iis/iis_http_service_request_queues.go

@@ -22,18 +22,19 @@ import (
 	"strings"
 
 	"github.com/prometheus-community/windows_exporter/internal/pdh"
+	pdhtypes "github.com/prometheus-community/windows_exporter/internal/pdh/types"
 	"github.com/prometheus-community/windows_exporter/internal/types"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
 type collectorHttpServiceRequestQueues struct {
-	perfDataCollectorHttpServiceRequestQueues *pdh.Collector[perfDataCounterValuesHttpServiceRequestQueues]
+	perfDataCollectorHttpServiceRequestQueues pdhtypes.Collector[perfDataCounterValuesHttpServiceRequestQueues]
 	perfDataObjectHttpServiceRequestQueues    []perfDataCounterValuesHttpServiceRequestQueues
 
-	httpRequestQueuesCurrentQueueSize     *prometheus.Desc
-	httpRequestQueuesTotalRejectedRequest *prometheus.Desc
-	httpRequestQueuesMaxQueueItemAge      *prometheus.Desc
-	httpRequestQueuesArrivalRate          *prometheus.Desc
+	httpRequestQueuesCurrentQueueSize *prometheus.Desc
+	httpRequestQueuesRejectedRequests *prometheus.Desc
+	httpRequestQueuesMaxQueueItemAge  *prometheus.Desc
+	httpRequestQueuesArrivedRequests  *prometheus.Desc
 }
 
 type perfDataCounterValuesHttpServiceRequestQueues struct {
@@ -59,32 +60,38 @@ func (c *Collector) buildHttpServiceRequestQueues() error {
 		return fmt.Errorf("failed to create Http Service collector: %w", err)
 	}
 
+	c.buildHttpServiceRequestQueuesDescriptors()
+
+	return nil
+}
+
+func (c *Collector) buildHttpServiceRequestQueuesDescriptors() {
+	// The named instances of HTTP Service Request Queues are the HTTP.sys
+	// request queues of the IIS application pools, so the label is "app".
 	c.httpRequestQueuesCurrentQueueSize = prometheus.NewDesc(
 		prometheus.BuildFQName(types.Namespace, Name, "http_requests_current_queue_size"),
-		"Http Request Current Queue Size",
-		[]string{"site"},
+		"Number of requests in the HTTP.sys request queue of the application pool (CurrentQueueSize)",
+		[]string{"app"},
 		nil,
 	)
-	c.httpRequestQueuesTotalRejectedRequest = prometheus.NewDesc(
-		prometheus.BuildFQName(types.Namespace, Name, "http_request_total_rejected_request"),
-		"Http Request Total Rejected Request",
-		[]string{"site"},
+	c.httpRequestQueuesRejectedRequests = prometheus.NewDesc(
+		prometheus.BuildFQName(types.Namespace, Name, "http_requests_rejected_total"),
+		"Total number of requests rejected by the HTTP.sys request queue of the application pool (RejectedRequests)",
+		[]string{"app"},
 		nil,
 	)
 	c.httpRequestQueuesMaxQueueItemAge = prometheus.NewDesc(
-		prometheus.BuildFQName(types.Namespace, Name, "http_requests_max_queue_item_age"),
-		"Http Request Max Queue Item Age. The values might be bogus if the queue is empty.",
-		[]string{"site"},
+		prometheus.BuildFQName(types.Namespace, Name, "http_requests_max_queue_item_age_seconds"),
+		"Age, in seconds, of the oldest request in the HTTP.sys request queue of the application pool. Not reported while the queue is empty (MaxQueueItemAge)",
+		[]string{"app"},
 		nil,
 	)
-	c.httpRequestQueuesArrivalRate = prometheus.NewDesc(
-		prometheus.BuildFQName(types.Namespace, Name, "http_requests_arrival_rate"),
-		"Http Request Arrival Rate",
-		[]string{"site"},
+	c.httpRequestQueuesArrivedRequests = prometheus.NewDesc(
+		prometheus.BuildFQName(types.Namespace, Name, "http_requests_arrived_total"),
+		"Total number of requests that arrived in the HTTP.sys request queue of the application pool (ArrivalRate)",
+		[]string{"app"},
 		nil,
 	)
-
-	return nil
 }
 
 func (c *Collector) collectHttpServiceRequestQueues(ch chan<- prometheus.Metric) error {
@@ -103,7 +110,7 @@ func (c *Collector) collectHttpServiceRequestQueues(ch chan<- prometheus.Metric)
 			continue
 		}
 
-		if c.config.SiteExclude.MatchString(data.Name) || !c.config.SiteInclude.MatchString(data.Name) {
+		if c.config.AppExclude.MatchString(data.Name) || !c.config.AppInclude.MatchString(data.Name) {
 			continue
 		}
 
@@ -115,22 +122,27 @@ func (c *Collector) collectHttpServiceRequestQueues(ch chan<- prometheus.Metric)
 		)
 
 		ch <- prometheus.MustNewConstMetric(
-			c.httpRequestQueuesTotalRejectedRequest,
-			prometheus.GaugeValue,
+			c.httpRequestQueuesRejectedRequests,
+			prometheus.CounterValue,
 			data.HttpRequestQueuesTotalRejectedRequests,
 			data.Name,
 		)
 
-		ch <- prometheus.MustNewConstMetric(
-			c.httpRequestQueuesMaxQueueItemAge,
-			prometheus.GaugeValue,
-			data.HttpRequestQueuesMaxQueueItemAge,
-			data.Name,
-		)
+		// MaxQueueItemAge is PERF_ELAPSED_TIME. On an empty queue its raw
+		// value is math.MaxInt64 instead of a start time, which gives a large
+		// negative elapsed time. That is no sample, so leave the series out.
+		if data.HttpRequestQueuesMaxQueueItemAge >= 0 {
+			ch <- prometheus.MustNewConstMetric(
+				c.httpRequestQueuesMaxQueueItemAge,
+				prometheus.GaugeValue,
+				data.HttpRequestQueuesMaxQueueItemAge,
+				data.Name,
+			)
+		}
 
 		ch <- prometheus.MustNewConstMetric(
-			c.httpRequestQueuesArrivalRate,
-			prometheus.GaugeValue,
+			c.httpRequestQueuesArrivedRequests,
+			prometheus.CounterValue,
 			data.HttpRequestQueuesArrivalRate,
 			data.Name,
 		)

internal/collector/iis/iis_w3svc_w3wp.go

@@ -29,7 +29,7 @@ import (
 
 type collectorW3SVCW3WP struct {
 	w3SVCW3WPPerfDataCollector   pdhtypes.Collector[perfDataCounterValuesW3SVCW3WP]
-	w3SVCW3WPPerfDataCollectorV8 *pdh.Collector[perfDataCounterValuesW3SVCW3WPV8]
+	w3SVCW3WPPerfDataCollectorV8 pdhtypes.Collector[perfDataCounterValuesW3SVCW3WPV8]
 	perfDataObjectW3SVCW3WP      []perfDataCounterValuesW3SVCW3WP
 	perfDataObjectW3SVCW3WPV8    []perfDataCounterValuesW3SVCW3WPV8
 
@@ -372,25 +372,25 @@ func (c *Collector) buildW3SVCW3WPDescriptors() {
 	)
 	c.w3SVCW3WPWebSocketRequestsActive = prometheus.NewDesc(
 		prometheus.BuildFQName(types.Namespace, Name, "worker_current_websocket_requests"),
-		"",
+		"Current number of WebSocket requests being processed by the worker process",
 		[]string{"app", "pid"},
 		nil,
 	)
 	c.w3SVCW3WPWebSocketConnectionAttempts = prometheus.NewDesc(
 		prometheus.BuildFQName(types.Namespace, Name, "worker_websocket_connection_attempts_total"),
-		"",
+		"Total number of WebSocket connection attempts since the worker process started",
 		[]string{"app", "pid"},
 		nil,
 	)
 	c.w3SVCW3WPWebSocketConnectionsAccepted = prometheus.NewDesc(
 		prometheus.BuildFQName(types.Namespace, Name, "worker_websocket_connection_accepted_total"),
-		"",
+		"Total number of WebSocket connections accepted since the worker process started",
 		[]string{"app", "pid"},
 		nil,
 	)
 	c.w3SVCW3WPWebSocketConnectionsRejected = prometheus.NewDesc(
 		prometheus.BuildFQName(types.Namespace, Name, "worker_websocket_connection_rejected_total"),
-		"",
+		"Total number of WebSocket connections rejected since the worker process started",
 		[]string{"app", "pid"},
 		nil,
 	)
@@ -470,7 +470,7 @@ func (c *Collector) collectW3SVCW3WPv8(ch chan<- prometheus.Metric) error {
 
 		ch <- prometheus.MustNewConstMetric(
 			c.w3SVCW3WPWebSocketRequestsActive,
-			prometheus.CounterValue,
+			prometheus.GaugeValue,
 			data.W3SVCW3WPWebSocketRequestsActive,
 			name,
 			pid,
@@ -533,7 +533,7 @@ func (c *Collector) collectW3SVCW3WPv7(ch chan<- prometheus.Metric) error {
 
 		ch <- prometheus.MustNewConstMetric(
 			c.w3SVCW3WPMaximumThreads,
-			prometheus.CounterValue,
+			prometheus.GaugeValue,
 			data.W3SVCW3WPMaximumThreads,
 			name,
 			pid,
@@ -549,7 +549,7 @@ func (c *Collector) collectW3SVCW3WPv7(ch chan<- prometheus.Metric) error {
 
 		ch <- prometheus.MustNewConstMetric(
 			c.w3SVCW3WPRequestsActive,
-			prometheus.CounterValue,
+			prometheus.GaugeValue,
 			data.W3SVCW3WPRequestsActive,
 			name,
 			pid,

internal/collector/iis/metric_types_test.go

@@ -0,0 +1,199 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build windows
+
+package iis
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/require"
+)
+
+type gatheredMetric struct {
+	kind   dto.MetricType
+	value  float64
+	labels map[string]string
+}
+
+// gatherCollect registers collect with a registry and returns the gathered
+// metrics by name, so the test checks names, types and labels as published.
+func gatherCollect(t *testing.T, collect func(ch chan<- prometheus.Metric) error) map[string][]gatheredMetric {
+	t.Helper()
+
+	ch := make(chan prometheus.Metric, 1000)
+	require.NoError(t, collect(ch))
+	close(ch)
+
+	collected := make([]prometheus.Metric, 0, len(ch))
+	for metric := range ch {
+		collected = append(collected, metric)
+	}
+
+	registry := prometheus.NewPedanticRegistry()
+	require.NoError(t, registry.Register(collectFunc(func(ch chan<- prometheus.Metric) {
+		for _, metric := range collected {
+			ch <- metric
+		}
+	})))
+
+	families, err := registry.Gather()
+	require.NoError(t, err)
+
+	result := make(map[string][]gatheredMetric)
+
+	for _, family := range families {
+		for _, metric := range family.GetMetric() {
+			labels := make(map[string]string)
+			for _, label := range metric.GetLabel() {
+				labels[label.GetName()] = label.GetValue()
+			}
+
+			var value float64
+
+			switch family.GetType() {
+			case dto.MetricType_COUNTER:
+				value = metric.GetCounter().GetValue()
+			case dto.MetricType_GAUGE:
+				value = metric.GetGauge().GetValue()
+			default:
+				t.Fatalf("unexpected type %s for %s", family.GetType(), family.GetName())
+			}
+
+			result[family.GetName()] = append(result[family.GetName()], gatheredMetric{kind: family.GetType(), value: value, labels: labels})
+		}
+	}
+
+	return result
+}
+
+// collectFunc is an unchecked collector: it describes no metrics, so the
+// registry validates the metrics it gathers instead of the descriptions.
+type collectFunc func(ch chan<- prometheus.Metric)
+
+func (collectFunc) Describe(chan<- *prometheus.Desc) {}
+
+func (f collectFunc) Collect(ch chan<- prometheus.Metric) { f(ch) }
+
+func requireMetric(t *testing.T, metrics map[string][]gatheredMetric, name string, kind dto.MetricType, value float64, labels map[string]string) {
+	t.Helper()
+
+	require.Len(t, metrics[name], 1, name)
+	require.Equal(t, kind, metrics[name][0].kind, name)
+	require.InDelta(t, value, metrics[name][0].value, 1e-9, name)
+
+	for key, expected := range labels {
+		require.Equal(t, expected, metrics[name][0].labels[key], "%s label %s", name, key)
+	}
+}
+
+func TestAppPoolWASUptimeMetrics(t *testing.T) {
+	t.Parallel()
+
+	c := New(nil)
+	c.buildAppPoolWASDescriptors()
+	c.perfDataCollectorAppPoolWAS = cacheFixture[perfDataCounterValuesAppPoolWAS]{sources: map[string]float64{
+		// internal/pdh turns the PERF_ELAPSED_TIME start times into elapsed seconds.
+		"Current Application Pool Uptime":        298,
+		"Total Application Pool Uptime":          600,
+		"Time Since Last Worker Process Failure": 298,
+		"Total Application Pool Recycles":        2,
+	}}
+
+	metrics := gatherCollect(t, c.collectAppPoolWAS)
+	app := map[string]string{"app": "123_Example"}
+
+	requireMetric(t, metrics, "windows_iis_current_application_pool_uptime_seconds", dto.MetricType_GAUGE, 298, app)
+	requireMetric(t, metrics, "windows_iis_total_application_pool_uptime_seconds", dto.MetricType_GAUGE, 600, app)
+	requireMetric(t, metrics, "windows_iis_time_since_last_worker_process_failure_seconds", dto.MetricType_GAUGE, 298, app)
+	requireMetric(t, metrics, "windows_iis_total_application_pool_recycles", dto.MetricType_COUNTER, 2, app)
+	require.NotContains(t, metrics, "windows_iis_current_application_pool_start_time")
+	require.NotContains(t, metrics, "windows_iis_total_application_pool_start_time")
+}
+
+func TestHttpServiceRequestQueuesMetrics(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		name       string
+		maxAge     float64
+		wantMaxAge bool
+	}{
+		// An empty queue reports math.MaxInt64 as start time, which internal/pdh
+		// turns into a large negative number of seconds.
+		{name: "empty queue", maxAge: -922337203685.4775, wantMaxAge: false},
+		{name: "waiting request", maxAge: 1.5, wantMaxAge: true},
+		{name: "request just queued", maxAge: 0, wantMaxAge: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			c := New(nil)
+			c.buildHttpServiceRequestQueuesDescriptors()
+			c.perfDataCollectorHttpServiceRequestQueues = cacheFixture[perfDataCounterValuesHttpServiceRequestQueues]{sources: map[string]float64{
+				"CurrentQueueSize": 1,
+				"RejectedRequests": 3,
+				"MaxQueueItemAge":  tc.maxAge,
+				"ArrivalRate":      67,
+			}}
+
+			metrics := gatherCollect(t, c.collectHttpServiceRequestQueues)
+			app := map[string]string{"app": "123_Example"}
+
+			requireMetric(t, metrics, "windows_iis_http_requests_current_queue_size", dto.MetricType_GAUGE, 1, app)
+			requireMetric(t, metrics, "windows_iis_http_requests_rejected_total", dto.MetricType_COUNTER, 3, app)
+			requireMetric(t, metrics, "windows_iis_http_requests_arrived_total", dto.MetricType_COUNTER, 67, app)
+
+			if tc.wantMaxAge {
+				requireMetric(t, metrics, "windows_iis_http_requests_max_queue_item_age_seconds", dto.MetricType_GAUGE, tc.maxAge, app)
+			} else {
+				require.NotContains(t, metrics, "windows_iis_http_requests_max_queue_item_age_seconds")
+			}
+		})
+	}
+}
+
+func TestW3SVCW3WPCurrentValuesAreGauges(t *testing.T) {
+	t.Parallel()
+
+	c := New(nil)
+	c.buildW3SVCW3WPDescriptors()
+
+	sources := map[string]float64{
+		"Maximum Threads Count":      256,
+		"Active Requests":            4,
+		"WebSocket Active Requests":  2,
+		"Total HTTP Requests Served": 10,
+	}
+	c.w3SVCW3WPPerfDataCollector = cacheFixture[perfDataCounterValuesW3SVCW3WP]{sources: sources}
+	c.w3SVCW3WPPerfDataCollectorV8 = cacheFixture[perfDataCounterValuesW3SVCW3WPV8]{sources: sources}
+
+	metrics := gatherCollect(t, func(ch chan<- prometheus.Metric) error {
+		if err := c.collectW3SVCW3WPv7(ch); err != nil {
+			return err
+		}
+
+		return c.collectW3SVCW3WPv8(ch)
+	})
+	labels := map[string]string{"app": "Example", "pid": "123"}
+
+	requireMetric(t, metrics, "windows_iis_worker_max_threads", dto.MetricType_GAUGE, 256, labels)
+	requireMetric(t, metrics, "windows_iis_worker_current_requests", dto.MetricType_GAUGE, 4, labels)
+	requireMetric(t, metrics, "windows_iis_worker_current_websocket_requests", dto.MetricType_GAUGE, 2, labels)
+	requireMetric(t, metrics, "windows_iis_worker_requests_total", dto.MetricType_COUNTER, 10, labels)
+}