sendnrw/windows_exporter
2
0
Fork 0
mirror of https://github.com/prometheus-community/windows_exporter.git synced 2026-10-20 11:39:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

collector: serialize collector runs and harden the scrape lifecycle (#2541)

Browse Source 
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>
This commit is contained in:
Jan-Otto Kröpke
2026-10-08 14:25:17 +02:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 16aa107c9f
commit 96a998772b
11 changed files with 1554 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/httphandler/httphandler.go
+58 -29
View File
@@ -18,8 +18,10 @@
package httphandler
import (
"context"
"fmt"
"log/slog"
"math"
"net/http"
"strconv"
"time"
@@ -34,7 +36,15 @@ import (
// Interface guard.
var _ http.Handler = (*MetricsHTTPHandler)(nil)
const defaultScrapeTimeout = 10.0
const (
// defaultScrapeTimeout applies if the request has no valid X-Prometheus-Scrape-Timeout-Seconds header.
defaultScrapeTimeout = 10 * time.Second
// minScrapeTimeout is the smallest time given to the collectors.
minScrapeTimeout = 10 * time.Millisecond
// maxScrapeTimeout is the largest time given to the collectors.
// The exporter's HTTP server ends responses after 5 minutes anyway.
maxScrapeTimeout = 5 * time.Minute
)
type MetricsHTTPHandler struct {
metricCollectors *collector.Collection
@@ -42,8 +52,8 @@ type MetricsHTTPHandler struct {
// the exporter itself.
exporterMetricsRegistry *prometheus.Registry
logger *slog.Logger
options Options
logger *slog.Logger
timeoutMargin time.Duration
}
type Options struct {
@@ -62,7 +72,7 @@ func New(logger *slog.Logger, metricCollectors *collector.Collection, options *O
handler := &MetricsHTTPHandler{
metricCollectors: metricCollectors,
logger: logger,
options: *options,
timeoutMargin: toDuration(options.TimeoutMargin),
}
if !options.DisableExporterMetrics {
@@ -82,9 +92,10 @@ func (c *MetricsHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
slog.String("remote", r.RemoteAddr),
)
// The scrape's deadline starts now, so time spent waiting for an earlier scrape counts against it.
scrapeTimeout := c.getScrapeTimeout(logger, r)
handler, err := c.handlerFactory(logger, scrapeTimeout, r.URL.Query()["collect[]"])
handler, err := c.handlerFactory(r.Context(), logger, scrapeTimeout, r.URL.Query()["collect[]"])
if err != nil {
logger.WarnContext(r.Context(), "Couldn't create filtered metrics handler",
slog.Any("err", err),
@@ -99,32 +110,51 @@ func (c *MetricsHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
handler.ServeHTTP(w, r)
}
// getScrapeTimeout returns the time the collectors get for this request.
// It is the client's X-Prometheus-Scrape-Timeout-Seconds minus the timeout margin,
// but the margin takes at most half of the client's timeout.
// The result is between minScrapeTimeout and maxScrapeTimeout.
func (c *MetricsHTTPHandler) getScrapeTimeout(logger *slog.Logger, r *http.Request) time.Duration {
var timeoutSeconds float64
timeout := defaultScrapeTimeout
if v := r.Header.Get("X-Prometheus-Scrape-Timeout-Seconds"); v != "" {
var err error
seconds, err := strconv.ParseFloat(v, 64)
timeoutSeconds, err = strconv.ParseFloat(v, 64)
if err != nil {
logger.WarnContext(r.Context(), fmt.Sprintf("Couldn't parse X-Prometheus-Scrape-Timeout-Seconds: %q. Defaulting timeout to %f", v, defaultScrapeTimeout))
switch {
case math.IsInf(seconds, 1):
// Includes values too large for a float64, which ParseFloat reports as an error.
timeout = maxScrapeTimeout
case err != nil, math.IsNaN(seconds), seconds <= 0:
logger.WarnContext(r.Context(), fmt.Sprintf("Invalid X-Prometheus-Scrape-Timeout-Seconds: %q. Defaulting timeout to %s", v, defaultScrapeTimeout))
default:
timeout = toDuration(seconds)
}
}
if timeoutSeconds == 0 {
timeoutSeconds = defaultScrapeTimeout
}
timeout = max(timeout-c.timeoutMargin, timeout/2)
timeoutSeconds -= c.options.TimeoutMargin
return time.Duration(timeoutSeconds*1e9) * time.Nanosecond
return min(max(timeout, minScrapeTimeout), maxScrapeTimeout)
}
func (c *MetricsHTTPHandler) handlerFactory(logger *slog.Logger, scrapeTimeout time.Duration, requestedCollectors []string) (http.Handler, error) {
// toDuration converts seconds to a duration between 0 and maxScrapeTimeout.
// NaN and negative values become 0.
func toDuration(seconds float64) time.Duration {
if math.IsNaN(seconds) || seconds <= 0 {
return 0
}
if seconds >= maxScrapeTimeout.Seconds() {
return maxScrapeTimeout
}
return time.Duration(seconds * float64(time.Second))
}
func (c *MetricsHTTPHandler) handlerFactory(ctx context.Context, logger *slog.Logger, scrapeTimeout time.Duration, requestedCollectors []string) (http.Handler, error) {
reg := prometheus.NewRegistry()
reg.MustRegister(version.NewCollector("windows_exporter"))
collectionHandler, err := c.metricCollectors.NewHandler(scrapeTimeout, c.logger, requestedCollectors)
collectionHandler, err := c.metricCollectors.NewHandlerWithContext(ctx, scrapeTimeout, logger, requestedCollectors)
if err != nil {
return nil, fmt.Errorf("couldn't create collector handler: %w", err)
}
@@ -133,17 +163,17 @@ func (c *MetricsHTTPHandler) handlerFactory(logger *slog.Logger, scrapeTimeout t
return nil, fmt.Errorf("couldn't register Prometheus collector: %w", err)
}
// Scrapes are serialized by the collection, so the handler doesn't limit requests in flight.
var regHandler http.Handler
if c.exporterMetricsRegistry != nil {
regHandler = promhttp.HandlerFor(
prometheus.Gatherers{c.exporterMetricsRegistry, reg},
promhttp.HandlerOpts{
ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
ErrorHandling: promhttp.ContinueOnError,
MaxRequestsInFlight: 1,
Registry: c.exporterMetricsRegistry,
EnableOpenMetrics: true,
ProcessStartTime: c.metricCollectors.GetStartTime(),
ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
ErrorHandling: promhttp.ContinueOnError,
Registry: c.exporterMetricsRegistry,
EnableOpenMetrics: true,
ProcessStartTime: c.metricCollectors.GetStartTime(),
},
)
@@ -156,11 +186,10 @@ func (c *MetricsHTTPHandler) handlerFactory(logger *slog.Logger, scrapeTimeout t
regHandler = promhttp.HandlerFor(
reg,
promhttp.HandlerOpts{
ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
ErrorHandling: promhttp.ContinueOnError,
MaxRequestsInFlight: 1,
EnableOpenMetrics: true,
ProcessStartTime: c.metricCollectors.GetStartTime(),
ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
ErrorHandling: promhttp.ContinueOnError,
EnableOpenMetrics: true,
ProcessStartTime: c.metricCollectors.GetStartTime(),
},
)
}
internal/httphandler/httphandler_test.go
+83
View File
@@ -18,8 +18,10 @@
package httphandler_test
import (
"context"
"errors"
"log/slog"
"math"
"net/http"
"net/http/httptest"
"testing"
@@ -154,6 +156,87 @@ func TestMetricsHTTPHandlerScrapeTimeout(t *testing.T) {
}
}
func TestMetricsHTTPHandlerScrapeBudget(t *testing.T) {
const maxScrapeTimeout = 5 * time.Minute
for _, tc := range []struct {
name string
header string
margin float64
want time.Duration
}{
{name: "missing header", margin: 0.5, want: 9500 * time.Millisecond},
{name: "invalid header", header: "invalid", margin: 0.5, want: 9500 * time.Millisecond},
{name: "zero", header: "0", margin: 0.5, want: 9500 * time.Millisecond},
{name: "negative", header: "-1", margin: 0.5, want: 9500 * time.Millisecond},
{name: "negative infinity", header: "-Inf", margin: 0.5, want: 9500 * time.Millisecond},
{name: "NaN", header: "NaN", margin: 0.5, want: 9500 * time.Millisecond},
{name: "infinity", header: "+Inf", margin: 0.5, want: maxScrapeTimeout - 500*time.Millisecond},
{name: "out of float range", header: "1e400", margin: 0.5, want: maxScrapeTimeout - 500*time.Millisecond},
{name: "huge", header: "1e12", margin: 0.5, want: maxScrapeTimeout - 500*time.Millisecond},
{name: "long", header: "120", margin: 0.5, want: 119500 * time.Millisecond},
{name: "regular", header: "2", margin: 0.5, want: 1500 * time.Millisecond},
{name: "margin larger than timeout", header: "0.3", margin: 0.5, want: 150 * time.Millisecond},
{name: "margin takes at most half", header: "0.8", margin: 0.5, want: 400 * time.Millisecond},
{name: "tiny", header: "0.000001", margin: 0.5, want: 10 * time.Millisecond},
{name: "no margin", header: "2", want: 2 * time.Second},
{name: "negative margin", header: "2", margin: -1, want: 2 * time.Second},
{name: "NaN margin", header: "2", margin: math.NaN(), want: 2 * time.Second},
} {
t.Run(tc.name, func(t *testing.T) {
// Synthetic time doesn't pass between the request and the collector's call.
synctest.Test(t, func(t *testing.T) {
test := &budgetCollector{}
collection := collector.New(collector.Map{"test": test})
handler := httphandler.New(slog.New(slog.DiscardHandler), collection, &httphandler.Options{DisableExporterMetrics: true, TimeoutMargin: tc.margin})
request := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics", nil)
if tc.header != "" {
request.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", tc.header)
}
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
require.Equal(t, http.StatusOK, response.Code)
require.Contains(t, response.Body.String(), `windows_exporter_collector_success{collector="test"} 1`)
require.Equal(t, tc.want, test.budget)
})
})
}
}
func TestMetricsHTTPHandlerCanceledRequest(t *testing.T) {
test := &budgetCollector{}
collection := collector.New(collector.Map{"test": test})
handler := httphandler.New(slog.New(slog.DiscardHandler), collection, &httphandler.Options{DisableExporterMetrics: true})
ctx, cancel := context.WithCancel(t.Context())
cancel()
response := httptest.NewRecorder()
handler.ServeHTTP(response, httptest.NewRequestWithContext(ctx, http.MethodGet, "/metrics", nil))
// The client is gone, so the collectors don't run.
require.False(t, test.called)
}
// budgetCollector records the time it is given.
type budgetCollector struct {
called bool
budget time.Duration
}
func (c *budgetCollector) GetName() string { return "test" }
func (c *budgetCollector) Build(_ *slog.Logger, _ *mi.Session) error { return nil }
func (c *budgetCollector) Close() error { return nil }
func (c *budgetCollector) Collect(_ chan<- prometheus.Metric, maxScrapeDuration time.Duration) error {
c.called = true
c.budget = maxScrapeDuration
return nil
}
type testCollector struct {
name string
err error