diff --git a/AGENTS.md b/AGENTS.md index f950327e..7a1cc820 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -174,9 +174,10 @@ or enabled collector set from your checkout. ``` Inspect `windows_exporter_collector_success`, -`windows_exporter_collector_duration_seconds` and -`windows_exporter_collector_timeout`, plus the affected metric families. HTTP 200 -alone does not mean every collector succeeded. PDH rate counters need a second +`windows_exporter_collector_duration_seconds`, +`windows_exporter_collector_timeout` and `windows_exporter_scrape_queue_timeout` +(the scrape timed out waiting for an earlier scrape, so no collector ran), plus the +affected metric families. HTTP 200 alone does not mean every collector succeeded. PDH rate counters need a second sample after initialization. To validate changed code, build a separate executable and run it on an unused diff --git a/UPGRADE.md b/UPGRADE.md index 00a07aca..7e6eabcf 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -398,6 +398,7 @@ The scrape timeout comes from the `X-Prometheus-Scrape-Timeout-Seconds` header, - The margin takes at most half of the client's timeout. A short `scrape_timeout` no longer leaves a negative or near-zero budget. - The timeout is limited to 5 minutes. Invalid header values fall back to 10 seconds. - If a collector is still running from an earlier, timed-out scrape, later scrapes skip it and report `windows_exporter_collector_timeout` as `1` until it finishes. In 0.31 a second copy of the collector started and ran in parallel. +- Scrapes run one at a time. In 0.31 concurrent scrapes ran in parallel: the in-flight limit applied to a handler that was created for each request, and the collection's concurrency channel was never used. A scrape whose timeout ends while it waits for an earlier scrape doesn't run any collector. It reports the new metric `windows_exporter_scrape_queue_timeout` as `1`, and `windows_exporter_collector_success` and `windows_exporter_collector_timeout` as `0` for every collector. `windows_exporter_collector_timeout` is `1` only for collectors that ran out of time or are still running from an earlier scrape. - A collector that failed to start, for example because its Windows feature isn't installed, reports `windows_exporter_collector_success` as `0` and is no longer called on every scrape. It logs one warning at startup. The exporter's own metrics now also include Go scheduler metrics (`go_sched_*`), unless `--web.disable-exporter-metrics` is set. diff --git a/contrib/mixin/dashboards/exporter.libsonnet b/contrib/mixin/dashboards/exporter.libsonnet index 70265f99..f91042e4 100644 --- a/contrib/mixin/dashboards/exporter.libsonnet +++ b/contrib/mixin/dashboards/exporter.libsonnet @@ -112,10 +112,13 @@ function(on) graphMode: 'area', }), - b.panel.new(94, 'Timed-out collectors', 'stat') - + b.panel.withDescription('Number of collectors that timed out on the latest scrape. The graph shows the count over the selected time range.') + b.panel.new(94, 'Timeouts', 'stat') + + b.panel.withDescription('Number of collectors that timed out on the latest scrape, and whether the latest scrape timed out while it waited for an earlier scrape, so that no collector ran. The graph shows the values over the selected time range.') + b.panel.withQueries([ - b.query.new('count(windows_exporter_collector_timeout{job=~"$job", instance="$instance"} == 1) or vector(0)', 'A'), + b.query.new('count(windows_exporter_collector_timeout{job=~"$job", instance="$instance"} == 1) or vector(0)', 'A') + + b.query.withLegendFormat('collectors'), + b.query.new('max(windows_exporter_scrape_queue_timeout{job=~"$job", instance="$instance"}) or vector(0)', 'B') + + b.query.withLegendFormat('queued scrape'), ]) + b.panel.withDefaults({ decimals: 0, diff --git a/dashboard/windows-exporter-dashboard.json b/dashboard/windows-exporter-dashboard.json index d0f2c9c7..bd13e566 100644 --- a/dashboard/windows-exporter-dashboard.json +++ b/dashboard/windows-exporter-dashboard.json @@ -25943,23 +25943,45 @@ "editorMode": "code", "expr": "count(windows_exporter_collector_timeout{job=~\"$job\", instance=\"$instance\"} == 1) or vector(0)", "instant": false, - "legendFormat": "__auto", + "legendFormat": "collectors", "range": true }, "version": "v0" }, "refId": "A" } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "max(windows_exporter_scrape_queue_timeout{job=~\"$job\", instance=\"$instance\"}) or vector(0)", + "instant": false, + "legendFormat": "queued scrape", + "range": true + }, + "version": "v0" + }, + "refId": "B" + } } ], "queryOptions": {}, "transformations": [] } }, - "description": "Number of collectors that timed out on the latest scrape. The graph shows the count over the selected time range.", + "description": "Number of collectors that timed out on the latest scrape, and whether the latest scrape timed out while it waited for an earlier scrape, so that no collector ran. The graph shows the values over the selected time range.", "id": 94, "links": [], - "title": "Timed-out collectors", + "title": "Timeouts", "vizConfig": { "group": "stat", "kind": "VizConfig", diff --git a/dashboard/windows-exporter-exporter.json b/dashboard/windows-exporter-exporter.json index dc184d74..c636e26c 100644 --- a/dashboard/windows-exporter-exporter.json +++ b/dashboard/windows-exporter-exporter.json @@ -1975,23 +1975,45 @@ "editorMode": "code", "expr": "count(windows_exporter_collector_timeout{job=~\"$job\", instance=\"$instance\"} == 1) or vector(0)", "instant": false, - "legendFormat": "__auto", + "legendFormat": "collectors", "range": true }, "version": "v0" }, "refId": "A" } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${datasource}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "max(windows_exporter_scrape_queue_timeout{job=~\"$job\", instance=\"$instance\"}) or vector(0)", + "instant": false, + "legendFormat": "queued scrape", + "range": true + }, + "version": "v0" + }, + "refId": "B" + } } ], "queryOptions": {}, "transformations": [] } }, - "description": "Number of collectors that timed out on the latest scrape. The graph shows the count over the selected time range.", + "description": "Number of collectors that timed out on the latest scrape, and whether the latest scrape timed out while it waited for an earlier scrape, so that no collector ran. The graph shows the values over the selected time range.", "id": 94, "links": [], - "title": "Timed-out collectors", + "title": "Timeouts", "vizConfig": { "group": "stat", "kind": "VizConfig", diff --git a/docs/runbooks/windowscollectorfailed.md b/docs/runbooks/windowscollectorfailed.md index ae59bd56..be1e794f 100644 --- a/docs/runbooks/windowscollectorfailed.md +++ b/docs/runbooks/windowscollectorfailed.md @@ -27,8 +27,14 @@ metrics may stop evaluating even though the exporter remains reachable. privileges, and performance counters. Confirm the required service or role exists and is running on the host. 4. Compare `windows_exporter_collector_duration_seconds` with the scrape timeout. - Test the relevant performance counters locally with Windows Performance + `windows_exporter_collector_timeout == 1` means the collector ran out of + time, or its call from an earlier scrape is still running. Test the relevant performance counters locally with Windows Performance Monitor. +5. If every collector of the host fails at the same time, check + `windows_exporter_scrape_queue_timeout`. A value of `1` means the scrape + ended while it waited for an earlier scrape, so no collector ran. Several + Prometheus servers or jobs that scrape the same exporter, or a scrape that + uses most of its timeout, cause this. ## Mitigation diff --git a/internal/httphandler/httphandler_test.go b/internal/httphandler/httphandler_test.go index a9f09f92..d685f881 100644 --- a/internal/httphandler/httphandler_test.go +++ b/internal/httphandler/httphandler_test.go @@ -194,6 +194,8 @@ func TestMetricsHTTPHandlerScrapeTimeout(t *testing.T) { handler.ServeHTTP(response, request) require.Equal(t, http.StatusOK, response.Code) + // A collector timeout is not a queued scrape. + require.Contains(t, response.Body.String(), "windows_exporter_scrape_queue_timeout 0") if tc.timeout { require.Contains(t, response.Body.String(), `windows_exporter_collector_timeout{collector="test"} 1`) diff --git a/pkg/collector/collect.go b/pkg/collector/collect.go index 67d0a280..4fff98bf 100644 --- a/pkg/collector/collect.go +++ b/pkg/collector/collect.go @@ -47,6 +47,9 @@ const ( failed // unavailable means the collector was skipped because its Build failed or the collection is closed. unavailable + // notStarted means the scrape ended before it started any collector, because an earlier scrape + // held the scrape slot until the scrape timed out or was canceled. The collector itself didn't time out. + notStarted ) // collectAll runs all collectors once. It returns when ctx is done, even if collectors are still running. @@ -54,7 +57,7 @@ const ( func (c *Collection) collectAll(ctx context.Context, ch chan<- prometheus.Metric, logger *slog.Logger) { collectorStartTime := time.Now() - if err := acquireScrapeSlot(ctx); err != nil { + if err := c.state.acquireScrapeSlot(ctx); err != nil { level := slog.LevelWarn if errors.Is(err, context.Canceled) { level = slog.LevelDebug @@ -65,15 +68,17 @@ func (c *Collection) collectAll(ctx context.Context, ch chan<- prometheus.Metric ) for name := range c.collectors { - c.sendCollectorStatus(ch, collectorStatus{name: name, statusCode: pending}) + c.sendCollectorStatus(ch, collectorStatus{name: name, statusCode: notStarted}) } + // A canceled scrape, for example at shutdown or after the client went away, did not time out. + c.sendScrapeQueueTimeout(ch, errors.Is(err, context.DeadlineExceeded)) c.sendScrapeDuration(ch, collectorStartTime) return } - defer releaseScrapeSlot() + defer c.state.releaseScrapeSlot() // WaitGroup to wait for all collectors to finish wg := sync.WaitGroup{} @@ -105,9 +110,24 @@ func (c *Collection) collectAll(ctx context.Context, ch chan<- prometheus.Metric c.sendCollectorStatus(ch, status) } + c.sendScrapeQueueTimeout(ch, false) c.sendScrapeDuration(ch, collectorStartTime) } +// sendScrapeQueueTimeout reports whether the scrape ended while it waited for an earlier scrape. +func (c *Collection) sendScrapeQueueTimeout(ch chan<- prometheus.Metric, timedOut bool) { + var value float64 + if timedOut { + value = 1.0 + } + + ch <- prometheus.MustNewConstMetric( + c.scrapeQueueTimeoutDesc, + prometheus.GaugeValue, + value, + ) +} + func (c *Collection) sendCollectorStatus(ch chan<- prometheus.Metric, status collectorStatus) { var successValue, timeoutValue float64 if status.statusCode == pending { diff --git a/pkg/collector/collect_test.go b/pkg/collector/collect_test.go index 50ad3e60..192de785 100644 --- a/pkg/collector/collect_test.go +++ b/pkg/collector/collect_test.go @@ -40,9 +40,9 @@ import ( "golang.org/x/sys/windows" ) -// The tests in this file share the package-level scrape slot, so they don't run in parallel. - func TestCollectErrorStatus(t *testing.T) { + t.Parallel() + failure := errors.New("collector failed") for _, tc := range []struct { @@ -106,6 +106,8 @@ func TestCollectErrorStatus(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { + t.Parallel() + logs := &recordHandler{} collection := collector.New(collector.Map{"test": &fakeCollector{name: "test", collectErr: tc.err}}) handler, err := collection.NewHandler(time.Second, slog.New(logs), nil) @@ -114,6 +116,7 @@ func TestCollectErrorStatus(t *testing.T) { out := scrape(t, handler) require.Contains(t, out, `windows_exporter_collector_success{collector="test"} `+tc.success) require.Contains(t, out, `windows_exporter_collector_timeout{collector="test"} 0`) + require.Contains(t, out, "windows_exporter_scrape_queue_timeout 0") require.Contains(t, out, "windows_test_test 42", "partial metrics must still be exported") warnings := 0 @@ -127,6 +130,8 @@ func TestCollectErrorStatus(t *testing.T) { } func TestCollectTimedOutCollectorDoesNotOverlap(t *testing.T) { + t.Parallel() + // The collector's delay and the scrape deadlines use synthetic time. synctest.Test(t, func(t *testing.T) { logs := &recordHandler{} @@ -155,6 +160,8 @@ func TestCollectTimedOutCollectorDoesNotOverlap(t *testing.T) { require.Contains(t, out, `windows_exporter_collector_timeout{collector="slow"} 1`) require.Contains(t, out, `windows_exporter_collector_success{collector="slow"} 0`) require.NotContains(t, out, "windows_test_slow") + // The scrape itself started, only the collector timed out. + require.Contains(t, out, "windows_exporter_scrape_queue_timeout 0") if handler == all { require.Contains(t, out, `windows_exporter_collector_success{collector="fast"} 1`) @@ -186,7 +193,11 @@ func TestCollectTimedOutCollectorDoesNotOverlap(t *testing.T) { } func TestCollectAdmission(t *testing.T) { + t.Parallel() + t.Run("canceled while queued", func(t *testing.T) { + t.Parallel() + running := newBlockingCollector() collection := collector.New(collector.Map{"test": running}) first := scrapeInBackground(t.Context(), collection, time.Minute) @@ -202,7 +213,7 @@ func TestCollectAdmission(t *testing.T) { select { case out := <-second: - require.Contains(t, out, `windows_exporter_collector_timeout{collector="test"} 1`) + requireNotStarted(t, out, "test", false) case <-time.After(10 * time.Second): t.Fatal("canceled scrape didn't return") } @@ -213,6 +224,8 @@ func TestCollectAdmission(t *testing.T) { }) t.Run("deadline passes while queued", func(t *testing.T) { + t.Parallel() + running := newBlockingCollector() collection := collector.New(collector.Map{"test": running}) first := scrapeInBackground(t.Context(), collection, time.Minute) @@ -224,18 +237,72 @@ func TestCollectAdmission(t *testing.T) { select { case out := <-second: - require.Contains(t, out, `windows_exporter_collector_timeout{collector="test"} 1`) + requireNotStarted(t, out, "test", true) require.Less(t, time.Since(start), 10*time.Second) case <-time.After(10 * time.Second): t.Fatal("queued scrape didn't return at its deadline") } close(running.block) - <-first + + out := <-first + require.Contains(t, out, `windows_exporter_collector_success{collector="test"} 1`) + require.Contains(t, out, "windows_exporter_scrape_queue_timeout 0") require.Equal(t, int32(1), running.calls.Load()) }) + t.Run("filtered view queues behind its collection", func(t *testing.T) { + t.Parallel() + + running := newBlockingCollector() + other := &fakeCollector{name: "other"} + collection := collector.New(collector.Map{"test": running, "other": other}) + first := scrapeInBackground(t.Context(), collection, time.Minute, "test") + + <-running.started + + // The view scrapes only a collector that is idle, but it shares the scrape slot with the collection. + second := scrapeInBackground(t.Context(), collection, 100*time.Millisecond, "other") + + select { + case out := <-second: + requireNotStarted(t, out, "other", true) + case <-time.After(10 * time.Second): + t.Fatal("queued scrape didn't return at its deadline") + } + + close(running.block) + <-first + require.Equal(t, int32(0), other.calls.Load()) + }) + + t.Run("independent collections don't queue", func(t *testing.T) { + t.Parallel() + + running := newBlockingCollector() + first := scrapeInBackground(t.Context(), collector.New(collector.Map{"test": running}), time.Minute) + + <-running.started + + independent := &fakeCollector{name: "test"} + second := scrapeInBackground(t.Context(), collector.New(collector.Map{"test": independent}), time.Minute) + + select { + case out := <-second: + require.Contains(t, out, `windows_exporter_collector_success{collector="test"} 1`) + require.Contains(t, out, "windows_exporter_scrape_queue_timeout 0") + case <-time.After(10 * time.Second): + t.Fatal("scrape of an independent collection waited for another collection") + } + + close(running.block) + <-first + require.Equal(t, int32(1), independent.calls.Load()) + }) + t.Run("canceled before admission", func(t *testing.T) { + t.Parallel() + test := &fakeCollector{name: "test"} collection := collector.New(collector.Map{"test": test}) @@ -245,11 +312,13 @@ func TestCollectAdmission(t *testing.T) { handler, err := collection.NewHandlerWithContext(ctx, time.Minute, slog.New(slog.DiscardHandler), nil) require.NoError(t, err) - require.Contains(t, scrape(t, handler), `windows_exporter_collector_timeout{collector="test"} 1`) + requireNotStarted(t, scrape(t, handler), "test", false) require.Equal(t, int32(0), test.calls.Load()) }) t.Run("deadline starts when the handler is created", func(t *testing.T) { + t.Parallel() + synctest.Test(t, func(t *testing.T) { test := &fakeCollector{name: "test"} collection := collector.New(collector.Map{"test": test}) @@ -266,6 +335,8 @@ func TestCollectAdmission(t *testing.T) { } func TestCollectFailedBuild(t *testing.T) { + t.Parallel() + for _, tc := range []struct { name string buildErr error @@ -277,6 +348,8 @@ func TestCollectFailedBuild(t *testing.T) { {name: "panic", panics: true, wantErr: "panic: build panic"}, } { t.Run(tc.name, func(t *testing.T) { + t.Parallel() + logs := &recordHandler{} logger := slog.New(logs) @@ -322,6 +395,8 @@ func TestCollectFailedBuild(t *testing.T) { } func TestBuildCanceled(t *testing.T) { + t.Parallel() + building := &fakeCollector{name: "building", buildBlock: make(chan struct{}), buildStarted: make(chan struct{}, 1)} collection := collector.New(collector.Map{"building": building}) @@ -358,7 +433,11 @@ func TestBuildCanceled(t *testing.T) { } func TestCollectionClose(t *testing.T) { + t.Parallel() + t.Run("without Build", func(t *testing.T) { + t.Parallel() + test := &fakeCollector{name: "test"} collection := collector.New(collector.Map{"test": test}) @@ -368,6 +447,8 @@ func TestCollectionClose(t *testing.T) { }) t.Run("idempotent", func(t *testing.T) { + t.Parallel() + test := &fakeCollector{name: "test"} collection := collector.New(collector.Map{"test": test}) require.NoError(t, collection.Build(t.Context(), slog.New(slog.DiscardHandler))) @@ -384,6 +465,8 @@ func TestCollectionClose(t *testing.T) { }) t.Run("waits for a running Collect", func(t *testing.T) { + t.Parallel() + synctest.Test(t, func(t *testing.T) { slow := &fakeCollector{name: "slow", delay: 2 * time.Second} collection := collector.New(collector.Map{"slow": slow}) @@ -400,6 +483,8 @@ func TestCollectionClose(t *testing.T) { }) t.Run("leaves a hung collector open", func(t *testing.T) { + t.Parallel() + synctest.Test(t, func(t *testing.T) { hung := &fakeCollector{name: "hung", delay: time.Minute} collection := collector.New(collector.Map{"hung": hung}) @@ -423,6 +508,7 @@ func TestCollectionClose(t *testing.T) { }) } +// TestBuildCloseCyclesDoNotLeakGoroutines doesn't run in parallel, because it counts the goroutines of the process. func TestBuildCloseCyclesDoNotLeakGoroutines(t *testing.T) { // The bubble also fails the test if any goroutine of it is still blocked at the end. synctest.Test(t, func(t *testing.T) { @@ -555,11 +641,27 @@ func (c *blockingCollector) Collect(ch chan<- prometheus.Metric, maxScrapeDurati return c.fakeCollector.Collect(ch, maxScrapeDuration) } +// requireNotStarted checks that a scrape didn't start the collectors and didn't blame them for it. +// timedOut tells whether the scrape timed out in the queue, rather than being canceled. +func requireNotStarted(t *testing.T, out, name string, timedOut bool) { + t.Helper() + + if timedOut { + require.Contains(t, out, "windows_exporter_scrape_queue_timeout 1") + } else { + require.Contains(t, out, "windows_exporter_scrape_queue_timeout 0") + } + + require.Contains(t, out, fmt.Sprintf(`windows_exporter_collector_success{collector=%q} 0`, name)) + require.Contains(t, out, fmt.Sprintf(`windows_exporter_collector_timeout{collector=%q} 0`, name)) +} + // scrapeInBackground starts a scrape with a handler bound to ctx and returns its output, or the error text. -func scrapeInBackground(ctx context.Context, collection *collector.Collection, timeout time.Duration) <-chan string { +// It scrapes the given collectors through a filtered view, or the whole collection if there are none. +func scrapeInBackground(ctx context.Context, collection *collector.Collection, timeout time.Duration, collectors ...string) <-chan string { out := make(chan string, 1) - handler, err := collection.NewHandlerWithContext(ctx, timeout, slog.New(slog.DiscardHandler), nil) + handler, err := collection.NewHandlerWithContext(ctx, timeout, slog.New(slog.DiscardHandler), collectors) if err != nil { out <- err.Error() diff --git a/pkg/collector/collection.go b/pkg/collector/collection.go index 33663725..7df29381 100644 --- a/pkg/collector/collection.go +++ b/pkg/collector/collection.go @@ -166,6 +166,12 @@ func New(collectors Map) *Collection { nil, nil, ), + scrapeQueueTimeoutDesc: prometheus.NewDesc( + prometheus.BuildFQName(types.Namespace, "exporter", "scrape_queue_timeout"), + "windows_exporter: Whether the scrape timed out while it waited for an earlier scrape, so that it didn't start the collectors.", + nil, + nil, + ), collectorScrapeDurationDesc: prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, "exporter", "collector_duration_seconds"), "windows_exporter: Duration of a collection.", @@ -401,6 +407,7 @@ func (c *Collection) WithCollectors(collectors []string) (*Collection, error) { state: c.state, isView: true, scrapeDurationDesc: c.scrapeDurationDesc, + scrapeQueueTimeoutDesc: c.scrapeQueueTimeoutDesc, collectorScrapeDurationDesc: c.collectorScrapeDurationDesc, collectorScrapeSuccessDesc: c.collectorScrapeSuccessDesc, collectorScrapeTimeoutDesc: c.collectorScrapeTimeoutDesc, diff --git a/pkg/collector/handler.go b/pkg/collector/handler.go index 86ff7c13..925f8305 100644 --- a/pkg/collector/handler.go +++ b/pkg/collector/handler.go @@ -29,13 +29,6 @@ import ( // Interface guard. var _ prometheus.Collector = (*Handler)(nil) -// scrapeSlot admits one scrape at a time, across all collections. -// We expose metrics directly from the memory region of the Win32 API. -// We should not allow more than one request at a time. -// -//nolint:gochecknoglobals -var scrapeSlot = make(chan struct{}, 1) - // Handler implements [prometheus.Collector] for a set of Windows Collection. type Handler struct { maxScrapeDuration time.Duration @@ -100,29 +93,3 @@ func (p *Handler) Collect(ch chan<- prometheus.Metric) { p.collection.collectAll(ctx, ch, p.logger) } - -// acquireScrapeSlot waits for the scrape slot until ctx is done. -func acquireScrapeSlot(ctx context.Context) error { - if err := ctx.Err(); err != nil { - return err - } - - select { - case scrapeSlot <- struct{}{}: - case <-ctx.Done(): - return ctx.Err() - } - - // Both cases may have been ready. Don't start a scrape that nobody waits for anymore. - if err := ctx.Err(); err != nil { - <-scrapeSlot - - return err - } - - return nil -} - -func releaseScrapeSlot() { - <-scrapeSlot -} diff --git a/pkg/collector/state.go b/pkg/collector/state.go index 6aaafd53..1541c127 100644 --- a/pkg/collector/state.go +++ b/pkg/collector/state.go @@ -40,12 +40,46 @@ type collectionState struct { miApp *mi.Application miSession *mi.Session + // scrapeSlot admits one scrape at a time to the collection and its filtered views. + // Independent collections don't share collector instances, so they don't wait for each other. + scrapeSlot chan struct{} + // closeMu serializes Close calls. closeMu sync.Mutex } func newCollectionState() *collectionState { - return &collectionState{collectors: make(map[string]*collectorState)} + return &collectionState{ + collectors: make(map[string]*collectorState), + scrapeSlot: make(chan struct{}, 1), + } +} + +// acquireScrapeSlot waits for the scrape slot until ctx is done. +func (s *collectionState) acquireScrapeSlot(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + + select { + case s.scrapeSlot <- struct{}{}: + case <-ctx.Done(): + return ctx.Err() + } + + // Both cases may have been ready. Don't start a scrape that nobody waits for anymore. + if err := ctx.Err(); err != nil { + <-s.scrapeSlot + + return err + } + + return nil +} + +// releaseScrapeSlot releases the slot acquired by acquireScrapeSlot. +func (s *collectionState) releaseScrapeSlot() { + <-s.scrapeSlot } // collector returns the state of the named collector, creating it on first use. diff --git a/pkg/collector/types.go b/pkg/collector/types.go index 1485ef46..91620ba2 100644 --- a/pkg/collector/types.go +++ b/pkg/collector/types.go @@ -38,6 +38,7 @@ type Collection struct { isView bool scrapeDurationDesc *prometheus.Desc + scrapeQueueTimeoutDesc *prometheus.Desc collectorScrapeDurationDesc *prometheus.Desc collectorScrapeSuccessDesc *prometheus.Desc collectorScrapeTimeoutDesc *prometheus.Desc diff --git a/tools/e2e-output.txt b/tools/e2e-output.txt index a7813e87..297c017c 100644 --- a/tools/e2e-output.txt +++ b/tools/e2e-output.txt @@ -147,6 +147,8 @@ windows_exporter_collector_timeout{collector="time"} 0 windows_exporter_collector_timeout{collector="udp"} 0 # HELP windows_exporter_scrape_duration_seconds windows_exporter: Total scrape duration. # TYPE windows_exporter_scrape_duration_seconds gauge +# HELP windows_exporter_scrape_queue_timeout windows_exporter: Whether the scrape timed out while it waited for an earlier scrape, so that it didn't start the collectors. +# TYPE windows_exporter_scrape_queue_timeout gauge # HELP windows_logical_disk_free_bytes Free space in bytes, updates every 10-15 min (LogicalDisk.PercentFreeSpace) # TYPE windows_logical_disk_free_bytes gauge # HELP windows_logical_disk_idle_seconds_total Seconds that the disk was idle (LogicalDisk.PercentIdleTime)