mirror of
https://github.com/prometheus-community/windows_exporter.git
synced 2026-10-18 02:29:04 +02:00
collector: serialize collector runs and harden the scrape lifecycle (#2541)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
16aa107c9f
commit
96a998772b
+155
-112
@@ -24,7 +24,6 @@ import (
|
||||
"log/slog"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus-community/windows_exporter/internal/pdh"
|
||||
@@ -41,17 +40,42 @@ type collectorStatus struct {
|
||||
type collectorStatusCode int
|
||||
|
||||
const (
|
||||
// pending means the collector timed out, or was skipped because an earlier call is still running.
|
||||
pending collectorStatusCode = iota
|
||||
success
|
||||
failed
|
||||
// unavailable means the collector was skipped because its Build failed or the collection is closed.
|
||||
unavailable
|
||||
)
|
||||
|
||||
func (c *Collection) collectAll(ch chan<- prometheus.Metric, logger *slog.Logger, maxScrapeDuration time.Duration) {
|
||||
// collectAll runs all collectors once. It returns when ctx is done, even if collectors are still running.
|
||||
// ctx must have a deadline.
|
||||
func (c *Collection) collectAll(ctx context.Context, ch chan<- prometheus.Metric, logger *slog.Logger) {
|
||||
collectorStartTime := time.Now()
|
||||
|
||||
if err := acquireScrapeSlot(ctx); err != nil {
|
||||
level := slog.LevelWarn
|
||||
if errors.Is(err, context.Canceled) {
|
||||
level = slog.LevelDebug
|
||||
}
|
||||
|
||||
logger.LogAttrs(ctx, level, "scrape not started, because an earlier scrape was still running",
|
||||
slog.Any("err", err),
|
||||
)
|
||||
|
||||
for name := range c.collectors {
|
||||
c.sendCollectorStatus(ch, collectorStatus{name: name, statusCode: pending})
|
||||
}
|
||||
|
||||
c.sendScrapeDuration(ch, collectorStartTime)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer releaseScrapeSlot()
|
||||
|
||||
// WaitGroup to wait for all collectors to finish
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(len(c.collectors))
|
||||
|
||||
// Using a channel to collect the status of each collector
|
||||
// A channel is safe to use concurrently while a map is not
|
||||
@@ -60,14 +84,12 @@ func (c *Collection) collectAll(ch chan<- prometheus.Metric, logger *slog.Logger
|
||||
// Execute all collectors concurrently
|
||||
// timeout handling is done in the execute function
|
||||
for name, metricsCollector := range c.collectors {
|
||||
go func(name string, metricsCollector Collector) {
|
||||
defer wg.Done()
|
||||
|
||||
wg.Go(func() {
|
||||
collectorStatusCh <- collectorStatus{
|
||||
name: name,
|
||||
statusCode: c.collectCollector(ch, logger, name, metricsCollector, maxScrapeDuration),
|
||||
statusCode: c.collectCollector(ctx, ch, logger, name, metricsCollector),
|
||||
}
|
||||
}(name, metricsCollector)
|
||||
})
|
||||
}
|
||||
|
||||
// Wait for all collectors to finish
|
||||
@@ -77,30 +99,38 @@ func (c *Collection) collectAll(ch chan<- prometheus.Metric, logger *slog.Logger
|
||||
close(collectorStatusCh)
|
||||
|
||||
for status := range collectorStatusCh {
|
||||
var successValue, timeoutValue float64
|
||||
if status.statusCode == pending {
|
||||
timeoutValue = 1.0
|
||||
}
|
||||
|
||||
if status.statusCode == success {
|
||||
successValue = 1.0
|
||||
}
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeSuccessDesc,
|
||||
prometheus.GaugeValue,
|
||||
successValue,
|
||||
status.name,
|
||||
)
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeTimeoutDesc,
|
||||
prometheus.GaugeValue,
|
||||
timeoutValue,
|
||||
status.name,
|
||||
)
|
||||
c.sendCollectorStatus(ch, status)
|
||||
}
|
||||
|
||||
c.sendScrapeDuration(ch, collectorStartTime)
|
||||
}
|
||||
|
||||
func (c *Collection) sendCollectorStatus(ch chan<- prometheus.Metric, status collectorStatus) {
|
||||
var successValue, timeoutValue float64
|
||||
if status.statusCode == pending {
|
||||
timeoutValue = 1.0
|
||||
}
|
||||
|
||||
if status.statusCode == success {
|
||||
successValue = 1.0
|
||||
}
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeSuccessDesc,
|
||||
prometheus.GaugeValue,
|
||||
successValue,
|
||||
status.name,
|
||||
)
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeTimeoutDesc,
|
||||
prometheus.GaugeValue,
|
||||
timeoutValue,
|
||||
status.name,
|
||||
)
|
||||
}
|
||||
|
||||
func (c *Collection) sendScrapeDuration(ch chan<- prometheus.Metric, collectorStartTime time.Time) {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.scrapeDurationDesc,
|
||||
prometheus.GaugeValue,
|
||||
@@ -108,105 +138,118 @@ func (c *Collection) collectAll(ch chan<- prometheus.Metric, logger *slog.Logger
|
||||
)
|
||||
}
|
||||
|
||||
func (c *Collection) collectCollector(ch chan<- prometheus.Metric, logger *slog.Logger, name string, collector Collector, maxScrapeDuration time.Duration) collectorStatusCode {
|
||||
var (
|
||||
err error
|
||||
numMetrics int
|
||||
duration time.Duration
|
||||
timeout atomic.Bool
|
||||
)
|
||||
// collectCollector runs one collector and forwards its metrics to ch until ctx is done.
|
||||
//
|
||||
// Only one Collect call runs per collector instance. A call that outlives ctx keeps running in the
|
||||
// background; its late metrics are dropped, and later scrapes skip the collector until the call returns.
|
||||
func (c *Collection) collectCollector(ctx context.Context, ch chan<- prometheus.Metric, logger *slog.Logger, name string, collector Collector) collectorStatusCode {
|
||||
deadline, _ := ctx.Deadline()
|
||||
|
||||
maxScrapeDuration := time.Until(deadline)
|
||||
if ctx.Err() != nil || maxScrapeDuration <= 0 {
|
||||
return pending
|
||||
}
|
||||
|
||||
state := c.state.collector(name)
|
||||
|
||||
switch state.tryCollect() {
|
||||
case started:
|
||||
case notAvailable:
|
||||
logger.LogAttrs(ctx, slog.LevelDebug, fmt.Sprintf("collector %s skipped, because it isn't initialized", name))
|
||||
|
||||
return unavailable
|
||||
case busy:
|
||||
level := slog.LevelDebug
|
||||
if state.logBusy() {
|
||||
level = slog.LevelWarn
|
||||
}
|
||||
|
||||
logger.LogAttrs(ctx, level, fmt.Sprintf("collector %s skipped, because its call from an earlier scrape is still running", name))
|
||||
|
||||
return pending
|
||||
}
|
||||
|
||||
// bufCh is a buffer channel to store the metrics
|
||||
// This is needed because once timeout is reached, the prometheus registry channel is closed.
|
||||
bufCh := make(chan prometheus.Metric, 1000)
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), maxScrapeDuration)
|
||||
defer cancel()
|
||||
t := time.Now()
|
||||
|
||||
// execute the collector
|
||||
// execute the collector. The goroutine owns the collector's slot until Collect returns.
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
errCh <- fmt.Errorf("panic in collector %s: %v. stack: %s", name, r,
|
||||
string(debug.Stack()),
|
||||
// Release the slot before closing bufCh, so the next scrape never sees this call as running.
|
||||
defer close(bufCh)
|
||||
defer state.release()
|
||||
|
||||
errCh <- collect(name, collector, bufCh, maxScrapeDuration)
|
||||
}()
|
||||
|
||||
numMetrics := 0
|
||||
|
||||
// Pass metrics to the prometheus registry until the collector finishes or the scrape ends.
|
||||
// This is the only goroutine that writes to ch, and it stops before collectCollector returns.
|
||||
for {
|
||||
select {
|
||||
case m, ok := <-bufCh:
|
||||
if ok {
|
||||
ch <- m
|
||||
|
||||
numMetrics++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
duration := time.Since(t)
|
||||
c.sendCollectorDuration(ch, name, duration)
|
||||
|
||||
return logCollectorResult(ctx, logger, name, <-errCh, duration, numMetrics)
|
||||
case <-ctx.Done():
|
||||
// Drain late metrics, so that Collect doesn't block on a full buffer.
|
||||
go func() {
|
||||
for range bufCh {
|
||||
}
|
||||
}()
|
||||
|
||||
duration := time.Since(t)
|
||||
c.sendCollectorDuration(ch, name, duration)
|
||||
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
logger.LogAttrs(ctx, slog.LevelWarn, fmt.Sprintf("collector %s timeouted after %s, resulting in %d metrics", name, duration, numMetrics))
|
||||
} else {
|
||||
logger.LogAttrs(ctx, slog.LevelDebug, fmt.Sprintf("collector %s canceled after %s, resulting in %d metrics", name, duration, numMetrics),
|
||||
slog.Any("err", ctx.Err()),
|
||||
)
|
||||
}
|
||||
|
||||
close(bufCh)
|
||||
}()
|
||||
return pending
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
errCh <- collector.Collect(bufCh, maxScrapeDuration)
|
||||
}()
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(1)
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
// This prevents a panic from race-condition when closing the ch channel too early.
|
||||
_ = recover()
|
||||
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
// Pass metrics to the prometheus registry
|
||||
// If timeout is reached, the channel is closed.
|
||||
// This will cause a panic if we try to write to it.
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case m, ok := <-bufCh:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if !timeout.Load() {
|
||||
ch <- m
|
||||
|
||||
numMetrics++
|
||||
}
|
||||
}
|
||||
// collect calls Collector.Collect and turns a panic into an error.
|
||||
func collect(name string, collector Collector, ch chan<- prometheus.Metric, maxScrapeDuration time.Duration) (err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("panic in collector %s: %v. stack: %s", name, r,
|
||||
string(debug.Stack()),
|
||||
)
|
||||
}
|
||||
}()
|
||||
|
||||
t := time.Now()
|
||||
return collector.Collect(ch, maxScrapeDuration)
|
||||
}
|
||||
|
||||
// Wait for the collector to finish or timeout
|
||||
select {
|
||||
case err = <-errCh:
|
||||
wg.Wait() // Wait for the buffer channel to be closed and empty
|
||||
|
||||
duration = time.Since(t)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeDurationDesc,
|
||||
prometheus.GaugeValue,
|
||||
duration.Seconds(),
|
||||
name,
|
||||
)
|
||||
case <-ctx.Done():
|
||||
timeout.Store(true)
|
||||
|
||||
duration = time.Since(t)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeDurationDesc,
|
||||
prometheus.GaugeValue,
|
||||
duration.Seconds(),
|
||||
name,
|
||||
)
|
||||
|
||||
logger.LogAttrs(ctx, slog.LevelWarn, fmt.Sprintf("collector %s timeouted after %s, resulting in %d metrics", name, maxScrapeDuration, numMetrics))
|
||||
|
||||
go func() {
|
||||
// Drain channel in case of premature return to not leak a goroutine.
|
||||
for range bufCh {
|
||||
}
|
||||
}()
|
||||
|
||||
return pending
|
||||
}
|
||||
func (c *Collection) sendCollectorDuration(ch chan<- prometheus.Metric, name string, duration time.Duration) {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeDurationDesc,
|
||||
prometheus.GaugeValue,
|
||||
duration.Seconds(),
|
||||
name,
|
||||
)
|
||||
}
|
||||
|
||||
func logCollectorResult(ctx context.Context, logger *slog.Logger, name string, err error, duration time.Duration, numMetrics int) collectorStatusCode {
|
||||
slogAttrs := make([]slog.Attr, 0)
|
||||
|
||||
result := "succeeded"
|
||||
|
||||
@@ -0,0 +1,574 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
// Copyright The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build windows
|
||||
|
||||
package collector_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus-community/windows_exporter/internal/mi"
|
||||
"github.com/prometheus-community/windows_exporter/internal/pdh"
|
||||
"github.com/prometheus-community/windows_exporter/pkg/collector"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/common/expfmt"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The tests in this file share the package-level scrape slot, so they don't run in parallel.
|
||||
|
||||
func TestCollectTimedOutCollectorDoesNotOverlap(t *testing.T) {
|
||||
// The collector's delay and the scrape deadlines use synthetic time.
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
logs := &recordHandler{}
|
||||
logger := slog.New(logs)
|
||||
|
||||
slow := &fakeCollector{name: "slow", delay: 2 * time.Second}
|
||||
fast := &fakeCollector{name: "fast"}
|
||||
collection := collector.New(collector.Map{"slow": slow, "fast": fast})
|
||||
|
||||
all, err := collection.NewHandler(300*time.Millisecond, logger, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
// A filtered view must share the running state with the full collection.
|
||||
filtered, err := collection.NewHandler(300*time.Millisecond, logger, []string{"slow"})
|
||||
require.NoError(t, err)
|
||||
|
||||
start := time.Now()
|
||||
|
||||
for i := range 5 {
|
||||
handler := all
|
||||
if i%2 == 1 {
|
||||
handler = filtered
|
||||
}
|
||||
|
||||
out := scrape(t, handler)
|
||||
require.Contains(t, out, `windows_exporter_collector_timeout{collector="slow"} 1`)
|
||||
require.Contains(t, out, `windows_exporter_collector_success{collector="slow"} 0`)
|
||||
require.NotContains(t, out, "windows_test_slow")
|
||||
|
||||
if handler == all {
|
||||
require.Contains(t, out, `windows_exporter_collector_success{collector="fast"} 1`)
|
||||
require.Contains(t, out, "windows_test_fast 42")
|
||||
}
|
||||
}
|
||||
|
||||
// Only the first scrape waited for its deadline, because the others skipped the running collector.
|
||||
require.Equal(t, 300*time.Millisecond, time.Since(start))
|
||||
|
||||
// The first call is still running, and no second call was started.
|
||||
require.Equal(t, int32(1), slow.calls.Load())
|
||||
require.Equal(t, int32(1), slow.maxActive.Load())
|
||||
require.Equal(t, 1, logs.warnings("timeouted"))
|
||||
require.Equal(t, 1, logs.warnings("is still running"))
|
||||
|
||||
// After the first call returns, the next scrape starts a new one.
|
||||
time.Sleep(2 * time.Second)
|
||||
synctest.Wait()
|
||||
|
||||
scrape(t, all)
|
||||
require.Equal(t, int32(2), slow.calls.Load())
|
||||
require.Equal(t, int32(1), slow.maxActive.Load())
|
||||
|
||||
// Let the second call finish before leaving the bubble.
|
||||
time.Sleep(2 * time.Second)
|
||||
synctest.Wait()
|
||||
})
|
||||
}
|
||||
|
||||
func TestCollectAdmission(t *testing.T) {
|
||||
t.Run("canceled while queued", func(t *testing.T) {
|
||||
running := newBlockingCollector()
|
||||
collection := collector.New(collector.Map{"test": running})
|
||||
first := scrapeInBackground(t.Context(), collection, time.Minute)
|
||||
|
||||
<-running.started
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
second := scrapeInBackground(ctx, collection, time.Minute)
|
||||
|
||||
// Give the second scrape time to queue behind the first one. The result is the same if it hasn't yet.
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case out := <-second:
|
||||
require.Contains(t, out, `windows_exporter_collector_timeout{collector="test"} 1`)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("canceled scrape didn't return")
|
||||
}
|
||||
|
||||
close(running.block)
|
||||
require.Contains(t, <-first, `windows_exporter_collector_success{collector="test"} 1`)
|
||||
require.Equal(t, int32(1), running.calls.Load())
|
||||
})
|
||||
|
||||
t.Run("deadline passes while queued", func(t *testing.T) {
|
||||
running := newBlockingCollector()
|
||||
collection := collector.New(collector.Map{"test": running})
|
||||
first := scrapeInBackground(t.Context(), collection, time.Minute)
|
||||
|
||||
<-running.started
|
||||
|
||||
start := time.Now()
|
||||
second := scrapeInBackground(t.Context(), collection, 100*time.Millisecond)
|
||||
|
||||
select {
|
||||
case out := <-second:
|
||||
require.Contains(t, out, `windows_exporter_collector_timeout{collector="test"} 1`)
|
||||
require.Less(t, time.Since(start), 10*time.Second)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("queued scrape didn't return at its deadline")
|
||||
}
|
||||
|
||||
close(running.block)
|
||||
<-first
|
||||
require.Equal(t, int32(1), running.calls.Load())
|
||||
})
|
||||
|
||||
t.Run("canceled before admission", func(t *testing.T) {
|
||||
test := &fakeCollector{name: "test"}
|
||||
collection := collector.New(collector.Map{"test": test})
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
handler, err := collection.NewHandlerWithContext(ctx, time.Minute, slog.New(slog.DiscardHandler), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Contains(t, scrape(t, handler), `windows_exporter_collector_timeout{collector="test"} 1`)
|
||||
require.Equal(t, int32(0), test.calls.Load())
|
||||
})
|
||||
|
||||
t.Run("deadline starts when the handler is created", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
test := &fakeCollector{name: "test"}
|
||||
collection := collector.New(collector.Map{"test": test})
|
||||
|
||||
handler, err := collection.NewHandlerWithContext(t.Context(), 10*time.Second, slog.New(slog.DiscardHandler), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
time.Sleep(4 * time.Second)
|
||||
|
||||
require.Contains(t, scrape(t, handler), `windows_exporter_collector_success{collector="test"} 1`)
|
||||
require.Equal(t, 6*time.Second, time.Duration(test.budget.Load()))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestCollectFailedBuild(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
buildErr error
|
||||
panics bool
|
||||
wantErr string
|
||||
}{
|
||||
{name: "tolerated error", buildErr: pdh.ErrNoData},
|
||||
{name: "error", buildErr: errors.New("build failed"), wantErr: "build failed"},
|
||||
{name: "panic", panics: true, wantErr: "panic: build panic"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
logs := &recordHandler{}
|
||||
logger := slog.New(logs)
|
||||
|
||||
failing := &fakeCollector{name: "failing", buildErr: tc.buildErr, buildPanic: tc.panics}
|
||||
working := &fakeCollector{name: "working"}
|
||||
collection := collector.New(collector.Map{"failing": failing, "working": working})
|
||||
|
||||
err := collection.Build(t.Context(), logger)
|
||||
if tc.wantErr == "" {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, logs.warnings("couldn't initialize collector"))
|
||||
} else {
|
||||
require.ErrorContains(t, err, tc.wantErr)
|
||||
}
|
||||
|
||||
if tc.panics {
|
||||
// The error carries the stack of the panic.
|
||||
require.ErrorContains(t, err, "runtime/debug.Stack")
|
||||
}
|
||||
|
||||
handler, err := collection.NewHandler(time.Minute, logger, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
logs.reset()
|
||||
|
||||
for range 3 {
|
||||
out := scrape(t, handler)
|
||||
require.Contains(t, out, `windows_exporter_collector_success{collector="failing"} 0`)
|
||||
require.Contains(t, out, `windows_exporter_collector_timeout{collector="failing"} 0`)
|
||||
require.Contains(t, out, `windows_exporter_collector_success{collector="working"} 1`)
|
||||
require.NotContains(t, out, "windows_test_failing")
|
||||
}
|
||||
|
||||
require.Equal(t, int32(0), failing.calls.Load())
|
||||
require.Equal(t, int32(3), working.calls.Load())
|
||||
require.Zero(t, logs.warnings(""), "a collector that failed to build must not log on every scrape")
|
||||
|
||||
// The collector may hold resources from its partial Build, so it is closed.
|
||||
require.NoError(t, collection.Close())
|
||||
require.Equal(t, int32(1), failing.closes.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildCanceled(t *testing.T) {
|
||||
building := &fakeCollector{name: "building", buildBlock: make(chan struct{}), buildStarted: make(chan struct{}, 1)}
|
||||
collection := collector.New(collector.Map{"building": building})
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
errCh <- collection.Build(ctx, slog.New(slog.DiscardHandler))
|
||||
}()
|
||||
|
||||
<-building.buildStarted
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Build didn't return after its context was canceled")
|
||||
}
|
||||
|
||||
// Close waits for the abandoned Build to return before it closes the collector.
|
||||
closeErr := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
closeErr <- collection.Close()
|
||||
}()
|
||||
|
||||
close(building.buildBlock)
|
||||
require.NoError(t, <-closeErr)
|
||||
require.Equal(t, int32(1), building.closes.Load())
|
||||
|
||||
require.ErrorIs(t, collection.Build(canceledContext(), slog.New(slog.DiscardHandler)), context.Canceled)
|
||||
}
|
||||
|
||||
func TestCollectionClose(t *testing.T) {
|
||||
t.Run("without Build", func(t *testing.T) {
|
||||
test := &fakeCollector{name: "test"}
|
||||
collection := collector.New(collector.Map{"test": test})
|
||||
|
||||
require.NoError(t, collection.Close())
|
||||
require.NoError(t, collection.Close())
|
||||
require.Equal(t, int32(0), test.closes.Load())
|
||||
})
|
||||
|
||||
t.Run("idempotent", func(t *testing.T) {
|
||||
test := &fakeCollector{name: "test"}
|
||||
collection := collector.New(collector.Map{"test": test})
|
||||
require.NoError(t, collection.Build(t.Context(), slog.New(slog.DiscardHandler)))
|
||||
|
||||
require.NoError(t, collection.Close())
|
||||
require.NoError(t, collection.Close())
|
||||
require.Equal(t, int32(1), test.closes.Load())
|
||||
|
||||
// A closed collector is not collected anymore.
|
||||
handler, err := collection.NewHandler(time.Minute, slog.New(slog.DiscardHandler), nil)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, scrape(t, handler), `windows_exporter_collector_success{collector="test"} 0`)
|
||||
require.Equal(t, int32(0), test.calls.Load())
|
||||
})
|
||||
|
||||
t.Run("waits for a running Collect", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
slow := &fakeCollector{name: "slow", delay: 2 * time.Second}
|
||||
collection := collector.New(collector.Map{"slow": slow})
|
||||
require.NoError(t, collection.Build(t.Context(), slog.New(slog.DiscardHandler)))
|
||||
|
||||
handler, err := collection.NewHandler(100*time.Millisecond, slog.New(slog.DiscardHandler), nil)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, scrape(t, handler), `windows_exporter_collector_timeout{collector="slow"} 1`)
|
||||
|
||||
require.NoError(t, collection.Close())
|
||||
require.Equal(t, int32(1), slow.closes.Load())
|
||||
require.Equal(t, int32(0), slow.closedWhileActive.Load())
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("leaves a hung collector open", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
hung := &fakeCollector{name: "hung", delay: time.Minute}
|
||||
collection := collector.New(collector.Map{"hung": hung})
|
||||
require.NoError(t, collection.Build(t.Context(), slog.New(slog.DiscardHandler)))
|
||||
|
||||
handler, err := collection.NewHandler(100*time.Millisecond, slog.New(slog.DiscardHandler), nil)
|
||||
require.NoError(t, err)
|
||||
scrape(t, handler)
|
||||
|
||||
require.ErrorContains(t, collection.Close(), "still running")
|
||||
require.Equal(t, int32(0), hung.closes.Load())
|
||||
|
||||
// Once the call returns, a later Close closes the collector.
|
||||
time.Sleep(time.Minute)
|
||||
synctest.Wait()
|
||||
|
||||
require.NoError(t, collection.Close())
|
||||
require.Equal(t, int32(1), hung.closes.Load())
|
||||
require.Equal(t, int32(0), hung.closedWhileActive.Load())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestBuildCloseCyclesDoNotLeakGoroutines(t *testing.T) {
|
||||
// The bubble also fails the test if any goroutine of it is still blocked at the end.
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
cycle := func() {
|
||||
slow := &fakeCollector{name: "slow", delay: time.Second}
|
||||
fast := &fakeCollector{name: "fast"}
|
||||
collection := collector.New(collector.Map{"slow": slow, "fast": fast})
|
||||
require.NoError(t, collection.Build(t.Context(), logger))
|
||||
|
||||
handler, err := collection.NewHandler(100*time.Millisecond, logger, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
out := scrape(t, handler)
|
||||
require.Contains(t, out, `windows_exporter_collector_timeout{collector="slow"} 1`)
|
||||
require.Contains(t, out, `windows_exporter_collector_success{collector="fast"} 1`)
|
||||
|
||||
require.NoError(t, collection.Close())
|
||||
require.Equal(t, int32(1), slow.closes.Load())
|
||||
require.Equal(t, int32(1), fast.closes.Load())
|
||||
}
|
||||
|
||||
cycle()
|
||||
synctest.Wait()
|
||||
|
||||
baseline := runtime.NumGoroutine()
|
||||
|
||||
for range 20 {
|
||||
cycle()
|
||||
}
|
||||
|
||||
synctest.Wait()
|
||||
|
||||
// One leaked goroutine per cycle would add 20.
|
||||
require.LessOrEqual(t, runtime.NumGoroutine(), baseline+2)
|
||||
})
|
||||
}
|
||||
|
||||
// fakeCollector is a collector whose Build and Collect behave as configured.
|
||||
type fakeCollector struct {
|
||||
name string
|
||||
buildErr error
|
||||
buildPanic bool
|
||||
buildStarted chan struct{}
|
||||
buildBlock chan struct{}
|
||||
delay time.Duration
|
||||
|
||||
calls atomic.Int32
|
||||
active atomic.Int32
|
||||
maxActive atomic.Int32
|
||||
closes atomic.Int32
|
||||
closedWhileActive atomic.Int32
|
||||
budget atomic.Int64
|
||||
}
|
||||
|
||||
func (c *fakeCollector) GetName() string { return c.name }
|
||||
|
||||
func (c *fakeCollector) Build(_ *slog.Logger, _ *mi.Session) error {
|
||||
if c.buildStarted != nil {
|
||||
c.buildStarted <- struct{}{}
|
||||
}
|
||||
|
||||
if c.buildBlock != nil {
|
||||
<-c.buildBlock
|
||||
}
|
||||
|
||||
if c.buildPanic {
|
||||
panic("build panic")
|
||||
}
|
||||
|
||||
return c.buildErr
|
||||
}
|
||||
|
||||
func (c *fakeCollector) Collect(ch chan<- prometheus.Metric, maxScrapeDuration time.Duration) error {
|
||||
c.calls.Add(1)
|
||||
c.budget.Store(int64(maxScrapeDuration))
|
||||
|
||||
active := c.active.Add(1)
|
||||
defer c.active.Add(-1)
|
||||
|
||||
for {
|
||||
maxActive := c.maxActive.Load()
|
||||
if active <= maxActive || c.maxActive.CompareAndSwap(maxActive, active) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(c.delay)
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(prometheus.NewDesc("windows_test_"+c.name, "Test metric", nil, nil), prometheus.GaugeValue, 42)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *fakeCollector) Close() error {
|
||||
c.closes.Add(1)
|
||||
|
||||
if c.active.Load() != 0 {
|
||||
c.closedWhileActive.Add(1)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// blockingCollector is a collector whose Collect signals started and waits until block is closed.
|
||||
type blockingCollector struct {
|
||||
*fakeCollector
|
||||
|
||||
started chan struct{}
|
||||
block chan struct{}
|
||||
}
|
||||
|
||||
func newBlockingCollector() *blockingCollector {
|
||||
c := &blockingCollector{
|
||||
started: make(chan struct{}, 1),
|
||||
block: make(chan struct{}),
|
||||
}
|
||||
c.fakeCollector = &fakeCollector{name: "test"}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *blockingCollector) Collect(ch chan<- prometheus.Metric, maxScrapeDuration time.Duration) error {
|
||||
c.started <- struct{}{}
|
||||
|
||||
<-c.block
|
||||
|
||||
return c.fakeCollector.Collect(ch, maxScrapeDuration)
|
||||
}
|
||||
|
||||
// scrapeInBackground starts a scrape with a handler bound to ctx and returns its output, or the error text.
|
||||
func scrapeInBackground(ctx context.Context, collection *collector.Collection, timeout time.Duration) <-chan string {
|
||||
out := make(chan string, 1)
|
||||
|
||||
handler, err := collection.NewHandlerWithContext(ctx, timeout, slog.New(slog.DiscardHandler), nil)
|
||||
if err != nil {
|
||||
out <- err.Error()
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
go func() {
|
||||
text, err := gather(handler)
|
||||
if err != nil {
|
||||
text = err.Error()
|
||||
}
|
||||
|
||||
out <- text
|
||||
}()
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func scrape(t *testing.T, handler *collector.Handler) string {
|
||||
t.Helper()
|
||||
|
||||
out, err := gather(handler)
|
||||
require.NoError(t, err)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// gather collects handler once and returns the metrics in the text format.
|
||||
func gather(handler *collector.Handler) (string, error) {
|
||||
registry := prometheus.NewRegistry()
|
||||
if err := registry.Register(handler); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
families, err := registry.Gather()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var out strings.Builder
|
||||
|
||||
for _, family := range families {
|
||||
if _, err := expfmt.MetricFamilyToText(&out, family); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
func canceledContext() context.Context {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
return ctx
|
||||
}
|
||||
|
||||
// recordHandler is a [slog.Handler] that records all messages.
|
||||
type recordHandler struct {
|
||||
mu sync.Mutex
|
||||
records []slog.Record
|
||||
}
|
||||
|
||||
func (h *recordHandler) Enabled(context.Context, slog.Level) bool { return true }
|
||||
|
||||
func (h *recordHandler) Handle(_ context.Context, record slog.Record) error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
h.records = append(h.records, record)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *recordHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
|
||||
|
||||
func (h *recordHandler) WithGroup(string) slog.Handler { return h }
|
||||
|
||||
// warnings returns the number of warnings whose message contains substr.
|
||||
func (h *recordHandler) warnings(substr string) int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
n := 0
|
||||
|
||||
for _, record := range h.records {
|
||||
if record.Level == slog.LevelWarn && strings.Contains(record.Message, substr) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
|
||||
return n
|
||||
}
|
||||
|
||||
func (h *recordHandler) reset() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
h.records = nil
|
||||
}
|
||||
+110
-47
@@ -24,7 +24,6 @@ import (
|
||||
"log/slog"
|
||||
"maps"
|
||||
"slices"
|
||||
"sync"
|
||||
gotime "time"
|
||||
|
||||
"github.com/alecthomas/kingpin/v2"
|
||||
@@ -158,8 +157,8 @@ func NewWithConfig(config Config) *Collection {
|
||||
// New To be called by the external libraries for collector initialization.
|
||||
func New(collectors Map) *Collection {
|
||||
return &Collection{
|
||||
collectors: collectors,
|
||||
concurrencyCh: make(chan struct{}, 1),
|
||||
collectors: collectors,
|
||||
state: newCollectionState(),
|
||||
scrapeDurationDesc: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(types.Namespace, "exporter", "scrape_duration_seconds"),
|
||||
"windows_exporter: Total scrape duration.",
|
||||
@@ -213,39 +212,68 @@ func (c *Collection) Disable(disabledCollectors []string) {
|
||||
}
|
||||
}
|
||||
|
||||
// closeTimeout bounds how long Close waits for running Build and Collect calls to return.
|
||||
const closeTimeout = 5 * gotime.Second
|
||||
|
||||
// Build To be called by the exporter for collector initialization.
|
||||
// Instead, fail fast, it will try to build all collectors and return all errors.
|
||||
// errors are joined with errors.Join.
|
||||
//
|
||||
// A collector whose Build fails is not collected. Scrapes report it with
|
||||
// windows_exporter_collector_success 0. A panic in a collector's Build is returned as an error.
|
||||
//
|
||||
// If ctx is done before all collectors are built, Build returns without waiting for them.
|
||||
// Collectors that are still building finish in the background, and Close waits for them.
|
||||
func (c *Collection) Build(ctx context.Context, logger *slog.Logger) error {
|
||||
c.startTime = gotime.Now()
|
||||
|
||||
err := c.initMI()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return fmt.Errorf("collector initialization aborted: %w", context.Cause(ctx))
|
||||
}
|
||||
|
||||
miSession, err := c.initMI()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error from initialize MI: %w", err)
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(len(c.collectors))
|
||||
type buildResult struct {
|
||||
name string
|
||||
err error
|
||||
}
|
||||
|
||||
errCh := make(chan error, len(c.collectors))
|
||||
// The channel is buffered, so that builds abandoned by a done ctx don't block.
|
||||
resultCh := make(chan buildResult, len(c.collectors))
|
||||
|
||||
for name, collector := range c.collectors {
|
||||
state := c.state.collector(name)
|
||||
|
||||
for _, collector := range c.collectors {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
if err := collector.Build(logger, c.miSession); err != nil {
|
||||
errCh <- fmt.Errorf("error build collector %s: %w", collector.GetName(), err)
|
||||
resultCh <- buildResult{
|
||||
name: collector.GetName(),
|
||||
err: state.build(ctx, logger, miSession, collector),
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
close(errCh)
|
||||
|
||||
errs := make([]error, 0, len(c.collectors))
|
||||
|
||||
for err := range errCh {
|
||||
for range len(c.collectors) {
|
||||
var result buildResult
|
||||
|
||||
select {
|
||||
case result = <-resultCh:
|
||||
case <-ctx.Done():
|
||||
errs = append(errs, fmt.Errorf("collector initialization aborted: %w", context.Cause(ctx)))
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
if result.err == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
err := fmt.Errorf("error build collector %s: %w", result.name, result.err)
|
||||
|
||||
if errors.Is(err, pdh.ErrNoData) ||
|
||||
errors.Is(err, winregistry.ErrNotExist) ||
|
||||
errors.Is(err, pdh.NewPdhError(pdh.CstatusNoObject)) ||
|
||||
@@ -253,7 +281,9 @@ func (c *Collection) Build(ctx context.Context, logger *slog.Logger) error {
|
||||
errors.Is(err, mi.MI_RESULT_INVALID_OPERATION_TIMEOUT) ||
|
||||
errors.Is(err, mi.MI_RESULT_INVALID_NAMESPACE) ||
|
||||
errors.Is(err, mi.MI_RESULT_INVALID_CLASS) {
|
||||
logger.LogAttrs(ctx, slog.LevelWarn, "couldn't initialize collector", slog.Any("err", err))
|
||||
logger.LogAttrs(ctx, slog.LevelWarn, "couldn't initialize collector",
|
||||
slog.Any("err", err),
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
@@ -265,65 +295,98 @@ func (c *Collection) Build(ctx context.Context, logger *slog.Logger) error {
|
||||
}
|
||||
|
||||
// Close To be called by the exporter for collector cleanup.
|
||||
//
|
||||
// Close waits up to 5 seconds for running Build and Collect calls to return.
|
||||
// A collector that is still running after that is left open and reported in the returned error,
|
||||
// because closing it would release resources that are still in use.
|
||||
// The MI session is then left open as well.
|
||||
//
|
||||
// Close is safe to call after a failed or partial Build and more than once.
|
||||
// Collectors that were never built are not closed. A closed collector is never built or collected again.
|
||||
func (c *Collection) Close() error {
|
||||
c.state.closeMu.Lock()
|
||||
defer c.state.closeMu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), closeTimeout)
|
||||
defer cancel()
|
||||
|
||||
errs := make([]error, 0, len(c.collectors))
|
||||
collectorBusy := false
|
||||
|
||||
for name, collector := range c.collectors {
|
||||
if err := c.state.collector(name).close(ctx, collector); err != nil {
|
||||
collectorBusy = collectorBusy || errors.Is(err, errCollectorBusy)
|
||||
|
||||
for _, collector := range c.collectors {
|
||||
if err := collector.Close(); err != nil {
|
||||
errs = append(errs, fmt.Errorf("error from close collector %s: %w", collector.GetName(), err))
|
||||
}
|
||||
}
|
||||
|
||||
app, err := c.miSession.GetApplication()
|
||||
if err != nil && !errors.Is(err, mi.ErrNotInitialized) {
|
||||
errs = append(errs, fmt.Errorf("error from get MI application: %w", err))
|
||||
}
|
||||
|
||||
if err := c.miSession.Close(); err != nil && !errors.Is(err, mi.ErrNotInitialized) {
|
||||
errs = append(errs, fmt.Errorf("error from close MI session: %w", err))
|
||||
}
|
||||
|
||||
if err := app.Close(); err != nil && !errors.Is(err, mi.ErrNotInitialized) {
|
||||
errs = append(errs, fmt.Errorf("error from close MI application: %w", err))
|
||||
if collectorBusy {
|
||||
// Closing the MI session would block on, or break, the operations of running collectors.
|
||||
errs = append(errs, errors.New("MI session left open, because collectors are still running"))
|
||||
} else {
|
||||
errs = append(errs, c.state.closeMI()...)
|
||||
}
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// initMI To be called by the exporter for collector initialization.
|
||||
func (c *Collection) initMI() error {
|
||||
// It returns the session of an earlier call, if there is one.
|
||||
func (c *Collection) initMI() (*mi.Session, error) {
|
||||
if session := c.state.session(); session != nil {
|
||||
return session, nil
|
||||
}
|
||||
|
||||
app, err := mi.ApplicationInitialize()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error from initialize MI application: %w", err)
|
||||
return nil, fmt.Errorf("error from initialize MI application: %w", err)
|
||||
}
|
||||
|
||||
destinationOptions, err := app.NewDestinationOptions()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error from create NewDestinationOptions: %w", err)
|
||||
_ = app.Close()
|
||||
|
||||
return nil, fmt.Errorf("error from create NewDestinationOptions: %w", err)
|
||||
}
|
||||
|
||||
if err = destinationOptions.SetLocale(mi.LocaleEnglish); err != nil {
|
||||
return fmt.Errorf("error from set locale: %w", err)
|
||||
}
|
||||
|
||||
if err = destinationOptions.SetTimeout(gotime.Second); err != nil {
|
||||
return fmt.Errorf("error from set timeout: %w", err)
|
||||
}
|
||||
|
||||
c.miSession, err = app.NewSession(destinationOptions)
|
||||
session, err := newMISession(app, destinationOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error from create NewSession: %w", err)
|
||||
_ = destinationOptions.Delete()
|
||||
_ = app.Close()
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil
|
||||
c.state.setMI(app, session)
|
||||
|
||||
return session, nil
|
||||
}
|
||||
|
||||
func newMISession(app *mi.Application, destinationOptions *mi.DestinationOptions) (*mi.Session, error) {
|
||||
if err := destinationOptions.SetLocale(mi.LocaleEnglish); err != nil {
|
||||
return nil, fmt.Errorf("error from set locale: %w", err)
|
||||
}
|
||||
|
||||
if err := destinationOptions.SetTimeout(gotime.Second); err != nil {
|
||||
return nil, fmt.Errorf("error from set timeout: %w", err)
|
||||
}
|
||||
|
||||
session, err := app.NewSession(destinationOptions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error from create NewSession: %w", err)
|
||||
}
|
||||
|
||||
return session, nil
|
||||
}
|
||||
|
||||
// WithCollectors To be called by the exporter for collector initialization.
|
||||
// The returned Collection shares the collector instances and their state with c,
|
||||
// so a collector never runs twice at the same time, whichever Collection scrapes it.
|
||||
func (c *Collection) WithCollectors(collectors []string) (*Collection, error) {
|
||||
metricCollectors := &Collection{
|
||||
miSession: c.miSession,
|
||||
startTime: c.startTime,
|
||||
concurrencyCh: c.concurrencyCh,
|
||||
state: c.state,
|
||||
scrapeDurationDesc: c.scrapeDurationDesc,
|
||||
collectorScrapeDurationDesc: c.collectorScrapeDurationDesc,
|
||||
collectorScrapeSuccessDesc: c.collectorScrapeSuccessDesc,
|
||||
|
||||
@@ -18,9 +18,9 @@
|
||||
package collector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
@@ -29,20 +29,27 @@ import (
|
||||
// Interface guard.
|
||||
var _ prometheus.Collector = (*Handler)(nil)
|
||||
|
||||
// scrapeSlot admits one scrape at a time, across all collections.
|
||||
// We are expose metrics directly from the memory region of the Win32 API.
|
||||
// We should not allow more than one request at a time.
|
||||
//
|
||||
//nolint:gochecknoglobals
|
||||
var concurrencyMu sync.Mutex
|
||||
var scrapeSlot = make(chan struct{}, 1)
|
||||
|
||||
// Handler implements [prometheus.Collector] for a set of Windows Collection.
|
||||
type Handler struct {
|
||||
maxScrapeDuration time.Duration
|
||||
logger *slog.Logger
|
||||
collection *Collection
|
||||
|
||||
// ctx and deadline are set by [Collection.NewHandlerWithContext] for a handler that serves one scrape.
|
||||
// [prometheus.Collector.Collect] has no context parameter, so the handler carries it.
|
||||
ctx context.Context //nolint:containedctx
|
||||
deadline time.Time
|
||||
}
|
||||
|
||||
// NewHandler returns a new Handler that implements a [prometheus.Collector] for the given metrics Collection.
|
||||
// Each call of Collect may take up to maxScrapeDuration, including the time it waits for an earlier scrape to finish.
|
||||
func (c *Collection) NewHandler(maxScrapeDuration time.Duration, logger *slog.Logger, collectors []string) (*Handler, error) {
|
||||
collection := c
|
||||
|
||||
@@ -62,12 +69,60 @@ func (c *Collection) NewHandler(maxScrapeDuration time.Duration, logger *slog.Lo
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewHandlerWithContext returns a Handler for a single scrape, such as one HTTP request.
|
||||
// The scrape's deadline is maxScrapeDuration from now, so the time Collect waits for an earlier scrape
|
||||
// to finish counts against it. Collect returns early once ctx is done, without starting collectors
|
||||
// that have not started yet.
|
||||
func (c *Collection) NewHandlerWithContext(ctx context.Context, maxScrapeDuration time.Duration, logger *slog.Logger, collectors []string) (*Handler, error) {
|
||||
handler, err := c.NewHandler(maxScrapeDuration, logger, collectors)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
handler.ctx = ctx
|
||||
handler.deadline = time.Now().Add(maxScrapeDuration)
|
||||
|
||||
return handler, nil
|
||||
}
|
||||
|
||||
func (p *Handler) Describe(_ chan<- *prometheus.Desc) {}
|
||||
|
||||
// Collect sends the collected metrics from each of the Collection to
|
||||
// prometheus.
|
||||
func (p *Handler) Collect(ch chan<- prometheus.Metric) {
|
||||
concurrencyMu.Lock()
|
||||
p.collection.collectAll(ch, p.logger, p.maxScrapeDuration)
|
||||
concurrencyMu.Unlock()
|
||||
ctx, deadline := p.ctx, p.deadline
|
||||
if ctx == nil {
|
||||
ctx, deadline = context.Background(), time.Now().Add(p.maxScrapeDuration)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithDeadline(ctx, deadline)
|
||||
defer cancel()
|
||||
|
||||
p.collection.collectAll(ctx, ch, p.logger)
|
||||
}
|
||||
|
||||
// acquireScrapeSlot waits for the scrape slot until ctx is done.
|
||||
func acquireScrapeSlot(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
select {
|
||||
case scrapeSlot <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// Both cases may have been ready. Don't start a scrape that nobody waits for anymore.
|
||||
if err := ctx.Err(); err != nil {
|
||||
<-scrapeSlot
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func releaseScrapeSlot() {
|
||||
<-scrapeSlot
|
||||
}
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
// Copyright The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build windows
|
||||
|
||||
package collector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
|
||||
"github.com/prometheus-community/windows_exporter/internal/mi"
|
||||
)
|
||||
|
||||
var (
|
||||
errCollectorBusy = errors.New("collector is still running")
|
||||
errCollectionClosed = errors.New("collection is closed")
|
||||
)
|
||||
|
||||
// collectionState is the state of a Collection that its filtered views share.
|
||||
type collectionState struct {
|
||||
mu sync.Mutex
|
||||
collectors map[string]*collectorState
|
||||
miApp *mi.Application
|
||||
miSession *mi.Session
|
||||
|
||||
// closeMu serializes Close calls.
|
||||
closeMu sync.Mutex
|
||||
}
|
||||
|
||||
func newCollectionState() *collectionState {
|
||||
return &collectionState{collectors: make(map[string]*collectorState)}
|
||||
}
|
||||
|
||||
// collector returns the state of the named collector, creating it on first use.
|
||||
func (s *collectionState) collector(name string) *collectorState {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
state, ok := s.collectors[name]
|
||||
if !ok {
|
||||
state = &collectorState{slot: make(chan struct{}, 1)}
|
||||
s.collectors[name] = state
|
||||
}
|
||||
|
||||
return state
|
||||
}
|
||||
|
||||
func (s *collectionState) session() *mi.Session {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
return s.miSession
|
||||
}
|
||||
|
||||
func (s *collectionState) setMI(app *mi.Application, session *mi.Session) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.miApp = app
|
||||
s.miSession = session
|
||||
}
|
||||
|
||||
// closeMI closes the MI session and application. Later calls do nothing.
|
||||
func (s *collectionState) closeMI() []error {
|
||||
s.mu.Lock()
|
||||
app, session := s.miApp, s.miSession
|
||||
s.miApp, s.miSession = nil, nil
|
||||
s.mu.Unlock()
|
||||
|
||||
var errs []error
|
||||
|
||||
if session != nil {
|
||||
if err := session.Close(); err != nil && !errors.Is(err, mi.ErrNotInitialized) {
|
||||
errs = append(errs, fmt.Errorf("error from close MI session: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
if app != nil {
|
||||
if err := app.Close(); err != nil && !errors.Is(err, mi.ErrNotInitialized) {
|
||||
errs = append(errs, fmt.Errorf("error from close MI application: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
type startResult int
|
||||
|
||||
const (
|
||||
started startResult = iota
|
||||
busy
|
||||
notAvailable
|
||||
)
|
||||
|
||||
// collectorState tracks the lifecycle of one collector instance.
|
||||
type collectorState struct {
|
||||
// slot is held while Build, Collect or Close of the collector runs.
|
||||
// A Collect call that outlives its scrape keeps the slot until it returns,
|
||||
// so no scrape starts a second call on the same instance.
|
||||
slot chan struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
// buildErr is the error of the last Build. A collector that failed to build is not collected.
|
||||
buildErr error
|
||||
// built is true once Build was called, so Close only closes collectors that may hold resources.
|
||||
built bool
|
||||
// closing is true once Close was called. No Build or Collect starts afterwards.
|
||||
closing bool
|
||||
// closed is true once Collector.Close was called.
|
||||
closed bool
|
||||
// busyLogged is true once a scrape logged that it skipped the current run.
|
||||
busyLogged bool
|
||||
}
|
||||
|
||||
// tryCollect acquires the slot for a Collect call without waiting.
|
||||
func (s *collectorState) tryCollect() startResult {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.closing || s.buildErr != nil {
|
||||
return notAvailable
|
||||
}
|
||||
|
||||
select {
|
||||
case s.slot <- struct{}{}:
|
||||
return started
|
||||
default:
|
||||
return busy
|
||||
}
|
||||
}
|
||||
|
||||
// release releases the slot acquired by tryCollect.
|
||||
func (s *collectorState) release() {
|
||||
s.mu.Lock()
|
||||
s.busyLogged = false
|
||||
s.mu.Unlock()
|
||||
|
||||
<-s.slot
|
||||
}
|
||||
|
||||
// logBusy reports whether a skipped scrape is the first one of the current run.
|
||||
func (s *collectorState) logBusy() bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
first := !s.busyLogged
|
||||
s.busyLogged = true
|
||||
|
||||
return first
|
||||
}
|
||||
|
||||
// build runs Collector.Build while it holds the slot and records the result.
|
||||
// A panic in Collector.Build is returned as an error.
|
||||
func (s *collectorState) build(ctx context.Context, logger *slog.Logger, miSession *mi.Session, collector Collector) (err error) {
|
||||
select {
|
||||
case s.slot <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
err = context.Cause(ctx)
|
||||
|
||||
s.mu.Lock()
|
||||
s.buildErr = err
|
||||
s.mu.Unlock()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
s.buildErr = err
|
||||
s.mu.Unlock()
|
||||
|
||||
<-s.slot
|
||||
}()
|
||||
|
||||
s.mu.Lock()
|
||||
closing := s.closing
|
||||
s.built = !closing
|
||||
s.mu.Unlock()
|
||||
|
||||
if closing {
|
||||
return errCollectionClosed
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("panic: %v. stack: %s", r, string(debug.Stack()))
|
||||
}
|
||||
}()
|
||||
|
||||
return collector.Build(logger, miSession)
|
||||
}
|
||||
|
||||
// close calls Collector.Close once the collector is idle. It waits until ctx is done for a running
|
||||
// Build or Collect call to return. If the call does not return in time, the collector is left open,
|
||||
// because closing it would release resources that are still in use.
|
||||
// After close, the collector is never built or collected again.
|
||||
func (s *collectorState) close(ctx context.Context, collector Collector) (err error) {
|
||||
s.mu.Lock()
|
||||
s.closing = true
|
||||
done := s.closed || !s.built
|
||||
s.mu.Unlock()
|
||||
|
||||
if done {
|
||||
return nil
|
||||
}
|
||||
|
||||
select {
|
||||
case s.slot <- struct{}{}:
|
||||
// The slot is never released, so no call starts after Close.
|
||||
case <-ctx.Done():
|
||||
return errCollectorBusy
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.closed = true
|
||||
s.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("panic: %v. stack: %s", r, string(debug.Stack()))
|
||||
}
|
||||
}()
|
||||
|
||||
return collector.Close()
|
||||
}
|
||||
@@ -29,10 +29,11 @@ import (
|
||||
const DefaultCollectors = "cpu,memory,logical_disk,physical_disk,net,os,service,system"
|
||||
|
||||
type Collection struct {
|
||||
collectors Map
|
||||
miSession *mi.Session
|
||||
startTime time.Time
|
||||
concurrencyCh chan struct{}
|
||||
collectors Map
|
||||
startTime time.Time
|
||||
|
||||
// state is shared with every Collection derived from this one by WithCollectors.
|
||||
state *collectionState
|
||||
|
||||
scrapeDurationDesc *prometheus.Desc
|
||||
collectorScrapeDurationDesc *prometheus.Desc
|
||||
|
||||
Reference in New Issue
Block a user