mirror of
https://github.com/prometheus-community/windows_exporter.git
synced 2026-10-17 18:19:04 +02:00
collector: serialize collector runs and harden the scrape lifecycle (#2541)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
16aa107c9f
commit
96a998772b
@@ -24,6 +24,7 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
@@ -40,8 +41,9 @@ var (
|
||||
// the exit code is sent to the service manager as well.
|
||||
exitCodeCh = make(chan int, 1)
|
||||
|
||||
// stopCh is a channel to send a signal to the service manager that the service is stopping.
|
||||
stopCh = make(chan struct{})
|
||||
// serviceStop is closed when the service manager asks the service to stop.
|
||||
// main cancels the root context then, so startup and the running exporter both observe it.
|
||||
serviceStop = newBroadcast()
|
||||
|
||||
// serviceManagerFinishedCh is a channel to send a signal to the main function that the service manager has stopped the service.
|
||||
serviceManagerFinishedCh = make(chan struct{}, 1)
|
||||
@@ -76,7 +78,11 @@ var IsService = func() bool {
|
||||
|
||||
defer func() {
|
||||
go func() {
|
||||
err := svc.Run(serviceName, &windowsExporterService{})
|
||||
err := svc.Run(serviceName, &windowsExporterService{
|
||||
stop: serviceStop,
|
||||
exitCodeCh: exitCodeCh,
|
||||
logEvent: logToEventToLog,
|
||||
})
|
||||
if err != nil {
|
||||
// https://github.com/open-telemetry/opentelemetry-collector/pull/9042
|
||||
if !errors.Is(err, windows.ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) {
|
||||
@@ -99,7 +105,37 @@ var IsService = func() bool {
|
||||
return true
|
||||
}()
|
||||
|
||||
type windowsExporterService struct{}
|
||||
// errServiceStop is the cause of the root context's cancellation when the service manager stops the service.
|
||||
var errServiceStop = errors.New("service stop requested")
|
||||
|
||||
// broadcast is a channel that is closed once to signal all receivers.
|
||||
type broadcast struct {
|
||||
once sync.Once
|
||||
ch chan struct{}
|
||||
}
|
||||
|
||||
func newBroadcast() *broadcast {
|
||||
return &broadcast{ch: make(chan struct{})}
|
||||
}
|
||||
|
||||
// Close closes the channel. Later calls do nothing.
|
||||
func (b *broadcast) Close() {
|
||||
b.once.Do(func() { close(b.ch) })
|
||||
}
|
||||
|
||||
// Done returns the channel that Close closes.
|
||||
func (b *broadcast) Done() <-chan struct{} {
|
||||
return b.ch
|
||||
}
|
||||
|
||||
type windowsExporterService struct {
|
||||
// stop is closed when a stop or shutdown request is received.
|
||||
stop *broadcast
|
||||
// exitCodeCh receives the exit code of the main function.
|
||||
exitCodeCh <-chan int
|
||||
// logEvent writes to the event log.
|
||||
logEvent func(eType uint16, msg string) error
|
||||
}
|
||||
|
||||
// Execute is the entry point for the Windows service manager.
|
||||
func (s *windowsExporterService) Execute(_ []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (bool, uint32) {
|
||||
@@ -109,11 +145,11 @@ func (s *windowsExporterService) Execute(_ []string, r <-chan svc.ChangeRequest,
|
||||
|
||||
for {
|
||||
select {
|
||||
case exitCodeCh := <-exitCodeCh:
|
||||
case exitCode := <-s.exitCodeCh:
|
||||
// Stop the service if an exit code from the main function is received.
|
||||
changes <- svc.Status{State: svc.StopPending}
|
||||
|
||||
return true, uint32(exitCodeCh)
|
||||
return serviceExitCode(exitCode)
|
||||
case c := <-r:
|
||||
// Handle the service control request.
|
||||
switch c.Cmd {
|
||||
@@ -121,22 +157,29 @@ func (s *windowsExporterService) Execute(_ []string, r <-chan svc.ChangeRequest,
|
||||
changes <- c.CurrentStatus
|
||||
case svc.Stop, svc.Shutdown:
|
||||
// Stop the service if a stop or shutdown request is received.
|
||||
_ = logToEventToLog(windows.EVENTLOG_INFORMATION_TYPE, "service stop received")
|
||||
_ = s.logEvent(windows.EVENTLOG_INFORMATION_TYPE, "service stop received")
|
||||
|
||||
changes <- svc.Status{State: svc.StopPending}
|
||||
|
||||
// Send a signal to the main function to stop the service.
|
||||
stopCh <- struct{}{}
|
||||
// Signal the main function to stop. This never blocks,
|
||||
// even if the main function is still starting or has already returned.
|
||||
s.stop.Close()
|
||||
|
||||
// Wait for the main function to stop the service.
|
||||
return false, uint32(<-exitCodeCh)
|
||||
return serviceExitCode(<-s.exitCodeCh)
|
||||
default:
|
||||
_ = logToEventToLog(windows.EVENTLOG_ERROR_TYPE, fmt.Sprintf("unexpected control request #%d", c))
|
||||
_ = s.logEvent(windows.EVENTLOG_ERROR_TYPE, fmt.Sprintf("unexpected control request #%d", c))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// serviceExitCode converts an exit code of the main function into the return values of [svc.Handler.Execute].
|
||||
// A non-zero exit code is reported as a service-specific exit code.
|
||||
func serviceExitCode(exitCode int) (bool, uint32) {
|
||||
return exitCode != 0, uint32(exitCode)
|
||||
}
|
||||
|
||||
// logToEventToLog logs a message to the Windows event log.
|
||||
func logToEventToLog(eType uint16, msg string) error {
|
||||
eventLog, err := eventlog.Open(serviceName)
|
||||
|
||||
@@ -48,11 +48,27 @@ import (
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
// collectionCloseTimeout bounds how long the exporter waits for the collectors to close on shutdown.
|
||||
const collectionCloseTimeout = 10 * time.Second
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill)
|
||||
|
||||
// A stop request from the service manager cancels the root context,
|
||||
// so that it aborts the startup as well as the running exporter.
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
|
||||
go func() {
|
||||
select {
|
||||
case <-serviceStop.Done():
|
||||
cancel(errServiceStop)
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
exitCode := run(ctx, os.Args[1:])
|
||||
|
||||
cancel(nil)
|
||||
stop()
|
||||
|
||||
// If we are running as a service, we need to signal the service control manager that we are done.
|
||||
@@ -178,12 +194,25 @@ func run(ctx context.Context, args []string) int {
|
||||
return 1
|
||||
}
|
||||
|
||||
var disabledCollectorList []string
|
||||
if *disabledCollectors != "" {
|
||||
collectors.Disable(slices.Compact(strings.Split(*disabledCollectors, ",")))
|
||||
disabledCollectorList = slices.Compact(strings.Split(*disabledCollectors, ","))
|
||||
collectors.Disable(disabledCollectorList)
|
||||
}
|
||||
|
||||
// Close the collectors on every return from here on. On a regular shutdown, this runs after the HTTP server has shut down.
|
||||
defer closeCollection(ctx, logger, collectors)
|
||||
|
||||
// Initialize collectors before loading
|
||||
if err = collectors.Build(ctx, logger); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
logger.LogAttrs(ctx, slog.LevelInfo, "windows_exporter startup aborted",
|
||||
slog.Any("reason", context.Cause(ctx)),
|
||||
)
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
for _, err := range utils.SplitError(err) {
|
||||
logger.LogAttrs(ctx, slog.LevelError, "couldn't initialize collector",
|
||||
slog.Any("err", err),
|
||||
@@ -193,7 +222,7 @@ func run(ctx context.Context, args []string) int {
|
||||
return 1
|
||||
}
|
||||
|
||||
logger.InfoContext(ctx, "Enabled collectors: "+strings.Join(enabledCollectorList, ", "))
|
||||
logger.InfoContext(ctx, "Enabled collectors: "+strings.Join(effectiveCollectors(enabledCollectorList, disabledCollectorList), ", "))
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("GET /health", httphandler.NewHealthHandler())
|
||||
@@ -233,9 +262,11 @@ func run(ctx context.Context, args []string) int {
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.LogAttrs(ctx, slog.LevelInfo, "Shutting down windows_exporter via kill signal")
|
||||
case <-stopCh:
|
||||
logger.LogAttrs(ctx, slog.LevelInfo, "Shutting down windows_exporter via service control")
|
||||
if errors.Is(context.Cause(ctx), errServiceStop) {
|
||||
logger.LogAttrs(ctx, slog.LevelInfo, "Shutting down windows_exporter via service control")
|
||||
} else {
|
||||
logger.LogAttrs(ctx, slog.LevelInfo, "Shutting down windows_exporter via kill signal")
|
||||
}
|
||||
case err := <-errCh:
|
||||
if err != nil {
|
||||
logger.LogAttrs(ctx, slog.LevelError, "Failed to start windows_exporter",
|
||||
@@ -321,6 +352,42 @@ func setPriorityWindows(ctx context.Context, logger *slog.Logger, pid int, prior
|
||||
return nil
|
||||
}
|
||||
|
||||
// closeCollection closes the collectors. It waits at most collectionCloseTimeout,
|
||||
// so that a collector that doesn't return can't block the shutdown.
|
||||
func closeCollection(ctx context.Context, logger *slog.Logger, collection *collector.Collection) {
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
//nolint:contextcheck // Close has no context parameter. The select below bounds the wait instead.
|
||||
go func() {
|
||||
errCh <- collection.Close()
|
||||
}()
|
||||
|
||||
timer := time.NewTimer(collectionCloseTimeout)
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, err := range utils.SplitError(err) {
|
||||
logger.LogAttrs(ctx, slog.LevelWarn, "couldn't close collector",
|
||||
slog.Any("err", err),
|
||||
)
|
||||
}
|
||||
case <-timer.C:
|
||||
logger.LogAttrs(ctx, slog.LevelWarn, fmt.Sprintf("collectors didn't close within %s", collectionCloseTimeout))
|
||||
}
|
||||
}
|
||||
|
||||
// effectiveCollectors returns the sorted list of enabled collectors without the disabled ones.
|
||||
func effectiveCollectors(enabled, disabled []string) []string {
|
||||
return slices.DeleteFunc(slices.Compact(slices.Sorted(slices.Values(enabled))), func(name string) bool {
|
||||
return slices.Contains(disabled, name)
|
||||
})
|
||||
}
|
||||
|
||||
func expandEnabledCollectors(enabled string) []string {
|
||||
expanded := strings.ReplaceAll(enabled, "[defaults]", collector.DefaultCollectors)
|
||||
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
// Copyright The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build windows
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sys/windows/svc"
|
||||
)
|
||||
|
||||
func TestServiceExecuteStop(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
exitCode int
|
||||
wantServiceSpecific bool
|
||||
wantExitCode uint32
|
||||
}{
|
||||
{name: "clean stop", exitCode: 0},
|
||||
// The main function fails, for example because the stop aborted its startup.
|
||||
{name: "failed stop", exitCode: 1, wantServiceSpecific: true, wantExitCode: 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
exitCodeCh := make(chan int, 1)
|
||||
service := &windowsExporterService{
|
||||
stop: newBroadcast(),
|
||||
exitCodeCh: exitCodeCh,
|
||||
logEvent: func(uint16, string) error { return nil },
|
||||
}
|
||||
|
||||
requests := make(chan svc.ChangeRequest)
|
||||
changes := make(chan svc.Status, 10)
|
||||
|
||||
type result struct {
|
||||
serviceSpecific bool
|
||||
exitCode uint32
|
||||
}
|
||||
|
||||
done := make(chan result, 1)
|
||||
|
||||
go func() {
|
||||
serviceSpecific, exitCode := service.Execute(nil, requests, changes)
|
||||
done <- result{serviceSpecific, exitCode}
|
||||
}()
|
||||
|
||||
requests <- svc.ChangeRequest{Cmd: svc.Stop}
|
||||
|
||||
// Execute signals the stop without waiting for the main function to receive it.
|
||||
select {
|
||||
case <-service.stop.Done():
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Execute didn't signal the stop")
|
||||
}
|
||||
|
||||
exitCodeCh <- tc.exitCode
|
||||
|
||||
select {
|
||||
case got := <-done:
|
||||
require.Equal(t, result{tc.wantServiceSpecific, tc.wantExitCode}, got)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Execute didn't return the exit code")
|
||||
}
|
||||
|
||||
require.Equal(t, svc.StartPending, (<-changes).State)
|
||||
require.Equal(t, svc.Running, (<-changes).State)
|
||||
require.Equal(t, svc.StopPending, (<-changes).State)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBroadcast(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := newBroadcast()
|
||||
|
||||
select {
|
||||
case <-b.Done():
|
||||
t.Fatal("broadcast is done before Close")
|
||||
default:
|
||||
}
|
||||
|
||||
b.Close()
|
||||
b.Close()
|
||||
|
||||
<-b.Done()
|
||||
}
|
||||
|
||||
func TestRunStartupCanceled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithCancelCause(t.Context())
|
||||
cancel(errServiceStop)
|
||||
|
||||
exitCodeCh := make(chan int, 1)
|
||||
|
||||
go func() {
|
||||
exitCodeCh <- run(ctx, []string{"--web.listen-address=127.0.0.1:0"})
|
||||
}()
|
||||
|
||||
select {
|
||||
case exitCode := <-exitCodeCh:
|
||||
// A stop during startup is a clean stop.
|
||||
require.Equal(t, 0, exitCode)
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("run didn't return after its context was canceled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveCollectors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
require.Equal(t,
|
||||
[]string{"cpu", "memory", "os"},
|
||||
effectiveCollectors([]string{"os", "cpu", "net", "memory", "cpu"}, []string{"net", "unknown"}),
|
||||
)
|
||||
require.Equal(t,
|
||||
[]string{"cpu", "net"},
|
||||
effectiveCollectors([]string{"net", "cpu"}, nil),
|
||||
)
|
||||
}
|
||||
@@ -18,8 +18,10 @@
|
||||
package httphandler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
@@ -34,7 +36,15 @@ import (
|
||||
// Interface guard.
|
||||
var _ http.Handler = (*MetricsHTTPHandler)(nil)
|
||||
|
||||
const defaultScrapeTimeout = 10.0
|
||||
const (
|
||||
// defaultScrapeTimeout applies if the request has no valid X-Prometheus-Scrape-Timeout-Seconds header.
|
||||
defaultScrapeTimeout = 10 * time.Second
|
||||
// minScrapeTimeout is the smallest time given to the collectors.
|
||||
minScrapeTimeout = 10 * time.Millisecond
|
||||
// maxScrapeTimeout is the largest time given to the collectors.
|
||||
// The exporter's HTTP server ends responses after 5 minutes anyway.
|
||||
maxScrapeTimeout = 5 * time.Minute
|
||||
)
|
||||
|
||||
type MetricsHTTPHandler struct {
|
||||
metricCollectors *collector.Collection
|
||||
@@ -42,8 +52,8 @@ type MetricsHTTPHandler struct {
|
||||
// the exporter itself.
|
||||
exporterMetricsRegistry *prometheus.Registry
|
||||
|
||||
logger *slog.Logger
|
||||
options Options
|
||||
logger *slog.Logger
|
||||
timeoutMargin time.Duration
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
@@ -62,7 +72,7 @@ func New(logger *slog.Logger, metricCollectors *collector.Collection, options *O
|
||||
handler := &MetricsHTTPHandler{
|
||||
metricCollectors: metricCollectors,
|
||||
logger: logger,
|
||||
options: *options,
|
||||
timeoutMargin: toDuration(options.TimeoutMargin),
|
||||
}
|
||||
|
||||
if !options.DisableExporterMetrics {
|
||||
@@ -82,9 +92,10 @@ func (c *MetricsHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
slog.String("remote", r.RemoteAddr),
|
||||
)
|
||||
|
||||
// The scrape's deadline starts now, so time spent waiting for an earlier scrape counts against it.
|
||||
scrapeTimeout := c.getScrapeTimeout(logger, r)
|
||||
|
||||
handler, err := c.handlerFactory(logger, scrapeTimeout, r.URL.Query()["collect[]"])
|
||||
handler, err := c.handlerFactory(r.Context(), logger, scrapeTimeout, r.URL.Query()["collect[]"])
|
||||
if err != nil {
|
||||
logger.WarnContext(r.Context(), "Couldn't create filtered metrics handler",
|
||||
slog.Any("err", err),
|
||||
@@ -99,32 +110,51 @@ func (c *MetricsHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// getScrapeTimeout returns the time the collectors get for this request.
|
||||
// It is the client's X-Prometheus-Scrape-Timeout-Seconds minus the timeout margin,
|
||||
// but the margin takes at most half of the client's timeout.
|
||||
// The result is between minScrapeTimeout and maxScrapeTimeout.
|
||||
func (c *MetricsHTTPHandler) getScrapeTimeout(logger *slog.Logger, r *http.Request) time.Duration {
|
||||
var timeoutSeconds float64
|
||||
timeout := defaultScrapeTimeout
|
||||
|
||||
if v := r.Header.Get("X-Prometheus-Scrape-Timeout-Seconds"); v != "" {
|
||||
var err error
|
||||
seconds, err := strconv.ParseFloat(v, 64)
|
||||
|
||||
timeoutSeconds, err = strconv.ParseFloat(v, 64)
|
||||
if err != nil {
|
||||
logger.WarnContext(r.Context(), fmt.Sprintf("Couldn't parse X-Prometheus-Scrape-Timeout-Seconds: %q. Defaulting timeout to %f", v, defaultScrapeTimeout))
|
||||
switch {
|
||||
case math.IsInf(seconds, 1):
|
||||
// Includes values too large for a float64, which ParseFloat reports as an error.
|
||||
timeout = maxScrapeTimeout
|
||||
case err != nil, math.IsNaN(seconds), seconds <= 0:
|
||||
logger.WarnContext(r.Context(), fmt.Sprintf("Invalid X-Prometheus-Scrape-Timeout-Seconds: %q. Defaulting timeout to %s", v, defaultScrapeTimeout))
|
||||
default:
|
||||
timeout = toDuration(seconds)
|
||||
}
|
||||
}
|
||||
|
||||
if timeoutSeconds == 0 {
|
||||
timeoutSeconds = defaultScrapeTimeout
|
||||
}
|
||||
timeout = max(timeout-c.timeoutMargin, timeout/2)
|
||||
|
||||
timeoutSeconds -= c.options.TimeoutMargin
|
||||
|
||||
return time.Duration(timeoutSeconds*1e9) * time.Nanosecond
|
||||
return min(max(timeout, minScrapeTimeout), maxScrapeTimeout)
|
||||
}
|
||||
|
||||
func (c *MetricsHTTPHandler) handlerFactory(logger *slog.Logger, scrapeTimeout time.Duration, requestedCollectors []string) (http.Handler, error) {
|
||||
// toDuration converts seconds to a duration between 0 and maxScrapeTimeout.
|
||||
// NaN and negative values become 0.
|
||||
func toDuration(seconds float64) time.Duration {
|
||||
if math.IsNaN(seconds) || seconds <= 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
if seconds >= maxScrapeTimeout.Seconds() {
|
||||
return maxScrapeTimeout
|
||||
}
|
||||
|
||||
return time.Duration(seconds * float64(time.Second))
|
||||
}
|
||||
|
||||
func (c *MetricsHTTPHandler) handlerFactory(ctx context.Context, logger *slog.Logger, scrapeTimeout time.Duration, requestedCollectors []string) (http.Handler, error) {
|
||||
reg := prometheus.NewRegistry()
|
||||
reg.MustRegister(version.NewCollector("windows_exporter"))
|
||||
|
||||
collectionHandler, err := c.metricCollectors.NewHandler(scrapeTimeout, c.logger, requestedCollectors)
|
||||
collectionHandler, err := c.metricCollectors.NewHandlerWithContext(ctx, scrapeTimeout, logger, requestedCollectors)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't create collector handler: %w", err)
|
||||
}
|
||||
@@ -133,17 +163,17 @@ func (c *MetricsHTTPHandler) handlerFactory(logger *slog.Logger, scrapeTimeout t
|
||||
return nil, fmt.Errorf("couldn't register Prometheus collector: %w", err)
|
||||
}
|
||||
|
||||
// Scrapes are serialized by the collection, so the handler doesn't limit requests in flight.
|
||||
var regHandler http.Handler
|
||||
if c.exporterMetricsRegistry != nil {
|
||||
regHandler = promhttp.HandlerFor(
|
||||
prometheus.Gatherers{c.exporterMetricsRegistry, reg},
|
||||
promhttp.HandlerOpts{
|
||||
ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
|
||||
ErrorHandling: promhttp.ContinueOnError,
|
||||
MaxRequestsInFlight: 1,
|
||||
Registry: c.exporterMetricsRegistry,
|
||||
EnableOpenMetrics: true,
|
||||
ProcessStartTime: c.metricCollectors.GetStartTime(),
|
||||
ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
|
||||
ErrorHandling: promhttp.ContinueOnError,
|
||||
Registry: c.exporterMetricsRegistry,
|
||||
EnableOpenMetrics: true,
|
||||
ProcessStartTime: c.metricCollectors.GetStartTime(),
|
||||
},
|
||||
)
|
||||
|
||||
@@ -156,11 +186,10 @@ func (c *MetricsHTTPHandler) handlerFactory(logger *slog.Logger, scrapeTimeout t
|
||||
regHandler = promhttp.HandlerFor(
|
||||
reg,
|
||||
promhttp.HandlerOpts{
|
||||
ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
|
||||
ErrorHandling: promhttp.ContinueOnError,
|
||||
MaxRequestsInFlight: 1,
|
||||
EnableOpenMetrics: true,
|
||||
ProcessStartTime: c.metricCollectors.GetStartTime(),
|
||||
ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
|
||||
ErrorHandling: promhttp.ContinueOnError,
|
||||
EnableOpenMetrics: true,
|
||||
ProcessStartTime: c.metricCollectors.GetStartTime(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -18,8 +18,10 @@
|
||||
package httphandler_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
@@ -154,6 +156,87 @@ func TestMetricsHTTPHandlerScrapeTimeout(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsHTTPHandlerScrapeBudget(t *testing.T) {
|
||||
const maxScrapeTimeout = 5 * time.Minute
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
header string
|
||||
margin float64
|
||||
want time.Duration
|
||||
}{
|
||||
{name: "missing header", margin: 0.5, want: 9500 * time.Millisecond},
|
||||
{name: "invalid header", header: "invalid", margin: 0.5, want: 9500 * time.Millisecond},
|
||||
{name: "zero", header: "0", margin: 0.5, want: 9500 * time.Millisecond},
|
||||
{name: "negative", header: "-1", margin: 0.5, want: 9500 * time.Millisecond},
|
||||
{name: "negative infinity", header: "-Inf", margin: 0.5, want: 9500 * time.Millisecond},
|
||||
{name: "NaN", header: "NaN", margin: 0.5, want: 9500 * time.Millisecond},
|
||||
{name: "infinity", header: "+Inf", margin: 0.5, want: maxScrapeTimeout - 500*time.Millisecond},
|
||||
{name: "out of float range", header: "1e400", margin: 0.5, want: maxScrapeTimeout - 500*time.Millisecond},
|
||||
{name: "huge", header: "1e12", margin: 0.5, want: maxScrapeTimeout - 500*time.Millisecond},
|
||||
{name: "long", header: "120", margin: 0.5, want: 119500 * time.Millisecond},
|
||||
{name: "regular", header: "2", margin: 0.5, want: 1500 * time.Millisecond},
|
||||
{name: "margin larger than timeout", header: "0.3", margin: 0.5, want: 150 * time.Millisecond},
|
||||
{name: "margin takes at most half", header: "0.8", margin: 0.5, want: 400 * time.Millisecond},
|
||||
{name: "tiny", header: "0.000001", margin: 0.5, want: 10 * time.Millisecond},
|
||||
{name: "no margin", header: "2", want: 2 * time.Second},
|
||||
{name: "negative margin", header: "2", margin: -1, want: 2 * time.Second},
|
||||
{name: "NaN margin", header: "2", margin: math.NaN(), want: 2 * time.Second},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// Synthetic time doesn't pass between the request and the collector's call.
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
test := &budgetCollector{}
|
||||
collection := collector.New(collector.Map{"test": test})
|
||||
handler := httphandler.New(slog.New(slog.DiscardHandler), collection, &httphandler.Options{DisableExporterMetrics: true, TimeoutMargin: tc.margin})
|
||||
request := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics", nil)
|
||||
|
||||
if tc.header != "" {
|
||||
request.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", tc.header)
|
||||
}
|
||||
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
require.Equal(t, http.StatusOK, response.Code)
|
||||
require.Contains(t, response.Body.String(), `windows_exporter_collector_success{collector="test"} 1`)
|
||||
require.Equal(t, tc.want, test.budget)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsHTTPHandlerCanceledRequest(t *testing.T) {
|
||||
test := &budgetCollector{}
|
||||
collection := collector.New(collector.Map{"test": test})
|
||||
handler := httphandler.New(slog.New(slog.DiscardHandler), collection, &httphandler.Options{DisableExporterMetrics: true})
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, httptest.NewRequestWithContext(ctx, http.MethodGet, "/metrics", nil))
|
||||
|
||||
// The client is gone, so the collectors don't run.
|
||||
require.False(t, test.called)
|
||||
}
|
||||
|
||||
// budgetCollector records the time it is given.
|
||||
type budgetCollector struct {
|
||||
called bool
|
||||
budget time.Duration
|
||||
}
|
||||
|
||||
func (c *budgetCollector) GetName() string { return "test" }
|
||||
func (c *budgetCollector) Build(_ *slog.Logger, _ *mi.Session) error { return nil }
|
||||
func (c *budgetCollector) Close() error { return nil }
|
||||
func (c *budgetCollector) Collect(_ chan<- prometheus.Metric, maxScrapeDuration time.Duration) error {
|
||||
c.called = true
|
||||
c.budget = maxScrapeDuration
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type testCollector struct {
|
||||
name string
|
||||
err error
|
||||
|
||||
+155
-112
@@ -24,7 +24,6 @@ import (
|
||||
"log/slog"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus-community/windows_exporter/internal/pdh"
|
||||
@@ -41,17 +40,42 @@ type collectorStatus struct {
|
||||
type collectorStatusCode int
|
||||
|
||||
const (
|
||||
// pending means the collector timed out, or was skipped because an earlier call is still running.
|
||||
pending collectorStatusCode = iota
|
||||
success
|
||||
failed
|
||||
// unavailable means the collector was skipped because its Build failed or the collection is closed.
|
||||
unavailable
|
||||
)
|
||||
|
||||
func (c *Collection) collectAll(ch chan<- prometheus.Metric, logger *slog.Logger, maxScrapeDuration time.Duration) {
|
||||
// collectAll runs all collectors once. It returns when ctx is done, even if collectors are still running.
|
||||
// ctx must have a deadline.
|
||||
func (c *Collection) collectAll(ctx context.Context, ch chan<- prometheus.Metric, logger *slog.Logger) {
|
||||
collectorStartTime := time.Now()
|
||||
|
||||
if err := acquireScrapeSlot(ctx); err != nil {
|
||||
level := slog.LevelWarn
|
||||
if errors.Is(err, context.Canceled) {
|
||||
level = slog.LevelDebug
|
||||
}
|
||||
|
||||
logger.LogAttrs(ctx, level, "scrape not started, because an earlier scrape was still running",
|
||||
slog.Any("err", err),
|
||||
)
|
||||
|
||||
for name := range c.collectors {
|
||||
c.sendCollectorStatus(ch, collectorStatus{name: name, statusCode: pending})
|
||||
}
|
||||
|
||||
c.sendScrapeDuration(ch, collectorStartTime)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer releaseScrapeSlot()
|
||||
|
||||
// WaitGroup to wait for all collectors to finish
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(len(c.collectors))
|
||||
|
||||
// Using a channel to collect the status of each collector
|
||||
// A channel is safe to use concurrently while a map is not
|
||||
@@ -60,14 +84,12 @@ func (c *Collection) collectAll(ch chan<- prometheus.Metric, logger *slog.Logger
|
||||
// Execute all collectors concurrently
|
||||
// timeout handling is done in the execute function
|
||||
for name, metricsCollector := range c.collectors {
|
||||
go func(name string, metricsCollector Collector) {
|
||||
defer wg.Done()
|
||||
|
||||
wg.Go(func() {
|
||||
collectorStatusCh <- collectorStatus{
|
||||
name: name,
|
||||
statusCode: c.collectCollector(ch, logger, name, metricsCollector, maxScrapeDuration),
|
||||
statusCode: c.collectCollector(ctx, ch, logger, name, metricsCollector),
|
||||
}
|
||||
}(name, metricsCollector)
|
||||
})
|
||||
}
|
||||
|
||||
// Wait for all collectors to finish
|
||||
@@ -77,30 +99,38 @@ func (c *Collection) collectAll(ch chan<- prometheus.Metric, logger *slog.Logger
|
||||
close(collectorStatusCh)
|
||||
|
||||
for status := range collectorStatusCh {
|
||||
var successValue, timeoutValue float64
|
||||
if status.statusCode == pending {
|
||||
timeoutValue = 1.0
|
||||
}
|
||||
|
||||
if status.statusCode == success {
|
||||
successValue = 1.0
|
||||
}
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeSuccessDesc,
|
||||
prometheus.GaugeValue,
|
||||
successValue,
|
||||
status.name,
|
||||
)
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeTimeoutDesc,
|
||||
prometheus.GaugeValue,
|
||||
timeoutValue,
|
||||
status.name,
|
||||
)
|
||||
c.sendCollectorStatus(ch, status)
|
||||
}
|
||||
|
||||
c.sendScrapeDuration(ch, collectorStartTime)
|
||||
}
|
||||
|
||||
func (c *Collection) sendCollectorStatus(ch chan<- prometheus.Metric, status collectorStatus) {
|
||||
var successValue, timeoutValue float64
|
||||
if status.statusCode == pending {
|
||||
timeoutValue = 1.0
|
||||
}
|
||||
|
||||
if status.statusCode == success {
|
||||
successValue = 1.0
|
||||
}
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeSuccessDesc,
|
||||
prometheus.GaugeValue,
|
||||
successValue,
|
||||
status.name,
|
||||
)
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeTimeoutDesc,
|
||||
prometheus.GaugeValue,
|
||||
timeoutValue,
|
||||
status.name,
|
||||
)
|
||||
}
|
||||
|
||||
func (c *Collection) sendScrapeDuration(ch chan<- prometheus.Metric, collectorStartTime time.Time) {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.scrapeDurationDesc,
|
||||
prometheus.GaugeValue,
|
||||
@@ -108,105 +138,118 @@ func (c *Collection) collectAll(ch chan<- prometheus.Metric, logger *slog.Logger
|
||||
)
|
||||
}
|
||||
|
||||
func (c *Collection) collectCollector(ch chan<- prometheus.Metric, logger *slog.Logger, name string, collector Collector, maxScrapeDuration time.Duration) collectorStatusCode {
|
||||
var (
|
||||
err error
|
||||
numMetrics int
|
||||
duration time.Duration
|
||||
timeout atomic.Bool
|
||||
)
|
||||
// collectCollector runs one collector and forwards its metrics to ch until ctx is done.
|
||||
//
|
||||
// Only one Collect call runs per collector instance. A call that outlives ctx keeps running in the
|
||||
// background; its late metrics are dropped, and later scrapes skip the collector until the call returns.
|
||||
func (c *Collection) collectCollector(ctx context.Context, ch chan<- prometheus.Metric, logger *slog.Logger, name string, collector Collector) collectorStatusCode {
|
||||
deadline, _ := ctx.Deadline()
|
||||
|
||||
maxScrapeDuration := time.Until(deadline)
|
||||
if ctx.Err() != nil || maxScrapeDuration <= 0 {
|
||||
return pending
|
||||
}
|
||||
|
||||
state := c.state.collector(name)
|
||||
|
||||
switch state.tryCollect() {
|
||||
case started:
|
||||
case notAvailable:
|
||||
logger.LogAttrs(ctx, slog.LevelDebug, fmt.Sprintf("collector %s skipped, because it isn't initialized", name))
|
||||
|
||||
return unavailable
|
||||
case busy:
|
||||
level := slog.LevelDebug
|
||||
if state.logBusy() {
|
||||
level = slog.LevelWarn
|
||||
}
|
||||
|
||||
logger.LogAttrs(ctx, level, fmt.Sprintf("collector %s skipped, because its call from an earlier scrape is still running", name))
|
||||
|
||||
return pending
|
||||
}
|
||||
|
||||
// bufCh is a buffer channel to store the metrics
|
||||
// This is needed because once timeout is reached, the prometheus registry channel is closed.
|
||||
bufCh := make(chan prometheus.Metric, 1000)
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), maxScrapeDuration)
|
||||
defer cancel()
|
||||
t := time.Now()
|
||||
|
||||
// execute the collector
|
||||
// execute the collector. The goroutine owns the collector's slot until Collect returns.
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
errCh <- fmt.Errorf("panic in collector %s: %v. stack: %s", name, r,
|
||||
string(debug.Stack()),
|
||||
// Release the slot before closing bufCh, so the next scrape never sees this call as running.
|
||||
defer close(bufCh)
|
||||
defer state.release()
|
||||
|
||||
errCh <- collect(name, collector, bufCh, maxScrapeDuration)
|
||||
}()
|
||||
|
||||
numMetrics := 0
|
||||
|
||||
// Pass metrics to the prometheus registry until the collector finishes or the scrape ends.
|
||||
// This is the only goroutine that writes to ch, and it stops before collectCollector returns.
|
||||
for {
|
||||
select {
|
||||
case m, ok := <-bufCh:
|
||||
if ok {
|
||||
ch <- m
|
||||
|
||||
numMetrics++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
duration := time.Since(t)
|
||||
c.sendCollectorDuration(ch, name, duration)
|
||||
|
||||
return logCollectorResult(ctx, logger, name, <-errCh, duration, numMetrics)
|
||||
case <-ctx.Done():
|
||||
// Drain late metrics, so that Collect doesn't block on a full buffer.
|
||||
go func() {
|
||||
for range bufCh {
|
||||
}
|
||||
}()
|
||||
|
||||
duration := time.Since(t)
|
||||
c.sendCollectorDuration(ch, name, duration)
|
||||
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
logger.LogAttrs(ctx, slog.LevelWarn, fmt.Sprintf("collector %s timeouted after %s, resulting in %d metrics", name, duration, numMetrics))
|
||||
} else {
|
||||
logger.LogAttrs(ctx, slog.LevelDebug, fmt.Sprintf("collector %s canceled after %s, resulting in %d metrics", name, duration, numMetrics),
|
||||
slog.Any("err", ctx.Err()),
|
||||
)
|
||||
}
|
||||
|
||||
close(bufCh)
|
||||
}()
|
||||
return pending
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
errCh <- collector.Collect(bufCh, maxScrapeDuration)
|
||||
}()
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(1)
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
// This prevents a panic from race-condition when closing the ch channel too early.
|
||||
_ = recover()
|
||||
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
// Pass metrics to the prometheus registry
|
||||
// If timeout is reached, the channel is closed.
|
||||
// This will cause a panic if we try to write to it.
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case m, ok := <-bufCh:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if !timeout.Load() {
|
||||
ch <- m
|
||||
|
||||
numMetrics++
|
||||
}
|
||||
}
|
||||
// collect calls Collector.Collect and turns a panic into an error.
|
||||
func collect(name string, collector Collector, ch chan<- prometheus.Metric, maxScrapeDuration time.Duration) (err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("panic in collector %s: %v. stack: %s", name, r,
|
||||
string(debug.Stack()),
|
||||
)
|
||||
}
|
||||
}()
|
||||
|
||||
t := time.Now()
|
||||
return collector.Collect(ch, maxScrapeDuration)
|
||||
}
|
||||
|
||||
// Wait for the collector to finish or timeout
|
||||
select {
|
||||
case err = <-errCh:
|
||||
wg.Wait() // Wait for the buffer channel to be closed and empty
|
||||
|
||||
duration = time.Since(t)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeDurationDesc,
|
||||
prometheus.GaugeValue,
|
||||
duration.Seconds(),
|
||||
name,
|
||||
)
|
||||
case <-ctx.Done():
|
||||
timeout.Store(true)
|
||||
|
||||
duration = time.Since(t)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeDurationDesc,
|
||||
prometheus.GaugeValue,
|
||||
duration.Seconds(),
|
||||
name,
|
||||
)
|
||||
|
||||
logger.LogAttrs(ctx, slog.LevelWarn, fmt.Sprintf("collector %s timeouted after %s, resulting in %d metrics", name, maxScrapeDuration, numMetrics))
|
||||
|
||||
go func() {
|
||||
// Drain channel in case of premature return to not leak a goroutine.
|
||||
for range bufCh {
|
||||
}
|
||||
}()
|
||||
|
||||
return pending
|
||||
}
|
||||
func (c *Collection) sendCollectorDuration(ch chan<- prometheus.Metric, name string, duration time.Duration) {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
c.collectorScrapeDurationDesc,
|
||||
prometheus.GaugeValue,
|
||||
duration.Seconds(),
|
||||
name,
|
||||
)
|
||||
}
|
||||
|
||||
func logCollectorResult(ctx context.Context, logger *slog.Logger, name string, err error, duration time.Duration, numMetrics int) collectorStatusCode {
|
||||
slogAttrs := make([]slog.Attr, 0)
|
||||
|
||||
result := "succeeded"
|
||||
|
||||
@@ -0,0 +1,574 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
// Copyright The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build windows
|
||||
|
||||
package collector_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus-community/windows_exporter/internal/mi"
|
||||
"github.com/prometheus-community/windows_exporter/internal/pdh"
|
||||
"github.com/prometheus-community/windows_exporter/pkg/collector"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/common/expfmt"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The tests in this file share the package-level scrape slot, so they don't run in parallel.
|
||||
|
||||
func TestCollectTimedOutCollectorDoesNotOverlap(t *testing.T) {
|
||||
// The collector's delay and the scrape deadlines use synthetic time.
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
logs := &recordHandler{}
|
||||
logger := slog.New(logs)
|
||||
|
||||
slow := &fakeCollector{name: "slow", delay: 2 * time.Second}
|
||||
fast := &fakeCollector{name: "fast"}
|
||||
collection := collector.New(collector.Map{"slow": slow, "fast": fast})
|
||||
|
||||
all, err := collection.NewHandler(300*time.Millisecond, logger, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
// A filtered view must share the running state with the full collection.
|
||||
filtered, err := collection.NewHandler(300*time.Millisecond, logger, []string{"slow"})
|
||||
require.NoError(t, err)
|
||||
|
||||
start := time.Now()
|
||||
|
||||
for i := range 5 {
|
||||
handler := all
|
||||
if i%2 == 1 {
|
||||
handler = filtered
|
||||
}
|
||||
|
||||
out := scrape(t, handler)
|
||||
require.Contains(t, out, `windows_exporter_collector_timeout{collector="slow"} 1`)
|
||||
require.Contains(t, out, `windows_exporter_collector_success{collector="slow"} 0`)
|
||||
require.NotContains(t, out, "windows_test_slow")
|
||||
|
||||
if handler == all {
|
||||
require.Contains(t, out, `windows_exporter_collector_success{collector="fast"} 1`)
|
||||
require.Contains(t, out, "windows_test_fast 42")
|
||||
}
|
||||
}
|
||||
|
||||
// Only the first scrape waited for its deadline, because the others skipped the running collector.
|
||||
require.Equal(t, 300*time.Millisecond, time.Since(start))
|
||||
|
||||
// The first call is still running, and no second call was started.
|
||||
require.Equal(t, int32(1), slow.calls.Load())
|
||||
require.Equal(t, int32(1), slow.maxActive.Load())
|
||||
require.Equal(t, 1, logs.warnings("timeouted"))
|
||||
require.Equal(t, 1, logs.warnings("is still running"))
|
||||
|
||||
// After the first call returns, the next scrape starts a new one.
|
||||
time.Sleep(2 * time.Second)
|
||||
synctest.Wait()
|
||||
|
||||
scrape(t, all)
|
||||
require.Equal(t, int32(2), slow.calls.Load())
|
||||
require.Equal(t, int32(1), slow.maxActive.Load())
|
||||
|
||||
// Let the second call finish before leaving the bubble.
|
||||
time.Sleep(2 * time.Second)
|
||||
synctest.Wait()
|
||||
})
|
||||
}
|
||||
|
||||
func TestCollectAdmission(t *testing.T) {
|
||||
t.Run("canceled while queued", func(t *testing.T) {
|
||||
running := newBlockingCollector()
|
||||
collection := collector.New(collector.Map{"test": running})
|
||||
first := scrapeInBackground(t.Context(), collection, time.Minute)
|
||||
|
||||
<-running.started
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
second := scrapeInBackground(ctx, collection, time.Minute)
|
||||
|
||||
// Give the second scrape time to queue behind the first one. The result is the same if it hasn't yet.
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case out := <-second:
|
||||
require.Contains(t, out, `windows_exporter_collector_timeout{collector="test"} 1`)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("canceled scrape didn't return")
|
||||
}
|
||||
|
||||
close(running.block)
|
||||
require.Contains(t, <-first, `windows_exporter_collector_success{collector="test"} 1`)
|
||||
require.Equal(t, int32(1), running.calls.Load())
|
||||
})
|
||||
|
||||
t.Run("deadline passes while queued", func(t *testing.T) {
|
||||
running := newBlockingCollector()
|
||||
collection := collector.New(collector.Map{"test": running})
|
||||
first := scrapeInBackground(t.Context(), collection, time.Minute)
|
||||
|
||||
<-running.started
|
||||
|
||||
start := time.Now()
|
||||
second := scrapeInBackground(t.Context(), collection, 100*time.Millisecond)
|
||||
|
||||
select {
|
||||
case out := <-second:
|
||||
require.Contains(t, out, `windows_exporter_collector_timeout{collector="test"} 1`)
|
||||
require.Less(t, time.Since(start), 10*time.Second)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("queued scrape didn't return at its deadline")
|
||||
}
|
||||
|
||||
close(running.block)
|
||||
<-first
|
||||
require.Equal(t, int32(1), running.calls.Load())
|
||||
})
|
||||
|
||||
t.Run("canceled before admission", func(t *testing.T) {
|
||||
test := &fakeCollector{name: "test"}
|
||||
collection := collector.New(collector.Map{"test": test})
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
handler, err := collection.NewHandlerWithContext(ctx, time.Minute, slog.New(slog.DiscardHandler), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Contains(t, scrape(t, handler), `windows_exporter_collector_timeout{collector="test"} 1`)
|
||||
require.Equal(t, int32(0), test.calls.Load())
|
||||
})
|
||||
|
||||
t.Run("deadline starts when the handler is created", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
test := &fakeCollector{name: "test"}
|
||||
collection := collector.New(collector.Map{"test": test})
|
||||
|
||||
handler, err := collection.NewHandlerWithContext(t.Context(), 10*time.Second, slog.New(slog.DiscardHandler), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
time.Sleep(4 * time.Second)
|
||||
|
||||
require.Contains(t, scrape(t, handler), `windows_exporter_collector_success{collector="test"} 1`)
|
||||
require.Equal(t, 6*time.Second, time.Duration(test.budget.Load()))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestCollectFailedBuild(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
buildErr error
|
||||
panics bool
|
||||
wantErr string
|
||||
}{
|
||||
{name: "tolerated error", buildErr: pdh.ErrNoData},
|
||||
{name: "error", buildErr: errors.New("build failed"), wantErr: "build failed"},
|
||||
{name: "panic", panics: true, wantErr: "panic: build panic"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
logs := &recordHandler{}
|
||||
logger := slog.New(logs)
|
||||
|
||||
failing := &fakeCollector{name: "failing", buildErr: tc.buildErr, buildPanic: tc.panics}
|
||||
working := &fakeCollector{name: "working"}
|
||||
collection := collector.New(collector.Map{"failing": failing, "working": working})
|
||||
|
||||
err := collection.Build(t.Context(), logger)
|
||||
if tc.wantErr == "" {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, logs.warnings("couldn't initialize collector"))
|
||||
} else {
|
||||
require.ErrorContains(t, err, tc.wantErr)
|
||||
}
|
||||
|
||||
if tc.panics {
|
||||
// The error carries the stack of the panic.
|
||||
require.ErrorContains(t, err, "runtime/debug.Stack")
|
||||
}
|
||||
|
||||
handler, err := collection.NewHandler(time.Minute, logger, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
logs.reset()
|
||||
|
||||
for range 3 {
|
||||
out := scrape(t, handler)
|
||||
require.Contains(t, out, `windows_exporter_collector_success{collector="failing"} 0`)
|
||||
require.Contains(t, out, `windows_exporter_collector_timeout{collector="failing"} 0`)
|
||||
require.Contains(t, out, `windows_exporter_collector_success{collector="working"} 1`)
|
||||
require.NotContains(t, out, "windows_test_failing")
|
||||
}
|
||||
|
||||
require.Equal(t, int32(0), failing.calls.Load())
|
||||
require.Equal(t, int32(3), working.calls.Load())
|
||||
require.Zero(t, logs.warnings(""), "a collector that failed to build must not log on every scrape")
|
||||
|
||||
// The collector may hold resources from its partial Build, so it is closed.
|
||||
require.NoError(t, collection.Close())
|
||||
require.Equal(t, int32(1), failing.closes.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildCanceled(t *testing.T) {
|
||||
building := &fakeCollector{name: "building", buildBlock: make(chan struct{}), buildStarted: make(chan struct{}, 1)}
|
||||
collection := collector.New(collector.Map{"building": building})
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
errCh <- collection.Build(ctx, slog.New(slog.DiscardHandler))
|
||||
}()
|
||||
|
||||
<-building.buildStarted
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Build didn't return after its context was canceled")
|
||||
}
|
||||
|
||||
// Close waits for the abandoned Build to return before it closes the collector.
|
||||
closeErr := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
closeErr <- collection.Close()
|
||||
}()
|
||||
|
||||
close(building.buildBlock)
|
||||
require.NoError(t, <-closeErr)
|
||||
require.Equal(t, int32(1), building.closes.Load())
|
||||
|
||||
require.ErrorIs(t, collection.Build(canceledContext(), slog.New(slog.DiscardHandler)), context.Canceled)
|
||||
}
|
||||
|
||||
func TestCollectionClose(t *testing.T) {
|
||||
t.Run("without Build", func(t *testing.T) {
|
||||
test := &fakeCollector{name: "test"}
|
||||
collection := collector.New(collector.Map{"test": test})
|
||||
|
||||
require.NoError(t, collection.Close())
|
||||
require.NoError(t, collection.Close())
|
||||
require.Equal(t, int32(0), test.closes.Load())
|
||||
})
|
||||
|
||||
t.Run("idempotent", func(t *testing.T) {
|
||||
test := &fakeCollector{name: "test"}
|
||||
collection := collector.New(collector.Map{"test": test})
|
||||
require.NoError(t, collection.Build(t.Context(), slog.New(slog.DiscardHandler)))
|
||||
|
||||
require.NoError(t, collection.Close())
|
||||
require.NoError(t, collection.Close())
|
||||
require.Equal(t, int32(1), test.closes.Load())
|
||||
|
||||
// A closed collector is not collected anymore.
|
||||
handler, err := collection.NewHandler(time.Minute, slog.New(slog.DiscardHandler), nil)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, scrape(t, handler), `windows_exporter_collector_success{collector="test"} 0`)
|
||||
require.Equal(t, int32(0), test.calls.Load())
|
||||
})
|
||||
|
||||
t.Run("waits for a running Collect", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
slow := &fakeCollector{name: "slow", delay: 2 * time.Second}
|
||||
collection := collector.New(collector.Map{"slow": slow})
|
||||
require.NoError(t, collection.Build(t.Context(), slog.New(slog.DiscardHandler)))
|
||||
|
||||
handler, err := collection.NewHandler(100*time.Millisecond, slog.New(slog.DiscardHandler), nil)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, scrape(t, handler), `windows_exporter_collector_timeout{collector="slow"} 1`)
|
||||
|
||||
require.NoError(t, collection.Close())
|
||||
require.Equal(t, int32(1), slow.closes.Load())
|
||||
require.Equal(t, int32(0), slow.closedWhileActive.Load())
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("leaves a hung collector open", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
hung := &fakeCollector{name: "hung", delay: time.Minute}
|
||||
collection := collector.New(collector.Map{"hung": hung})
|
||||
require.NoError(t, collection.Build(t.Context(), slog.New(slog.DiscardHandler)))
|
||||
|
||||
handler, err := collection.NewHandler(100*time.Millisecond, slog.New(slog.DiscardHandler), nil)
|
||||
require.NoError(t, err)
|
||||
scrape(t, handler)
|
||||
|
||||
require.ErrorContains(t, collection.Close(), "still running")
|
||||
require.Equal(t, int32(0), hung.closes.Load())
|
||||
|
||||
// Once the call returns, a later Close closes the collector.
|
||||
time.Sleep(time.Minute)
|
||||
synctest.Wait()
|
||||
|
||||
require.NoError(t, collection.Close())
|
||||
require.Equal(t, int32(1), hung.closes.Load())
|
||||
require.Equal(t, int32(0), hung.closedWhileActive.Load())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestBuildCloseCyclesDoNotLeakGoroutines(t *testing.T) {
|
||||
// The bubble also fails the test if any goroutine of it is still blocked at the end.
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
cycle := func() {
|
||||
slow := &fakeCollector{name: "slow", delay: time.Second}
|
||||
fast := &fakeCollector{name: "fast"}
|
||||
collection := collector.New(collector.Map{"slow": slow, "fast": fast})
|
||||
require.NoError(t, collection.Build(t.Context(), logger))
|
||||
|
||||
handler, err := collection.NewHandler(100*time.Millisecond, logger, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
out := scrape(t, handler)
|
||||
require.Contains(t, out, `windows_exporter_collector_timeout{collector="slow"} 1`)
|
||||
require.Contains(t, out, `windows_exporter_collector_success{collector="fast"} 1`)
|
||||
|
||||
require.NoError(t, collection.Close())
|
||||
require.Equal(t, int32(1), slow.closes.Load())
|
||||
require.Equal(t, int32(1), fast.closes.Load())
|
||||
}
|
||||
|
||||
cycle()
|
||||
synctest.Wait()
|
||||
|
||||
baseline := runtime.NumGoroutine()
|
||||
|
||||
for range 20 {
|
||||
cycle()
|
||||
}
|
||||
|
||||
synctest.Wait()
|
||||
|
||||
// One leaked goroutine per cycle would add 20.
|
||||
require.LessOrEqual(t, runtime.NumGoroutine(), baseline+2)
|
||||
})
|
||||
}
|
||||
|
||||
// fakeCollector is a collector whose Build and Collect behave as configured.
|
||||
type fakeCollector struct {
|
||||
name string
|
||||
buildErr error
|
||||
buildPanic bool
|
||||
buildStarted chan struct{}
|
||||
buildBlock chan struct{}
|
||||
delay time.Duration
|
||||
|
||||
calls atomic.Int32
|
||||
active atomic.Int32
|
||||
maxActive atomic.Int32
|
||||
closes atomic.Int32
|
||||
closedWhileActive atomic.Int32
|
||||
budget atomic.Int64
|
||||
}
|
||||
|
||||
func (c *fakeCollector) GetName() string { return c.name }
|
||||
|
||||
func (c *fakeCollector) Build(_ *slog.Logger, _ *mi.Session) error {
|
||||
if c.buildStarted != nil {
|
||||
c.buildStarted <- struct{}{}
|
||||
}
|
||||
|
||||
if c.buildBlock != nil {
|
||||
<-c.buildBlock
|
||||
}
|
||||
|
||||
if c.buildPanic {
|
||||
panic("build panic")
|
||||
}
|
||||
|
||||
return c.buildErr
|
||||
}
|
||||
|
||||
func (c *fakeCollector) Collect(ch chan<- prometheus.Metric, maxScrapeDuration time.Duration) error {
|
||||
c.calls.Add(1)
|
||||
c.budget.Store(int64(maxScrapeDuration))
|
||||
|
||||
active := c.active.Add(1)
|
||||
defer c.active.Add(-1)
|
||||
|
||||
for {
|
||||
maxActive := c.maxActive.Load()
|
||||
if active <= maxActive || c.maxActive.CompareAndSwap(maxActive, active) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(c.delay)
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(prometheus.NewDesc("windows_test_"+c.name, "Test metric", nil, nil), prometheus.GaugeValue, 42)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *fakeCollector) Close() error {
|
||||
c.closes.Add(1)
|
||||
|
||||
if c.active.Load() != 0 {
|
||||
c.closedWhileActive.Add(1)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// blockingCollector is a collector whose Collect signals started and waits until block is closed.
|
||||
type blockingCollector struct {
|
||||
*fakeCollector
|
||||
|
||||
started chan struct{}
|
||||
block chan struct{}
|
||||
}
|
||||
|
||||
func newBlockingCollector() *blockingCollector {
|
||||
c := &blockingCollector{
|
||||
started: make(chan struct{}, 1),
|
||||
block: make(chan struct{}),
|
||||
}
|
||||
c.fakeCollector = &fakeCollector{name: "test"}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *blockingCollector) Collect(ch chan<- prometheus.Metric, maxScrapeDuration time.Duration) error {
|
||||
c.started <- struct{}{}
|
||||
|
||||
<-c.block
|
||||
|
||||
return c.fakeCollector.Collect(ch, maxScrapeDuration)
|
||||
}
|
||||
|
||||
// scrapeInBackground starts a scrape with a handler bound to ctx and returns its output, or the error text.
|
||||
func scrapeInBackground(ctx context.Context, collection *collector.Collection, timeout time.Duration) <-chan string {
|
||||
out := make(chan string, 1)
|
||||
|
||||
handler, err := collection.NewHandlerWithContext(ctx, timeout, slog.New(slog.DiscardHandler), nil)
|
||||
if err != nil {
|
||||
out <- err.Error()
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
go func() {
|
||||
text, err := gather(handler)
|
||||
if err != nil {
|
||||
text = err.Error()
|
||||
}
|
||||
|
||||
out <- text
|
||||
}()
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func scrape(t *testing.T, handler *collector.Handler) string {
|
||||
t.Helper()
|
||||
|
||||
out, err := gather(handler)
|
||||
require.NoError(t, err)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// gather collects handler once and returns the metrics in the text format.
|
||||
func gather(handler *collector.Handler) (string, error) {
|
||||
registry := prometheus.NewRegistry()
|
||||
if err := registry.Register(handler); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
families, err := registry.Gather()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var out strings.Builder
|
||||
|
||||
for _, family := range families {
|
||||
if _, err := expfmt.MetricFamilyToText(&out, family); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
func canceledContext() context.Context {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
return ctx
|
||||
}
|
||||
|
||||
// recordHandler is a [slog.Handler] that records all messages.
|
||||
type recordHandler struct {
|
||||
mu sync.Mutex
|
||||
records []slog.Record
|
||||
}
|
||||
|
||||
func (h *recordHandler) Enabled(context.Context, slog.Level) bool { return true }
|
||||
|
||||
func (h *recordHandler) Handle(_ context.Context, record slog.Record) error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
h.records = append(h.records, record)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *recordHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
|
||||
|
||||
func (h *recordHandler) WithGroup(string) slog.Handler { return h }
|
||||
|
||||
// warnings returns the number of warnings whose message contains substr.
|
||||
func (h *recordHandler) warnings(substr string) int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
n := 0
|
||||
|
||||
for _, record := range h.records {
|
||||
if record.Level == slog.LevelWarn && strings.Contains(record.Message, substr) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
|
||||
return n
|
||||
}
|
||||
|
||||
func (h *recordHandler) reset() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
h.records = nil
|
||||
}
|
||||
+110
-47
@@ -24,7 +24,6 @@ import (
|
||||
"log/slog"
|
||||
"maps"
|
||||
"slices"
|
||||
"sync"
|
||||
gotime "time"
|
||||
|
||||
"github.com/alecthomas/kingpin/v2"
|
||||
@@ -158,8 +157,8 @@ func NewWithConfig(config Config) *Collection {
|
||||
// New To be called by the external libraries for collector initialization.
|
||||
func New(collectors Map) *Collection {
|
||||
return &Collection{
|
||||
collectors: collectors,
|
||||
concurrencyCh: make(chan struct{}, 1),
|
||||
collectors: collectors,
|
||||
state: newCollectionState(),
|
||||
scrapeDurationDesc: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(types.Namespace, "exporter", "scrape_duration_seconds"),
|
||||
"windows_exporter: Total scrape duration.",
|
||||
@@ -213,39 +212,68 @@ func (c *Collection) Disable(disabledCollectors []string) {
|
||||
}
|
||||
}
|
||||
|
||||
// closeTimeout bounds how long Close waits for running Build and Collect calls to return.
|
||||
const closeTimeout = 5 * gotime.Second
|
||||
|
||||
// Build To be called by the exporter for collector initialization.
|
||||
// Instead, fail fast, it will try to build all collectors and return all errors.
|
||||
// errors are joined with errors.Join.
|
||||
//
|
||||
// A collector whose Build fails is not collected. Scrapes report it with
|
||||
// windows_exporter_collector_success 0. A panic in a collector's Build is returned as an error.
|
||||
//
|
||||
// If ctx is done before all collectors are built, Build returns without waiting for them.
|
||||
// Collectors that are still building finish in the background, and Close waits for them.
|
||||
func (c *Collection) Build(ctx context.Context, logger *slog.Logger) error {
|
||||
c.startTime = gotime.Now()
|
||||
|
||||
err := c.initMI()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return fmt.Errorf("collector initialization aborted: %w", context.Cause(ctx))
|
||||
}
|
||||
|
||||
miSession, err := c.initMI()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error from initialize MI: %w", err)
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(len(c.collectors))
|
||||
type buildResult struct {
|
||||
name string
|
||||
err error
|
||||
}
|
||||
|
||||
errCh := make(chan error, len(c.collectors))
|
||||
// The channel is buffered, so that builds abandoned by a done ctx don't block.
|
||||
resultCh := make(chan buildResult, len(c.collectors))
|
||||
|
||||
for name, collector := range c.collectors {
|
||||
state := c.state.collector(name)
|
||||
|
||||
for _, collector := range c.collectors {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
if err := collector.Build(logger, c.miSession); err != nil {
|
||||
errCh <- fmt.Errorf("error build collector %s: %w", collector.GetName(), err)
|
||||
resultCh <- buildResult{
|
||||
name: collector.GetName(),
|
||||
err: state.build(ctx, logger, miSession, collector),
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
close(errCh)
|
||||
|
||||
errs := make([]error, 0, len(c.collectors))
|
||||
|
||||
for err := range errCh {
|
||||
for range len(c.collectors) {
|
||||
var result buildResult
|
||||
|
||||
select {
|
||||
case result = <-resultCh:
|
||||
case <-ctx.Done():
|
||||
errs = append(errs, fmt.Errorf("collector initialization aborted: %w", context.Cause(ctx)))
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
if result.err == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
err := fmt.Errorf("error build collector %s: %w", result.name, result.err)
|
||||
|
||||
if errors.Is(err, pdh.ErrNoData) ||
|
||||
errors.Is(err, winregistry.ErrNotExist) ||
|
||||
errors.Is(err, pdh.NewPdhError(pdh.CstatusNoObject)) ||
|
||||
@@ -253,7 +281,9 @@ func (c *Collection) Build(ctx context.Context, logger *slog.Logger) error {
|
||||
errors.Is(err, mi.MI_RESULT_INVALID_OPERATION_TIMEOUT) ||
|
||||
errors.Is(err, mi.MI_RESULT_INVALID_NAMESPACE) ||
|
||||
errors.Is(err, mi.MI_RESULT_INVALID_CLASS) {
|
||||
logger.LogAttrs(ctx, slog.LevelWarn, "couldn't initialize collector", slog.Any("err", err))
|
||||
logger.LogAttrs(ctx, slog.LevelWarn, "couldn't initialize collector",
|
||||
slog.Any("err", err),
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
@@ -265,65 +295,98 @@ func (c *Collection) Build(ctx context.Context, logger *slog.Logger) error {
|
||||
}
|
||||
|
||||
// Close To be called by the exporter for collector cleanup.
|
||||
//
|
||||
// Close waits up to 5 seconds for running Build and Collect calls to return.
|
||||
// A collector that is still running after that is left open and reported in the returned error,
|
||||
// because closing it would release resources that are still in use.
|
||||
// The MI session is then left open as well.
|
||||
//
|
||||
// Close is safe to call after a failed or partial Build and more than once.
|
||||
// Collectors that were never built are not closed. A closed collector is never built or collected again.
|
||||
func (c *Collection) Close() error {
|
||||
c.state.closeMu.Lock()
|
||||
defer c.state.closeMu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), closeTimeout)
|
||||
defer cancel()
|
||||
|
||||
errs := make([]error, 0, len(c.collectors))
|
||||
collectorBusy := false
|
||||
|
||||
for name, collector := range c.collectors {
|
||||
if err := c.state.collector(name).close(ctx, collector); err != nil {
|
||||
collectorBusy = collectorBusy || errors.Is(err, errCollectorBusy)
|
||||
|
||||
for _, collector := range c.collectors {
|
||||
if err := collector.Close(); err != nil {
|
||||
errs = append(errs, fmt.Errorf("error from close collector %s: %w", collector.GetName(), err))
|
||||
}
|
||||
}
|
||||
|
||||
app, err := c.miSession.GetApplication()
|
||||
if err != nil && !errors.Is(err, mi.ErrNotInitialized) {
|
||||
errs = append(errs, fmt.Errorf("error from get MI application: %w", err))
|
||||
}
|
||||
|
||||
if err := c.miSession.Close(); err != nil && !errors.Is(err, mi.ErrNotInitialized) {
|
||||
errs = append(errs, fmt.Errorf("error from close MI session: %w", err))
|
||||
}
|
||||
|
||||
if err := app.Close(); err != nil && !errors.Is(err, mi.ErrNotInitialized) {
|
||||
errs = append(errs, fmt.Errorf("error from close MI application: %w", err))
|
||||
if collectorBusy {
|
||||
// Closing the MI session would block on, or break, the operations of running collectors.
|
||||
errs = append(errs, errors.New("MI session left open, because collectors are still running"))
|
||||
} else {
|
||||
errs = append(errs, c.state.closeMI()...)
|
||||
}
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// initMI To be called by the exporter for collector initialization.
|
||||
func (c *Collection) initMI() error {
|
||||
// It returns the session of an earlier call, if there is one.
|
||||
func (c *Collection) initMI() (*mi.Session, error) {
|
||||
if session := c.state.session(); session != nil {
|
||||
return session, nil
|
||||
}
|
||||
|
||||
app, err := mi.ApplicationInitialize()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error from initialize MI application: %w", err)
|
||||
return nil, fmt.Errorf("error from initialize MI application: %w", err)
|
||||
}
|
||||
|
||||
destinationOptions, err := app.NewDestinationOptions()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error from create NewDestinationOptions: %w", err)
|
||||
_ = app.Close()
|
||||
|
||||
return nil, fmt.Errorf("error from create NewDestinationOptions: %w", err)
|
||||
}
|
||||
|
||||
if err = destinationOptions.SetLocale(mi.LocaleEnglish); err != nil {
|
||||
return fmt.Errorf("error from set locale: %w", err)
|
||||
}
|
||||
|
||||
if err = destinationOptions.SetTimeout(gotime.Second); err != nil {
|
||||
return fmt.Errorf("error from set timeout: %w", err)
|
||||
}
|
||||
|
||||
c.miSession, err = app.NewSession(destinationOptions)
|
||||
session, err := newMISession(app, destinationOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error from create NewSession: %w", err)
|
||||
_ = destinationOptions.Delete()
|
||||
_ = app.Close()
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil
|
||||
c.state.setMI(app, session)
|
||||
|
||||
return session, nil
|
||||
}
|
||||
|
||||
func newMISession(app *mi.Application, destinationOptions *mi.DestinationOptions) (*mi.Session, error) {
|
||||
if err := destinationOptions.SetLocale(mi.LocaleEnglish); err != nil {
|
||||
return nil, fmt.Errorf("error from set locale: %w", err)
|
||||
}
|
||||
|
||||
if err := destinationOptions.SetTimeout(gotime.Second); err != nil {
|
||||
return nil, fmt.Errorf("error from set timeout: %w", err)
|
||||
}
|
||||
|
||||
session, err := app.NewSession(destinationOptions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error from create NewSession: %w", err)
|
||||
}
|
||||
|
||||
return session, nil
|
||||
}
|
||||
|
||||
// WithCollectors To be called by the exporter for collector initialization.
|
||||
// The returned Collection shares the collector instances and their state with c,
|
||||
// so a collector never runs twice at the same time, whichever Collection scrapes it.
|
||||
func (c *Collection) WithCollectors(collectors []string) (*Collection, error) {
|
||||
metricCollectors := &Collection{
|
||||
miSession: c.miSession,
|
||||
startTime: c.startTime,
|
||||
concurrencyCh: c.concurrencyCh,
|
||||
state: c.state,
|
||||
scrapeDurationDesc: c.scrapeDurationDesc,
|
||||
collectorScrapeDurationDesc: c.collectorScrapeDurationDesc,
|
||||
collectorScrapeSuccessDesc: c.collectorScrapeSuccessDesc,
|
||||
|
||||
@@ -18,9 +18,9 @@
|
||||
package collector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
@@ -29,20 +29,27 @@ import (
|
||||
// Interface guard.
|
||||
var _ prometheus.Collector = (*Handler)(nil)
|
||||
|
||||
// scrapeSlot admits one scrape at a time, across all collections.
|
||||
// We are expose metrics directly from the memory region of the Win32 API.
|
||||
// We should not allow more than one request at a time.
|
||||
//
|
||||
//nolint:gochecknoglobals
|
||||
var concurrencyMu sync.Mutex
|
||||
var scrapeSlot = make(chan struct{}, 1)
|
||||
|
||||
// Handler implements [prometheus.Collector] for a set of Windows Collection.
|
||||
type Handler struct {
|
||||
maxScrapeDuration time.Duration
|
||||
logger *slog.Logger
|
||||
collection *Collection
|
||||
|
||||
// ctx and deadline are set by [Collection.NewHandlerWithContext] for a handler that serves one scrape.
|
||||
// [prometheus.Collector.Collect] has no context parameter, so the handler carries it.
|
||||
ctx context.Context //nolint:containedctx
|
||||
deadline time.Time
|
||||
}
|
||||
|
||||
// NewHandler returns a new Handler that implements a [prometheus.Collector] for the given metrics Collection.
|
||||
// Each call of Collect may take up to maxScrapeDuration, including the time it waits for an earlier scrape to finish.
|
||||
func (c *Collection) NewHandler(maxScrapeDuration time.Duration, logger *slog.Logger, collectors []string) (*Handler, error) {
|
||||
collection := c
|
||||
|
||||
@@ -62,12 +69,60 @@ func (c *Collection) NewHandler(maxScrapeDuration time.Duration, logger *slog.Lo
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewHandlerWithContext returns a Handler for a single scrape, such as one HTTP request.
|
||||
// The scrape's deadline is maxScrapeDuration from now, so the time Collect waits for an earlier scrape
|
||||
// to finish counts against it. Collect returns early once ctx is done, without starting collectors
|
||||
// that have not started yet.
|
||||
func (c *Collection) NewHandlerWithContext(ctx context.Context, maxScrapeDuration time.Duration, logger *slog.Logger, collectors []string) (*Handler, error) {
|
||||
handler, err := c.NewHandler(maxScrapeDuration, logger, collectors)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
handler.ctx = ctx
|
||||
handler.deadline = time.Now().Add(maxScrapeDuration)
|
||||
|
||||
return handler, nil
|
||||
}
|
||||
|
||||
func (p *Handler) Describe(_ chan<- *prometheus.Desc) {}
|
||||
|
||||
// Collect sends the collected metrics from each of the Collection to
|
||||
// prometheus.
|
||||
func (p *Handler) Collect(ch chan<- prometheus.Metric) {
|
||||
concurrencyMu.Lock()
|
||||
p.collection.collectAll(ch, p.logger, p.maxScrapeDuration)
|
||||
concurrencyMu.Unlock()
|
||||
ctx, deadline := p.ctx, p.deadline
|
||||
if ctx == nil {
|
||||
ctx, deadline = context.Background(), time.Now().Add(p.maxScrapeDuration)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithDeadline(ctx, deadline)
|
||||
defer cancel()
|
||||
|
||||
p.collection.collectAll(ctx, ch, p.logger)
|
||||
}
|
||||
|
||||
// acquireScrapeSlot waits for the scrape slot until ctx is done.
|
||||
func acquireScrapeSlot(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
select {
|
||||
case scrapeSlot <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// Both cases may have been ready. Don't start a scrape that nobody waits for anymore.
|
||||
if err := ctx.Err(); err != nil {
|
||||
<-scrapeSlot
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func releaseScrapeSlot() {
|
||||
<-scrapeSlot
|
||||
}
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
// Copyright The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build windows
|
||||
|
||||
package collector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
|
||||
"github.com/prometheus-community/windows_exporter/internal/mi"
|
||||
)
|
||||
|
||||
var (
|
||||
errCollectorBusy = errors.New("collector is still running")
|
||||
errCollectionClosed = errors.New("collection is closed")
|
||||
)
|
||||
|
||||
// collectionState is the state of a Collection that its filtered views share.
|
||||
type collectionState struct {
|
||||
mu sync.Mutex
|
||||
collectors map[string]*collectorState
|
||||
miApp *mi.Application
|
||||
miSession *mi.Session
|
||||
|
||||
// closeMu serializes Close calls.
|
||||
closeMu sync.Mutex
|
||||
}
|
||||
|
||||
func newCollectionState() *collectionState {
|
||||
return &collectionState{collectors: make(map[string]*collectorState)}
|
||||
}
|
||||
|
||||
// collector returns the state of the named collector, creating it on first use.
|
||||
func (s *collectionState) collector(name string) *collectorState {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
state, ok := s.collectors[name]
|
||||
if !ok {
|
||||
state = &collectorState{slot: make(chan struct{}, 1)}
|
||||
s.collectors[name] = state
|
||||
}
|
||||
|
||||
return state
|
||||
}
|
||||
|
||||
func (s *collectionState) session() *mi.Session {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
return s.miSession
|
||||
}
|
||||
|
||||
func (s *collectionState) setMI(app *mi.Application, session *mi.Session) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.miApp = app
|
||||
s.miSession = session
|
||||
}
|
||||
|
||||
// closeMI closes the MI session and application. Later calls do nothing.
|
||||
func (s *collectionState) closeMI() []error {
|
||||
s.mu.Lock()
|
||||
app, session := s.miApp, s.miSession
|
||||
s.miApp, s.miSession = nil, nil
|
||||
s.mu.Unlock()
|
||||
|
||||
var errs []error
|
||||
|
||||
if session != nil {
|
||||
if err := session.Close(); err != nil && !errors.Is(err, mi.ErrNotInitialized) {
|
||||
errs = append(errs, fmt.Errorf("error from close MI session: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
if app != nil {
|
||||
if err := app.Close(); err != nil && !errors.Is(err, mi.ErrNotInitialized) {
|
||||
errs = append(errs, fmt.Errorf("error from close MI application: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
type startResult int
|
||||
|
||||
const (
|
||||
started startResult = iota
|
||||
busy
|
||||
notAvailable
|
||||
)
|
||||
|
||||
// collectorState tracks the lifecycle of one collector instance.
|
||||
type collectorState struct {
|
||||
// slot is held while Build, Collect or Close of the collector runs.
|
||||
// A Collect call that outlives its scrape keeps the slot until it returns,
|
||||
// so no scrape starts a second call on the same instance.
|
||||
slot chan struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
// buildErr is the error of the last Build. A collector that failed to build is not collected.
|
||||
buildErr error
|
||||
// built is true once Build was called, so Close only closes collectors that may hold resources.
|
||||
built bool
|
||||
// closing is true once Close was called. No Build or Collect starts afterwards.
|
||||
closing bool
|
||||
// closed is true once Collector.Close was called.
|
||||
closed bool
|
||||
// busyLogged is true once a scrape logged that it skipped the current run.
|
||||
busyLogged bool
|
||||
}
|
||||
|
||||
// tryCollect acquires the slot for a Collect call without waiting.
|
||||
func (s *collectorState) tryCollect() startResult {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.closing || s.buildErr != nil {
|
||||
return notAvailable
|
||||
}
|
||||
|
||||
select {
|
||||
case s.slot <- struct{}{}:
|
||||
return started
|
||||
default:
|
||||
return busy
|
||||
}
|
||||
}
|
||||
|
||||
// release releases the slot acquired by tryCollect.
|
||||
func (s *collectorState) release() {
|
||||
s.mu.Lock()
|
||||
s.busyLogged = false
|
||||
s.mu.Unlock()
|
||||
|
||||
<-s.slot
|
||||
}
|
||||
|
||||
// logBusy reports whether a skipped scrape is the first one of the current run.
|
||||
func (s *collectorState) logBusy() bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
first := !s.busyLogged
|
||||
s.busyLogged = true
|
||||
|
||||
return first
|
||||
}
|
||||
|
||||
// build runs Collector.Build while it holds the slot and records the result.
|
||||
// A panic in Collector.Build is returned as an error.
|
||||
func (s *collectorState) build(ctx context.Context, logger *slog.Logger, miSession *mi.Session, collector Collector) (err error) {
|
||||
select {
|
||||
case s.slot <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
err = context.Cause(ctx)
|
||||
|
||||
s.mu.Lock()
|
||||
s.buildErr = err
|
||||
s.mu.Unlock()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
s.buildErr = err
|
||||
s.mu.Unlock()
|
||||
|
||||
<-s.slot
|
||||
}()
|
||||
|
||||
s.mu.Lock()
|
||||
closing := s.closing
|
||||
s.built = !closing
|
||||
s.mu.Unlock()
|
||||
|
||||
if closing {
|
||||
return errCollectionClosed
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("panic: %v. stack: %s", r, string(debug.Stack()))
|
||||
}
|
||||
}()
|
||||
|
||||
return collector.Build(logger, miSession)
|
||||
}
|
||||
|
||||
// close calls Collector.Close once the collector is idle. It waits until ctx is done for a running
|
||||
// Build or Collect call to return. If the call does not return in time, the collector is left open,
|
||||
// because closing it would release resources that are still in use.
|
||||
// After close, the collector is never built or collected again.
|
||||
func (s *collectorState) close(ctx context.Context, collector Collector) (err error) {
|
||||
s.mu.Lock()
|
||||
s.closing = true
|
||||
done := s.closed || !s.built
|
||||
s.mu.Unlock()
|
||||
|
||||
if done {
|
||||
return nil
|
||||
}
|
||||
|
||||
select {
|
||||
case s.slot <- struct{}{}:
|
||||
// The slot is never released, so no call starts after Close.
|
||||
case <-ctx.Done():
|
||||
return errCollectorBusy
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.closed = true
|
||||
s.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("panic: %v. stack: %s", r, string(debug.Stack()))
|
||||
}
|
||||
}()
|
||||
|
||||
return collector.Close()
|
||||
}
|
||||
@@ -29,10 +29,11 @@ import (
|
||||
const DefaultCollectors = "cpu,memory,logical_disk,physical_disk,net,os,service,system"
|
||||
|
||||
type Collection struct {
|
||||
collectors Map
|
||||
miSession *mi.Session
|
||||
startTime time.Time
|
||||
concurrencyCh chan struct{}
|
||||
collectors Map
|
||||
startTime time.Time
|
||||
|
||||
// state is shared with every Collection derived from this one by WithCollectors.
|
||||
state *collectionState
|
||||
|
||||
scrapeDurationDesc *prometheus.Desc
|
||||
collectorScrapeDurationDesc *prometheus.Desc
|
||||
|
||||
Reference in New Issue
Block a user