mirror of
https://github.com/prometheus-community/windows_exporter.git
synced 2026-10-26 05:29:03 +01:00
chore: reuse per-scrape buffers (#2802)
Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>
This commit is contained in:
@@ -176,6 +176,9 @@ exercised a live Windows role.
|
|||||||
the original `Collection` owns collectors and MI resources. A timed-out call
|
the original `Collection` owns collectors and MI resources. A timed-out call
|
||||||
may still be running: prevent overlap, drain late metrics and do not free its
|
may still be running: prevent overlap, drain late metrics and do not free its
|
||||||
resources while it runs. Honor the supplied scrape budget in blocking APIs.
|
resources while it runs. Honor the supplied scrape budget in blocking APIs.
|
||||||
|
The collection reuses a collector's metrics channel for every call, so
|
||||||
|
`Collect` must not send after it returned, for example from a goroutine it
|
||||||
|
doesn't wait for: the metric would show up in the next scrape.
|
||||||
`Build` has no budget: bound its MI probe queries with `mi.BuildQueryTimeout`.
|
`Build` has no budget: bound its MI probe queries with `mi.BuildQueryTimeout`.
|
||||||
A non-positive MI query timeout selects `mi.DefaultQueryTimeout` (5s), which
|
A non-positive MI query timeout selects `mi.DefaultQueryTimeout` (5s), which
|
||||||
is too short for slow providers.
|
is too short for slow providers.
|
||||||
|
|||||||
@@ -92,6 +92,12 @@ func parseGPUCounterInstanceString(s string) Instance {
|
|||||||
type instanceCache struct {
|
type instanceCache struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
entries map[string]Instance
|
entries map[string]Instance
|
||||||
|
// spare is the map that commit replaced. The next scrape reuses it, so that
|
||||||
|
// the two maps alternate instead of one being allocated per scrape.
|
||||||
|
spare map[string]Instance
|
||||||
|
// active counts the scrapes between begin and commit. A replaced map is
|
||||||
|
// only kept as spare if no scrape may still read it.
|
||||||
|
active int
|
||||||
}
|
}
|
||||||
|
|
||||||
// instanceScrape parses the instance names of one scrape.
|
// instanceScrape parses the instance names of one scrape.
|
||||||
@@ -107,15 +113,24 @@ type instanceScrape struct {
|
|||||||
|
|
||||||
func (c *instanceCache) begin() *instanceScrape {
|
func (c *instanceCache) begin() *instanceScrape {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
prev := c.entries
|
prev, next := c.entries, c.spare
|
||||||
|
c.spare = nil
|
||||||
|
c.active++
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
|
|
||||||
return &instanceScrape{cache: c, prev: prev, next: make(map[string]Instance, len(prev))}
|
if next == nil {
|
||||||
|
next = make(map[string]Instance, len(prev))
|
||||||
|
} else {
|
||||||
|
clear(next)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &instanceScrape{cache: c, prev: prev, next: next}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *instanceCache) reset() {
|
func (c *instanceCache) reset() {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
c.entries = nil
|
c.entries = nil
|
||||||
|
c.spare = nil
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -154,7 +169,16 @@ func (s *instanceScrape) commit() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
s.cache.mu.Lock()
|
c := s.cache
|
||||||
s.cache.entries = next
|
|
||||||
s.cache.mu.Unlock()
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
c.active--
|
||||||
|
|
||||||
|
if c.active == 0 {
|
||||||
|
c.spare = c.entries
|
||||||
|
}
|
||||||
|
|
||||||
|
c.entries = next
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
package gpu
|
package gpu
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -109,3 +110,61 @@ func TestInstanceCache(t *testing.T) {
|
|||||||
cache.reset()
|
cache.reset()
|
||||||
require.Empty(t, cache.entries)
|
require.Empty(t, cache.entries)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInstanceCacheReusesMaps(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var cache instanceCache
|
||||||
|
|
||||||
|
name := "pid_1234_luid_0x00000000_0x00005678_phys_0_eng_0_engtype_3D"
|
||||||
|
want := parseGPUCounterInstanceString(name)
|
||||||
|
|
||||||
|
scrape := cache.begin()
|
||||||
|
scrape.parse(name)
|
||||||
|
scrape.commit()
|
||||||
|
|
||||||
|
first := cache.entries
|
||||||
|
|
||||||
|
scrape = cache.begin()
|
||||||
|
require.Equal(t, want, scrape.parse(name))
|
||||||
|
scrape.commit()
|
||||||
|
|
||||||
|
// The map of the first scrape is kept for the next one.
|
||||||
|
require.Equal(t, reflect.ValueOf(first).Pointer(), reflect.ValueOf(cache.spare).Pointer())
|
||||||
|
|
||||||
|
scrape = cache.begin()
|
||||||
|
require.Equal(t, reflect.ValueOf(first).Pointer(), reflect.ValueOf(scrape.next).Pointer())
|
||||||
|
require.Empty(t, scrape.next)
|
||||||
|
require.Equal(t, want, scrape.parse(name))
|
||||||
|
scrape.commit()
|
||||||
|
|
||||||
|
require.Equal(t, map[string]Instance{name: want}, cache.entries)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInstanceCacheOverlappingScrapes(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var cache instanceCache
|
||||||
|
|
||||||
|
name := "luid_0x00000000_0x00005678_phys_0"
|
||||||
|
|
||||||
|
scrape := cache.begin()
|
||||||
|
scrape.parse(name)
|
||||||
|
scrape.commit()
|
||||||
|
|
||||||
|
// a still reads the map that b's commit replaces, so the map must not be reused.
|
||||||
|
a := cache.begin()
|
||||||
|
b := cache.begin()
|
||||||
|
b.parse(name)
|
||||||
|
b.commit()
|
||||||
|
|
||||||
|
require.Nil(t, cache.spare)
|
||||||
|
|
||||||
|
c := cache.begin()
|
||||||
|
|
||||||
|
require.Equal(t, parseGPUCounterInstanceString(name), a.parse(name))
|
||||||
|
require.Equal(t, map[string]Instance{name: parseGPUCounterInstanceString(name)}, a.prev)
|
||||||
|
|
||||||
|
a.commit()
|
||||||
|
c.commit()
|
||||||
|
}
|
||||||
|
|||||||
@@ -56,10 +56,37 @@ type MetricsHTTPHandler struct {
|
|||||||
// the exporter itself.
|
// the exporter itself.
|
||||||
exporterMetricsRegistry *prometheus.Registry
|
exporterMetricsRegistry *prometheus.Registry
|
||||||
|
|
||||||
|
// registries holds idle registries for scrapes of all collectors. Since
|
||||||
|
// prometheus/client_golang#2170 (not in v1.25.0), a registry sizes the buffers of a
|
||||||
|
// Gather by the number of metrics of its previous Gather, so reusing it saves growing
|
||||||
|
// them on every scrape. Filtered scrapes use a new registry each.
|
||||||
|
registries chan *scrapeRegistry
|
||||||
|
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
timeoutMargin time.Duration
|
timeoutMargin time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// maxIdleRegistries is the number of idle registries kept for reuse. More are only needed
|
||||||
|
// while scrapes overlap.
|
||||||
|
const maxIdleRegistries = 4
|
||||||
|
|
||||||
|
// scrapeRegistry is a registry that serves one scrape at a time.
|
||||||
|
type scrapeRegistry struct {
|
||||||
|
reg *prometheus.Registry
|
||||||
|
collector scrapeCollector
|
||||||
|
}
|
||||||
|
|
||||||
|
// scrapeCollector collects the metrics of the scrape that currently uses its registry.
|
||||||
|
type scrapeCollector struct {
|
||||||
|
handler *collector.Handler
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *scrapeCollector) Describe(_ chan<- *prometheus.Desc) {}
|
||||||
|
|
||||||
|
func (c *scrapeCollector) Collect(ch chan<- prometheus.Metric) {
|
||||||
|
c.handler.Collect(ch)
|
||||||
|
}
|
||||||
|
|
||||||
type Options struct {
|
type Options struct {
|
||||||
DisableExporterMetrics bool
|
DisableExporterMetrics bool
|
||||||
TimeoutMargin float64
|
TimeoutMargin float64
|
||||||
@@ -75,6 +102,7 @@ func New(logger *slog.Logger, metricCollectors *collector.Collection, options *O
|
|||||||
|
|
||||||
handler := &MetricsHTTPHandler{
|
handler := &MetricsHTTPHandler{
|
||||||
metricCollectors: metricCollectors,
|
metricCollectors: metricCollectors,
|
||||||
|
registries: make(chan *scrapeRegistry, maxIdleRegistries),
|
||||||
logger: logger,
|
logger: logger,
|
||||||
timeoutMargin: toDuration(options.TimeoutMargin),
|
timeoutMargin: toDuration(options.TimeoutMargin),
|
||||||
}
|
}
|
||||||
@@ -105,7 +133,27 @@ func (c *MetricsHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
// The scrape's deadline starts now, so time spent waiting for an earlier scrape counts against it.
|
// The scrape's deadline starts now, so time spent waiting for an earlier scrape counts against it.
|
||||||
scrapeTimeout := c.getScrapeTimeout(logger, r)
|
scrapeTimeout := c.getScrapeTimeout(logger, r)
|
||||||
|
|
||||||
handler, err := c.handlerFactory(r.Context(), logger, scrapeTimeout, r.URL.Query()["collect[]"])
|
requestedCollectors := r.URL.Query()["collect[]"]
|
||||||
|
|
||||||
|
collectionHandler, err := c.metricCollectors.NewHandlerWithContext(r.Context(), scrapeTimeout, logger, requestedCollectors)
|
||||||
|
if err != nil {
|
||||||
|
err = fmt.Errorf("couldn't create collector handler: %w", err)
|
||||||
|
} else if len(requestedCollectors) == 0 {
|
||||||
|
sr := c.getRegistry()
|
||||||
|
defer c.putRegistry(sr)
|
||||||
|
|
||||||
|
sr.collector.handler = collectionHandler
|
||||||
|
|
||||||
|
c.newHTTPHandler(logger, sr.reg).ServeHTTP(w, r)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var handler http.Handler
|
||||||
|
if err == nil {
|
||||||
|
handler, err = c.handlerFactory(logger, collectionHandler)
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.WarnContext(r.Context(), "Couldn't create filtered metrics handler",
|
logger.WarnContext(r.Context(), "Couldn't create filtered metrics handler",
|
||||||
slog.Any("err", err),
|
slog.Any("err", err),
|
||||||
@@ -163,19 +211,44 @@ func toDuration(seconds float64) time.Duration {
|
|||||||
return time.Duration(seconds * float64(time.Second))
|
return time.Duration(seconds * float64(time.Second))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *MetricsHTTPHandler) handlerFactory(ctx context.Context, logger *slog.Logger, scrapeTimeout time.Duration, requestedCollectors []string) (http.Handler, error) {
|
// getRegistry returns an idle registry for a scrape of all collectors, or a new one.
|
||||||
|
func (c *MetricsHTTPHandler) getRegistry() *scrapeRegistry {
|
||||||
|
select {
|
||||||
|
case sr := <-c.registries:
|
||||||
|
return sr
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
sr := &scrapeRegistry{reg: prometheus.NewRegistry()}
|
||||||
|
sr.reg.MustRegister(version.NewCollector("windows_exporter"), &sr.collector)
|
||||||
|
|
||||||
|
return sr
|
||||||
|
}
|
||||||
|
|
||||||
|
// putRegistry keeps a registry whose scrape is done for reuse.
|
||||||
|
func (c *MetricsHTTPHandler) putRegistry(sr *scrapeRegistry) {
|
||||||
|
sr.collector.handler = nil
|
||||||
|
|
||||||
|
select {
|
||||||
|
case c.registries <- sr:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handlerFactory returns the handler of a filtered scrape, which uses a new registry.
|
||||||
|
func (c *MetricsHTTPHandler) handlerFactory(logger *slog.Logger, collectionHandler *collector.Handler) (http.Handler, error) {
|
||||||
reg := prometheus.NewRegistry()
|
reg := prometheus.NewRegistry()
|
||||||
reg.MustRegister(version.NewCollector("windows_exporter"))
|
reg.MustRegister(version.NewCollector("windows_exporter"))
|
||||||
|
|
||||||
collectionHandler, err := c.metricCollectors.NewHandlerWithContext(ctx, scrapeTimeout, logger, requestedCollectors)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("couldn't create collector handler: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := reg.Register(collectionHandler); err != nil {
|
if err := reg.Register(collectionHandler); err != nil {
|
||||||
return nil, fmt.Errorf("couldn't register Prometheus collector: %w", err)
|
return nil, fmt.Errorf("couldn't register Prometheus collector: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return c.newHTTPHandler(logger, reg), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newHTTPHandler returns the handler that serves the metrics of reg and of the exporter itself.
|
||||||
|
func (c *MetricsHTTPHandler) newHTTPHandler(logger *slog.Logger, reg *prometheus.Registry) http.Handler {
|
||||||
// Scrapes are serialized by the collection, so the handler doesn't limit requests in flight.
|
// Scrapes are serialized by the collection, so the handler doesn't limit requests in flight.
|
||||||
opts := promhttp.HandlerOpts{
|
opts := promhttp.HandlerOpts{
|
||||||
ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
|
ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
|
||||||
@@ -185,7 +258,7 @@ func (c *MetricsHTTPHandler) handlerFactory(ctx context.Context, logger *slog.Lo
|
|||||||
}
|
}
|
||||||
|
|
||||||
if c.exporterMetricsRegistry == nil {
|
if c.exporterMetricsRegistry == nil {
|
||||||
return canceledScrapeHandler(logger, reg, opts), nil
|
return canceledScrapeHandler(logger, reg, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Note that we have to use h.exporterMetricsRegistry here to
|
// Note that we have to use h.exporterMetricsRegistry here to
|
||||||
@@ -195,7 +268,7 @@ func (c *MetricsHTTPHandler) handlerFactory(ctx context.Context, logger *slog.Lo
|
|||||||
return promhttp.InstrumentMetricHandler(
|
return promhttp.InstrumentMetricHandler(
|
||||||
c.exporterMetricsRegistry,
|
c.exporterMetricsRegistry,
|
||||||
canceledScrapeHandler(logger, mergedGatherer{first: c.exporterMetricsRegistry, second: reg}, opts),
|
canceledScrapeHandler(logger, mergedGatherer{first: c.exporterMetricsRegistry, second: reg}, opts),
|
||||||
), nil
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// canceledScrapeHandler serves the metrics of gatherer like [promhttp.HandlerFor], but fails the
|
// canceledScrapeHandler serves the metrics of gatherer like [promhttp.HandlerFor], but fails the
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ import (
|
|||||||
"math"
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/synctest"
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
@@ -33,6 +35,7 @@ import (
|
|||||||
"github.com/prometheus-community/windows_exporter/internal/types"
|
"github.com/prometheus-community/windows_exporter/internal/types"
|
||||||
"github.com/prometheus-community/windows_exporter/pkg/collector"
|
"github.com/prometheus-community/windows_exporter/pkg/collector"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -86,6 +89,46 @@ func TestMetricsHTTPHandler(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Scrapes of all collectors reuse registries. Repeated and overlapping scrapes,
|
||||||
|
// mixed with filtered ones, must each serve the metrics once.
|
||||||
|
func TestMetricsHTTPHandlerReusesRegistries(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
collection := collector.New(collector.Map{
|
||||||
|
"first": &testCollector{name: "first"},
|
||||||
|
"second": &testCollector{name: "second"},
|
||||||
|
})
|
||||||
|
handler := httphandler.New(slog.New(slog.DiscardHandler), collection, nil)
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
for i := range 20 {
|
||||||
|
wg.Go(func() {
|
||||||
|
query := ""
|
||||||
|
if i%5 == 0 {
|
||||||
|
query = "?collect[]=first"
|
||||||
|
}
|
||||||
|
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(response, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics"+query, nil))
|
||||||
|
|
||||||
|
body := response.Body.String()
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusOK, response.Code)
|
||||||
|
assert.Equal(t, 1, strings.Count(body, "\nwindows_test_first 42\n"))
|
||||||
|
assert.Equal(t, 1, strings.Count(body, "\nwindows_exporter_build_info{"))
|
||||||
|
|
||||||
|
if query == "" {
|
||||||
|
assert.Equal(t, 1, strings.Count(body, "\nwindows_test_second 42\n"))
|
||||||
|
} else {
|
||||||
|
assert.NotContains(t, body, "windows_test_second")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
func TestMetricsHTTPHandlerCollectorStatus(t *testing.T) {
|
func TestMetricsHTTPHandlerCollectorStatus(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -143,6 +143,11 @@ type Scrape struct {
|
|||||||
committed bool
|
committed bool
|
||||||
// used is set by the first MustNewConstMetric call.
|
// used is set by the first MustNewConstMetric call.
|
||||||
used atomic.Bool
|
used atomic.Bool
|
||||||
|
|
||||||
|
// counters and gauges hold the metrics the scrape hands out. They are
|
||||||
|
// guarded by the lock of the cache.
|
||||||
|
counters chunk[counterMetric]
|
||||||
|
gauges chunk[gaugeMetric]
|
||||||
}
|
}
|
||||||
|
|
||||||
// Begin starts a scrape. Call Commit when it is done.
|
// Begin starts a scrape. Call Commit when it is done.
|
||||||
@@ -343,9 +348,10 @@ func mustNewConstMetric[K comparable](s *Scrape, m *seriesMap[K], k K, desc *pro
|
|||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
if c.resets == s.resets && !s.committed {
|
if c.resets == s.resets && !s.committed {
|
||||||
if labels, ok := m.get(k, s.generation); ok {
|
if labels, ok := m.get(k, s.generation); ok {
|
||||||
|
metric := s.newConstMetric(desc, valueType, value, labels)
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
|
|
||||||
return newConstMetric(desc, valueType, value, labels)
|
return metric
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
@@ -368,7 +374,7 @@ func mustNewConstMetric[K comparable](s *Scrape, m *seriesMap[K], k K, desc *pro
|
|||||||
|
|
||||||
// Another goroutine of the scrape may have added the series meanwhile.
|
// Another goroutine of the scrape may have added the series meanwhile.
|
||||||
if labels, ok := m.get(k, s.generation); ok {
|
if labels, ok := m.get(k, s.generation); ok {
|
||||||
return newConstMetric(desc, valueType, value, labels)
|
return s.newConstMetric(desc, valueType, value, labels)
|
||||||
}
|
}
|
||||||
|
|
||||||
if *m == nil {
|
if *m == nil {
|
||||||
@@ -378,7 +384,7 @@ func mustNewConstMetric[K comparable](s *Scrape, m *seriesMap[K], k K, desc *pro
|
|||||||
labels := c.intern(out.GetLabel())
|
labels := c.intern(out.GetLabel())
|
||||||
(*m)[k] = series{labels: labels, seen: s.generation}
|
(*m)[k] = series{labels: labels, seen: s.generation}
|
||||||
|
|
||||||
return newConstMetric(desc, valueType, value, labels)
|
return s.newConstMetric(desc, valueType, value, labels)
|
||||||
}
|
}
|
||||||
|
|
||||||
func knownValueType(valueType prometheus.ValueType) bool {
|
func knownValueType(valueType prometheus.ValueType) bool {
|
||||||
@@ -390,20 +396,50 @@ func knownValueType(valueType prometheus.ValueType) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Metrics of a scrape are allocated in chunks, which grow from minChunk to
|
||||||
|
// maxChunk metrics, so a collector with few series doesn't allocate more
|
||||||
|
// than it needs. A metric that is kept keeps its chunk alive, which is why
|
||||||
|
// chunks stay small.
|
||||||
|
const (
|
||||||
|
minChunk = 8
|
||||||
|
maxChunk = 128
|
||||||
|
)
|
||||||
|
|
||||||
|
// chunk hands out the metrics of one type of a scrape.
|
||||||
|
type chunk[T any] struct {
|
||||||
|
free []T
|
||||||
|
size int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *chunk[T]) next() *T {
|
||||||
|
if len(c.free) == 0 {
|
||||||
|
c.size = min(max(2*c.size, minChunk), maxChunk)
|
||||||
|
c.free = make([]T, c.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
m := &c.free[0]
|
||||||
|
c.free = c.free[1:]
|
||||||
|
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
// newConstMetric returns a const metric with shared label pairs. The value
|
// newConstMetric returns a const metric with shared label pairs. The value
|
||||||
// message lives in the metric, so Write doesn't allocate. Label pairs are
|
// message lives in the metric, so Write doesn't allocate. Label pairs are
|
||||||
// immutable once written, like those of prometheus.NewConstMetric.
|
// immutable once written, like those of prometheus.NewConstMetric.
|
||||||
|
// The caller holds the lock of the cache, which guards the chunks.
|
||||||
//
|
//
|
||||||
//nolint:ireturn // Const metrics have no exported type.
|
//nolint:ireturn // Const metrics have no exported type.
|
||||||
func newConstMetric(desc *prometheus.Desc, valueType prometheus.ValueType, value float64, labels []*dto.LabelPair) prometheus.Metric {
|
func (s *Scrape) newConstMetric(desc *prometheus.Desc, valueType prometheus.ValueType, value float64, labels []*dto.LabelPair) prometheus.Metric {
|
||||||
switch valueType {
|
switch valueType {
|
||||||
case prometheus.CounterValue:
|
case prometheus.CounterValue:
|
||||||
m := &counterMetric{desc: desc, labels: labels, value: value}
|
m := s.counters.next()
|
||||||
|
*m = counterMetric{desc: desc, labels: labels, value: value}
|
||||||
m.counter.Value = &m.value
|
m.counter.Value = &m.value
|
||||||
|
|
||||||
return m
|
return m
|
||||||
case prometheus.GaugeValue:
|
case prometheus.GaugeValue:
|
||||||
m := &gaugeMetric{desc: desc, labels: labels, value: value}
|
m := s.gauges.next()
|
||||||
|
*m = gaugeMetric{desc: desc, labels: labels, value: value}
|
||||||
m.gauge.Value = &m.value
|
m.gauge.Value = &m.value
|
||||||
|
|
||||||
return m
|
return m
|
||||||
|
|||||||
@@ -82,6 +82,38 @@ func TestMustNewConstMetric(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Metrics share chunks, so each must keep its own value, also across chunks
|
||||||
|
// and after the next scrape reuses the series.
|
||||||
|
func TestChunkedMetricsKeepTheirValues(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var cache labelcache.Cache
|
||||||
|
|
||||||
|
const n = 300
|
||||||
|
|
||||||
|
for scrapeNumber := range 2 {
|
||||||
|
scrape := cache.Begin()
|
||||||
|
metrics := make([]prometheus.Metric, 0, 2*n)
|
||||||
|
|
||||||
|
for i := range n {
|
||||||
|
value := float64(scrapeNumber*1000 + i)
|
||||||
|
metrics = append(metrics,
|
||||||
|
scrape.MustNewConstMetric(counterDesc, prometheus.CounterValue, value, "p", strconv.Itoa(i), "user"),
|
||||||
|
scrape.MustNewConstMetric(gaugeDesc, prometheus.GaugeValue, -value, "p", strconv.Itoa(i)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
scrape.Commit()
|
||||||
|
|
||||||
|
for i := range n {
|
||||||
|
value := float64(scrapeNumber*1000 + i)
|
||||||
|
require.InDelta(t, value, write(t, metrics[2*i]).GetCounter().GetValue(), 0)
|
||||||
|
require.InDelta(t, -value, write(t, metrics[2*i+1]).GetGauge().GetValue(), 0)
|
||||||
|
require.Equal(t, strconv.Itoa(i), write(t, metrics[2*i+1]).GetLabel()[1].GetValue())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// From the second scrape on, a series reuses the label pairs of the previous
|
// From the second scrape on, a series reuses the label pairs of the previous
|
||||||
// one; series that a scrape didn't report get new pairs.
|
// one; series that a scrape didn't report get new pairs.
|
||||||
func TestLabelReuse(t *testing.T) {
|
func TestLabelReuse(t *testing.T) {
|
||||||
|
|||||||
+39
-23
@@ -176,7 +176,9 @@ func (c *Collection) collectCollector(ctx context.Context, ch chan<- prometheus.
|
|||||||
|
|
||||||
state := c.state.collector(name)
|
state := c.state.collector(name)
|
||||||
|
|
||||||
switch state.tryCollect() {
|
result, bufCh, doneCh := state.tryCollect()
|
||||||
|
|
||||||
|
switch result {
|
||||||
case started:
|
case started:
|
||||||
case notAvailable:
|
case notAvailable:
|
||||||
logger.LogAttrs(ctx, slog.LevelDebug, "collector skipped, because it isn't initialized",
|
logger.LogAttrs(ctx, slog.LevelDebug, "collector skipped, because it isn't initialized",
|
||||||
@@ -197,20 +199,12 @@ func (c *Collection) collectCollector(ctx context.Context, ch chan<- prometheus.
|
|||||||
return pending
|
return pending
|
||||||
}
|
}
|
||||||
|
|
||||||
// bufCh is a buffer channel to store the metrics
|
// bufCh buffers the metrics, because the prometheus registry closes ch once the scrape ends,
|
||||||
// This is needed because once timeout is reached, the prometheus registry channel is closed.
|
// while the collector may still send. The collector state reuses bufCh and doneCh for every call.
|
||||||
bufCh := make(chan prometheus.Metric, 1000)
|
|
||||||
errCh := make(chan error, 1)
|
|
||||||
|
|
||||||
t := time.Now()
|
t := time.Now()
|
||||||
|
|
||||||
// execute the collector. The goroutine owns the collector's slot until Collect returns.
|
|
||||||
go func() {
|
go func() {
|
||||||
// Release the slot before closing bufCh, so the next scrape never sees this call as running.
|
doneCh <- collect(name, collector, bufCh, maxScrapeDuration)
|
||||||
defer close(bufCh)
|
|
||||||
defer state.release()
|
|
||||||
|
|
||||||
errCh <- collect(name, collector, bufCh, maxScrapeDuration)
|
|
||||||
}()
|
}()
|
||||||
|
|
||||||
numMetrics := 0
|
numMetrics := 0
|
||||||
@@ -219,25 +213,29 @@ func (c *Collection) collectCollector(ctx context.Context, ch chan<- prometheus.
|
|||||||
// This is the only goroutine that writes to ch, and it stops before collectCollector returns.
|
// This is the only goroutine that writes to ch, and it stops before collectCollector returns.
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case m, ok := <-bufCh:
|
case m := <-bufCh:
|
||||||
if ok {
|
ch <- m
|
||||||
ch <- m
|
|
||||||
|
numMetrics++
|
||||||
|
case err := <-doneCh:
|
||||||
|
// Collect sent its metrics before its result, so the rest is buffered.
|
||||||
|
for len(bufCh) > 0 {
|
||||||
|
ch <- <-bufCh
|
||||||
|
|
||||||
numMetrics++
|
numMetrics++
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Release the slot only once bufCh is empty, so the next call starts with an empty buffer.
|
||||||
|
state.release()
|
||||||
|
|
||||||
duration := time.Since(t)
|
duration := time.Since(t)
|
||||||
c.sendCollectorDuration(ch, name, duration)
|
c.sendCollectorDuration(ch, name, duration)
|
||||||
|
|
||||||
return logCollectorResult(ctx, logger, name, <-errCh, duration, numMetrics)
|
return logCollectorResult(ctx, logger, name, err, duration, numMetrics)
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
// Drain late metrics, so that Collect doesn't block on a full buffer.
|
// Drain late metrics, so that Collect doesn't block on a full buffer,
|
||||||
go func() {
|
// and release the slot once the call returned and bufCh is empty.
|
||||||
for range bufCh {
|
go drainCollect(state, bufCh, doneCh)
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
duration := time.Since(t)
|
duration := time.Since(t)
|
||||||
c.sendCollectorDuration(ch, name, duration)
|
c.sendCollectorDuration(ch, name, duration)
|
||||||
@@ -262,6 +260,24 @@ func (c *Collection) collectCollector(ctx context.Context, ch chan<- prometheus.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// drainCollect drops the metrics of a Collect call that outlived its scrape. Once the call returned
|
||||||
|
// and the buffer is empty, it releases the slot of the collector.
|
||||||
|
func drainCollect(state *collectorState, bufCh <-chan prometheus.Metric, doneCh <-chan error) {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-bufCh:
|
||||||
|
case <-doneCh: //nolint:errcheck // The scrape already reported the call as timed out.
|
||||||
|
for len(bufCh) > 0 {
|
||||||
|
<-bufCh
|
||||||
|
}
|
||||||
|
|
||||||
|
state.release()
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// collect calls Collector.Collect and turns a panic into an error.
|
// collect calls Collector.Collect and turns a panic into an error.
|
||||||
// It begins a scrape of the collector's caches before and ends it after the call.
|
// It begins a scrape of the collector's caches before and ends it after the call.
|
||||||
func collect(name string, collector Collector, ch chan<- prometheus.Metric, maxScrapeDuration time.Duration) (err error) {
|
func collect(name string, collector Collector, ch chan<- prometheus.Metric, maxScrapeDuration time.Duration) (err error) {
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"runtime"
|
"runtime"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -192,6 +193,59 @@ func TestCollectTimedOutCollectorDoesNotOverlap(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The metrics buffer of a collector is reused, so the late metrics of a timed-out call must not reach
|
||||||
|
// the next scrape.
|
||||||
|
func TestCollectLateMetricsDoNotLeak(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
// More metrics than the buffer holds, so the late call blocks until they are drained.
|
||||||
|
late := &countingCollector{fakeCollector: &fakeCollector{name: "late"}, metrics: 1500, firstDelay: 2 * time.Second}
|
||||||
|
collection := collector.New(collector.Map{"late": late})
|
||||||
|
|
||||||
|
handler, err := collection.NewHandler(300*time.Millisecond, slog.New(slog.DiscardHandler), nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
out := scrape(t, handler)
|
||||||
|
require.Contains(t, out, `windows_exporter_collector_timeout{collector="late"} 1`)
|
||||||
|
require.NotContains(t, out, "windows_test_late")
|
||||||
|
|
||||||
|
// The first call sends its metrics into the buffer after the scrape ended.
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
for call := 2; call <= 3; call++ {
|
||||||
|
out = scrape(t, handler)
|
||||||
|
require.Contains(t, out, `windows_exporter_collector_success{collector="late"} 1`)
|
||||||
|
require.Equal(t, late.metrics, strings.Count(out, "windows_test_late{"))
|
||||||
|
require.Equal(t, late.metrics, strings.Count(out, "} "+strconv.Itoa(call)+"\n"), "scrape %d got metrics of another call", call)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// countingCollector sends metrics whose value is the number of the call. Only the first call waits for firstDelay.
|
||||||
|
type countingCollector struct {
|
||||||
|
*fakeCollector
|
||||||
|
|
||||||
|
metrics int
|
||||||
|
firstDelay time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingCollector) Collect(ch chan<- prometheus.Metric, _ time.Duration) error {
|
||||||
|
call := c.calls.Add(1)
|
||||||
|
if call == 1 {
|
||||||
|
time.Sleep(c.firstDelay)
|
||||||
|
}
|
||||||
|
|
||||||
|
desc := prometheus.NewDesc("windows_test_"+c.name, "Test metric", []string{"i"}, nil)
|
||||||
|
|
||||||
|
for i := range c.metrics {
|
||||||
|
ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(call), strconv.Itoa(i))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func TestCollectAdmission(t *testing.T) {
|
func TestCollectAdmission(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
+21
-4
@@ -27,6 +27,7 @@ import (
|
|||||||
|
|
||||||
"github.com/prometheus-community/windows_exporter/internal/mi"
|
"github.com/prometheus-community/windows_exporter/internal/mi"
|
||||||
"github.com/prometheus-community/windows_exporter/internal/scrapecache"
|
"github.com/prometheus-community/windows_exporter/internal/scrapecache"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -162,23 +163,39 @@ type collectorState struct {
|
|||||||
closed bool
|
closed bool
|
||||||
// busyLogged is true once a scrape logged that it skipped the current run.
|
// busyLogged is true once a scrape logged that it skipped the current run.
|
||||||
busyLogged bool
|
busyLogged bool
|
||||||
|
|
||||||
|
// metrics buffers the metrics of a Collect call, and done receives its result.
|
||||||
|
// Only the holder of the slot uses them, so every call reuses them.
|
||||||
|
// The holder releases the slot once done was received and metrics is empty.
|
||||||
|
metrics chan prometheus.Metric
|
||||||
|
done chan error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// metricsBufferSize is the capacity of collectorState.metrics.
|
||||||
|
const metricsBufferSize = 1000
|
||||||
|
|
||||||
// tryCollect acquires the slot for a Collect call without waiting.
|
// tryCollect acquires the slot for a Collect call without waiting.
|
||||||
func (s *collectorState) tryCollect() startResult {
|
// On success, it returns the buffers of the call.
|
||||||
|
func (s *collectorState) tryCollect() (startResult, chan prometheus.Metric, chan error) {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
if s.closing || s.buildErr != nil {
|
if s.closing || s.buildErr != nil {
|
||||||
return notAvailable
|
return notAvailable, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case s.slot <- struct{}{}:
|
case s.slot <- struct{}{}:
|
||||||
return started
|
|
||||||
default:
|
default:
|
||||||
return busy
|
return busy, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if s.metrics == nil {
|
||||||
|
s.metrics = make(chan prometheus.Metric, metricsBufferSize)
|
||||||
|
s.done = make(chan error, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return started, s.metrics, s.done
|
||||||
}
|
}
|
||||||
|
|
||||||
// release releases the slot acquired by tryCollect.
|
// release releases the slot acquired by tryCollect.
|
||||||
|
|||||||
Reference in New Issue
Block a user