mirror of
https://github.com/prometheus-community/windows_exporter.git
synced 2026-10-17 01:59:04 +02:00
mscluster: reuse metrics of unchanged cluster objects between scrapes (#2681)
Co-authored-by: Claude <noreply@anthropic.com> Signed-off-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
e6d1f00d1f
commit
2224698b36
@@ -44,3 +44,10 @@
|
||||
- Use `ClusterOpenEnum` and the `Open*Ex` functions (Windows Server 2008 R2+),
|
||||
not `ClusterOpenEnumEx` (Windows Server 2016+), to keep Windows Server
|
||||
2012 R2 support.
|
||||
- Cluster, network, node, group and resource metrics are reused between
|
||||
scrapes through `internal/metriccache` when every input is unchanged. A
|
||||
cache value must hold every input of the cached series: field values and
|
||||
whether each field is published (`appendObjectFieldsKey`, which resolves the
|
||||
build-dependent rules of `objectField`), owner node and node names. Objects
|
||||
with missing fields are not cached, so they report the same errors on every
|
||||
scrape. Shared volume sizes change constantly and are not cached.
|
||||
|
||||
@@ -129,10 +129,10 @@ func (c *Collector) Close() error {
|
||||
return c.closeSources()
|
||||
}
|
||||
|
||||
// closeSources releases the ClusAPI sources. A source whose Close fails is
|
||||
// dropped as well: the clusapi.Cluster has already released its handle, and a
|
||||
// later Build must be able to create new sources (Grafana Alloy rebuilds
|
||||
// collectors in place).
|
||||
// closeSources releases the ClusAPI sources and drops the cached metrics. A
|
||||
// source whose Close fails is dropped as well: the clusapi.Cluster has already
|
||||
// released its handle, and a later Build must be able to create new sources
|
||||
// (Grafana Alloy rebuilds collectors in place).
|
||||
func (c *Collector) closeSources() error {
|
||||
var errs []error
|
||||
|
||||
@@ -155,6 +155,12 @@ func (c *Collector) closeSources() error {
|
||||
closeSource(c.resourceGroupSource, func() { c.resourceGroupSource = nil })
|
||||
closeSource(c.sharedVolumesSource, func() { c.sharedVolumesSource = nil })
|
||||
|
||||
c.clusterMetricCache.Reset()
|
||||
c.networkMetricCache.Reset()
|
||||
c.nodeMetricCache.Reset()
|
||||
c.resourceMetricCache.Reset()
|
||||
c.resourceGroupMetricCache.Reset()
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,10 @@ type clusterSource interface {
|
||||
type collectorCluster struct {
|
||||
clusterSource clusterSource
|
||||
|
||||
// clusterMetricCache reuses the field metrics of the cluster; see
|
||||
// objectMetricCache.
|
||||
clusterMetricCache objectMetricCache
|
||||
|
||||
clusterAddEvictDelay *prometheus.Desc
|
||||
clusterAdminAccessPoint *prometheus.Desc
|
||||
clusterAutoAssignNodeSite *prometheus.Desc
|
||||
@@ -675,6 +679,9 @@ func (c *Collector) buildClusterDescriptors() {
|
||||
[]string{"name"},
|
||||
nil,
|
||||
)
|
||||
|
||||
// Cached metrics refer to the descriptors of an earlier build.
|
||||
c.clusterMetricCache.Reset()
|
||||
}
|
||||
|
||||
func (c *Collector) collectCluster(ch chan<- prometheus.Metric, maxScrapeDuration time.Duration) error {
|
||||
@@ -688,7 +695,11 @@ func (c *Collector) collectCluster(ch chan<- prometheus.Metric, maxScrapeDuratio
|
||||
}
|
||||
|
||||
func (c *Collector) publishCluster(ch chan<- prometheus.Metric, cluster clusapi.Object, build uint16, resultErr error) error {
|
||||
scrape := c.clusterMetricCache.Begin()
|
||||
|
||||
if cluster.Name == "" {
|
||||
commitScrape(scrape, 0, resultErr)
|
||||
|
||||
return resultErr
|
||||
}
|
||||
|
||||
@@ -786,5 +797,8 @@ func (c *Collector) publishCluster(ch chan<- prometheus.Metric, cluster clusapi.
|
||||
{name: "DetectedCloudPlatform", desc: c.clusterDetectedCloudPlatform, minBuild: ws2022, older: neverPublish},
|
||||
}
|
||||
|
||||
return publishObjectFields(ch, "cluster", cluster, fields, build, resultErr)
|
||||
_, resultErr = publishCachedObjectFields(ch, scrape, nil, "cluster", cluster, fields, build, resultErr)
|
||||
commitScrape(scrape, 1, resultErr)
|
||||
|
||||
return resultErr
|
||||
}
|
||||
|
||||
@@ -35,6 +35,10 @@ type networkSource interface {
|
||||
type collectorNetwork struct {
|
||||
networkSource networkSource
|
||||
|
||||
// networkMetricCache reuses the field metrics of unchanged networks; see
|
||||
// objectMetricCache.
|
||||
networkMetricCache objectMetricCache
|
||||
|
||||
networkCharacteristics *prometheus.Desc
|
||||
networkFlags *prometheus.Desc
|
||||
networkMetric *prometheus.Desc
|
||||
@@ -98,6 +102,9 @@ func (c *Collector) buildNetworkDescriptors() {
|
||||
[]string{"name"},
|
||||
nil,
|
||||
)
|
||||
|
||||
// Cached metrics refer to the descriptors of an earlier build.
|
||||
c.networkMetricCache.Reset()
|
||||
}
|
||||
|
||||
// Collect sends the metric values for each metric
|
||||
@@ -121,6 +128,10 @@ func (c *Collector) publishNetworks(ch chan<- prometheus.Metric, networks []clus
|
||||
{name: "State", desc: c.networkState},
|
||||
}
|
||||
|
||||
scrape := c.networkMetricCache.Begin()
|
||||
|
||||
var key []byte
|
||||
|
||||
for _, network := range networks {
|
||||
if network.Name == "" {
|
||||
continue
|
||||
@@ -128,8 +139,10 @@ func (c *Collector) publishNetworks(ch chan<- prometheus.Metric, networks []clus
|
||||
|
||||
// All network properties exist since Windows Server 2012, so the
|
||||
// build does not matter.
|
||||
resultErr = publishObjectFields(ch, "network", network, fields, 0, resultErr)
|
||||
key, resultErr = publishCachedObjectFields(ch, scrape, key, "network", network, fields, 0, resultErr)
|
||||
}
|
||||
|
||||
commitScrape(scrape, len(networks), resultErr)
|
||||
|
||||
return resultErr
|
||||
}
|
||||
|
||||
@@ -36,6 +36,10 @@ type nodeSource interface {
|
||||
type collectorNode struct {
|
||||
nodeSource nodeSource
|
||||
|
||||
// nodeMetricCache reuses the field metrics of unchanged nodes; see
|
||||
// objectMetricCache.
|
||||
nodeMetricCache objectMetricCache
|
||||
|
||||
nodeBuildNumber *prometheus.Desc
|
||||
nodeCharacteristics *prometheus.Desc
|
||||
nodeDetectedCloudPlatform *prometheus.Desc
|
||||
@@ -171,6 +175,9 @@ func (c *Collector) buildNodeDescriptors() {
|
||||
[]string{"name"},
|
||||
nil,
|
||||
)
|
||||
|
||||
// Cached metrics refer to the descriptors of an earlier build.
|
||||
c.nodeMetricCache.Reset()
|
||||
}
|
||||
|
||||
// Collect sends the metric values for each metric
|
||||
@@ -207,15 +214,20 @@ func (c *Collector) publishNodes(ch chan<- prometheus.Metric, nodes []clusapi.Ob
|
||||
}
|
||||
|
||||
nodeNames := make([]string, 0, len(nodes))
|
||||
scrape := c.nodeMetricCache.Begin()
|
||||
|
||||
var key []byte
|
||||
|
||||
for _, node := range nodes {
|
||||
if node.Name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
resultErr = publishObjectFields(ch, "node", node, fields, build, resultErr)
|
||||
key, resultErr = publishCachedObjectFields(ch, scrape, key, "node", node, fields, build, resultErr)
|
||||
nodeNames = append(nodeNames, node.Name)
|
||||
}
|
||||
|
||||
commitScrape(scrape, len(nodes), resultErr)
|
||||
|
||||
return nodeNames, resultErr
|
||||
}
|
||||
|
||||
@@ -18,10 +18,12 @@
|
||||
package mscluster
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/prometheus-community/windows_exporter/internal/headers/clusapi"
|
||||
"github.com/prometheus-community/windows_exporter/internal/metriccache"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
@@ -57,36 +59,137 @@ type objectField struct {
|
||||
absent *uint32
|
||||
}
|
||||
|
||||
// resolve returns the raw value that publishObjectFields publishes for field,
|
||||
// whether it publishes the field, and whether the field is missing: not
|
||||
// reported by the cluster service although it should be, which is an error.
|
||||
func (field objectField) resolve(object clusapi.Object, build uint16) (uint32, bool, bool) {
|
||||
raw, exists := object.Values[field.name]
|
||||
older := build < field.minBuild
|
||||
|
||||
switch {
|
||||
case older && field.older == neverPublish:
|
||||
return 0, false, false
|
||||
case exists:
|
||||
return raw, true, false
|
||||
case field.absent != nil:
|
||||
return *field.absent, true, false
|
||||
case older && field.older == zeroIfMissing:
|
||||
return 0, true, false
|
||||
case older:
|
||||
return 0, false, false
|
||||
default:
|
||||
return 0, false, true
|
||||
}
|
||||
}
|
||||
|
||||
// metric builds the metric of field for a raw value of the object name.
|
||||
//
|
||||
//nolint:ireturn // Const metrics have no exported type.
|
||||
func (field objectField) metric(raw uint32, name string) prometheus.Metric {
|
||||
value := float64(raw)
|
||||
if field.signed {
|
||||
value = float64(int32(raw))
|
||||
}
|
||||
|
||||
return prometheus.MustNewConstMetric(field.desc, prometheus.GaugeValue, value, name)
|
||||
}
|
||||
|
||||
// publishObjectFields publishes the fields of one object with its name as the
|
||||
// only label. Missing values are omitted and reported, never published as 0.
|
||||
func publishObjectFields(ch chan<- prometheus.Metric, kind string, object clusapi.Object, fields []objectField, build uint16, resultErr error) error {
|
||||
for _, field := range fields {
|
||||
raw, exists := object.Values[field.name]
|
||||
older := build < field.minBuild
|
||||
|
||||
switch {
|
||||
case older && field.older == neverPublish:
|
||||
continue
|
||||
case exists:
|
||||
case field.absent != nil:
|
||||
raw = *field.absent
|
||||
case older && field.older == zeroIfMissing:
|
||||
raw = 0
|
||||
case older:
|
||||
continue
|
||||
default:
|
||||
raw, publish, missing := field.resolve(object, build)
|
||||
if missing {
|
||||
resultErr = errors.Join(resultErr, fmt.Errorf("%s %q: missing property %s", kind, object.Name, field.name))
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
value := float64(raw)
|
||||
if field.signed {
|
||||
value = float64(int32(raw))
|
||||
if publish {
|
||||
ch <- field.metric(raw, object.Name)
|
||||
}
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(field.desc, prometheus.GaugeValue, value, object.Name)
|
||||
}
|
||||
|
||||
return resultErr
|
||||
}
|
||||
|
||||
// objectMetricCache reuses the field metrics of cluster objects whose values
|
||||
// did not change since the last scrape. Building the const metrics is most of
|
||||
// the allocations of a scrape, while the values are still read through ClusAPI
|
||||
// on every scrape. The key is the object name. The value encodes every input
|
||||
// of the cached metrics: the published fields with their raw values (see
|
||||
// appendObjectFieldsKey) and, for groups, the owner node and the node names.
|
||||
type objectMetricCache = metriccache.Cache[string, string]
|
||||
|
||||
// appendObjectFieldsKey appends the published fields of object and their raw
|
||||
// values to key. It returns false if a field is missing: such an object
|
||||
// returns an error on every scrape and is not cached.
|
||||
func appendObjectFieldsKey(key []byte, object clusapi.Object, fields []objectField, build uint16) ([]byte, bool) {
|
||||
for _, field := range fields {
|
||||
raw, publish, missing := field.resolve(object, build)
|
||||
|
||||
switch {
|
||||
case missing:
|
||||
return key, false
|
||||
case publish:
|
||||
key = append(key, 1)
|
||||
key = binary.LittleEndian.AppendUint32(key, raw)
|
||||
default:
|
||||
key = append(key, 0)
|
||||
}
|
||||
}
|
||||
|
||||
return key, true
|
||||
}
|
||||
|
||||
// appendKeyString appends s with its length to key, so that consecutive
|
||||
// strings cannot run into each other.
|
||||
func appendKeyString(key []byte, s string) []byte {
|
||||
key = binary.AppendUvarint(key, uint64(len(s)))
|
||||
|
||||
return append(key, s...)
|
||||
}
|
||||
|
||||
// objectFieldMetrics builds the field metrics of an object without missing
|
||||
// fields, with room for extra metrics.
|
||||
func objectFieldMetrics(object clusapi.Object, fields []objectField, build uint16, extra int) []prometheus.Metric {
|
||||
metrics := make([]prometheus.Metric, 0, len(fields)+extra)
|
||||
|
||||
for _, field := range fields {
|
||||
if raw, publish, _ := field.resolve(object, build); publish {
|
||||
metrics = append(metrics, field.metric(raw, object.Name))
|
||||
}
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
// publishCachedObjectFields publishes the fields of object like
|
||||
// publishObjectFields and reuses the metrics of the last scrape if they were
|
||||
// built from the same values. It returns key for reuse as a buffer.
|
||||
func publishCachedObjectFields(ch chan<- prometheus.Metric, scrape *metriccache.Scrape[string, string], key []byte, kind string, object clusapi.Object, fields []objectField, build uint16, resultErr error) ([]byte, error) {
|
||||
key, ok := appendObjectFieldsKey(key[:0], object, fields, build)
|
||||
if !ok {
|
||||
return key, publishObjectFields(ch, kind, object, fields, build, resultErr)
|
||||
}
|
||||
|
||||
metrics := scrape.Get(object.Name, string(key), func() []prometheus.Metric {
|
||||
return objectFieldMetrics(object, fields, build, 0)
|
||||
})
|
||||
|
||||
for _, metric := range metrics {
|
||||
ch <- metric
|
||||
}
|
||||
|
||||
return key, resultErr
|
||||
}
|
||||
|
||||
// commitScrape commits a scrape unless it collected nothing because reading
|
||||
// the objects failed, so a failed read keeps the cached metrics.
|
||||
func commitScrape[K, V comparable](scrape *metriccache.Scrape[K, V], collected int, err error) {
|
||||
if collected == 0 && err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
scrape.Commit()
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/prometheus-community/windows_exporter/internal/headers/clusapi"
|
||||
"github.com/prometheus-community/windows_exporter/internal/metriccache"
|
||||
"github.com/prometheus-community/windows_exporter/internal/types"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
@@ -37,6 +38,12 @@ type resourceSource interface {
|
||||
type collectorResource struct {
|
||||
resourceSource resourceSource
|
||||
|
||||
// resourceMetricCache reuses the metrics of unchanged resources. Each
|
||||
// resource publishes 16 series plus one owner node series per node, and
|
||||
// building them is most of the allocations of a scrape. The values are
|
||||
// still read through ClusAPI on every scrape.
|
||||
resourceMetricCache metriccache.Cache[string, resourceCacheValue]
|
||||
|
||||
resourceCharacteristics *prometheus.Desc
|
||||
resourceDeadlockTimeout *prometheus.Desc
|
||||
resourceEmbeddedFailureAction *prometheus.Desc
|
||||
@@ -203,6 +210,9 @@ func (c *Collector) buildResourceDescriptors() {
|
||||
[]string{"type", "owner_group", "name"},
|
||||
nil,
|
||||
)
|
||||
|
||||
// Cached metrics refer to the descriptors of an earlier build.
|
||||
c.resourceMetricCache.Reset()
|
||||
}
|
||||
|
||||
// Collect sends the metric values for each metric
|
||||
@@ -217,58 +227,146 @@ func (c *Collector) collectResource(ch chan<- prometheus.Metric, maxScrapeDurati
|
||||
return c.publishResources(ch, resources, nodeNames, resultErr)
|
||||
}
|
||||
|
||||
// resourceFieldCount is the number of DWORD properties of each resource.
|
||||
const resourceFieldCount = 15
|
||||
|
||||
// resourceCacheValue holds every input of the cached metrics of a resource
|
||||
// besides its name, which is the key.
|
||||
type resourceCacheValue struct {
|
||||
resourceType string
|
||||
ownerGroup string
|
||||
ownerNode string
|
||||
// NodeNames encodes the node names with appendKeyString.
|
||||
nodeNames string
|
||||
values [resourceFieldCount]uint32
|
||||
}
|
||||
|
||||
func (c *Collector) publishResources(ch chan<- prometheus.Metric, resources []clusapi.Resource, nodeNames []string, resultErr error) error {
|
||||
fieldNames := [resourceFieldCount]string{
|
||||
"Characteristics",
|
||||
"DeadlockTimeout",
|
||||
"EmbeddedFailureAction",
|
||||
"Flags",
|
||||
"IsAlivePollInterval",
|
||||
"LooksAlivePollInterval",
|
||||
"MonitorProcessId",
|
||||
"PendingTimeout",
|
||||
"ResourceClass",
|
||||
"RestartAction",
|
||||
"RestartDelay",
|
||||
"RestartPeriod",
|
||||
"RestartThreshold",
|
||||
"RetryPeriodOnFailure",
|
||||
"State",
|
||||
}
|
||||
fieldDescs := [resourceFieldCount]*prometheus.Desc{
|
||||
c.resourceCharacteristics,
|
||||
c.resourceDeadlockTimeout,
|
||||
c.resourceEmbeddedFailureAction,
|
||||
c.resourceFlags,
|
||||
c.resourceIsAlivePollInterval,
|
||||
c.resourceLooksAlivePollInterval,
|
||||
c.resourceMonitorProcessId,
|
||||
c.resourcePendingTimeout,
|
||||
c.resourceResourceClass,
|
||||
c.resourceRestartAction,
|
||||
c.resourceRestartDelay,
|
||||
c.resourceRestartPeriod,
|
||||
c.resourceRestartThreshold,
|
||||
c.resourceRetryPeriodOnFailure,
|
||||
c.resourceState,
|
||||
}
|
||||
|
||||
var nodeNamesKey []byte
|
||||
for _, nodeName := range nodeNames {
|
||||
nodeNamesKey = appendKeyString(nodeNamesKey, nodeName)
|
||||
}
|
||||
|
||||
nodeNamesValue := string(nodeNamesKey)
|
||||
scrape := c.resourceMetricCache.Begin()
|
||||
|
||||
for _, resource := range resources {
|
||||
if !resource.IdentityValid {
|
||||
continue
|
||||
}
|
||||
|
||||
fields := []struct {
|
||||
name string
|
||||
desc *prometheus.Desc
|
||||
}{
|
||||
{"Characteristics", c.resourceCharacteristics},
|
||||
{"DeadlockTimeout", c.resourceDeadlockTimeout},
|
||||
{"EmbeddedFailureAction", c.resourceEmbeddedFailureAction},
|
||||
{"Flags", c.resourceFlags},
|
||||
{"IsAlivePollInterval", c.resourceIsAlivePollInterval},
|
||||
{"LooksAlivePollInterval", c.resourceLooksAlivePollInterval},
|
||||
{"MonitorProcessId", c.resourceMonitorProcessId},
|
||||
{"PendingTimeout", c.resourcePendingTimeout},
|
||||
{"ResourceClass", c.resourceResourceClass},
|
||||
{"RestartAction", c.resourceRestartAction},
|
||||
{"RestartDelay", c.resourceRestartDelay},
|
||||
{"RestartPeriod", c.resourceRestartPeriod},
|
||||
{"RestartThreshold", c.resourceRestartThreshold},
|
||||
{"RetryPeriodOnFailure", c.resourceRetryPeriodOnFailure},
|
||||
{"State", c.resourceState},
|
||||
value := resourceCacheValue{
|
||||
resourceType: resource.Type,
|
||||
ownerGroup: resource.OwnerGroup,
|
||||
ownerNode: resource.OwnerNode,
|
||||
nodeNames: nodeNamesValue,
|
||||
}
|
||||
for _, field := range fields {
|
||||
value, exists := resource.Values[field.name]
|
||||
|
||||
complete := true
|
||||
|
||||
for index, name := range fieldNames {
|
||||
raw, exists := resource.Values[name]
|
||||
if !exists {
|
||||
resultErr = errors.Join(resultErr, fmt.Errorf("resource %q: missing DWORD property %s", resource.Name, field.name))
|
||||
complete = false
|
||||
|
||||
continue
|
||||
break
|
||||
}
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(field.desc, prometheus.GaugeValue, float64(value), resource.Type, resource.OwnerGroup, resource.Name)
|
||||
value.values[index] = raw
|
||||
}
|
||||
|
||||
// MSCluster_Resource.Subclass is NULL for every resource (seen on
|
||||
// Windows Server 2022, also for IP Address resources whose class
|
||||
// information carries CLUS_RESSUBCLASS_NETWORK_INTERNET_PROTOCOL), so
|
||||
// the WMI collector always published 0. Keep that series unchanged.
|
||||
ch <- prometheus.MustNewConstMetric(c.resourceSubClass, prometheus.GaugeValue, 0, resource.Type, resource.OwnerGroup, resource.Name)
|
||||
if !complete {
|
||||
// A resource with missing fields is not cached, so that it
|
||||
// reports its errors on every scrape.
|
||||
for index, name := range fieldNames {
|
||||
raw, exists := resource.Values[name]
|
||||
if !exists {
|
||||
resultErr = errors.Join(resultErr, fmt.Errorf("resource %q: missing DWORD property %s", resource.Name, name))
|
||||
|
||||
for _, nodeName := range nodeNames {
|
||||
value := 0.0
|
||||
if resource.OwnerNode == nodeName {
|
||||
value = 1
|
||||
continue
|
||||
}
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(fieldDescs[index], prometheus.GaugeValue, float64(raw), resource.Type, resource.OwnerGroup, resource.Name)
|
||||
}
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(c.resourceOwnerNode, prometheus.GaugeValue, value, resource.Type, resource.OwnerGroup, nodeName, resource.Name)
|
||||
for _, metric := range c.resourceNodeMetrics(nil, resource, nodeNames) {
|
||||
ch <- metric
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
metrics := scrape.Get(resource.Name, value, func() []prometheus.Metric {
|
||||
metrics := make([]prometheus.Metric, 0, resourceFieldCount+1+len(nodeNames))
|
||||
for index, raw := range value.values {
|
||||
metrics = append(metrics, prometheus.MustNewConstMetric(fieldDescs[index], prometheus.GaugeValue, float64(raw), resource.Type, resource.OwnerGroup, resource.Name))
|
||||
}
|
||||
|
||||
return c.resourceNodeMetrics(metrics, resource, nodeNames)
|
||||
})
|
||||
|
||||
for _, metric := range metrics {
|
||||
ch <- metric
|
||||
}
|
||||
}
|
||||
|
||||
commitScrape(scrape, len(resources), resultErr)
|
||||
|
||||
return resultErr
|
||||
}
|
||||
|
||||
// resourceNodeMetrics appends the subclass metric and the owner node metrics
|
||||
// of a resource, one per node, to metrics.
|
||||
func (c *Collector) resourceNodeMetrics(metrics []prometheus.Metric, resource clusapi.Resource, nodeNames []string) []prometheus.Metric {
|
||||
// MSCluster_Resource.Subclass is NULL for every resource (seen on
|
||||
// Windows Server 2022, also for IP Address resources whose class
|
||||
// information carries CLUS_RESSUBCLASS_NETWORK_INTERNET_PROTOCOL), so
|
||||
// the WMI collector always published 0. Keep that series unchanged.
|
||||
metrics = append(metrics, prometheus.MustNewConstMetric(c.resourceSubClass, prometheus.GaugeValue, 0, resource.Type, resource.OwnerGroup, resource.Name))
|
||||
|
||||
for _, nodeName := range nodeNames {
|
||||
value := 0.0
|
||||
if resource.OwnerNode == nodeName {
|
||||
value = 1
|
||||
}
|
||||
|
||||
metrics = append(metrics, prometheus.MustNewConstMetric(c.resourceOwnerNode, prometheus.GaugeValue, value, resource.Type, resource.OwnerGroup, nodeName, resource.Name))
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
@@ -299,6 +299,82 @@ func BenchmarkResourcePublication(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceMetricCache(t *testing.T) {
|
||||
c := New(&Config{CollectorsEnabled: []string{subCollectorResource}})
|
||||
c.buildResourceDescriptors()
|
||||
|
||||
publish := func(resource clusapi.Resource, nodeNames ...string) ([]prometheus.Metric, error) {
|
||||
ch := make(chan prometheus.Metric, 100)
|
||||
err := c.publishResources(ch, []clusapi.Resource{resource}, nodeNames, nil)
|
||||
|
||||
close(ch)
|
||||
|
||||
var metrics []prometheus.Metric
|
||||
for metric := range ch {
|
||||
metrics = append(metrics, metric)
|
||||
}
|
||||
|
||||
return metrics, err
|
||||
}
|
||||
|
||||
resource := resourceFixture()
|
||||
|
||||
first, err := publish(resource, "nodeA", "nodeB")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
second, err := publish(resource, "nodeA", "nodeB")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(first) != 18 || len(second) != len(first) || first[0] != second[0] {
|
||||
t.Fatal("did not reuse the metrics of an unchanged resource")
|
||||
}
|
||||
|
||||
// A changed value, owner node or node list builds new metrics.
|
||||
resource.Values = maps.Clone(resource.Values)
|
||||
resource.Values["State"] = 2
|
||||
resource.OwnerNode = "nodeC"
|
||||
|
||||
third, err := publish(resource, "nodeA", "nodeB", "nodeC")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
families, err := gatherPublished(t, func(ch chan<- prometheus.Metric) error {
|
||||
for _, metric := range third {
|
||||
ch <- metric
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if families["windows_mscluster_resource_state"].GetMetric()[0].GetGauge().GetValue() != 2 {
|
||||
t.Fatal("reused the metrics of a changed resource")
|
||||
}
|
||||
|
||||
for _, metric := range families["windows_mscluster_resource_owner_node"].GetMetric() {
|
||||
if hosted := metricLabels(metric)["node_name"] == "nodeC"; (metric.GetGauge().GetValue() == 1) != hosted {
|
||||
t.Fatal("reused the owner node of a changed resource")
|
||||
}
|
||||
}
|
||||
|
||||
// A resource with missing fields reports them on every scrape.
|
||||
delete(resource.Values, "Flags")
|
||||
|
||||
for range 2 {
|
||||
metrics, err := publish(resource, "nodeA")
|
||||
if err == nil || len(metrics) != 16 {
|
||||
t.Fatalf("missing field: %d metrics, %v", len(metrics), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceLargeClusterGather(t *testing.T) {
|
||||
resources := make([]clusapi.Resource, 81)
|
||||
for index := range resources {
|
||||
|
||||
@@ -36,6 +36,10 @@ type resourceGroupSource interface {
|
||||
type collectorResourceGroup struct {
|
||||
resourceGroupSource resourceGroupSource
|
||||
|
||||
// resourceGroupMetricCache reuses the field and owner node metrics of
|
||||
// unchanged groups; see objectMetricCache.
|
||||
resourceGroupMetricCache objectMetricCache
|
||||
|
||||
resourceGroupAutoFailbackType *prometheus.Desc
|
||||
resourceGroupCharacteristics *prometheus.Desc
|
||||
resourceGroupColdStartSetting *prometheus.Desc
|
||||
@@ -171,6 +175,9 @@ func (c *Collector) buildResourceGroupDescriptors() {
|
||||
[]string{"name"},
|
||||
nil,
|
||||
)
|
||||
|
||||
// Cached metrics refer to the descriptors of an earlier build.
|
||||
c.resourceGroupMetricCache.Reset()
|
||||
}
|
||||
|
||||
// Collect sends the metric values for each metric
|
||||
@@ -202,26 +209,74 @@ func (c *Collector) publishResourceGroups(ch chan<- prometheus.Metric, groups []
|
||||
{name: "State", desc: c.resourceGroupState},
|
||||
}
|
||||
|
||||
scrape := c.resourceGroupMetricCache.Begin()
|
||||
|
||||
var nodeNamesKey []byte
|
||||
for _, nodeName := range nodeNames {
|
||||
nodeNamesKey = appendKeyString(nodeNamesKey, nodeName)
|
||||
}
|
||||
|
||||
var key []byte
|
||||
|
||||
for _, group := range groups {
|
||||
if group.Name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
resultErr = publishObjectFields(ch, "group", group, fields, build, resultErr)
|
||||
var ok bool
|
||||
|
||||
key, ok = appendObjectFieldsKey(key[:0], group, fields, build)
|
||||
if !ok {
|
||||
// A group with missing fields is not cached, so that it reports
|
||||
// its errors on every scrape.
|
||||
resultErr = publishObjectFields(ch, "group", group, fields, build, resultErr)
|
||||
|
||||
for _, metric := range c.resourceGroupOwnerNodeMetrics(nil, group, nodeNames) {
|
||||
ch <- metric
|
||||
}
|
||||
|
||||
if !group.OwnerNodeValid {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, nodeName := range nodeNames {
|
||||
isCurrentState := 0.0
|
||||
if group.OwnerNode == nodeName {
|
||||
isCurrentState = 1.0
|
||||
}
|
||||
if group.OwnerNodeValid {
|
||||
key = append(key, 1)
|
||||
key = appendKeyString(key, group.OwnerNode)
|
||||
key = append(key, nodeNamesKey...)
|
||||
} else {
|
||||
key = append(key, 0)
|
||||
}
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(c.resourceGroupOwnerNode, prometheus.GaugeValue, isCurrentState, nodeName, group.Name)
|
||||
metrics := scrape.Get(group.Name, string(key), func() []prometheus.Metric {
|
||||
metrics := objectFieldMetrics(group, fields, build, len(nodeNames))
|
||||
|
||||
return c.resourceGroupOwnerNodeMetrics(metrics, group, nodeNames)
|
||||
})
|
||||
|
||||
for _, metric := range metrics {
|
||||
ch <- metric
|
||||
}
|
||||
}
|
||||
|
||||
commitScrape(scrape, len(groups), resultErr)
|
||||
|
||||
return resultErr
|
||||
}
|
||||
|
||||
// resourceGroupOwnerNodeMetrics appends the owner node metrics of a group, one
|
||||
// per node, to metrics. A group without a valid owner node has none.
|
||||
func (c *Collector) resourceGroupOwnerNodeMetrics(metrics []prometheus.Metric, group clusapi.Object, nodeNames []string) []prometheus.Metric {
|
||||
if !group.OwnerNodeValid {
|
||||
return metrics
|
||||
}
|
||||
|
||||
for _, nodeName := range nodeNames {
|
||||
isCurrentState := 0.0
|
||||
if group.OwnerNode == nodeName {
|
||||
isCurrentState = 1.0
|
||||
}
|
||||
|
||||
metrics = append(metrics, prometheus.MustNewConstMetric(c.resourceGroupOwnerNode, prometheus.GaugeValue, isCurrentState, nodeName, group.Name))
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user