mscluster: reuse metrics of unchanged cluster objects between scrapes (#2681)

Co-authored-by: Claude <noreply@anthropic.com>
Signed-off-by: Claude <noreply@anthropic.com>
This commit is contained in:
Jan-Otto Kröpke
2026-10-11 16:28:48 +02:00
committed by GitHub
co-authored by Claude
parent e6d1f00d1f
commit 2224698b36
9 changed files with 452 additions and 68 deletions
+7
View File
@@ -44,3 +44,10 @@
- Use `ClusterOpenEnum` and the `Open*Ex` functions (Windows Server 2008 R2+),
not `ClusterOpenEnumEx` (Windows Server 2016+), to keep Windows Server
2012 R2 support.
- Cluster, network, node, group and resource metrics are reused between
scrapes through `internal/metriccache` when every input is unchanged. A
cache value must hold every input of the cached series: field values and
whether each field is published (`appendObjectFieldsKey`, which resolves the
build-dependent rules of `objectField`), owner node and node names. Objects
with missing fields are not cached, so they report the same errors on every
scrape. Shared volume sizes change constantly and are not cached.
+10 -4
View File
@@ -129,10 +129,10 @@ func (c *Collector) Close() error {
return c.closeSources()
}
// closeSources releases the ClusAPI sources. A source whose Close fails is
// dropped as well: the clusapi.Cluster has already released its handle, and a
// later Build must be able to create new sources (Grafana Alloy rebuilds
// collectors in place).
// closeSources releases the ClusAPI sources and drops the cached metrics. A
// source whose Close fails is dropped as well: the clusapi.Cluster has already
// released its handle, and a later Build must be able to create new sources
// (Grafana Alloy rebuilds collectors in place).
func (c *Collector) closeSources() error {
var errs []error
@@ -155,6 +155,12 @@ func (c *Collector) closeSources() error {
closeSource(c.resourceGroupSource, func() { c.resourceGroupSource = nil })
closeSource(c.sharedVolumesSource, func() { c.sharedVolumesSource = nil })
c.clusterMetricCache.Reset()
c.networkMetricCache.Reset()
c.nodeMetricCache.Reset()
c.resourceMetricCache.Reset()
c.resourceGroupMetricCache.Reset()
return errors.Join(errs...)
}
@@ -36,6 +36,10 @@ type clusterSource interface {
type collectorCluster struct {
clusterSource clusterSource
// clusterMetricCache reuses the field metrics of the cluster; see
// objectMetricCache.
clusterMetricCache objectMetricCache
clusterAddEvictDelay *prometheus.Desc
clusterAdminAccessPoint *prometheus.Desc
clusterAutoAssignNodeSite *prometheus.Desc
@@ -675,6 +679,9 @@ func (c *Collector) buildClusterDescriptors() {
[]string{"name"},
nil,
)
// Cached metrics refer to the descriptors of an earlier build.
c.clusterMetricCache.Reset()
}
func (c *Collector) collectCluster(ch chan<- prometheus.Metric, maxScrapeDuration time.Duration) error {
@@ -688,7 +695,11 @@ func (c *Collector) collectCluster(ch chan<- prometheus.Metric, maxScrapeDuratio
}
func (c *Collector) publishCluster(ch chan<- prometheus.Metric, cluster clusapi.Object, build uint16, resultErr error) error {
scrape := c.clusterMetricCache.Begin()
if cluster.Name == "" {
commitScrape(scrape, 0, resultErr)
return resultErr
}
@@ -786,5 +797,8 @@ func (c *Collector) publishCluster(ch chan<- prometheus.Metric, cluster clusapi.
{name: "DetectedCloudPlatform", desc: c.clusterDetectedCloudPlatform, minBuild: ws2022, older: neverPublish},
}
return publishObjectFields(ch, "cluster", cluster, fields, build, resultErr)
_, resultErr = publishCachedObjectFields(ch, scrape, nil, "cluster", cluster, fields, build, resultErr)
commitScrape(scrape, 1, resultErr)
return resultErr
}
@@ -35,6 +35,10 @@ type networkSource interface {
type collectorNetwork struct {
networkSource networkSource
// networkMetricCache reuses the field metrics of unchanged networks; see
// objectMetricCache.
networkMetricCache objectMetricCache
networkCharacteristics *prometheus.Desc
networkFlags *prometheus.Desc
networkMetric *prometheus.Desc
@@ -98,6 +102,9 @@ func (c *Collector) buildNetworkDescriptors() {
[]string{"name"},
nil,
)
// Cached metrics refer to the descriptors of an earlier build.
c.networkMetricCache.Reset()
}
// Collect sends the metric values for each metric
@@ -121,6 +128,10 @@ func (c *Collector) publishNetworks(ch chan<- prometheus.Metric, networks []clus
{name: "State", desc: c.networkState},
}
scrape := c.networkMetricCache.Begin()
var key []byte
for _, network := range networks {
if network.Name == "" {
continue
@@ -128,8 +139,10 @@ func (c *Collector) publishNetworks(ch chan<- prometheus.Metric, networks []clus
// All network properties exist since Windows Server 2012, so the
// build does not matter.
resultErr = publishObjectFields(ch, "network", network, fields, 0, resultErr)
key, resultErr = publishCachedObjectFields(ch, scrape, key, "network", network, fields, 0, resultErr)
}
commitScrape(scrape, len(networks), resultErr)
return resultErr
}
+13 -1
View File
@@ -36,6 +36,10 @@ type nodeSource interface {
type collectorNode struct {
nodeSource nodeSource
// nodeMetricCache reuses the field metrics of unchanged nodes; see
// objectMetricCache.
nodeMetricCache objectMetricCache
nodeBuildNumber *prometheus.Desc
nodeCharacteristics *prometheus.Desc
nodeDetectedCloudPlatform *prometheus.Desc
@@ -171,6 +175,9 @@ func (c *Collector) buildNodeDescriptors() {
[]string{"name"},
nil,
)
// Cached metrics refer to the descriptors of an earlier build.
c.nodeMetricCache.Reset()
}
// Collect sends the metric values for each metric
@@ -207,15 +214,20 @@ func (c *Collector) publishNodes(ch chan<- prometheus.Metric, nodes []clusapi.Ob
}
nodeNames := make([]string, 0, len(nodes))
scrape := c.nodeMetricCache.Begin()
var key []byte
for _, node := range nodes {
if node.Name == "" {
continue
}
resultErr = publishObjectFields(ch, "node", node, fields, build, resultErr)
key, resultErr = publishCachedObjectFields(ch, scrape, key, "node", node, fields, build, resultErr)
nodeNames = append(nodeNames, node.Name)
}
commitScrape(scrape, len(nodes), resultErr)
return nodeNames, resultErr
}
+122 -19
View File
@@ -18,10 +18,12 @@
package mscluster
import (
"encoding/binary"
"errors"
"fmt"
"github.com/prometheus-community/windows_exporter/internal/headers/clusapi"
"github.com/prometheus-community/windows_exporter/internal/metriccache"
"github.com/prometheus/client_golang/prometheus"
)
@@ -57,36 +59,137 @@ type objectField struct {
absent *uint32
}
// resolve returns the raw value that publishObjectFields publishes for field,
// whether it publishes the field, and whether the field is missing: not
// reported by the cluster service although it should be, which is an error.
func (field objectField) resolve(object clusapi.Object, build uint16) (uint32, bool, bool) {
raw, exists := object.Values[field.name]
older := build < field.minBuild
switch {
case older && field.older == neverPublish:
return 0, false, false
case exists:
return raw, true, false
case field.absent != nil:
return *field.absent, true, false
case older && field.older == zeroIfMissing:
return 0, true, false
case older:
return 0, false, false
default:
return 0, false, true
}
}
// metric builds the metric of field for a raw value of the object name.
//
//nolint:ireturn // Const metrics have no exported type.
func (field objectField) metric(raw uint32, name string) prometheus.Metric {
value := float64(raw)
if field.signed {
value = float64(int32(raw))
}
return prometheus.MustNewConstMetric(field.desc, prometheus.GaugeValue, value, name)
}
// publishObjectFields publishes the fields of one object with its name as the
// only label. Missing values are omitted and reported, never published as 0.
func publishObjectFields(ch chan<- prometheus.Metric, kind string, object clusapi.Object, fields []objectField, build uint16, resultErr error) error {
for _, field := range fields {
raw, exists := object.Values[field.name]
older := build < field.minBuild
switch {
case older && field.older == neverPublish:
continue
case exists:
case field.absent != nil:
raw = *field.absent
case older && field.older == zeroIfMissing:
raw = 0
case older:
continue
default:
raw, publish, missing := field.resolve(object, build)
if missing {
resultErr = errors.Join(resultErr, fmt.Errorf("%s %q: missing property %s", kind, object.Name, field.name))
continue
}
value := float64(raw)
if field.signed {
value = float64(int32(raw))
if publish {
ch <- field.metric(raw, object.Name)
}
ch <- prometheus.MustNewConstMetric(field.desc, prometheus.GaugeValue, value, object.Name)
}
return resultErr
}
// objectMetricCache reuses the field metrics of cluster objects whose values
// did not change since the last scrape. Building the const metrics is most of
// the allocations of a scrape, while the values are still read through ClusAPI
// on every scrape. The key is the object name. The value encodes every input
// of the cached metrics: the published fields with their raw values (see
// appendObjectFieldsKey) and, for groups, the owner node and the node names.
type objectMetricCache = metriccache.Cache[string, string]
// appendObjectFieldsKey appends the published fields of object and their raw
// values to key. It returns false if a field is missing: such an object
// returns an error on every scrape and is not cached.
func appendObjectFieldsKey(key []byte, object clusapi.Object, fields []objectField, build uint16) ([]byte, bool) {
for _, field := range fields {
raw, publish, missing := field.resolve(object, build)
switch {
case missing:
return key, false
case publish:
key = append(key, 1)
key = binary.LittleEndian.AppendUint32(key, raw)
default:
key = append(key, 0)
}
}
return key, true
}
// appendKeyString appends s with its length to key, so that consecutive
// strings cannot run into each other.
func appendKeyString(key []byte, s string) []byte {
key = binary.AppendUvarint(key, uint64(len(s)))
return append(key, s...)
}
// objectFieldMetrics builds the field metrics of an object without missing
// fields, with room for extra metrics.
func objectFieldMetrics(object clusapi.Object, fields []objectField, build uint16, extra int) []prometheus.Metric {
metrics := make([]prometheus.Metric, 0, len(fields)+extra)
for _, field := range fields {
if raw, publish, _ := field.resolve(object, build); publish {
metrics = append(metrics, field.metric(raw, object.Name))
}
}
return metrics
}
// publishCachedObjectFields publishes the fields of object like
// publishObjectFields and reuses the metrics of the last scrape if they were
// built from the same values. It returns key for reuse as a buffer.
func publishCachedObjectFields(ch chan<- prometheus.Metric, scrape *metriccache.Scrape[string, string], key []byte, kind string, object clusapi.Object, fields []objectField, build uint16, resultErr error) ([]byte, error) {
key, ok := appendObjectFieldsKey(key[:0], object, fields, build)
if !ok {
return key, publishObjectFields(ch, kind, object, fields, build, resultErr)
}
metrics := scrape.Get(object.Name, string(key), func() []prometheus.Metric {
return objectFieldMetrics(object, fields, build, 0)
})
for _, metric := range metrics {
ch <- metric
}
return key, resultErr
}
// commitScrape commits a scrape unless it collected nothing because reading
// the objects failed, so a failed read keeps the cached metrics.
func commitScrape[K, V comparable](scrape *metriccache.Scrape[K, V], collected int, err error) {
if collected == 0 && err != nil {
return
}
scrape.Commit()
}
@@ -23,6 +23,7 @@ import (
"time"
"github.com/prometheus-community/windows_exporter/internal/headers/clusapi"
"github.com/prometheus-community/windows_exporter/internal/metriccache"
"github.com/prometheus-community/windows_exporter/internal/types"
"github.com/prometheus/client_golang/prometheus"
)
@@ -37,6 +38,12 @@ type resourceSource interface {
type collectorResource struct {
resourceSource resourceSource
// resourceMetricCache reuses the metrics of unchanged resources. Each
// resource publishes 16 series plus one owner node series per node, and
// building them is most of the allocations of a scrape. The values are
// still read through ClusAPI on every scrape.
resourceMetricCache metriccache.Cache[string, resourceCacheValue]
resourceCharacteristics *prometheus.Desc
resourceDeadlockTimeout *prometheus.Desc
resourceEmbeddedFailureAction *prometheus.Desc
@@ -203,6 +210,9 @@ func (c *Collector) buildResourceDescriptors() {
[]string{"type", "owner_group", "name"},
nil,
)
// Cached metrics refer to the descriptors of an earlier build.
c.resourceMetricCache.Reset()
}
// Collect sends the metric values for each metric
@@ -217,58 +227,146 @@ func (c *Collector) collectResource(ch chan<- prometheus.Metric, maxScrapeDurati
return c.publishResources(ch, resources, nodeNames, resultErr)
}
// resourceFieldCount is the number of DWORD properties of each resource.
const resourceFieldCount = 15
// resourceCacheValue holds every input of the cached metrics of a resource
// besides its name, which is the key.
type resourceCacheValue struct {
resourceType string
ownerGroup string
ownerNode string
// NodeNames encodes the node names with appendKeyString.
nodeNames string
values [resourceFieldCount]uint32
}
func (c *Collector) publishResources(ch chan<- prometheus.Metric, resources []clusapi.Resource, nodeNames []string, resultErr error) error {
fieldNames := [resourceFieldCount]string{
"Characteristics",
"DeadlockTimeout",
"EmbeddedFailureAction",
"Flags",
"IsAlivePollInterval",
"LooksAlivePollInterval",
"MonitorProcessId",
"PendingTimeout",
"ResourceClass",
"RestartAction",
"RestartDelay",
"RestartPeriod",
"RestartThreshold",
"RetryPeriodOnFailure",
"State",
}
fieldDescs := [resourceFieldCount]*prometheus.Desc{
c.resourceCharacteristics,
c.resourceDeadlockTimeout,
c.resourceEmbeddedFailureAction,
c.resourceFlags,
c.resourceIsAlivePollInterval,
c.resourceLooksAlivePollInterval,
c.resourceMonitorProcessId,
c.resourcePendingTimeout,
c.resourceResourceClass,
c.resourceRestartAction,
c.resourceRestartDelay,
c.resourceRestartPeriod,
c.resourceRestartThreshold,
c.resourceRetryPeriodOnFailure,
c.resourceState,
}
var nodeNamesKey []byte
for _, nodeName := range nodeNames {
nodeNamesKey = appendKeyString(nodeNamesKey, nodeName)
}
nodeNamesValue := string(nodeNamesKey)
scrape := c.resourceMetricCache.Begin()
for _, resource := range resources {
if !resource.IdentityValid {
continue
}
fields := []struct {
name string
desc *prometheus.Desc
}{
{"Characteristics", c.resourceCharacteristics},
{"DeadlockTimeout", c.resourceDeadlockTimeout},
{"EmbeddedFailureAction", c.resourceEmbeddedFailureAction},
{"Flags", c.resourceFlags},
{"IsAlivePollInterval", c.resourceIsAlivePollInterval},
{"LooksAlivePollInterval", c.resourceLooksAlivePollInterval},
{"MonitorProcessId", c.resourceMonitorProcessId},
{"PendingTimeout", c.resourcePendingTimeout},
{"ResourceClass", c.resourceResourceClass},
{"RestartAction", c.resourceRestartAction},
{"RestartDelay", c.resourceRestartDelay},
{"RestartPeriod", c.resourceRestartPeriod},
{"RestartThreshold", c.resourceRestartThreshold},
{"RetryPeriodOnFailure", c.resourceRetryPeriodOnFailure},
{"State", c.resourceState},
value := resourceCacheValue{
resourceType: resource.Type,
ownerGroup: resource.OwnerGroup,
ownerNode: resource.OwnerNode,
nodeNames: nodeNamesValue,
}
for _, field := range fields {
value, exists := resource.Values[field.name]
complete := true
for index, name := range fieldNames {
raw, exists := resource.Values[name]
if !exists {
resultErr = errors.Join(resultErr, fmt.Errorf("resource %q: missing DWORD property %s", resource.Name, field.name))
complete = false
continue
break
}
ch <- prometheus.MustNewConstMetric(field.desc, prometheus.GaugeValue, float64(value), resource.Type, resource.OwnerGroup, resource.Name)
value.values[index] = raw
}
// MSCluster_Resource.Subclass is NULL for every resource (seen on
// Windows Server 2022, also for IP Address resources whose class
// information carries CLUS_RESSUBCLASS_NETWORK_INTERNET_PROTOCOL), so
// the WMI collector always published 0. Keep that series unchanged.
ch <- prometheus.MustNewConstMetric(c.resourceSubClass, prometheus.GaugeValue, 0, resource.Type, resource.OwnerGroup, resource.Name)
if !complete {
// A resource with missing fields is not cached, so that it
// reports its errors on every scrape.
for index, name := range fieldNames {
raw, exists := resource.Values[name]
if !exists {
resultErr = errors.Join(resultErr, fmt.Errorf("resource %q: missing DWORD property %s", resource.Name, name))
for _, nodeName := range nodeNames {
value := 0.0
if resource.OwnerNode == nodeName {
value = 1
continue
}
ch <- prometheus.MustNewConstMetric(fieldDescs[index], prometheus.GaugeValue, float64(raw), resource.Type, resource.OwnerGroup, resource.Name)
}
ch <- prometheus.MustNewConstMetric(c.resourceOwnerNode, prometheus.GaugeValue, value, resource.Type, resource.OwnerGroup, nodeName, resource.Name)
for _, metric := range c.resourceNodeMetrics(nil, resource, nodeNames) {
ch <- metric
}
continue
}
metrics := scrape.Get(resource.Name, value, func() []prometheus.Metric {
metrics := make([]prometheus.Metric, 0, resourceFieldCount+1+len(nodeNames))
for index, raw := range value.values {
metrics = append(metrics, prometheus.MustNewConstMetric(fieldDescs[index], prometheus.GaugeValue, float64(raw), resource.Type, resource.OwnerGroup, resource.Name))
}
return c.resourceNodeMetrics(metrics, resource, nodeNames)
})
for _, metric := range metrics {
ch <- metric
}
}
commitScrape(scrape, len(resources), resultErr)
return resultErr
}
// resourceNodeMetrics appends the subclass metric and the owner node metrics
// of a resource, one per node, to metrics.
func (c *Collector) resourceNodeMetrics(metrics []prometheus.Metric, resource clusapi.Resource, nodeNames []string) []prometheus.Metric {
// MSCluster_Resource.Subclass is NULL for every resource (seen on
// Windows Server 2022, also for IP Address resources whose class
// information carries CLUS_RESSUBCLASS_NETWORK_INTERNET_PROTOCOL), so
// the WMI collector always published 0. Keep that series unchanged.
metrics = append(metrics, prometheus.MustNewConstMetric(c.resourceSubClass, prometheus.GaugeValue, 0, resource.Type, resource.OwnerGroup, resource.Name))
for _, nodeName := range nodeNames {
value := 0.0
if resource.OwnerNode == nodeName {
value = 1
}
metrics = append(metrics, prometheus.MustNewConstMetric(c.resourceOwnerNode, prometheus.GaugeValue, value, resource.Type, resource.OwnerGroup, nodeName, resource.Name))
}
return metrics
}
@@ -299,6 +299,82 @@ func BenchmarkResourcePublication(b *testing.B) {
}
}
func TestResourceMetricCache(t *testing.T) {
c := New(&Config{CollectorsEnabled: []string{subCollectorResource}})
c.buildResourceDescriptors()
publish := func(resource clusapi.Resource, nodeNames ...string) ([]prometheus.Metric, error) {
ch := make(chan prometheus.Metric, 100)
err := c.publishResources(ch, []clusapi.Resource{resource}, nodeNames, nil)
close(ch)
var metrics []prometheus.Metric
for metric := range ch {
metrics = append(metrics, metric)
}
return metrics, err
}
resource := resourceFixture()
first, err := publish(resource, "nodeA", "nodeB")
if err != nil {
t.Fatal(err)
}
second, err := publish(resource, "nodeA", "nodeB")
if err != nil {
t.Fatal(err)
}
if len(first) != 18 || len(second) != len(first) || first[0] != second[0] {
t.Fatal("did not reuse the metrics of an unchanged resource")
}
// A changed value, owner node or node list builds new metrics.
resource.Values = maps.Clone(resource.Values)
resource.Values["State"] = 2
resource.OwnerNode = "nodeC"
third, err := publish(resource, "nodeA", "nodeB", "nodeC")
if err != nil {
t.Fatal(err)
}
families, err := gatherPublished(t, func(ch chan<- prometheus.Metric) error {
for _, metric := range third {
ch <- metric
}
return nil
})
if err != nil {
t.Fatal(err)
}
if families["windows_mscluster_resource_state"].GetMetric()[0].GetGauge().GetValue() != 2 {
t.Fatal("reused the metrics of a changed resource")
}
for _, metric := range families["windows_mscluster_resource_owner_node"].GetMetric() {
if hosted := metricLabels(metric)["node_name"] == "nodeC"; (metric.GetGauge().GetValue() == 1) != hosted {
t.Fatal("reused the owner node of a changed resource")
}
}
// A resource with missing fields reports them on every scrape.
delete(resource.Values, "Flags")
for range 2 {
metrics, err := publish(resource, "nodeA")
if err == nil || len(metrics) != 16 {
t.Fatalf("missing field: %d metrics, %v", len(metrics), err)
}
}
}
func TestResourceLargeClusterGather(t *testing.T) {
resources := make([]clusapi.Resource, 81)
for index := range resources {
@@ -36,6 +36,10 @@ type resourceGroupSource interface {
type collectorResourceGroup struct {
resourceGroupSource resourceGroupSource
// resourceGroupMetricCache reuses the field and owner node metrics of
// unchanged groups; see objectMetricCache.
resourceGroupMetricCache objectMetricCache
resourceGroupAutoFailbackType *prometheus.Desc
resourceGroupCharacteristics *prometheus.Desc
resourceGroupColdStartSetting *prometheus.Desc
@@ -171,6 +175,9 @@ func (c *Collector) buildResourceGroupDescriptors() {
[]string{"name"},
nil,
)
// Cached metrics refer to the descriptors of an earlier build.
c.resourceGroupMetricCache.Reset()
}
// Collect sends the metric values for each metric
@@ -202,26 +209,74 @@ func (c *Collector) publishResourceGroups(ch chan<- prometheus.Metric, groups []
{name: "State", desc: c.resourceGroupState},
}
scrape := c.resourceGroupMetricCache.Begin()
var nodeNamesKey []byte
for _, nodeName := range nodeNames {
nodeNamesKey = appendKeyString(nodeNamesKey, nodeName)
}
var key []byte
for _, group := range groups {
if group.Name == "" {
continue
}
resultErr = publishObjectFields(ch, "group", group, fields, build, resultErr)
var ok bool
key, ok = appendObjectFieldsKey(key[:0], group, fields, build)
if !ok {
// A group with missing fields is not cached, so that it reports
// its errors on every scrape.
resultErr = publishObjectFields(ch, "group", group, fields, build, resultErr)
for _, metric := range c.resourceGroupOwnerNodeMetrics(nil, group, nodeNames) {
ch <- metric
}
if !group.OwnerNodeValid {
continue
}
for _, nodeName := range nodeNames {
isCurrentState := 0.0
if group.OwnerNode == nodeName {
isCurrentState = 1.0
}
if group.OwnerNodeValid {
key = append(key, 1)
key = appendKeyString(key, group.OwnerNode)
key = append(key, nodeNamesKey...)
} else {
key = append(key, 0)
}
ch <- prometheus.MustNewConstMetric(c.resourceGroupOwnerNode, prometheus.GaugeValue, isCurrentState, nodeName, group.Name)
metrics := scrape.Get(group.Name, string(key), func() []prometheus.Metric {
metrics := objectFieldMetrics(group, fields, build, len(nodeNames))
return c.resourceGroupOwnerNodeMetrics(metrics, group, nodeNames)
})
for _, metric := range metrics {
ch <- metric
}
}
commitScrape(scrape, len(groups), resultErr)
return resultErr
}
// resourceGroupOwnerNodeMetrics appends the owner node metrics of a group, one
// per node, to metrics. A group without a valid owner node has none.
func (c *Collector) resourceGroupOwnerNodeMetrics(metrics []prometheus.Metric, group clusapi.Object, nodeNames []string) []prometheus.Metric {
if !group.OwnerNodeValid {
return metrics
}
for _, nodeName := range nodeNames {
isCurrentState := 0.0
if group.OwnerNode == nodeName {
isCurrentState = 1.0
}
metrics = append(metrics, prometheus.MustNewConstMetric(c.resourceGroupOwnerNode, prometheus.GaugeValue, isCurrentState, nodeName, group.Name))
}
return metrics
}