From df0618e64d9404c1d1b8a8fbbc68fbd07e702730 Mon Sep 17 00:00:00 2001 From: Ben Reedy Date: Thu, 26 Nov 2020 21:45:55 +1000 Subject: [PATCH 1/7] Add DFSR collectors Signed-off-by: Ben Reedy --- README.md | 1 + collector/dfsr.go | 777 +++++++++++++++++++++++++++++++++++++++++ docs/README.md | 1 + docs/collector.dfsr.md | 72 ++++ 4 files changed, 851 insertions(+) create mode 100644 collector/dfsr.go create mode 100644 docs/collector.dfsr.md diff --git a/README.md b/README.md index ede25eb3..7d0bd5f5 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Name | Description | Enabled by default [cpu](docs/collector.cpu.md) | CPU usage | ✓ [cs](docs/collector.cs.md) | "Computer System" metrics (system properties, num cpus/total memory) | ✓ [container](docs/collector.container.md) | Container metrics | +[dfsr](docs/collector.dfsr.md) | DFSR metrics | [dhcp](docs/collector.dhcp.md) | DHCP Server | [dns](docs/collector.dns.md) | DNS Server | [exchange](docs/collector.exchange.md) | Exchange metrics | diff --git a/collector/dfsr.go b/collector/dfsr.go new file mode 100644 index 00000000..88e9c462 --- /dev/null +++ b/collector/dfsr.go @@ -0,0 +1,777 @@ +// +build windows + +package collector + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/log" +) + +const ( + dfsrVolumeSubsystem = "dfsr_volume" + dfsrConnectionSubsystem = "dfsr_connection" + dfsrFoldersSubsystem = "dfsr_folder" +) + +func init() { + registerCollector(dfsrConnectionSubsystem, NewDFSRConnectionCollector, "DFS Replication Service Connections") + registerCollector(dfsrFoldersSubsystem, NewDFSRConnectionCollector, "DFS Replication Service Folders") + registerCollector(dfsrVolumeSubsystem, NewDFSRConnectionCollector, "DFS Replication Service Volumes") +} + +type DFSRConnectionCollector struct { + BandwidthSavingsUsingDFSReplicationTotal *prometheus.Desc + BytesReceivedTotal *prometheus.Desc + CompressedSizeOfFilesReceivedTotal *prometheus.Desc + FilesReceivedTotal *prometheus.Desc + RDCBytesReceivedTotal *prometheus.Desc + RDCCompressedSizeOfFilesReceivedTotal *prometheus.Desc + RDCSizeOfFilesReceivedTotal *prometheus.Desc + RDCNumberofFilesReceivedTotal *prometheus.Desc + SizeOfFilesReceivedTotal *prometheus.Desc +} + +func NewDFSRConnectionCollector() (Collector, error) { + return &DFSRConnectionCollector{ + BandwidthSavingsUsingDFSReplicationTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "bandwidth_savings_using_dfs_replication_total"), + "Total amount of bandwidth savings using DFS Replication for this connection, in bytes", + []string{"name"}, + nil, + ), + + BytesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "bytes_received_total"), + "Total bytes received for connection", + []string{"name"}, + nil, + ), + + CompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "compressed_size_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + FilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "files_received_total"), + "Total number of files receieved for connection", + []string{"name"}, + nil, + ), + + RDCBytesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_bytes_received_total"), + "", + []string{"name"}, + nil, + ), + + RDCCompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_compressed_size_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + RDCNumberofFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_number_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + RDCSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_size_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + SizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "size_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + }, nil +} + +// Collect sends the metric values for each metric +// to the provided prometheus Metric channel. +func (c *DFSRConnectionCollector) Collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) error { + if desc, err := c.collect(ctx, ch); err != nil { + log.Error("failed collecting dfsr_connection metrics:", desc, err) + return err + } + return nil +} + +// Perflib: "DFS Replication Service Connections" +type PerflibDFSRConnection struct { + Name string + + BandwidthSavingsUsingDFSReplicationTotal float64 `perflib:"Bandwidth Savings Using DFS Replication"` + BytesReceivedTotal float64 `perflib:"Total Bytes Received"` + CompressedSizeOfFilesReceivedTotal float64 `perflib:"Compressed Size of Files Received"` + FilesReceivedTotal float64 `perflib:"Total Files Received"` + RDCBytesReceivedTotal float64 `perflib:"RDC Bytes Received"` + RDCCompressedSizeOfFilesReceivedTotal float64 `perflib:"RDC Compressed Size of Files Received"` + RDCNumberofFilesReceivedTotal float64 `perflib:"RDC Number of Files Received"` + RDCSizeOfFilesReceivedTotal float64 `perflib:"RDC Size of Files Received"` + SizeOfFilesReceivedTotal float64 `perflib:"Size of Files Received"` +} + +func (c *DFSRConnectionCollector) collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) { + var dst []PerflibDFSRConnection + if err := unmarshalObject(ctx.perfObjects["DFS Replication Connections"], &dst); err != nil { + return nil, err + } + + for _, connection := range dst { + ch <- prometheus.MustNewConstMetric( + c.BandwidthSavingsUsingDFSReplicationTotal, + prometheus.CounterValue, + connection.BandwidthSavingsUsingDFSReplicationTotal, + connection.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.BytesReceivedTotal, + prometheus.CounterValue, + connection.BytesReceivedTotal, + connection.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.CompressedSizeOfFilesReceivedTotal, + prometheus.CounterValue, + connection.CompressedSizeOfFilesReceivedTotal, + connection.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.FilesReceivedTotal, + prometheus.CounterValue, + connection.FilesReceivedTotal, + connection.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.RDCBytesReceivedTotal, + prometheus.CounterValue, + connection.RDCBytesReceivedTotal, + connection.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.RDCCompressedSizeOfFilesReceivedTotal, + prometheus.CounterValue, + connection.RDCCompressedSizeOfFilesReceivedTotal, + connection.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.RDCSizeOfFilesReceivedTotal, + prometheus.CounterValue, + connection.RDCSizeOfFilesReceivedTotal, + connection.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.RDCNumberofFilesReceivedTotal, + prometheus.CounterValue, + connection.RDCNumberofFilesReceivedTotal, + connection.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.SizeOfFilesReceivedTotal, + prometheus.CounterValue, + connection.SizeOfFilesReceivedTotal, + connection.Name, + ) + + } + return nil, nil +} + +type DFSRVolumeCollector struct { + DatabaseLookupsTotal *prometheus.Desc + DatabaseCommitsTotal *prometheus.Desc + USNJournalUnreadPercentage *prometheus.Desc + USNJournalRecordsAcceptedTotal *prometheus.Desc + USNJournalRecordsReadTotal *prometheus.Desc +} + +func NewDFSRVolumeCollector() (Collector, error) { + return &DFSRVolumeCollector{ + DatabaseCommitsTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "database_commits_total"), + "Total number of DFSR Volume database commits", + []string{"name"}, + nil, + ), + + DatabaseLookupsTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "database_lookups_total"), + "Total number of DFSR Volume database lookups", + []string{"name"}, + nil, + ), + + USNJournalUnreadPercentage: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "usn_journal_unread_percentage"), + "Percentage of DFSR Volume USN journal records that are unread", + []string{"name"}, + nil, + ), + + USNJournalRecordsAcceptedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "usn_journal_records_accepted_total"), + "Total number of USN journal records accepted", + []string{"name"}, + nil, + ), + + USNJournalRecordsReadTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "usn_journal_records_read_total"), + "Total number of DFSR Volume USN journal records read", + []string{"name"}, + nil, + ), + }, nil +} + +// Collect sends the metric values for each metric +// to the provided prometheus Metric channel. +func (c *DFSRVolumeCollector) Collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) error { + if desc, err := c.collect(ctx, ch); err != nil { + log.Error("failed collecting dfsr_volume metrics:", desc, err) + return err + } + return nil +} + +// Perflib: "DFS Replication Service Volumes" +type PerflibDFSRVolume struct { + Name string + + DatabaseCommitsTotal float64 `perflib:"Database Commits"` + DatabaseLookupsTotal float64 `perflib:"Database Lookups"` + USNJournalRecordsReadTotal float64 `perflib:"USN Journal Records Read"` + USNJournalRecordsAcceptedTotal float64 `perflib:"USN Journal Records Accepted"` + USNJournalUnreadPercentage float64 `perflib:"USN Journal Records Unread Percentage"` +} + +func (c *DFSRVolumeCollector) collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) { + var dst []PerflibDFSRVolume + if err := unmarshalObject(ctx.perfObjects["DFS Replication Service Volumes"], &dst); err != nil { + return nil, err + } + + for _, volume := range dst { + ch <- prometheus.MustNewConstMetric( + c.DatabaseLookupsTotal, + prometheus.CounterValue, + volume.DatabaseLookupsTotal, + volume.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.DatabaseCommitsTotal, + prometheus.CounterValue, + volume.DatabaseCommitsTotal, + volume.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.USNJournalRecordsAcceptedTotal, + prometheus.CounterValue, + volume.USNJournalRecordsAcceptedTotal, + volume.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.USNJournalRecordsReadTotal, + prometheus.CounterValue, + volume.USNJournalRecordsReadTotal, + volume.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.USNJournalUnreadPercentage, + prometheus.GaugeValue, + volume.USNJournalUnreadPercentage, + volume.Name, + ) + + } + return nil, nil +} + +type DFSRReplicatedFoldersCollector struct { + BandwidthSavingsUsingDFSReplicationTotal *prometheus.Desc + CompressedSizeOfFilesReceivedTotal *prometheus.Desc + ConflictBytesCleanedupTotal *prometheus.Desc + ConflictBytesGeneratedTotal *prometheus.Desc + ConflictFilesCleanedUpTotal *prometheus.Desc + ConflictFilesGeneratedTotal *prometheus.Desc + ConflictFolderCleanupsCompletedTotal *prometheus.Desc + ConflictSpaceInUse *prometheus.Desc + DeletedSpaceInUse *prometheus.Desc + DeletedBytesCleanedUpTotal *prometheus.Desc + DeletedBytesGeneratedTotal *prometheus.Desc + DeletedFilesCleanedUpTotal *prometheus.Desc + DeletedFilesGeneratedTotal *prometheus.Desc + FileInstallsRetriedTotal *prometheus.Desc + FileInstallsSucceededTotal *prometheus.Desc + FilesReceivedTotal *prometheus.Desc + RDCBytesReceivedTotal *prometheus.Desc + RDCCompressedSizeOfFilesReceivedTotal *prometheus.Desc + RDCNumberofFilesReceivedTotal *prometheus.Desc + RDCSizeOfFilesReceivedTotal *prometheus.Desc + SizeOfFilesReceivedTotal *prometheus.Desc + StagingSpaceInUse *prometheus.Desc + StagingBytesCleanedUpTotal *prometheus.Desc + StagingBytesGeneratedTotal *prometheus.Desc + StagingFilesCleanedUpTotal *prometheus.Desc + StagingFilesGeneratedTotal *prometheus.Desc + UpdatesDroppedTotal *prometheus.Desc +} + +func NewDFSRReplicatedFoldersCollector() (Collector, error) { + return &DFSRReplicatedFoldersCollector{ + BandwidthSavingsUsingDFSReplicationTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "bandwidth_savings_using_dfs_replication_total"), + "", + []string{"name"}, + nil, + ), + + CompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "compressed_size_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + ConflictBytesCleanedupTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "conflict_bytes_cleaned_up_total"), + "", + []string{"name"}, + nil, + ), + + ConflictBytesGeneratedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "conflict_bytes_generated_total"), + "", + []string{"name"}, + nil, + ), + + ConflictFilesCleanedUpTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "conflict_files_cleaned_up_total"), + "", + []string{"name"}, + nil, + ), + + ConflictFilesGeneratedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "conflict_files_generated_total"), + "", + []string{"name"}, + nil, + ), + + ConflictFolderCleanupsCompletedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "conflict_folder_cleanups_total"), + "", + []string{"name"}, + nil, + ), + + ConflictSpaceInUse: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "conflict_space_in_use"), + "", + []string{"name"}, + nil, + ), + + DeletedSpaceInUse: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "deleted_space_in_use"), + "", + []string{"name"}, + nil, + ), + + DeletedBytesCleanedUpTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "deleted_bytes_cleaned_up_total"), + "", + []string{"name"}, + nil, + ), + + DeletedBytesGeneratedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "deleted_bytes_generated_total"), + "", + []string{"name"}, + nil, + ), + + DeletedFilesCleanedUpTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "deleted_files_cleaned_up_total"), + "", + []string{"name"}, + nil, + ), + + DeletedFilesGeneratedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "deleted_files_generated_total"), + "", + []string{"name"}, + nil, + ), + + FileInstallsRetriedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "file_installs_retried_total"), + "", + []string{"name"}, + nil, + ), + + FileInstallsSucceededTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "file_installs_succeeded_total"), + "", + []string{"name"}, + nil, + ), + + FilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "files_received_total"), + "", + []string{"name"}, + nil, + ), + + RDCBytesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_bytes_received_total"), + "", + []string{"name"}, + nil, + ), + + RDCCompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_compressed_size_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + RDCNumberofFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_number_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + RDCSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_size_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + SizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "size_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + StagingSpaceInUse: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "staging_space_in_use"), + "", + []string{"name"}, + nil, + ), + + StagingBytesCleanedUpTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "staging_bytes_cleaned_up_total"), + "", + []string{"name"}, + nil, + ), + + StagingBytesGeneratedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "staging_bytes_generated_total"), + "", + []string{"name"}, + nil, + ), + + StagingFilesCleanedUpTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "staging_files_cleaned_up_total"), + "", + []string{"name"}, + nil, + ), + + StagingFilesGeneratedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "staging_files_generated_total"), + "", + []string{"name"}, + nil, + ), + + UpdatesDroppedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "updates_dropped_total"), + "", + []string{"name"}, + nil, + ), + }, nil +} + +// Collect sends the metric values for each metric +// to the provided prometheus Metric channel. +func (c *DFSRReplicatedFoldersCollector) Collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) error { + if desc, err := c.collect(ctx, ch); err != nil { + log.Error("failed collecting dfsr_folder metrics:", desc, err) + return err + } + return nil +} + +// Perflib: "DFS Replicated Folder" +type PerflibDFSRFolder struct { + Name string + + BandwidthSavingsUsingDFSReplicationTotal float64 `perflib:"Bandwidth Savings Using DFS Replication"` + CompressedSizeOfFilesReceivedTotal float64 `perflib:"Compressed Size of Files Received"` + ConflictBytesCleanedupTotal float64 `perflib:"Conflict Bytes Cleaned Up"` + ConflictBytesGeneratedTotal float64 `perflib:"Conflict Bytes Generated"` + ConflictFilesCleanedUpTotal float64 `perflib:"Conflict Files Cleaned Up"` + ConflictFilesGeneratedTotal float64 `perflib:"Conflict Files Generated"` + ConflictFolderCleanupsCompletedTotal float64 `perflib:"Conflict Folder Cleanups Completed"` + ConflictSpaceInUse float64 `perflib:"Conflict Space In Use"` + DeletedSpaceInUse float64 `perflib:"Deleted Space In Use"` + DeletedBytesCleanedUpTotal float64 `perflib:"Deleted Bytes Cleaned Up"` + DeletedBytesGeneratedTotal float64 `perflib:"Deleted Bytes Generated"` + DeletedFilesCleanedUpTotal float64 `perflib:"Deleted Files Cleaned Up"` + DeletedFilesGeneratedTotal float64 `perflib:"Deleted Files Generated"` + FileInstallsRetriedTotal float64 `perflib:"File Installs Retried"` + FileInstallsSucceededTotal float64 `perflib:"File Installs Succeeded"` + FilesReceivedTotal float64 `perflib:"Total Files Received"` + RDCBytesReceivedTotal float64 `perflib:"RDC Bytes Received"` + RDCCompressedSizeOfFilesReceivedTotal float64 `perflib:"RDC Compressed Size of Files Received"` + RDCNumberofFilesReceivedTotal float64 `perflib:"RDC Number of Files Received"` + RDCSizeOfFilesReceivedTotal float64 `perflib:"RDC Size of Files Received"` + SizeOfFilesReceivedTotal float64 `perflib:"Size of Files Received"` + StagingSpaceInUse float64 `perflib:"Staging Space In Use"` + StagingBytesCleanedUpTotal float64 `perflib:"Staging Bytes Cleaned Up"` + StagingBytesGeneratedTotal float64 `perflib:"Staging Bytes Generated"` + StagingFilesCleanedUpTotal float64 `perflib:"Staging Files Cleaned Up"` + StagingFilesGeneratedTotal float64 `perflib:"Staging Files Generated"` + UpdatesDroppedTotal float64 `perflib:"Updates Dropped"` +} + +func (c *DFSRReplicatedFoldersCollector) collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) { + var dst []PerflibDFSRFolder + if err := unmarshalObject(ctx.perfObjects["DFS Replicated Folders"], &dst); err != nil { + return nil, err + } + + for _, folder := range dst { + ch <- prometheus.MustNewConstMetric( + c.BandwidthSavingsUsingDFSReplicationTotal, + prometheus.CounterValue, + folder.BandwidthSavingsUsingDFSReplicationTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.CompressedSizeOfFilesReceivedTotal, + prometheus.CounterValue, + folder.CompressedSizeOfFilesReceivedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.ConflictBytesCleanedupTotal, + prometheus.CounterValue, + folder.ConflictBytesCleanedupTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.ConflictBytesGeneratedTotal, + prometheus.CounterValue, + folder.ConflictBytesGeneratedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.ConflictFilesCleanedUpTotal, + prometheus.CounterValue, + folder.ConflictFilesCleanedUpTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.ConflictFilesGeneratedTotal, + prometheus.CounterValue, + folder.ConflictFilesGeneratedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.ConflictFolderCleanupsCompletedTotal, + prometheus.CounterValue, + folder.ConflictFolderCleanupsCompletedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.ConflictSpaceInUse, + prometheus.GaugeValue, + folder.ConflictSpaceInUse, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.DeletedSpaceInUse, + prometheus.GaugeValue, + folder.DeletedSpaceInUse, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.DeletedBytesCleanedUpTotal, + prometheus.CounterValue, + folder.DeletedBytesCleanedUpTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.DeletedBytesGeneratedTotal, + prometheus.CounterValue, + folder.DeletedBytesGeneratedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.DeletedFilesCleanedUpTotal, + prometheus.CounterValue, + folder.DeletedFilesCleanedUpTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.DeletedFilesGeneratedTotal, + prometheus.CounterValue, + folder.DeletedFilesGeneratedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.FileInstallsRetriedTotal, + prometheus.CounterValue, + folder.FileInstallsRetriedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.FileInstallsSucceededTotal, + prometheus.CounterValue, + folder.FileInstallsSucceededTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.FilesReceivedTotal, + prometheus.CounterValue, + folder.FilesReceivedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.RDCBytesReceivedTotal, + prometheus.CounterValue, + folder.RDCBytesReceivedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.RDCCompressedSizeOfFilesReceivedTotal, + prometheus.CounterValue, + folder.RDCCompressedSizeOfFilesReceivedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.RDCNumberofFilesReceivedTotal, + prometheus.CounterValue, + folder.RDCNumberofFilesReceivedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.RDCSizeOfFilesReceivedTotal, + prometheus.CounterValue, + folder.RDCSizeOfFilesReceivedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.SizeOfFilesReceivedTotal, + prometheus.CounterValue, + folder.SizeOfFilesReceivedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.StagingSpaceInUse, + prometheus.GaugeValue, + folder.StagingSpaceInUse, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.StagingBytesCleanedUpTotal, + prometheus.CounterValue, + folder.StagingBytesCleanedUpTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.StagingBytesGeneratedTotal, + prometheus.CounterValue, + folder.StagingBytesGeneratedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.StagingFilesCleanedUpTotal, + prometheus.CounterValue, + folder.StagingFilesCleanedUpTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.StagingFilesGeneratedTotal, + prometheus.CounterValue, + folder.StagingFilesGeneratedTotal, + folder.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.UpdatesDroppedTotal, + prometheus.CounterValue, + folder.UpdatesDroppedTotal, + folder.Name, + ) + } + return nil, nil +} diff --git a/docs/README.md b/docs/README.md index aba1deaf..08526d9a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,6 +6,7 @@ This directory contains documentation of the collectors in the windows_exporter, - [`adfs`](collector.adfs.md) - [`cpu`](collector.cpu.md) - [`cs`](collector.cs.md) +- [`dfsr`](collector.dfsr.md) - [`dhcp`](collector.dhcp.md) - [`dns`](collector.dns.md) - [`hyperv`](collector.hyperv.md) diff --git a/docs/collector.dfsr.md b/docs/collector.dfsr.md new file mode 100644 index 00000000..01faef70 --- /dev/null +++ b/docs/collector.dfsr.md @@ -0,0 +1,72 @@ +# dfsr collectors + +The dfsr collectors expose metrics for [DFSR](https://docs.microsoft.com/en-us/windows-server/storage/dfs-replication/dfsr-overview). + +||| +-|- +Metric name prefix | `dfsr_connection`, `dfsr_folder`, `dfsr_volume` +Data source | Perflib +Enabled by default? | No + +## Flags + +None + +## Metrics + +Name | Description | Type | Labels +-----|-------------|------|------- +`dfsr_connection_bandwidth_savings_using_dfs_replication_total` | | counter | None +`dfsr_connection_bytes_received_total` | | counter | None +`dfsr_connection_compressed_size_of_files_received_total` | | counter | None +`dfsr_connection_files_received_total` | | counter | None +`dfsr_connection_rdc_bytes_received_total` | | counter | None +`dfsr_connection_rdc_compressed_size_of_files_received_total` | | counter | None +`dfsr_connection_rdc_number_of_files_received_total` | | counter | None +`dfsr_connection_rdc_size_of_files_received_total` | | counter | None +`dfsr_connection_size_of_files_received_total` | | counter | None +`dfsr_folder_bandwidth_savings_using_dfs_replication_total` | | counter | None +`dfsr_folder_compressed_size_of_files_received_total` | | counter | None +`dfsr_folder_conflict_bytes_cleaned_up_total` | | counter | None +`dfsr_folder_conflict_bytes_generated_total` | | counter | None +`dfsr_folder_conflict_files_cleaned_up_total` | | counter | None +`dfsr_folder_conflict_files_generated_total` | | counter | None +`dfsr_folder_conflict_folder_cleanups_total` | | counter | None +`dfsr_folder_conflict_space_in_use` | | gauge | None +`dfsr_folder_deleted_space_in_use` | | gauge | None +`dfsr_folder_deleted_bytes_cleaned_up_total` | | counter | None +`dfsr_folder_deleted_bytes_generated_total` | | counter | None +`dfsr_folder_deleted_files_cleaned_up_total` | | counter | None +`dfsr_folder_deleted_files_generated_total` | | counter | None +`dfsr_folder_file_installs_retried_total` | | counter | None +`dfsr_folder_file_installs_succeeded_total` | | counter | None +`dfsr_folder_files_received_total` | | counter | None +`dfsr_folder_rdc_bytes_received_total` | | counter | None +`dfsr_folder_rdc_compressed_size_of_files_received_total` | | counter | None +`dfsr_folder_rdc_number_of_files_received_total` | | counter | None +`dfsr_folder_rdc_size_of_files_received_total` | | counter | None +`dfsr_folder_size_of_files_received_total` | | counter | None +`dfsr_folder_staging_space_in_use` | | gauge | None +`dfsr_folder_staging_bytes_cleaned_up_total` | | counter | None +`dfsr_folder_staging_bytes_generated_total` | | counter | None +`dfsr_folder_staging_files_cleaned_up_total` | | counter | None +`dfsr_folder_staging_files_generated_total` | | counter | None +`dfsr_folder_updates_dropped_total` | | counter | None +`dfsr_volume_bandwidth_savings_using_dfs_replication_total` | | counter | None +`dfsr_volume_bytes_received_total` | | counter | None +`dfsr_volume_compressed_size_of_files_received_total` | | counter | None +`dfsr_volume_files_received_total` | | counter | None +`dfsr_volume_rdc_bytes_received_total` | | counter | None +`dfsr_volume_rdc_compressed_size_of_files_received_total` | | counter | None +`dfsr_volume_rdc_number_of_files_received_total` | | counter | None +`dfsr_volume_rdc_size_of_files_received_total` | | counter | None +`dfsr_volume_size_of_files_received_total` | | counter | None + +### Example metric +_This collector does not yet have explained examples, we would appreciate your help adding them!_ + +## Useful queries +_This collector does not yet have any useful queries added, we would appreciate your help adding them!_ + +## Alerting examples +_This collector does not yet have alerting examples, we would appreciate your help adding them!_ From ccac306c2de80519ee1cadb99ec85fd9ae5370dd Mon Sep 17 00:00:00 2001 From: Ben Reedy Date: Thu, 3 Dec 2020 16:59:15 +1000 Subject: [PATCH 2/7] Rewrite DFSR collector to use sub collectors DFSR collector now follow similar structure to the MSSQL collector, where several 'child' collectors are run concurrently, depending on user input from the `--collectors.dfsr.sources-enabled` flag. Signed-off-by: Ben Reedy --- collector/dfsr.go | 966 +++++++++++++++++++++++------------------ docs/collector.dfsr.md | 103 ++--- 2 files changed, 591 insertions(+), 478 deletions(-) diff --git a/collector/dfsr.go b/collector/dfsr.go index 88e9c462..b331b16d 100644 --- a/collector/dfsr.go +++ b/collector/dfsr.go @@ -3,111 +3,508 @@ package collector import ( + "errors" + "strings" + "sync" + "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/log" + "gopkg.in/alecthomas/kingpin.v2" ) -const ( - dfsrVolumeSubsystem = "dfsr_volume" - dfsrConnectionSubsystem = "dfsr_connection" - dfsrFoldersSubsystem = "dfsr_folder" -) +var dfsrEnabledCollectors = kingpin.Flag("collectors.dfsr.sources-enabled", "Comma-seperated list of DFSR Perflib sources to use.").Default("connection,folder,volume").String() func init() { - registerCollector(dfsrConnectionSubsystem, NewDFSRConnectionCollector, "DFS Replication Service Connections") - registerCollector(dfsrFoldersSubsystem, NewDFSRConnectionCollector, "DFS Replication Service Folders") - registerCollector(dfsrVolumeSubsystem, NewDFSRConnectionCollector, "DFS Replication Service Volumes") + // Perflib sources are dynamic, depending on the enabled child collectors + var perflibDependencies []string + for _, source := range expandEnabledChildCollectors(*dfsrEnabledCollectors) { + perflibDependencies = append(perflibDependencies, dfsrGetPerfObjectName(source)) + } + + registerCollector("dfsr", NewDFSRCollector, perflibDependencies...) } -type DFSRConnectionCollector struct { - BandwidthSavingsUsingDFSReplicationTotal *prometheus.Desc - BytesReceivedTotal *prometheus.Desc - CompressedSizeOfFilesReceivedTotal *prometheus.Desc - FilesReceivedTotal *prometheus.Desc - RDCBytesReceivedTotal *prometheus.Desc - RDCCompressedSizeOfFilesReceivedTotal *prometheus.Desc - RDCSizeOfFilesReceivedTotal *prometheus.Desc - RDCNumberofFilesReceivedTotal *prometheus.Desc - SizeOfFilesReceivedTotal *prometheus.Desc +// DFSRCollector contains the metric and state data of the DFSR collectors. +type DFSRCollector struct { + // Meta + dfsrScrapeDurationDesc *prometheus.Desc + dfsrScrapeSuccessDesc *prometheus.Desc + + // Connection source + ConnectionBandwidthSavingsUsingDFSReplicationTotal *prometheus.Desc + ConnectionBytesReceivedTotal *prometheus.Desc + ConnectionCompressedSizeOfFilesReceivedTotal *prometheus.Desc + ConnectionFilesReceivedTotal *prometheus.Desc + ConnectionRDCBytesReceivedTotal *prometheus.Desc + ConnectionRDCCompressedSizeOfFilesReceivedTotal *prometheus.Desc + ConnectionRDCSizeOfFilesReceivedTotal *prometheus.Desc + ConnectionRDCNumberofFilesReceivedTotal *prometheus.Desc + ConnectionSizeOfFilesReceivedTotal *prometheus.Desc + + // Folder source + FolderBandwidthSavingsUsingDFSReplicationTotal *prometheus.Desc + FolderCompressedSizeOfFilesReceivedTotal *prometheus.Desc + FolderConflictBytesCleanedupTotal *prometheus.Desc + FolderConflictBytesGeneratedTotal *prometheus.Desc + FolderConflictFilesCleanedUpTotal *prometheus.Desc + FolderConflictFilesGeneratedTotal *prometheus.Desc + FolderConflictFolderCleanupsCompletedTotal *prometheus.Desc + FolderConflictSpaceInUse *prometheus.Desc + FolderDeletedSpaceInUse *prometheus.Desc + FolderDeletedBytesCleanedUpTotal *prometheus.Desc + FolderDeletedBytesGeneratedTotal *prometheus.Desc + FolderDeletedFilesCleanedUpTotal *prometheus.Desc + FolderDeletedFilesGeneratedTotal *prometheus.Desc + FolderFileInstallsRetriedTotal *prometheus.Desc + FolderFileInstallsSucceededTotal *prometheus.Desc + FolderFilesReceivedTotal *prometheus.Desc + FolderRDCBytesReceivedTotal *prometheus.Desc + FolderRDCCompressedSizeOfFilesReceivedTotal *prometheus.Desc + FolderRDCNumberofFilesReceivedTotal *prometheus.Desc + FolderRDCSizeOfFilesReceivedTotal *prometheus.Desc + FolderSizeOfFilesReceivedTotal *prometheus.Desc + FolderStagingSpaceInUse *prometheus.Desc + FolderStagingBytesCleanedUpTotal *prometheus.Desc + FolderStagingBytesGeneratedTotal *prometheus.Desc + FolderStagingFilesCleanedUpTotal *prometheus.Desc + FolderStagingFilesGeneratedTotal *prometheus.Desc + FolderUpdatesDroppedTotal *prometheus.Desc + + // Volume source + VolumeDatabaseLookupsTotal *prometheus.Desc + VolumeDatabaseCommitsTotal *prometheus.Desc + VolumeUSNJournalUnreadPercentage *prometheus.Desc + VolumeUSNJournalRecordsAcceptedTotal *prometheus.Desc + VolumeUSNJournalRecordsReadTotal *prometheus.Desc + + // Map of child collector functions used during collection + dfsrChildCollectors dfsrCollectorMap + // Internal counter for number of child collector failures during collection + dfsrChildCollectorFailure int } -func NewDFSRConnectionCollector() (Collector, error) { - return &DFSRConnectionCollector{ - BandwidthSavingsUsingDFSReplicationTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "bandwidth_savings_using_dfs_replication_total"), +type dfsrCollectorMap map[string]dfsrCollectorFunc + +type dfsrCollectorFunc func(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) + +// Split provided perflib sources and deduplicate +func dfsrExpandEnabledSources(enabled string) []string { + separated := strings.Split(enabled, ",") + unique := map[string]bool{} + for _, s := range separated { + if s != "" { + unique[s] = true + } + } + result := make([]string, 0, len(unique)) + for s := range unique { + result = append(result, s) + } + return result +} + +// Map Perflib sources to DFSR collector names +// E.G. volume -> DFS Replication Service Volumes +func dfsrGetPerfObjectName(collector string) string { + prefix := "DFS " + suffix := "" + switch collector { + case "connection": + suffix = "Replication Connections" + case "folder": + suffix = "Replicated Folders" + case "volume": + suffix = "Replication Service Volumes" + } + return (prefix + suffix) +} + +// NewDFSRCollector is registered +func NewDFSRCollector() (Collector, error) { + const subsystem = "dfsr" + + enabled := dfsrExpandEnabledSources(*dfsrEnabledCollectors) + perfCounters := make([]string, 0, len(enabled)) + for _, c := range enabled { + perfCounters = append(perfCounters, dfsrGetPerfObjectName(c)) + } + addPerfCounterDependencies(subsystem, perfCounters) + + dfsrCollector := DFSRCollector{ + // meta + dfsrScrapeDurationDesc: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "collector_duration_seconds"), + "windows_exporter: Duration of an dfsr child collection.", + []string{"collector"}, + nil, + ), + dfsrScrapeSuccessDesc: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "collector_success"), + "windows_exporter: Whether a dfsr child collector was successful.", + []string{"collector"}, + nil, + ), + + // Connection + ConnectionBandwidthSavingsUsingDFSReplicationTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "bandwidth_savings_using_dfs_replication_total"), "Total amount of bandwidth savings using DFS Replication for this connection, in bytes", []string{"name"}, nil, ), - BytesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "bytes_received_total"), + ConnectionBytesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "bytes_received_total"), "Total bytes received for connection", []string{"name"}, nil, ), - CompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "compressed_size_of_files_received_total"), + ConnectionCompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "compressed_size_of_files_received_total"), "", []string{"name"}, nil, ), - FilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "files_received_total"), + ConnectionFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "files_received_total"), "Total number of files receieved for connection", []string{"name"}, nil, ), - RDCBytesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_bytes_received_total"), + ConnectionRDCBytesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "rdc_bytes_received_total"), "", []string{"name"}, nil, ), - RDCCompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_compressed_size_of_files_received_total"), + ConnectionRDCCompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "rdc_compressed_size_of_files_received_total"), "", []string{"name"}, nil, ), - RDCNumberofFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_number_of_files_received_total"), + ConnectionRDCNumberofFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "rdc_number_of_files_received_total"), "", []string{"name"}, nil, ), - RDCSizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_size_of_files_received_total"), + ConnectionRDCSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "rdc_size_of_files_received_total"), "", []string{"name"}, nil, ), - SizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "size_of_files_received_total"), + ConnectionSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "size_of_files_received_total"), "", []string{"name"}, nil, ), - }, nil + + // Folder + FolderBandwidthSavingsUsingDFSReplicationTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "bandwidth_savings_using_dfs_replication_total"), + "", + []string{"name"}, + nil, + ), + + FolderCompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "compressed_size_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + FolderConflictBytesCleanedupTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "conflict_bytes_cleaned_up_total"), + "", + []string{"name"}, + nil, + ), + + FolderConflictBytesGeneratedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "conflict_bytes_generated_total"), + "", + []string{"name"}, + nil, + ), + + FolderConflictFilesCleanedUpTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "conflict_files_cleaned_up_total"), + "", + []string{"name"}, + nil, + ), + + FolderConflictFilesGeneratedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "conflict_files_generated_total"), + "", + []string{"name"}, + nil, + ), + + FolderConflictFolderCleanupsCompletedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "conflict_folder_cleanups_total"), + "", + []string{"name"}, + nil, + ), + + FolderConflictSpaceInUse: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "conflict_space_in_use"), + "", + []string{"name"}, + nil, + ), + + FolderDeletedSpaceInUse: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "deleted_space_in_use"), + "", + []string{"name"}, + nil, + ), + + FolderDeletedBytesCleanedUpTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "deleted_bytes_cleaned_up_total"), + "", + []string{"name"}, + nil, + ), + + FolderDeletedBytesGeneratedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "deleted_bytes_generated_total"), + "", + []string{"name"}, + nil, + ), + + FolderDeletedFilesCleanedUpTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "deleted_files_cleaned_up_total"), + "", + []string{"name"}, + nil, + ), + + FolderDeletedFilesGeneratedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "deleted_files_generated_total"), + "", + []string{"name"}, + nil, + ), + + FolderFileInstallsRetriedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "file_installs_retried_total"), + "", + []string{"name"}, + nil, + ), + + FolderFileInstallsSucceededTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "file_installs_succeeded_total"), + "", + []string{"name"}, + nil, + ), + + FolderFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "files_received_total"), + "", + []string{"name"}, + nil, + ), + + FolderRDCBytesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "rdc_bytes_received_total"), + "", + []string{"name"}, + nil, + ), + + FolderRDCCompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "rdc_compressed_size_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + FolderRDCNumberofFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "rdc_number_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + FolderRDCSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "rdc_size_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + FolderSizeOfFilesReceivedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "size_of_files_received_total"), + "", + []string{"name"}, + nil, + ), + + FolderStagingSpaceInUse: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "staging_space_in_use"), + "", + []string{"name"}, + nil, + ), + + FolderStagingBytesCleanedUpTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "staging_bytes_cleaned_up_total"), + "", + []string{"name"}, + nil, + ), + + FolderStagingBytesGeneratedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "staging_bytes_generated_total"), + "", + []string{"name"}, + nil, + ), + + FolderStagingFilesCleanedUpTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "staging_files_cleaned_up_total"), + "", + []string{"name"}, + nil, + ), + + FolderStagingFilesGeneratedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "staging_files_generated_total"), + "", + []string{"name"}, + nil, + ), + + FolderUpdatesDroppedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "updates_dropped_total"), + "", + []string{"name"}, + nil, + ), + + // Volume + VolumeDatabaseCommitsTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "database_commits_total"), + "Total number of DFSR Volume database commits", + []string{"name"}, + nil, + ), + + VolumeDatabaseLookupsTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "database_lookups_total"), + "Total number of DFSR Volume database lookups", + []string{"name"}, + nil, + ), + + VolumeUSNJournalUnreadPercentage: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "usn_journal_unread_percentage"), + "Percentage of DFSR Volume USN journal records that are unread", + []string{"name"}, + nil, + ), + + VolumeUSNJournalRecordsAcceptedTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "usn_journal_records_accepted_total"), + "Total number of USN journal records accepted", + []string{"name"}, + nil, + ), + + VolumeUSNJournalRecordsReadTotal: prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "usn_journal_records_read_total"), + "Total number of DFSR Volume USN journal records read", + []string{"name"}, + nil, + ), + } + + dfsrCollector.dfsrChildCollectors = dfsrCollector.getDFSRChildCollectors() + + return &dfsrCollector, nil } -// Collect sends the metric values for each metric -// to the provided prometheus Metric channel. -func (c *DFSRConnectionCollector) Collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) error { - if desc, err := c.collect(ctx, ch); err != nil { - log.Error("failed collecting dfsr_connection metrics:", desc, err) - return err +// Maps child collectors names to their relevant collection function, +// for use in DFSRCollector.Collect() +func (c *DFSRCollector) getDFSRChildCollectors() dfsrCollectorMap { + dfsrCollectors := make(dfsrCollectorMap) + dfsrCollectors["connection"] = c.collectConnection + dfsrCollectors["folder"] = c.collectFolder + dfsrCollectors["volume"] = c.collectVolume + + return dfsrCollectors +} + +// Collect implements the Collector interface. +// Sends metric values for each metric to the provided prometheus Metric channel. +func (c *DFSRCollector) Collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) error { + wg := sync.WaitGroup{} + + for name, function := range c.dfsrChildCollectors { + wg.Add(1) + go c.execute(ctx, name, function, ch, &wg) + + } + wg.Wait() + + if c.dfsrChildCollectorFailure > 0 { + return errors.New("at least one child collector failed") } return nil } +// Child-specific functions are provided to this function and executed concurrently. +// Child collector metrics & results are reported. +func (c *DFSRCollector) execute(ctx *ScrapeContext, name string, fn dfsrCollectorFunc, ch chan<- prometheus.Metric, wg *sync.WaitGroup) { + defer wg.Done() + + begin := time.Now() + // Child collector function called here, sends metric data back through channel + _, err := fn(ctx, ch) + duration := time.Since(begin) + var success float64 + + if err != nil { + log.Errorf("dfsr class collector %s failed after %fs: %s", name, duration.Seconds(), err) + success = 0 + c.dfsrChildCollectorFailure++ + } else { + log.Debugf("dfsr class collector %s succeeded after %fs.", name, duration.Seconds()) + success = 1 + } + + ch <- prometheus.MustNewConstMetric( + c.dfsrScrapeDurationDesc, + prometheus.GaugeValue, + duration.Seconds(), + name, + ) + ch <- prometheus.MustNewConstMetric( + c.dfsrScrapeSuccessDesc, + prometheus.GaugeValue, + success, + name, + ) +} + // Perflib: "DFS Replication Service Connections" type PerflibDFSRConnection struct { Name string @@ -123,7 +520,7 @@ type PerflibDFSRConnection struct { SizeOfFilesReceivedTotal float64 `perflib:"Size of Files Received"` } -func (c *DFSRConnectionCollector) collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) { +func (c *DFSRCollector) collectConnection(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) { var dst []PerflibDFSRConnection if err := unmarshalObject(ctx.perfObjects["DFS Replication Connections"], &dst); err != nil { return nil, err @@ -131,63 +528,63 @@ func (c *DFSRConnectionCollector) collect(ctx *ScrapeContext, ch chan<- promethe for _, connection := range dst { ch <- prometheus.MustNewConstMetric( - c.BandwidthSavingsUsingDFSReplicationTotal, + c.ConnectionBandwidthSavingsUsingDFSReplicationTotal, prometheus.CounterValue, connection.BandwidthSavingsUsingDFSReplicationTotal, connection.Name, ) ch <- prometheus.MustNewConstMetric( - c.BytesReceivedTotal, + c.ConnectionBytesReceivedTotal, prometheus.CounterValue, connection.BytesReceivedTotal, connection.Name, ) ch <- prometheus.MustNewConstMetric( - c.CompressedSizeOfFilesReceivedTotal, + c.ConnectionCompressedSizeOfFilesReceivedTotal, prometheus.CounterValue, connection.CompressedSizeOfFilesReceivedTotal, connection.Name, ) ch <- prometheus.MustNewConstMetric( - c.FilesReceivedTotal, + c.ConnectionFilesReceivedTotal, prometheus.CounterValue, connection.FilesReceivedTotal, connection.Name, ) ch <- prometheus.MustNewConstMetric( - c.RDCBytesReceivedTotal, + c.ConnectionRDCBytesReceivedTotal, prometheus.CounterValue, connection.RDCBytesReceivedTotal, connection.Name, ) ch <- prometheus.MustNewConstMetric( - c.RDCCompressedSizeOfFilesReceivedTotal, + c.ConnectionRDCCompressedSizeOfFilesReceivedTotal, prometheus.CounterValue, connection.RDCCompressedSizeOfFilesReceivedTotal, connection.Name, ) ch <- prometheus.MustNewConstMetric( - c.RDCSizeOfFilesReceivedTotal, + c.ConnectionRDCSizeOfFilesReceivedTotal, prometheus.CounterValue, connection.RDCSizeOfFilesReceivedTotal, connection.Name, ) ch <- prometheus.MustNewConstMetric( - c.RDCNumberofFilesReceivedTotal, + c.ConnectionRDCNumberofFilesReceivedTotal, prometheus.CounterValue, connection.RDCNumberofFilesReceivedTotal, connection.Name, ) ch <- prometheus.MustNewConstMetric( - c.SizeOfFilesReceivedTotal, + c.ConnectionSizeOfFilesReceivedTotal, prometheus.CounterValue, connection.SizeOfFilesReceivedTotal, connection.Name, @@ -195,353 +592,7 @@ func (c *DFSRConnectionCollector) collect(ctx *ScrapeContext, ch chan<- promethe } return nil, nil -} -type DFSRVolumeCollector struct { - DatabaseLookupsTotal *prometheus.Desc - DatabaseCommitsTotal *prometheus.Desc - USNJournalUnreadPercentage *prometheus.Desc - USNJournalRecordsAcceptedTotal *prometheus.Desc - USNJournalRecordsReadTotal *prometheus.Desc -} - -func NewDFSRVolumeCollector() (Collector, error) { - return &DFSRVolumeCollector{ - DatabaseCommitsTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "database_commits_total"), - "Total number of DFSR Volume database commits", - []string{"name"}, - nil, - ), - - DatabaseLookupsTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "database_lookups_total"), - "Total number of DFSR Volume database lookups", - []string{"name"}, - nil, - ), - - USNJournalUnreadPercentage: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "usn_journal_unread_percentage"), - "Percentage of DFSR Volume USN journal records that are unread", - []string{"name"}, - nil, - ), - - USNJournalRecordsAcceptedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "usn_journal_records_accepted_total"), - "Total number of USN journal records accepted", - []string{"name"}, - nil, - ), - - USNJournalRecordsReadTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "usn_journal_records_read_total"), - "Total number of DFSR Volume USN journal records read", - []string{"name"}, - nil, - ), - }, nil -} - -// Collect sends the metric values for each metric -// to the provided prometheus Metric channel. -func (c *DFSRVolumeCollector) Collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) error { - if desc, err := c.collect(ctx, ch); err != nil { - log.Error("failed collecting dfsr_volume metrics:", desc, err) - return err - } - return nil -} - -// Perflib: "DFS Replication Service Volumes" -type PerflibDFSRVolume struct { - Name string - - DatabaseCommitsTotal float64 `perflib:"Database Commits"` - DatabaseLookupsTotal float64 `perflib:"Database Lookups"` - USNJournalRecordsReadTotal float64 `perflib:"USN Journal Records Read"` - USNJournalRecordsAcceptedTotal float64 `perflib:"USN Journal Records Accepted"` - USNJournalUnreadPercentage float64 `perflib:"USN Journal Records Unread Percentage"` -} - -func (c *DFSRVolumeCollector) collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) { - var dst []PerflibDFSRVolume - if err := unmarshalObject(ctx.perfObjects["DFS Replication Service Volumes"], &dst); err != nil { - return nil, err - } - - for _, volume := range dst { - ch <- prometheus.MustNewConstMetric( - c.DatabaseLookupsTotal, - prometheus.CounterValue, - volume.DatabaseLookupsTotal, - volume.Name, - ) - - ch <- prometheus.MustNewConstMetric( - c.DatabaseCommitsTotal, - prometheus.CounterValue, - volume.DatabaseCommitsTotal, - volume.Name, - ) - - ch <- prometheus.MustNewConstMetric( - c.USNJournalRecordsAcceptedTotal, - prometheus.CounterValue, - volume.USNJournalRecordsAcceptedTotal, - volume.Name, - ) - - ch <- prometheus.MustNewConstMetric( - c.USNJournalRecordsReadTotal, - prometheus.CounterValue, - volume.USNJournalRecordsReadTotal, - volume.Name, - ) - - ch <- prometheus.MustNewConstMetric( - c.USNJournalUnreadPercentage, - prometheus.GaugeValue, - volume.USNJournalUnreadPercentage, - volume.Name, - ) - - } - return nil, nil -} - -type DFSRReplicatedFoldersCollector struct { - BandwidthSavingsUsingDFSReplicationTotal *prometheus.Desc - CompressedSizeOfFilesReceivedTotal *prometheus.Desc - ConflictBytesCleanedupTotal *prometheus.Desc - ConflictBytesGeneratedTotal *prometheus.Desc - ConflictFilesCleanedUpTotal *prometheus.Desc - ConflictFilesGeneratedTotal *prometheus.Desc - ConflictFolderCleanupsCompletedTotal *prometheus.Desc - ConflictSpaceInUse *prometheus.Desc - DeletedSpaceInUse *prometheus.Desc - DeletedBytesCleanedUpTotal *prometheus.Desc - DeletedBytesGeneratedTotal *prometheus.Desc - DeletedFilesCleanedUpTotal *prometheus.Desc - DeletedFilesGeneratedTotal *prometheus.Desc - FileInstallsRetriedTotal *prometheus.Desc - FileInstallsSucceededTotal *prometheus.Desc - FilesReceivedTotal *prometheus.Desc - RDCBytesReceivedTotal *prometheus.Desc - RDCCompressedSizeOfFilesReceivedTotal *prometheus.Desc - RDCNumberofFilesReceivedTotal *prometheus.Desc - RDCSizeOfFilesReceivedTotal *prometheus.Desc - SizeOfFilesReceivedTotal *prometheus.Desc - StagingSpaceInUse *prometheus.Desc - StagingBytesCleanedUpTotal *prometheus.Desc - StagingBytesGeneratedTotal *prometheus.Desc - StagingFilesCleanedUpTotal *prometheus.Desc - StagingFilesGeneratedTotal *prometheus.Desc - UpdatesDroppedTotal *prometheus.Desc -} - -func NewDFSRReplicatedFoldersCollector() (Collector, error) { - return &DFSRReplicatedFoldersCollector{ - BandwidthSavingsUsingDFSReplicationTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "bandwidth_savings_using_dfs_replication_total"), - "", - []string{"name"}, - nil, - ), - - CompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "compressed_size_of_files_received_total"), - "", - []string{"name"}, - nil, - ), - - ConflictBytesCleanedupTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "conflict_bytes_cleaned_up_total"), - "", - []string{"name"}, - nil, - ), - - ConflictBytesGeneratedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "conflict_bytes_generated_total"), - "", - []string{"name"}, - nil, - ), - - ConflictFilesCleanedUpTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "conflict_files_cleaned_up_total"), - "", - []string{"name"}, - nil, - ), - - ConflictFilesGeneratedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "conflict_files_generated_total"), - "", - []string{"name"}, - nil, - ), - - ConflictFolderCleanupsCompletedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "conflict_folder_cleanups_total"), - "", - []string{"name"}, - nil, - ), - - ConflictSpaceInUse: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "conflict_space_in_use"), - "", - []string{"name"}, - nil, - ), - - DeletedSpaceInUse: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "deleted_space_in_use"), - "", - []string{"name"}, - nil, - ), - - DeletedBytesCleanedUpTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "deleted_bytes_cleaned_up_total"), - "", - []string{"name"}, - nil, - ), - - DeletedBytesGeneratedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "deleted_bytes_generated_total"), - "", - []string{"name"}, - nil, - ), - - DeletedFilesCleanedUpTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "deleted_files_cleaned_up_total"), - "", - []string{"name"}, - nil, - ), - - DeletedFilesGeneratedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "deleted_files_generated_total"), - "", - []string{"name"}, - nil, - ), - - FileInstallsRetriedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "file_installs_retried_total"), - "", - []string{"name"}, - nil, - ), - - FileInstallsSucceededTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "file_installs_succeeded_total"), - "", - []string{"name"}, - nil, - ), - - FilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "files_received_total"), - "", - []string{"name"}, - nil, - ), - - RDCBytesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_bytes_received_total"), - "", - []string{"name"}, - nil, - ), - - RDCCompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_compressed_size_of_files_received_total"), - "", - []string{"name"}, - nil, - ), - - RDCNumberofFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_number_of_files_received_total"), - "", - []string{"name"}, - nil, - ), - - RDCSizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "rdc_size_of_files_received_total"), - "", - []string{"name"}, - nil, - ), - - SizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "size_of_files_received_total"), - "", - []string{"name"}, - nil, - ), - - StagingSpaceInUse: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "staging_space_in_use"), - "", - []string{"name"}, - nil, - ), - - StagingBytesCleanedUpTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "staging_bytes_cleaned_up_total"), - "", - []string{"name"}, - nil, - ), - - StagingBytesGeneratedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "staging_bytes_generated_total"), - "", - []string{"name"}, - nil, - ), - - StagingFilesCleanedUpTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "staging_files_cleaned_up_total"), - "", - []string{"name"}, - nil, - ), - - StagingFilesGeneratedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "staging_files_generated_total"), - "", - []string{"name"}, - nil, - ), - - UpdatesDroppedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, dfsrVolumeSubsystem, "updates_dropped_total"), - "", - []string{"name"}, - nil, - ), - }, nil -} - -// Collect sends the metric values for each metric -// to the provided prometheus Metric channel. -func (c *DFSRReplicatedFoldersCollector) Collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) error { - if desc, err := c.collect(ctx, ch); err != nil { - log.Error("failed collecting dfsr_folder metrics:", desc, err) - return err - } - return nil } // Perflib: "DFS Replicated Folder" @@ -577,7 +628,7 @@ type PerflibDFSRFolder struct { UpdatesDroppedTotal float64 `perflib:"Updates Dropped"` } -func (c *DFSRReplicatedFoldersCollector) collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) { +func (c *DFSRCollector) collectFolder(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) { var dst []PerflibDFSRFolder if err := unmarshalObject(ctx.perfObjects["DFS Replicated Folders"], &dst); err != nil { return nil, err @@ -585,189 +636,189 @@ func (c *DFSRReplicatedFoldersCollector) collect(ctx *ScrapeContext, ch chan<- p for _, folder := range dst { ch <- prometheus.MustNewConstMetric( - c.BandwidthSavingsUsingDFSReplicationTotal, + c.FolderBandwidthSavingsUsingDFSReplicationTotal, prometheus.CounterValue, folder.BandwidthSavingsUsingDFSReplicationTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.CompressedSizeOfFilesReceivedTotal, + c.FolderCompressedSizeOfFilesReceivedTotal, prometheus.CounterValue, folder.CompressedSizeOfFilesReceivedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.ConflictBytesCleanedupTotal, + c.FolderConflictBytesCleanedupTotal, prometheus.CounterValue, folder.ConflictBytesCleanedupTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.ConflictBytesGeneratedTotal, + c.FolderConflictBytesGeneratedTotal, prometheus.CounterValue, folder.ConflictBytesGeneratedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.ConflictFilesCleanedUpTotal, + c.FolderConflictFilesCleanedUpTotal, prometheus.CounterValue, folder.ConflictFilesCleanedUpTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.ConflictFilesGeneratedTotal, + c.FolderConflictFilesGeneratedTotal, prometheus.CounterValue, folder.ConflictFilesGeneratedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.ConflictFolderCleanupsCompletedTotal, + c.FolderConflictFolderCleanupsCompletedTotal, prometheus.CounterValue, folder.ConflictFolderCleanupsCompletedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.ConflictSpaceInUse, + c.FolderConflictSpaceInUse, prometheus.GaugeValue, folder.ConflictSpaceInUse, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.DeletedSpaceInUse, + c.FolderDeletedSpaceInUse, prometheus.GaugeValue, folder.DeletedSpaceInUse, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.DeletedBytesCleanedUpTotal, + c.FolderDeletedBytesCleanedUpTotal, prometheus.CounterValue, folder.DeletedBytesCleanedUpTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.DeletedBytesGeneratedTotal, + c.FolderDeletedBytesGeneratedTotal, prometheus.CounterValue, folder.DeletedBytesGeneratedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.DeletedFilesCleanedUpTotal, + c.FolderDeletedFilesCleanedUpTotal, prometheus.CounterValue, folder.DeletedFilesCleanedUpTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.DeletedFilesGeneratedTotal, + c.FolderDeletedFilesGeneratedTotal, prometheus.CounterValue, folder.DeletedFilesGeneratedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.FileInstallsRetriedTotal, + c.FolderFileInstallsRetriedTotal, prometheus.CounterValue, folder.FileInstallsRetriedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.FileInstallsSucceededTotal, + c.FolderFileInstallsSucceededTotal, prometheus.CounterValue, folder.FileInstallsSucceededTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.FilesReceivedTotal, + c.FolderFilesReceivedTotal, prometheus.CounterValue, folder.FilesReceivedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.RDCBytesReceivedTotal, + c.FolderRDCBytesReceivedTotal, prometheus.CounterValue, folder.RDCBytesReceivedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.RDCCompressedSizeOfFilesReceivedTotal, + c.FolderRDCCompressedSizeOfFilesReceivedTotal, prometheus.CounterValue, folder.RDCCompressedSizeOfFilesReceivedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.RDCNumberofFilesReceivedTotal, + c.FolderRDCNumberofFilesReceivedTotal, prometheus.CounterValue, folder.RDCNumberofFilesReceivedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.RDCSizeOfFilesReceivedTotal, + c.FolderRDCSizeOfFilesReceivedTotal, prometheus.CounterValue, folder.RDCSizeOfFilesReceivedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.SizeOfFilesReceivedTotal, + c.FolderSizeOfFilesReceivedTotal, prometheus.CounterValue, folder.SizeOfFilesReceivedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.StagingSpaceInUse, + c.FolderStagingSpaceInUse, prometheus.GaugeValue, folder.StagingSpaceInUse, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.StagingBytesCleanedUpTotal, + c.FolderStagingBytesCleanedUpTotal, prometheus.CounterValue, folder.StagingBytesCleanedUpTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.StagingBytesGeneratedTotal, + c.FolderStagingBytesGeneratedTotal, prometheus.CounterValue, folder.StagingBytesGeneratedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.StagingFilesCleanedUpTotal, + c.FolderStagingFilesCleanedUpTotal, prometheus.CounterValue, folder.StagingFilesCleanedUpTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.StagingFilesGeneratedTotal, + c.FolderStagingFilesGeneratedTotal, prometheus.CounterValue, folder.StagingFilesGeneratedTotal, folder.Name, ) ch <- prometheus.MustNewConstMetric( - c.UpdatesDroppedTotal, + c.FolderUpdatesDroppedTotal, prometheus.CounterValue, folder.UpdatesDroppedTotal, folder.Name, @@ -775,3 +826,60 @@ func (c *DFSRReplicatedFoldersCollector) collect(ctx *ScrapeContext, ch chan<- p } return nil, nil } + +// Perflib: "DFS Replication Service Volumes" +type PerflibDFSRVolume struct { + Name string + + DatabaseCommitsTotal float64 `perflib:"Database Commits"` + DatabaseLookupsTotal float64 `perflib:"Database Lookups"` + USNJournalRecordsReadTotal float64 `perflib:"USN Journal Records Read"` + USNJournalRecordsAcceptedTotal float64 `perflib:"USN Journal Records Accepted"` + USNJournalUnreadPercentage float64 `perflib:"USN Journal Records Unread Percentage"` +} + +func (c *DFSRCollector) collectVolume(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) { + var dst []PerflibDFSRVolume + if err := unmarshalObject(ctx.perfObjects["DFS Replication Service Volumes"], &dst); err != nil { + return nil, err + } + + for _, volume := range dst { + ch <- prometheus.MustNewConstMetric( + c.VolumeDatabaseLookupsTotal, + prometheus.CounterValue, + volume.DatabaseLookupsTotal, + volume.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.VolumeDatabaseCommitsTotal, + prometheus.CounterValue, + volume.DatabaseCommitsTotal, + volume.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.VolumeUSNJournalRecordsAcceptedTotal, + prometheus.CounterValue, + volume.USNJournalRecordsAcceptedTotal, + volume.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.VolumeUSNJournalRecordsReadTotal, + prometheus.CounterValue, + volume.USNJournalRecordsReadTotal, + volume.Name, + ) + + ch <- prometheus.MustNewConstMetric( + c.VolumeUSNJournalUnreadPercentage, + prometheus.GaugeValue, + volume.USNJournalUnreadPercentage, + volume.Name, + ) + + } + return nil, nil +} diff --git a/docs/collector.dfsr.md b/docs/collector.dfsr.md index 01faef70..9369048d 100644 --- a/docs/collector.dfsr.md +++ b/docs/collector.dfsr.md @@ -1,66 +1,71 @@ -# dfsr collectors +# dfsr collector -The dfsr collectors expose metrics for [DFSR](https://docs.microsoft.com/en-us/windows-server/storage/dfs-replication/dfsr-overview). +The dfsr collector exposes metrics for [DFSR](https://docs.microsoft.com/en-us/windows-server/storage/dfs-replication/dfsr-overview). ||| -|- -Metric name prefix | `dfsr_connection`, `dfsr_folder`, `dfsr_volume` +Metric name prefix | `dfsr` Data source | Perflib Enabled by default? | No ## Flags -None +### `--collectors.dfsr.sources-enabled` + +Comma-separated list of DFSR Perflib sources to use. Supported values are `connection`, `folder` and `volume`. +All sources are enabled by default ## Metrics Name | Description | Type | Labels -----|-------------|------|------- -`dfsr_connection_bandwidth_savings_using_dfs_replication_total` | | counter | None -`dfsr_connection_bytes_received_total` | | counter | None -`dfsr_connection_compressed_size_of_files_received_total` | | counter | None -`dfsr_connection_files_received_total` | | counter | None -`dfsr_connection_rdc_bytes_received_total` | | counter | None -`dfsr_connection_rdc_compressed_size_of_files_received_total` | | counter | None -`dfsr_connection_rdc_number_of_files_received_total` | | counter | None -`dfsr_connection_rdc_size_of_files_received_total` | | counter | None -`dfsr_connection_size_of_files_received_total` | | counter | None -`dfsr_folder_bandwidth_savings_using_dfs_replication_total` | | counter | None -`dfsr_folder_compressed_size_of_files_received_total` | | counter | None -`dfsr_folder_conflict_bytes_cleaned_up_total` | | counter | None -`dfsr_folder_conflict_bytes_generated_total` | | counter | None -`dfsr_folder_conflict_files_cleaned_up_total` | | counter | None -`dfsr_folder_conflict_files_generated_total` | | counter | None -`dfsr_folder_conflict_folder_cleanups_total` | | counter | None -`dfsr_folder_conflict_space_in_use` | | gauge | None -`dfsr_folder_deleted_space_in_use` | | gauge | None -`dfsr_folder_deleted_bytes_cleaned_up_total` | | counter | None -`dfsr_folder_deleted_bytes_generated_total` | | counter | None -`dfsr_folder_deleted_files_cleaned_up_total` | | counter | None -`dfsr_folder_deleted_files_generated_total` | | counter | None -`dfsr_folder_file_installs_retried_total` | | counter | None -`dfsr_folder_file_installs_succeeded_total` | | counter | None -`dfsr_folder_files_received_total` | | counter | None -`dfsr_folder_rdc_bytes_received_total` | | counter | None -`dfsr_folder_rdc_compressed_size_of_files_received_total` | | counter | None -`dfsr_folder_rdc_number_of_files_received_total` | | counter | None -`dfsr_folder_rdc_size_of_files_received_total` | | counter | None -`dfsr_folder_size_of_files_received_total` | | counter | None -`dfsr_folder_staging_space_in_use` | | gauge | None -`dfsr_folder_staging_bytes_cleaned_up_total` | | counter | None -`dfsr_folder_staging_bytes_generated_total` | | counter | None -`dfsr_folder_staging_files_cleaned_up_total` | | counter | None -`dfsr_folder_staging_files_generated_total` | | counter | None -`dfsr_folder_updates_dropped_total` | | counter | None -`dfsr_volume_bandwidth_savings_using_dfs_replication_total` | | counter | None -`dfsr_volume_bytes_received_total` | | counter | None -`dfsr_volume_compressed_size_of_files_received_total` | | counter | None -`dfsr_volume_files_received_total` | | counter | None -`dfsr_volume_rdc_bytes_received_total` | | counter | None -`dfsr_volume_rdc_compressed_size_of_files_received_total` | | counter | None -`dfsr_volume_rdc_number_of_files_received_total` | | counter | None -`dfsr_volume_rdc_size_of_files_received_total` | | counter | None -`dfsr_volume_size_of_files_received_total` | | counter | None +`windows_dfsr_collector_duration_seconds` | The time taken for each sub-collector to return | gauge | collector +`windows_dfsr_collector_success` | 1 if sub-collector succeeded, 0 otherwise | gauge | collector +`windows_dfsr_connection_bandwidth_savings_using_dfs_replication_total` | | counter | name +`windows_dfsr_connection_bytes_received_total` | | counter | name +`windows_dfsr_connection_compressed_size_of_files_received_total` | | counter | name +`windows_dfsr_connection_files_received_total` | | counter | name +`windows_dfsr_connection_rdc_bytes_received_total` | | counter | name +`windows_dfsr_connection_rdc_compressed_size_of_files_received_total` | | counter | name +`windows_dfsr_connection_rdc_number_of_files_received_total` | | counter | name +`windows_dfsr_connection_rdc_size_of_files_received_total` | | counter | name +`windows_dfsr_connection_size_of_files_received_total` | | counter | name +`windows_dfsr_folder_bandwidth_savings_using_dfs_replication_total` | | counter | name +`windows_dfsr_folder_compressed_size_of_files_received_total` | | counter | name +`windows_dfsr_folder_conflict_bytes_cleaned_up_total` | | counter | name +`windows_dfsr_folder_conflict_bytes_generated_total` | | counter | name +`windows_dfsr_folder_conflict_files_cleaned_up_total` | | counter | name +`windows_dfsr_folder_conflict_files_generated_total` | | counter | name +`windows_dfsr_folder_conflict_folder_cleanups_total` | | counter | name +`windows_dfsr_folder_conflict_space_in_use` | | gauge | name +`windows_dfsr_folder_deleted_space_in_use` | | gauge | name +`windows_dfsr_folder_deleted_bytes_cleaned_up_total` | | counter | name +`windows_dfsr_folder_deleted_bytes_generated_total` | | counter | name +`windows_dfsr_folder_deleted_files_cleaned_up_total` | | counter | name +`windows_dfsr_folder_deleted_files_generated_total` | | counter | name +`windows_dfsr_folder_file_installs_retried_total` | | counter | name +`windows_dfsr_folder_file_installs_succeeded_total` | | counter | name +`windows_dfsr_folder_files_received_total` | | counter | name +`windows_dfsr_folder_rdc_bytes_received_total` | | counter | name +`windows_dfsr_folder_rdc_compressed_size_of_files_received_total` | | counter | name +`windows_dfsr_folder_rdc_number_of_files_received_total` | | counter | name +`windows_dfsr_folder_rdc_size_of_files_received_total` | | counter | name +`windows_dfsr_folder_size_of_files_received_total` | | counter | name +`windows_dfsr_folder_staging_space_in_use` | | gauge | name +`windows_dfsr_folder_staging_bytes_cleaned_up_total` | | counter | name +`windows_dfsr_folder_staging_bytes_generated_total` | | counter | name +`windows_dfsr_folder_staging_files_cleaned_up_total` | | counter | name +`windows_dfsr_folder_staging_files_generated_total` | | counter | name +`windows_dfsr_folder_updates_dropped_total` | | counter | name +`windows_dfsr_volume_bandwidth_savings_using_dfs_replication_total` | | counter | name +`windows_dfsr_volume_bytes_received_total` | | counter | name +`windows_dfsr_volume_compressed_size_of_files_received_total` | | counter | name +`windows_dfsr_volume_files_received_total` | | counter | name +`windows_dfsr_volume_rdc_bytes_received_total` | | counter | name +`windows_dfsr_volume_rdc_compressed_size_of_files_received_total` | | counter | name +`windows_dfsr_volume_rdc_number_of_files_received_total` | | counter | name +`windows_dfsr_volume_rdc_size_of_files_received_total` | | counter | name +`windows_dfsr_volume_size_of_files_received_total` | | counter | name ### Example metric _This collector does not yet have explained examples, we would appreciate your help adding them!_ From b5ce53fdac3c6925d916f79c85fabf877f03200a Mon Sep 17 00:00:00 2001 From: Ben Reedy Date: Thu, 3 Dec 2020 20:20:28 +1000 Subject: [PATCH 3/7] Merge mssql and dfsr expandEnabledCollectors func Move to common collector.go and add function test. Signed-off-by: Ben Reedy --- collector/collector.go | 20 ++++++++++++++++++++ collector/collector_test.go | 34 ++++++++++++++++++++++++++++++++++ collector/dfsr.go | 19 +------------------ collector/mssql.go | 19 ++----------------- 4 files changed, 57 insertions(+), 35 deletions(-) create mode 100644 collector/collector_test.go diff --git a/collector/collector.go b/collector/collector.go index 089b5674..f7973ed3 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -2,6 +2,7 @@ package collector import ( "fmt" + "sort" "strconv" "strings" @@ -128,3 +129,22 @@ func find(slice []string, val string) bool { } return false } + +// Used by more complex collectors where user input specifies enabled child collectors. +// Splits provided child collectors and deduplicate. +func expandEnabledChildCollectors(enabled string) []string { + separated := strings.Split(enabled, ",") + unique := map[string]bool{} + for _, s := range separated { + if s != "" { + unique[s] = true + } + } + result := make([]string, 0, len(unique)) + for s := range unique { + result = append(result, s) + } + // Ensure result is ordered, to prevent test failure + sort.Strings(result) + return result +} diff --git a/collector/collector_test.go b/collector/collector_test.go new file mode 100644 index 00000000..bafeb8f9 --- /dev/null +++ b/collector/collector_test.go @@ -0,0 +1,34 @@ +package collector + +import ( + "reflect" + "testing" +) + +func TestExpandChildCollectors(t *testing.T) { + cases := []struct { + name string + input string + expectedOutput []string + }{ + { + name: "simple", + input: "testing1,testing2,testing3", + expectedOutput: []string{"testing1", "testing2", "testing3"}, + }, + { + name: "duplicate", + input: "testing1,testing2,testing2,testing3", + expectedOutput: []string{"testing1", "testing2", "testing3"}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + output := expandEnabledChildCollectors(c.input) + if !reflect.DeepEqual(output, c.expectedOutput) { + t.Errorf("Output mismatch, expected %+v, got %+v", c.expectedOutput, output) + } + }) + } +} diff --git a/collector/dfsr.go b/collector/dfsr.go index b331b16d..aefeafcf 100644 --- a/collector/dfsr.go +++ b/collector/dfsr.go @@ -4,7 +4,6 @@ package collector import ( "errors" - "strings" "sync" "time" @@ -88,22 +87,6 @@ type dfsrCollectorMap map[string]dfsrCollectorFunc type dfsrCollectorFunc func(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) -// Split provided perflib sources and deduplicate -func dfsrExpandEnabledSources(enabled string) []string { - separated := strings.Split(enabled, ",") - unique := map[string]bool{} - for _, s := range separated { - if s != "" { - unique[s] = true - } - } - result := make([]string, 0, len(unique)) - for s := range unique { - result = append(result, s) - } - return result -} - // Map Perflib sources to DFSR collector names // E.G. volume -> DFS Replication Service Volumes func dfsrGetPerfObjectName(collector string) string { @@ -124,7 +107,7 @@ func dfsrGetPerfObjectName(collector string) string { func NewDFSRCollector() (Collector, error) { const subsystem = "dfsr" - enabled := dfsrExpandEnabledSources(*dfsrEnabledCollectors) + enabled := expandEnabledChildCollectors(*dfsrEnabledCollectors) perfCounters := make([]string, 0, len(enabled)) for _, c := range enabled { perfCounters = append(perfCounters, dfsrGetPerfObjectName(c)) diff --git a/collector/mssql.go b/collector/mssql.go index 477dc26b..4f746146 100644 --- a/collector/mssql.go +++ b/collector/mssql.go @@ -90,21 +90,6 @@ func (c *MSSQLCollector) getMSSQLCollectors() mssqlCollectorsMap { return mssqlCollectors } -func mssqlExpandEnabledCollectors(enabled string) []string { - separated := strings.Split(enabled, ",") - unique := map[string]bool{} - for _, s := range separated { - if s != "" { - unique[s] = true - } - } - result := make([]string, 0, len(unique)) - for s := range unique { - result = append(result, s) - } - return result -} - // mssqlGetPerfObjectName - Returns the name of the Windows Performance // Counter object for the given SQL instance and collector. func mssqlGetPerfObjectName(sqlInstance string, collector string) string { @@ -407,7 +392,7 @@ func NewMSSQLCollector() (Collector, error) { const subsystem = "mssql" - enabled := mssqlExpandEnabledCollectors(*mssqlEnabledCollectors) + enabled := expandEnabledChildCollectors(*mssqlEnabledCollectors) mssqlInstances := getMSSQLInstances() perfCounters := make([]string, 0, len(mssqlInstances)*len(enabled)) for instance := range mssqlInstances { @@ -1857,7 +1842,7 @@ func (c *MSSQLCollector) execute(ctx *ScrapeContext, name string, fn mssqlCollec func (c *MSSQLCollector) Collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) error { wg := sync.WaitGroup{} - enabled := mssqlExpandEnabledCollectors(*mssqlEnabledCollectors) + enabled := expandEnabledChildCollectors(*mssqlEnabledCollectors) for sqlInstance := range c.mssqlInstances { for _, name := range enabled { function := c.mssqlCollectors[name] From 769b15eb8699106667d8b61379b846ac57dfe876 Mon Sep 17 00:00:00 2001 From: Ben Reedy Date: Wed, 9 Dec 2020 14:45:26 +1000 Subject: [PATCH 4/7] Execute DFSR child collectors serially Previous concurrent setup was not required due to speed of Perflib collectors. Signed-off-by: Ben Reedy --- collector/dfsr.go | 100 ++++++++++++++++------------------------------ 1 file changed, 34 insertions(+), 66 deletions(-) diff --git a/collector/dfsr.go b/collector/dfsr.go index aefeafcf..0096ef8d 100644 --- a/collector/dfsr.go +++ b/collector/dfsr.go @@ -3,12 +3,7 @@ package collector import ( - "errors" - "sync" - "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/log" "gopkg.in/alecthomas/kingpin.v2" ) @@ -78,14 +73,10 @@ type DFSRCollector struct { VolumeUSNJournalRecordsReadTotal *prometheus.Desc // Map of child collector functions used during collection - dfsrChildCollectors dfsrCollectorMap - // Internal counter for number of child collector failures during collection - dfsrChildCollectorFailure int + dfsrChildCollectors []dfsrCollectorFunc } -type dfsrCollectorMap map[string]dfsrCollectorFunc - -type dfsrCollectorFunc func(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) +type dfsrCollectorFunc func(ctx *ScrapeContext, ch chan<- prometheus.Metric) error // Map Perflib sources to DFSR collector names // E.G. volume -> DFS Replication Service Volumes @@ -420,18 +411,25 @@ func NewDFSRCollector() (Collector, error) { ), } - dfsrCollector.dfsrChildCollectors = dfsrCollector.getDFSRChildCollectors() + dfsrCollector.dfsrChildCollectors = dfsrCollector.getDFSRChildCollectors(enabled) return &dfsrCollector, nil } -// Maps child collectors names to their relevant collection function, +// Maps enabled child collectors names to their relevant collection function, // for use in DFSRCollector.Collect() -func (c *DFSRCollector) getDFSRChildCollectors() dfsrCollectorMap { - dfsrCollectors := make(dfsrCollectorMap) - dfsrCollectors["connection"] = c.collectConnection - dfsrCollectors["folder"] = c.collectFolder - dfsrCollectors["volume"] = c.collectVolume +func (c *DFSRCollector) getDFSRChildCollectors(enabledCollectors []string) []dfsrCollectorFunc { + var dfsrCollectors []dfsrCollectorFunc + for _, collector := range enabledCollectors { + switch collector { + case "connection": + dfsrCollectors = append(dfsrCollectors, c.collectConnection) + case "folder": + dfsrCollectors = append(dfsrCollectors, c.collectFolder) + case "volume": + dfsrCollectors = append(dfsrCollectors, c.collectVolume) + } + } return dfsrCollectors } @@ -439,53 +437,24 @@ func (c *DFSRCollector) getDFSRChildCollectors() dfsrCollectorMap { // Collect implements the Collector interface. // Sends metric values for each metric to the provided prometheus Metric channel. func (c *DFSRCollector) Collect(ctx *ScrapeContext, ch chan<- prometheus.Metric) error { - wg := sync.WaitGroup{} - - for name, function := range c.dfsrChildCollectors { - wg.Add(1) - go c.execute(ctx, name, function, ch, &wg) - - } - wg.Wait() - - if c.dfsrChildCollectorFailure > 0 { - return errors.New("at least one child collector failed") + for _, fn := range c.dfsrChildCollectors { + err := fn(ctx, ch) + if err != nil { + return err + } } return nil } // Child-specific functions are provided to this function and executed concurrently. // Child collector metrics & results are reported. -func (c *DFSRCollector) execute(ctx *ScrapeContext, name string, fn dfsrCollectorFunc, ch chan<- prometheus.Metric, wg *sync.WaitGroup) { - defer wg.Done() - - begin := time.Now() +func (c *DFSRCollector) execute(ctx *ScrapeContext, name string, fn dfsrCollectorFunc, ch chan<- prometheus.Metric) error { // Child collector function called here, sends metric data back through channel - _, err := fn(ctx, ch) - duration := time.Since(begin) - var success float64 - + err := fn(ctx, ch) if err != nil { - log.Errorf("dfsr class collector %s failed after %fs: %s", name, duration.Seconds(), err) - success = 0 - c.dfsrChildCollectorFailure++ - } else { - log.Debugf("dfsr class collector %s succeeded after %fs.", name, duration.Seconds()) - success = 1 + return err } - - ch <- prometheus.MustNewConstMetric( - c.dfsrScrapeDurationDesc, - prometheus.GaugeValue, - duration.Seconds(), - name, - ) - ch <- prometheus.MustNewConstMetric( - c.dfsrScrapeSuccessDesc, - prometheus.GaugeValue, - success, - name, - ) + return err } // Perflib: "DFS Replication Service Connections" @@ -503,10 +472,10 @@ type PerflibDFSRConnection struct { SizeOfFilesReceivedTotal float64 `perflib:"Size of Files Received"` } -func (c *DFSRCollector) collectConnection(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) { +func (c *DFSRCollector) collectConnection(ctx *ScrapeContext, ch chan<- prometheus.Metric) error { var dst []PerflibDFSRConnection if err := unmarshalObject(ctx.perfObjects["DFS Replication Connections"], &dst); err != nil { - return nil, err + return err } for _, connection := range dst { @@ -574,8 +543,7 @@ func (c *DFSRCollector) collectConnection(ctx *ScrapeContext, ch chan<- promethe ) } - return nil, nil - + return nil } // Perflib: "DFS Replicated Folder" @@ -611,10 +579,10 @@ type PerflibDFSRFolder struct { UpdatesDroppedTotal float64 `perflib:"Updates Dropped"` } -func (c *DFSRCollector) collectFolder(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) { +func (c *DFSRCollector) collectFolder(ctx *ScrapeContext, ch chan<- prometheus.Metric) error { var dst []PerflibDFSRFolder if err := unmarshalObject(ctx.perfObjects["DFS Replicated Folders"], &dst); err != nil { - return nil, err + return err } for _, folder := range dst { @@ -807,7 +775,7 @@ func (c *DFSRCollector) collectFolder(ctx *ScrapeContext, ch chan<- prometheus.M folder.Name, ) } - return nil, nil + return nil } // Perflib: "DFS Replication Service Volumes" @@ -821,10 +789,10 @@ type PerflibDFSRVolume struct { USNJournalUnreadPercentage float64 `perflib:"USN Journal Records Unread Percentage"` } -func (c *DFSRCollector) collectVolume(ctx *ScrapeContext, ch chan<- prometheus.Metric) (*prometheus.Desc, error) { +func (c *DFSRCollector) collectVolume(ctx *ScrapeContext, ch chan<- prometheus.Metric) error { var dst []PerflibDFSRVolume if err := unmarshalObject(ctx.perfObjects["DFS Replication Service Volumes"], &dst); err != nil { - return nil, err + return err } for _, volume := range dst { @@ -864,5 +832,5 @@ func (c *DFSRCollector) collectVolume(ctx *ScrapeContext, ch chan<- prometheus.M ) } - return nil, nil + return nil } From a1a986f4d0fe9dab352f84e42f853cc584c6e0a7 Mon Sep 17 00:00:00 2001 From: Ben Reedy Date: Wed, 9 Dec 2020 14:47:53 +1000 Subject: [PATCH 5/7] Reset mssql child failure counter on each scrape Resolves issue where collector would always return a failure after an inital failure, as the counter was not reset. Signed-off-by: Ben Reedy --- collector/mssql.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/collector/mssql.go b/collector/mssql.go index 4f746146..17bf8c4e 100644 --- a/collector/mssql.go +++ b/collector/mssql.go @@ -1808,6 +1808,8 @@ func NewMSSQLCollector() (Collector, error) { type mssqlCollectorFunc func(ctx *ScrapeContext, ch chan<- prometheus.Metric, sqlInstance string) (*prometheus.Desc, error) func (c *MSSQLCollector) execute(ctx *ScrapeContext, name string, fn mssqlCollectorFunc, ch chan<- prometheus.Metric, sqlInstance string, wg *sync.WaitGroup) { + // Reset failure counter on each scrape + c.mssqlChildCollectorFailure = 0 defer wg.Done() begin := time.Now() From 2837bdfb502c2feb2473620347ba46c73e698afc Mon Sep 17 00:00:00 2001 From: Ben Reedy Date: Wed, 9 Dec 2020 19:49:32 +1000 Subject: [PATCH 6/7] Add/move DFSR metric units to end of metric name Signed-off-by: Ben Reedy --- collector/dfsr.go | 66 +++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/collector/dfsr.go b/collector/dfsr.go index 0096ef8d..b94ed8d6 100644 --- a/collector/dfsr.go +++ b/collector/dfsr.go @@ -122,7 +122,7 @@ func NewDFSRCollector() (Collector, error) { // Connection ConnectionBandwidthSavingsUsingDFSReplicationTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "bandwidth_savings_using_dfs_replication_total"), + prometheus.BuildFQName(Namespace, subsystem, "bandwidth_savings_using_dfs_replication_bytes_total"), "Total amount of bandwidth savings using DFS Replication for this connection, in bytes", []string{"name"}, nil, @@ -143,14 +143,14 @@ func NewDFSRCollector() (Collector, error) { ), ConnectionFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "files_received_total"), + prometheus.BuildFQName(Namespace, subsystem, "received_files_total"), "Total number of files receieved for connection", []string{"name"}, nil, ), ConnectionRDCBytesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "rdc_bytes_received_total"), + prometheus.BuildFQName(Namespace, subsystem, "rdc_received_bytes_total"), "", []string{"name"}, nil, @@ -164,22 +164,22 @@ func NewDFSRCollector() (Collector, error) { ), ConnectionRDCNumberofFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "rdc_number_of_files_received_total"), - "", + prometheus.BuildFQName(Namespace, subsystem, "rdc_received_files_total"), + "Total number of Remote Differential Compression files received", []string{"name"}, nil, ), ConnectionRDCSizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "rdc_size_of_files_received_total"), - "", + prometheus.BuildFQName(Namespace, subsystem, "rdc_size_of_received_files_bytes_total"), + "Total size of received Remote Differential Compression files, in bytes.", []string{"name"}, nil, ), ConnectionSizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "size_of_files_received_total"), - "", + prometheus.BuildFQName(Namespace, subsystem, "files_received_bytes_total"), + "Total size of files received, in bytes", []string{"name"}, nil, ), @@ -200,28 +200,28 @@ func NewDFSRCollector() (Collector, error) { ), FolderConflictBytesCleanedupTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "conflict_bytes_cleaned_up_total"), + prometheus.BuildFQName(Namespace, subsystem, "conflict_cleaned_up_bytes_total"), "", []string{"name"}, nil, ), FolderConflictBytesGeneratedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "conflict_bytes_generated_total"), + prometheus.BuildFQName(Namespace, subsystem, "conflict_generated_bytes_total"), "", []string{"name"}, nil, ), FolderConflictFilesCleanedUpTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "conflict_files_cleaned_up_total"), + prometheus.BuildFQName(Namespace, subsystem, "conflict_cleaned_up_files_total"), "", []string{"name"}, nil, ), FolderConflictFilesGeneratedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "conflict_files_generated_total"), + prometheus.BuildFQName(Namespace, subsystem, "conflict_generated_files_total"), "", []string{"name"}, nil, @@ -235,42 +235,42 @@ func NewDFSRCollector() (Collector, error) { ), FolderConflictSpaceInUse: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "conflict_space_in_use"), + prometheus.BuildFQName(Namespace, subsystem, "conflict_space_in_use_bytes"), "", []string{"name"}, nil, ), FolderDeletedSpaceInUse: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "deleted_space_in_use"), + prometheus.BuildFQName(Namespace, subsystem, "deleted_space_in_use_bytes"), "", []string{"name"}, nil, ), FolderDeletedBytesCleanedUpTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "deleted_bytes_cleaned_up_total"), + prometheus.BuildFQName(Namespace, subsystem, "deleted_cleaned_up_bytes_total"), "", []string{"name"}, nil, ), FolderDeletedBytesGeneratedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "deleted_bytes_generated_total"), + prometheus.BuildFQName(Namespace, subsystem, "deleted_generated_bytes_total"), "", []string{"name"}, nil, ), FolderDeletedFilesCleanedUpTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "deleted_files_cleaned_up_total"), + prometheus.BuildFQName(Namespace, subsystem, "deleted_cleaned_up_files_total"), "", []string{"name"}, nil, ), FolderDeletedFilesGeneratedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "deleted_files_generated_total"), + prometheus.BuildFQName(Namespace, subsystem, "deleted_generated_files_total"), "", []string{"name"}, nil, @@ -291,84 +291,84 @@ func NewDFSRCollector() (Collector, error) { ), FolderFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "files_received_total"), + prometheus.BuildFQName(Namespace, subsystem, "received_files_total"), "", []string{"name"}, nil, ), FolderRDCBytesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "rdc_bytes_received_total"), + prometheus.BuildFQName(Namespace, subsystem, "rdc_received_bytes_total"), "", []string{"name"}, nil, ), FolderRDCCompressedSizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "rdc_compressed_size_of_files_received_total"), + prometheus.BuildFQName(Namespace, subsystem, "rdc_compressed_size_of_files_received_bytes_total"), "", []string{"name"}, nil, ), FolderRDCNumberofFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "rdc_number_of_files_received_total"), + prometheus.BuildFQName(Namespace, subsystem, "rdc_received_files_total"), "", []string{"name"}, nil, ), FolderRDCSizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "rdc_size_of_files_received_total"), + prometheus.BuildFQName(Namespace, subsystem, "rdc_files_received_bytes_total"), "", []string{"name"}, nil, ), FolderSizeOfFilesReceivedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "size_of_files_received_total"), + prometheus.BuildFQName(Namespace, subsystem, "files_received_bytes_total"), "", []string{"name"}, nil, ), FolderStagingSpaceInUse: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "staging_space_in_use"), + prometheus.BuildFQName(Namespace, subsystem, "staging_space_in_use_bytes"), "", []string{"name"}, nil, ), FolderStagingBytesCleanedUpTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "staging_bytes_cleaned_up_total"), + prometheus.BuildFQName(Namespace, subsystem, "staging_cleaned_up_bytes_total"), "", []string{"name"}, nil, ), FolderStagingBytesGeneratedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "staging_bytes_generated_total"), + prometheus.BuildFQName(Namespace, subsystem, "staging_generated_bytes_total"), "", []string{"name"}, nil, ), FolderStagingFilesCleanedUpTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "staging_files_cleaned_up_total"), + prometheus.BuildFQName(Namespace, subsystem, "staging_cleaned_up_files_total"), "", []string{"name"}, nil, ), FolderStagingFilesGeneratedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "staging_files_generated_total"), + prometheus.BuildFQName(Namespace, subsystem, "staging_generated_files_total"), "", []string{"name"}, nil, ), FolderUpdatesDroppedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "updates_dropped_total"), + prometheus.BuildFQName(Namespace, subsystem, "dropped_updates_total"), "", []string{"name"}, nil, @@ -397,14 +397,14 @@ func NewDFSRCollector() (Collector, error) { ), VolumeUSNJournalRecordsAcceptedTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "usn_journal_records_accepted_total"), + prometheus.BuildFQName(Namespace, subsystem, "usn_journal_accepted_records_total"), "Total number of USN journal records accepted", []string{"name"}, nil, ), VolumeUSNJournalRecordsReadTotal: prometheus.NewDesc( - prometheus.BuildFQName(Namespace, subsystem, "usn_journal_records_read_total"), + prometheus.BuildFQName(Namespace, subsystem, "usn_journal_read_records_total"), "Total number of DFSR Volume USN journal records read", []string{"name"}, nil, From 9d03debcb63a6d3eda03419f17a4ce92842c6cea Mon Sep 17 00:00:00 2001 From: Ben Reedy Date: Sun, 20 Dec 2020 12:09:10 +1000 Subject: [PATCH 7/7] Add experimental notice to dfsr collector Signed-off-by: Ben Reedy --- collector/dfsr.go | 2 ++ collector/smtp.go | 1 + docs/collector.dfsr.md | 2 ++ 3 files changed, 5 insertions(+) diff --git a/collector/dfsr.go b/collector/dfsr.go index b94ed8d6..2573ca94 100644 --- a/collector/dfsr.go +++ b/collector/dfsr.go @@ -4,12 +4,14 @@ package collector import ( "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/log" "gopkg.in/alecthomas/kingpin.v2" ) var dfsrEnabledCollectors = kingpin.Flag("collectors.dfsr.sources-enabled", "Comma-seperated list of DFSR Perflib sources to use.").Default("connection,folder,volume").String() func init() { + log.Info("dfsr collector is in an experimental state! Metrics for this collector have not been tested.") // Perflib sources are dynamic, depending on the enabled child collectors var perflibDependencies []string for _, source := range expandEnabledChildCollectors(*dfsrEnabledCollectors) { diff --git a/collector/smtp.go b/collector/smtp.go index df236edc..70eb6b3a 100644 --- a/collector/smtp.go +++ b/collector/smtp.go @@ -11,6 +11,7 @@ import ( ) func init() { + log.Info("smtp collector is in an experimental state! Metrics for this collector have not been tested.") registerCollector("smtp", NewSMTPCollector, "SMTP Server") } diff --git a/docs/collector.dfsr.md b/docs/collector.dfsr.md index 9369048d..e018f107 100644 --- a/docs/collector.dfsr.md +++ b/docs/collector.dfsr.md @@ -2,6 +2,8 @@ The dfsr collector exposes metrics for [DFSR](https://docs.microsoft.com/en-us/windows-server/storage/dfs-replication/dfsr-overview). +**Collector is currently in an experimental state and testing of metrics has not been undertaken.** Feedback on this collector is welcome. + ||| -|- Metric name prefix | `dfsr`