Add cpu metrics based on newer and more accurate perflib sources

This change adds 4 new CPU related metrics:

 * process_mperf_total
 * processor_rtc_total
 * processor_utility_total
 * processor_privileged_utility_total

and renames the existing process_performance to
processor_performance_total, since it was previously misunderstood and
was unlikely to be have been useful without the above new metrics

The data sources for these are not particularly well understood, and the
examples show that in some cases, arbitrary scaling factors are required
to actually make them useful, but in my testing on hundreds of systems
with a broad range of CPUs and operating systems from 2012r2 through to
2019 has proved out that we can use them to accurately display actual
CPU frequencies and CPU utilisation as it is represented in taskmgr.

Things I don't particularly like and would like input on:

 * I would have preferred to do the scaling of processor_mperf_total in
the code, but there isn't an elegant way of doing this right now.
 * Maybe processor_mperf_total should be called
processor_mperformance_total.

See #787 for discussion.

Signed-off-by: Steffen Higel <higels@valvesoftware.com>

collector/cpu.go

@@ -37,6 +37,10 @@ type cpuCollectorFull struct {
 	ProcessorFrequencyMHz    *prometheus.Desc
 	ProcessorMaxFrequencyMHz *prometheus.Desc
 	ProcessorPerformance     *prometheus.Desc
+	ProcessorMPerf           *prometheus.Desc
+	ProcessorRTC             *prometheus.Desc
+	ProcessorUtility         *prometheus.Desc
+	ProcessorPrivUtility     *prometheus.Desc
 }
 
 // newCPUCollector constructs a new cpuCollector, appropriate for the running OS
@@ -129,11 +133,35 @@ func newCPUCollector() (Collector, error) {
 			nil,
 		),
 		ProcessorPerformance: prometheus.NewDesc(
-			prometheus.BuildFQName(Namespace, subsystem, "processor_performance"),
+			prometheus.BuildFQName(Namespace, subsystem, "processor_performance_total"),
 			"Processor Performance is the average performance of the processor while it is executing instructions, as a percentage of the nominal performance of the processor. On some processors, Processor Performance may exceed 100%",
 			[]string{"core"},
 			nil,
 		),
+		ProcessorMPerf: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "processor_mperf_total"),
+			"Processor MPerf is the number of TSC ticks incremented while executing instructions",
+			[]string{"core"},
+			nil,
+		),
+		ProcessorRTC: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "processor_rtc_total"),
+			"Processor RTC represents the number of RTC ticks made since the system booted. It should consistently be 64e6, and can be used to properly derive Processor Utility Rate",
+			[]string{"core"},
+			nil,
+		),
+		ProcessorUtility: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "processor_utility_total"),
+			"Processor Utility represents is the amount of time the core spends executing instructions",
+			[]string{"core"},
+			nil,
+		),
+		ProcessorPrivUtility: prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "processor_privileged_utility_total"),
+			"Processor Privilieged Utility represents is the amount of time the core has spent executing instructions inside the kernel",
+			[]string{"core"},
+			nil,
+		),
 	}, nil
 }
 
@@ -258,8 +286,10 @@ type perflibProcessorInformation struct {
 	PrivilegedUtilitySeconds float64 `perflib:"% Privileged Utility"`
 	ProcessorFrequencyMHz    float64 `perflib:"Processor Frequency"`
 	ProcessorPerformance     float64 `perflib:"% Processor Performance"`
+	ProcessorMPerf           float64 `perflib:"% Processor Performance,secondvalue"`
 	ProcessorTimeSeconds     float64 `perflib:"% Processor Time"`
 	ProcessorUtilityRate     float64 `perflib:"% Processor Utility"`
+	ProcessorRTC             float64 `perflib:"% Processor Utility,secondvalue"`
 	UserTimeSeconds          float64 `perflib:"% User Time"`
 }
 
@@ -366,10 +396,34 @@ func (c *cpuCollectorFull) Collect(ctx *ScrapeContext, ch chan<- prometheus.Metr
 		)
 		ch <- prometheus.MustNewConstMetric(
 			c.ProcessorPerformance,
-			prometheus.GaugeValue,
+			prometheus.CounterValue,
 			cpu.ProcessorPerformance,
 			core,
 		)
+		ch <- prometheus.MustNewConstMetric(
+			c.ProcessorMPerf,
+			prometheus.CounterValue,
+			cpu.ProcessorMPerf,
+			core,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			c.ProcessorRTC,
+			prometheus.CounterValue,
+			cpu.ProcessorRTC,
+			core,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			c.ProcessorUtility,
+			prometheus.CounterValue,
+			cpu.ProcessorUtilityRate,
+			core,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			c.ProcessorPrivUtility,
+			prometheus.CounterValue,
+			cpu.PrivilegedUtilitySeconds,
+			core,
+		)
 	}
 
 	return nil

collector/perflib.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"reflect"
 	"strconv"
+	"strings"
 
 	perflibCollector "github.com/leoluk/perflib_exporter/collector"
 	"github.com/leoluk/perflib_exporter/perflib"
@@ -67,6 +68,16 @@ func unmarshalObject(obj *perflib.PerfObject, vs interface{}) error {
 			if tag == "" {
 				continue
 			}
+			secondValue := false
+
+			st := strings.Split(tag, ",")
+			tag = st[0]
+
+			for _, t := range st {
+				if t == "secondvalue" {
+					secondValue = true
+				}
+			}
 
 			ctr, found := counters[tag]
 			if !found {
@@ -80,6 +91,14 @@ func unmarshalObject(obj *perflib.PerfObject, vs interface{}) error {
 				return fmt.Errorf("tagged field %v has wrong type %v, must be float64", f.Name, fieldType)
 			}
 
+			if secondValue {
+				if !ctr.Def.HasSecondValue {
+					return fmt.Errorf("tagged field %v expected a SecondValue, which was not present", f.Name)
+				}
+				target.Field(i).SetFloat(float64(ctr.SecondValue))
+				continue
+			}
+
 			switch ctr.Def.CounterType {
 			case perflibCollector.PERF_ELAPSED_TIME:
 				target.Field(i).SetFloat(float64(ctr.Value-windowsEpoch) / float64(obj.Frequency))

collector/perflib_test.go

@@ -11,6 +11,7 @@ import (
 type simple struct {
 	ValA float64 `perflib:"Something"`
 	ValB float64 `perflib:"Something Else"`
+	ValC float64 `perflib:"Something Else,secondvalue"`
 }
 
 func TestUnmarshalPerflib(t *testing.T) {
@@ -62,16 +63,18 @@ func TestUnmarshalPerflib(t *testing.T) {
 							},
 							{
 								Def: &perflib.PerfCounterDef{
-									Name:        "Something Else",
-									CounterType: perflibCollector.PERF_COUNTER_COUNTER,
+									Name:           "Something Else",
+									CounterType:    perflibCollector.PERF_COUNTER_COUNTER,
+									HasSecondValue: true,
 								},
-								Value: 256,
+								Value:       256,
+								SecondValue: 222,
 							},
 						},
 					},
 				},
 			},
-			expectedOutput: []simple{{ValA: 123, ValB: 256}},
+			expectedOutput: []simple{{ValA: 123, ValB: 256, ValC: 222}},
 			expectError:    false,
 		},
 		{