service: add delayed_start and trigger_start labels to windows_service_info (#2768)

Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>

internal/collector/service/AGENTS.md

@@ -9,6 +9,19 @@
   `RegNotifyChangeKeyValue` instead. Arm it with `REG_NOTIFY_THREAD_AGNOSTIC`:
   without it the registration ends when the arming OS thread exits, and Go
   goroutines are not bound to one.
+- The delayed auto-start flag and the triggers are queried with
+  `QueryServiceConfig2` on the same handle and cached with the configuration.
+  The SCM ignores the delayed flag unless the start type is auto, so it is only
+  queried for auto-start services. `x/sys/windows` has no
+  `SERVICE_TRIGGER_INFO`; the mirror in `service.go` only reads `cTriggers`.
+- The delayed flag is the `DelayedAutostart` value of the service key, so a
+  change updates the key's last write time. Triggers are stored in the
+  `TriggerInfo` subkey. Adding the first trigger creates it, which updates the
+  last write time of the service key (`TestCollectorPicksUpConfigChanges`
+  checks this). That a later trigger change made with `sc triggerinfo` also
+  does is assumed, not verified: the SCM is believed to rewrite the whole
+  `TriggerInfo` subkey. If it only changes deeper keys, the cache picks the
+  change up after at most `configCacheMaxAge`.
 - The subtree notification fires every few seconds on a normal machine (W32Time,
   Tcpip, VSS, driver `Enum` keys, Hyper-V). Treat it as a hint only and compare
   the last write time of each service's own key, which changes only for its own

internal/collector/service/config_cache.go

@@ -26,7 +26,6 @@ import (
 
 	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/registry"
-	"golang.org/x/sys/windows/svc/mgr"
 )
 
 const (
@@ -54,7 +53,7 @@ var errConfigCacheClosed = errors.New("service configuration cache is closed")
 
 // cachedServiceConfig is the result of queryServiceConfig.
 type cachedServiceConfig struct {
-	config    mgr.Config
+	config    serviceConfig
 	available bool
 
 	// lastWrite is the last write time of the service's registry key, read
@@ -280,7 +279,7 @@ func (cache *serviceConfigCache) close() error {
 
 // getCachedServiceConfig returns the configuration of a service from the cache,
 // or queries and caches it on a miss.
-func (c *Collector) getCachedServiceConfig(serviceName string) (mgr.Config, bool, error) {
+func (c *Collector) getCachedServiceConfig(serviceName string) (serviceConfig, bool, error) {
 	entry, generation, ok := c.configCache.get(serviceName)
 	if ok {
 		return entry.config, entry.available, nil
@@ -296,7 +295,7 @@ func (c *Collector) getCachedServiceConfig(serviceName string) (mgr.Config, bool
 
 	config, available, err := c.queryServiceConfig(serviceName)
 	if err != nil {
-		return mgr.Config{}, false, err
+		return serviceConfig{}, false, err
 	}
 
 	switch {
@@ -308,6 +307,8 @@ func (c *Collector) getCachedServiceConfig(serviceName string) (mgr.Config, bool
 	case !available:
 		// An unavailable configuration, for example a display name from a missing
 		// MUI file, can become available without a change to the service key.
+	case !config.extendedAvailable:
+		// The delayed auto-start flag and triggers are queried again on the next scrape.
 	case time.Since(lastWrite) < configCacheMinKeyAge:
 		// Cached on a later scrape, once a second write in the same tick is ruled out.
 	default:

internal/collector/service/config_cache_internal_test.go

@@ -25,7 +25,6 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sys/windows/registry"
-	"golang.org/x/sys/windows/svc/mgr"
 )
 
 // newTestConfigCache returns a cache that watches a temporary key below
@@ -68,7 +67,7 @@ func newTestConfigCache(t *testing.T) (*serviceConfigCache, registry.Key) {
 
 		_, generation, _ := cache.get(name)
 		cache.put(name, generation, cachedServiceConfig{
-			config:    mgr.Config{DisplayName: name},
+			config:    serviceConfig{DisplayName: name},
 			available: true,
 			lastWrite: lastWrite,
 		})

internal/collector/service/service.go

@@ -95,6 +95,32 @@ type Collector struct {
 	metricCache metriccache.Cache[string, serviceValues]
 }
 
+// serviceConfig is the configuration of a service that the collector publishes.
+// Of the embedded [mgr.Config], only BinaryPathName, DisplayName, StartType,
+// ServiceStartName and DelayedAutoStart are set. DelayedAutoStart is only
+// queried for auto-start services and false for the others.
+type serviceConfig struct {
+	mgr.Config
+
+	// triggerStart is true if the service has triggers, which start or stop it
+	// on events such as a device arrival or a network change.
+	triggerStart bool
+
+	// extendedAvailable is true if DelayedAutoStart and triggerStart were queried
+	// successfully. Otherwise both are false and their labels stay empty.
+	extendedAvailable bool
+}
+
+// serviceTriggerInfo mirrors SERVICE_TRIGGER_INFO, which QueryServiceConfig2
+// returns for SERVICE_CONFIG_TRIGGER_INFO. golang.org/x/sys/windows doesn't define it.
+// Only cTriggers is read; the pointers point into the same buffer.
+// https://learn.microsoft.com/en-us/windows/win32/api/winsvc/ns-winsvc-service_trigger_info
+type serviceTriggerInfo struct {
+	triggers uint32  // cTriggers
+	trigger  uintptr // pTriggers
+	_        uintptr // pReserved
+}
+
 // serviceValues holds the inputs of the metrics of a service besides its name.
 type serviceValues struct {
 	state       uint32
@@ -102,7 +128,10 @@ type serviceValues struct {
 	displayName string
 	runAs       string
 	pathName    string
-	processID   uint32
+	// delayedStart and triggerStart are the label values "true" or "false".
+	delayedStart string
+	triggerStart string
+	processID    uint32
 	// processStartTime is the creation time of the process in nanoseconds,
 	// or 0 if it is unknown and the process metric is not published.
 	processStartTime uint64
@@ -187,7 +216,9 @@ func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error {
 
 	c.serviceConfigPoolBytes = sync.Pool{
 		New: func() any {
-			return new([]byte)
+			buf := make([]byte, 1024)
+
+			return &buf
 		},
 	}
 
@@ -196,7 +227,7 @@ func (c *Collector) Build(logger *slog.Logger, _ *mi.Session) error {
 	c.info = prometheus.NewDesc(
 		prometheus.BuildFQName(types.Namespace, Name, "info"),
 		"A metric with a constant '1' value labeled with service information",
-		[]string{"service", "display_name", "run_as", "path_name"},
+		[]string{"service", "display_name", "run_as", "path_name", "delayed_start", "trigger_start"},
 		nil,
 	)
 	c.state = prometheus.NewDesc(
@@ -408,6 +439,13 @@ func (c *Collector) collectService(ch chan<- prometheus.Metric, scrape *metricca
 		processID:   service.status.ProcessId,
 	}
 
+	// The labels stay empty if the service configuration or the delayed
+	// auto-start flag and triggers are not available.
+	if configAvailable && serviceConfig.extendedAvailable {
+		values.delayedStart = strconv.FormatBool(serviceConfig.DelayedAutoStart)
+		values.triggerStart = strconv.FormatBool(serviceConfig.triggerStart)
+	}
+
 	if values.processID != 0 {
 		values.processStartTime = c.processStartTime(serviceName, values.processID)
 	}
@@ -435,6 +473,8 @@ func (c *Collector) serviceMetrics(serviceName string, values serviceValues) []p
 		values.displayName,
 		values.runAs,
 		values.pathName,
+		values.delayedStart,
+		values.triggerStart,
 	))
 
 	var (
@@ -588,16 +628,16 @@ func (c *Collector) queryAllServices() ([]serviceEntry, error) {
 
 // queryServiceConfig opens the service and queries its configuration.
 // configAvailable is false if the service has no readable configuration.
-func (c *Collector) queryServiceConfig(serviceName string) (mgr.Config, bool, error) {
+func (c *Collector) queryServiceConfig(serviceName string) (serviceConfig, bool, error) {
 	serviceNamePtr, err := windows.UTF16PtrFromString(serviceName)
 	if err != nil {
-		return mgr.Config{}, false, fmt.Errorf("invalid service name: %w", err)
+		return serviceConfig{}, false, fmt.Errorf("invalid service name: %w", err)
 	}
 
 	// Open connection for service handler.
 	serviceHandle, err := windows.OpenService(c.serviceManagerHandle.Handle, serviceNamePtr, windows.SERVICE_QUERY_CONFIG)
 	if err != nil {
-		return mgr.Config{}, false, fmt.Errorf("failed to open service: %w", err)
+		return serviceConfig{}, false, fmt.Errorf("failed to open service: %w", err)
 	}
 
 	// Create handle for each service.
@@ -612,10 +652,10 @@ func (c *Collector) queryServiceConfig(serviceName string) (mgr.Config, bool, er
 	}(serviceManager)
 
 	// Get Service Configuration.
-	serviceConfig, err := c.getServiceConfig(serviceManager)
+	config, err := c.getServiceConfig(serviceManager)
 	if err != nil {
 		if !errors.Is(err, windows.ERROR_FILE_NOT_FOUND) && !errors.Is(err, windows.ERROR_MUI_FILE_NOT_FOUND) {
-			return mgr.Config{}, false, fmt.Errorf("failed to get service configuration: %w", err)
+			return serviceConfig{}, false, fmt.Errorf("failed to get service configuration: %w", err)
 		}
 
 		c.logger.Log(context.Background(), slog.LevelDebug, "failed collecting service config",
@@ -623,10 +663,10 @@ func (c *Collector) queryServiceConfig(serviceName string) (mgr.Config, bool, er
 			slog.String("service", serviceName),
 		)
 
-		return mgr.Config{}, false, nil
+		return serviceConfig{}, false, nil
 	}
 
-	return serviceConfig, true, nil
+	return config, true, nil
 }
 
 func (c *Collector) getProcessStartTime(pid uint32) (uint64, error) {
@@ -657,45 +697,100 @@ func (c *Collector) getProcessStartTime(pid uint32) (uint64, error) {
 	return uint64(creation.Nanoseconds()), nil
 }
 
-// getServiceConfig is an optimized variant of [mgr.Service] that only
+// getServiceConfig is an optimized variant of [mgr.Service.Config] that only
 // retrieves the necessary information.
-func (c *Collector) getServiceConfig(service *mgr.Service) (mgr.Config, error) {
-	var serviceConfig *windows.QUERY_SERVICE_CONFIG
+func (c *Collector) getServiceConfig(service *mgr.Service) (serviceConfig, error) {
+	buf := c.serviceConfigPoolBytes.Get().(*[]byte) //nolint:forcetypeassert // The pool only holds *[]byte.
+	defer c.serviceConfigPoolBytes.Put(buf)
 
-	bytesNeeded := uint32(1024)
-
-	buf, ok := c.serviceConfigPoolBytes.Get().(*[]byte)
-	if !ok || len(*buf) == 0 {
-		*buf = make([]byte, bytesNeeded)
-	} else {
-		bytesNeeded = uint32(cap(*buf))
+	err := queryServiceConfigBuffer(buf, func(data *byte, size uint32, bytesNeeded *uint32) error {
+		return windows.QueryServiceConfig(service.Handle, (*windows.QUERY_SERVICE_CONFIG)(unsafe.Pointer(data)), size, bytesNeeded)
+	})
+	if err != nil {
+		return serviceConfig{}, err
 	}
 
-	for {
-		serviceConfig = (*windows.QUERY_SERVICE_CONFIG)(unsafe.Pointer(&(*buf)[0]))
+	queryServiceConfig := (*windows.QUERY_SERVICE_CONFIG)(unsafe.Pointer(&(*buf)[0]))
 
-		err := windows.QueryServiceConfig(service.Handle, serviceConfig, bytesNeeded, &bytesNeeded)
+	config := serviceConfig{
+		BinaryPathName:   windows.UTF16PtrToString(queryServiceConfig.BinaryPathName),
+		DisplayName:      windows.UTF16PtrToString(queryServiceConfig.DisplayName),
+		StartType:        queryServiceConfig.StartType,
+		ServiceStartName: windows.UTF16PtrToString(queryServiceConfig.ServiceStartName),
+	}
+
+	// A failure only leaves the delayed_start and trigger_start labels empty.
+	// It must not hide the rest of the configuration.
+	if err := queryExtendedServiceConfig(service.Handle, buf, &config); err != nil {
+		c.logger.Log(context.Background(), slog.LevelDebug, "failed to query the delayed auto-start flag and triggers of the service",
+			slog.Any("err", err),
+			slog.String("service", service.Name),
+		)
+
+		config.DelayedAutoStart = false
+		config.triggerStart = false
+
+		return config, nil
+	}
+
+	config.extendedAvailable = true
+
+	return config, nil
+}
+
+// queryExtendedServiceConfig queries the delayed auto-start flag and the
+// triggers of a service with QueryServiceConfig2 into config. buf must not be
+// empty; the configuration strings must already be copied out of it.
+func queryExtendedServiceConfig(handle windows.Handle, buf *[]byte, config *serviceConfig) error {
+	// The service control manager ignores the delayed auto-start flag of services
+	// that don't start automatically, so it is only queried for them.
+	if config.StartType == windows.SERVICE_AUTO_START {
+		var (
+			delayed     windows.SERVICE_DELAYED_AUTO_START_INFO
+			bytesNeeded uint32
+		)
+
+		err := windows.QueryServiceConfig2(handle, windows.SERVICE_CONFIG_DELAYED_AUTO_START_INFO,
+			(*byte)(unsafe.Pointer(&delayed)), uint32(unsafe.Sizeof(delayed)), &bytesNeeded)
+		if err != nil {
+			return fmt.Errorf("failed to query the delayed auto-start flag: %w", err)
+		}
+
+		config.DelayedAutoStart = delayed.IsDelayedAutoStartUp != 0
+	}
+
+	err := queryServiceConfigBuffer(buf, func(data *byte, size uint32, bytesNeeded *uint32) error {
+		return windows.QueryServiceConfig2(handle, windows.SERVICE_CONFIG_TRIGGER_INFO, data, size, bytesNeeded)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to query the service triggers: %w", err)
+	}
+
+	config.triggerStart = (*serviceTriggerInfo)(unsafe.Pointer(&(*buf)[0])).triggers > 0
+
+	return nil
+}
+
+// queryServiceConfigBuffer calls query with the buffer and grows the buffer
+// until the result fits. query wraps QueryServiceConfig or QueryServiceConfig2.
+// The buffer must not be empty.
+func queryServiceConfigBuffer(buf *[]byte, query func(data *byte, size uint32, bytesNeeded *uint32) error) error {
+	for {
+		var bytesNeeded uint32
+
+		err := query(&(*buf)[0], uint32(len(*buf)), &bytesNeeded)
 		if err == nil {
-			break
+			return nil
 		}
 
 		if !errors.Is(err, windows.ERROR_INSUFFICIENT_BUFFER) && !errors.Is(err, windows.ERROR_MORE_DATA) {
-			return mgr.Config{}, err
+			return err
 		}
 
 		if bytesNeeded <= uint32(len(*buf)) {
-			return mgr.Config{}, fmt.Errorf("win32 reports buffer too small (%d), but buffer is large enough (%d): %w", uint32(cap(*buf)), bytesNeeded, err)
+			return fmt.Errorf("win32 reports the buffer as too small (%d bytes needed, %d available): %w", bytesNeeded, len(*buf), err)
 		}
 
 		*buf = make([]byte, bytesNeeded)
 	}
-
-	defer c.serviceConfigPoolBytes.Put(buf)
-
-	return mgr.Config{
-		BinaryPathName:   windows.UTF16PtrToString(serviceConfig.BinaryPathName),
-		DisplayName:      windows.UTF16PtrToString(serviceConfig.DisplayName),
-		StartType:        serviceConfig.StartType,
-		ServiceStartName: windows.UTF16PtrToString(serviceConfig.ServiceStartName),
-	}, nil
 }

internal/collector/service/service_internal_test.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"testing"
 	"time"
+	"unsafe"
 
 	"github.com/prometheus-community/windows_exporter/internal/scrapecache"
 	"github.com/prometheus/client_golang/prometheus"
@@ -98,6 +99,40 @@ func serviceMetrics(t assert.TestingT, c *Collector, name string) map[string]flo
 	return metrics
 }
 
+// setStartTrigger gives a service a trigger that starts it once the first IP
+// address becomes available. golang.org/x/sys/windows/svc/mgr can't set triggers.
+func setStartTrigger(t *testing.T, service *mgr.Service) {
+	t.Helper()
+
+	// NETWORK_MANAGER_FIRST_IP_ADDRESS_ARRIVAL_GUID.
+	subtype := windows.GUID{Data1: 0x4f27f2de, Data2: 0x14e2, Data3: 0x430b, Data4: [8]byte{0xa5, 0x49, 0x7c, 0xd4, 0x8c, 0xbc, 0x82, 0x45}}
+
+	// SERVICE_TRIGGER.
+	trigger := struct {
+		triggerType uint32
+		action      uint32
+		subtype     *windows.GUID
+		dataItems   uint32
+		data        uintptr
+	}{
+		triggerType: 3, // SERVICE_TRIGGER_TYPE_IP_ADDRESS_AVAILABILITY
+		action:      1, // SERVICE_TRIGGER_ACTION_SERVICE_START
+		subtype:     &subtype,
+	}
+
+	// SERVICE_TRIGGER_INFO.
+	info := struct {
+		triggers uint32
+		trigger  unsafe.Pointer
+		reserved uintptr
+	}{
+		triggers: 1,
+		trigger:  unsafe.Pointer(&trigger),
+	}
+
+	require.NoError(t, windows.ChangeServiceConfig2(service.Handle, windows.SERVICE_CONFIG_TRIGGER_INFO, (*byte)(unsafe.Pointer(&info))))
+}
+
 // TestCollectorPicksUpConfigChanges creates a service, changes its configuration
 // through the service control manager and checks that the cached configuration
 // is replaced. It needs an elevated process, which the Windows CI runners are.
@@ -148,7 +183,7 @@ func TestCollectorPicksUpConfigChanges(t *testing.T) {
 
 	metrics := serviceMetrics(t, c, name)
 	assert.InDelta(t, 1.0, metrics["windows_service_start_mode{start_mode=manual}"], 0)
-	assert.Contains(t, metrics, `windows_service_info{display_name=before,path_name=C:\Windows\System32\cmd.exe,run_as=LocalSystem}`)
+	assert.Contains(t, metrics, `windows_service_info{delayed_start=false,display_name=before,path_name=C:\Windows\System32\cmd.exe,run_as=LocalSystem,trigger_start=false}`)
 
 	_, _, ok := c.configCache.get(name)
 	require.True(t, ok, "the configuration must be cached after a scrape")
@@ -156,16 +191,45 @@ func TestCollectorPicksUpConfigChanges(t *testing.T) {
 	config, err := service.Config()
 	require.NoError(t, err)
 
-	config.StartType = mgr.StartDisabled
+	// The service is never started. It is switched back to manual start below,
+	// so that a killed test doesn't leave an auto-start service behind.
+	config.StartType = mgr.StartAutomatic
+	config.DelayedAutoStart = true
 	config.DisplayName = "after"
 	require.NoError(t, service.UpdateConfig(config))
 
 	// The change notification is delivered asynchronously.
 	require.EventuallyWithT(t, func(collect *assert.CollectT) {
 		metrics := serviceMetrics(collect, c, name)
-		assert.InDelta(collect, 1.0, metrics["windows_service_start_mode{start_mode=disabled}"], 0)
+		assert.InDelta(collect, 1.0, metrics["windows_service_start_mode{start_mode=auto}"], 0)
 		assert.InDelta(collect, 0.0, metrics["windows_service_start_mode{start_mode=manual}"], 0)
-		assert.Contains(collect, metrics, `windows_service_info{display_name=after,path_name=C:\Windows\System32\cmd.exe,run_as=LocalSystem}`)
+		assert.Contains(collect, metrics, `windows_service_info{delayed_start=true,display_name=after,path_name=C:\Windows\System32\cmd.exe,run_as=LocalSystem,trigger_start=false}`)
+	}, 5*time.Second, 50*time.Millisecond)
+
+	// Windows ignores the delayed auto-start flag of a manual service, and so does the label.
+	config.StartType = mgr.StartManual
+	require.NoError(t, service.UpdateConfig(config))
+
+	require.EventuallyWithT(t, func(collect *assert.CollectT) {
+		metrics := serviceMetrics(collect, c, name)
+		assert.InDelta(collect, 1.0, metrics["windows_service_start_mode{start_mode=manual}"], 0)
+		assert.Contains(collect, metrics, `windows_service_info{delayed_start=false,display_name=after,path_name=C:\Windows\System32\cmd.exe,run_as=LocalSystem,trigger_start=false}`)
+	}, 5*time.Second, 50*time.Millisecond)
+
+	// Cache the changed configuration, so that the trigger must invalidate it.
+	time.Sleep(configCacheMinKeyAge + 100*time.Millisecond)
+	serviceMetrics(t, c, name)
+
+	_, _, ok = c.configCache.get(name)
+	require.True(t, ok, "the changed configuration must be cached after a scrape")
+
+	// Triggers are stored in the TriggerInfo subkey of the service key. Creating
+	// it changes the last write time of the service key.
+	setStartTrigger(t, service)
+
+	require.EventuallyWithT(t, func(collect *assert.CollectT) {
+		metrics := serviceMetrics(collect, c, name)
+		assert.Contains(collect, metrics, `windows_service_info{delayed_start=false,display_name=after,path_name=C:\Windows\System32\cmd.exe,run_as=LocalSystem,trigger_start=true}`)
 	}, 5*time.Second, 50*time.Millisecond)
 
 	// The service control manager removes the service once the last handle is closed.
@@ -182,3 +246,18 @@ func TestCollectorPicksUpConfigChanges(t *testing.T) {
 		assert.False(collect, ok, "the configuration of a deleted service must be dropped")
 	}, 5*time.Second, 50*time.Millisecond)
 }
+
+// TestServiceTriggerInfoLayout checks the mirror of SERVICE_TRIGGER_INFO
+// (winsvc.h): DWORD cTriggers, PSERVICE_TRIGGER pTriggers and PBYTE pReserved.
+// The pointers are pointer-aligned, so cTriggers is padded to a pointer word.
+func TestServiceTriggerInfoLayout(t *testing.T) {
+	t.Parallel()
+
+	var info serviceTriggerInfo
+
+	pointerSize := unsafe.Sizeof(uintptr(0))
+
+	assert.Equal(t, uintptr(0), unsafe.Offsetof(info.triggers))
+	assert.Equal(t, pointerSize, unsafe.Offsetof(info.trigger))
+	assert.Equal(t, 3*pointerSize, unsafe.Sizeof(info))
+}