diff --git a/client/internal/engine.go b/client/internal/engine.go index 452075da8..d6d125582 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -82,6 +82,12 @@ const ( PeerConnectionTimeoutMax = 45000 // ms PeerConnectionTimeoutMin = 30000 // ms disableAutoUpdate = "disabled" + + // systemInfoTimeout bounds how long the sync loop waits for system info / posture + // check gathering. The gathering runs uncancellable system calls (process scan, + // exec, os.Stat); without this bound a single stuck call freezes handleSync, and + // thus syncMsgMux, for as long as the call hangs (observed multi-minute freezes). + systemInfoTimeout = 15 * time.Second ) var ErrResetConnection = fmt.Errorf("reset connection") @@ -1066,11 +1072,23 @@ func (e *Engine) updateChecksIfNew(checks []*mgmProto.Checks) error { } e.checks = checks - info, err := system.GetInfoWithChecks(e.ctx, checks) - if err != nil { - log.Warnf("failed to get system info with checks: %v", err) - info = system.GetInfo(e.ctx) + info, ok := system.GetInfoWithChecksTimeout(e.ctx, systemInfoTimeout, checks) + if !ok { + // Gathering timed out; skip the meta sync this cycle rather than blocking the + // sync loop (and syncMsgMux) on a stuck system call. A later sync will retry. + return nil } + e.applyInfoFlags(info) + + if err := e.mgmClient.SyncMeta(info); err != nil { + log.Errorf("could not sync meta: error %s", err) + return err + } + return nil +} + +// applyInfoFlags sets the engine's config-derived feature flags on the gathered system info. +func (e *Engine) applyInfoFlags(info *system.Info) { info.SetFlags( e.config.RosenpassEnabled, e.config.RosenpassPermissive, @@ -1089,12 +1107,6 @@ func (e *Engine) updateChecksIfNew(checks []*mgmProto.Checks) error { e.config.EnableSSHRemotePortForwarding, e.config.DisableSSHAuth, ) - - if err := e.mgmClient.SyncMeta(info); err != nil { - log.Errorf("could not sync meta: error %s", err) - return err - } - return nil } func (e *Engine) updateConfig(conf *mgmProto.PeerConfig) error { @@ -1240,31 +1252,15 @@ func (e *Engine) receiveManagementEvents() { e.shutdownWg.Add(1) go func() { defer e.shutdownWg.Done() - info, err := system.GetInfoWithChecks(e.ctx, e.checks) - if err != nil { - log.Warnf("failed to get system info with checks: %v", err) + info, ok := system.GetInfoWithChecksTimeout(e.ctx, systemInfoTimeout, e.checks) + if !ok { + // Gathering timed out; connect the stream with base info so management + // connectivity still comes up rather than blocking here. info = system.GetInfo(e.ctx) } - info.SetFlags( - e.config.RosenpassEnabled, - e.config.RosenpassPermissive, - &e.config.ServerSSHAllowed, - e.config.DisableClientRoutes, - e.config.DisableServerRoutes, - e.config.DisableDNS, - e.config.DisableFirewall, - e.config.BlockLANAccess, - e.config.BlockInbound, - e.config.DisableIPv6, - e.config.LazyConnectionEnabled, - e.config.EnableSSHRoot, - e.config.EnableSSHSFTP, - e.config.EnableSSHLocalPortForwarding, - e.config.EnableSSHRemotePortForwarding, - e.config.DisableSSHAuth, - ) + e.applyInfoFlags(info) - err = e.mgmClient.Sync(e.ctx, info, e.handleSync) + err := e.mgmClient.Sync(e.ctx, info, e.handleSync) if err != nil { // happens if management is unavailable for a long time. // We want to cancel the operation of the whole client diff --git a/client/system/info.go b/client/system/info.go index 477d5162b..3f4a0aab7 100644 --- a/client/system/info.go +++ b/client/system/info.go @@ -4,6 +4,7 @@ import ( "context" "net/netip" "strings" + "time" log "github.com/sirupsen/logrus" "google.golang.org/grpc/metadata" @@ -166,3 +167,34 @@ func GetInfoWithChecks(ctx context.Context, checks []*proto.Checks) (*Info, erro log.Debugf("all system information gathered successfully") return info, nil } + +// GetInfoWithChecksTimeout is GetInfoWithChecks bounded by timeout. Posture-check gathering +// runs uncancellable system calls (process enumeration, os.Stat), so calling it inline can +// block the caller for as long as such a call hangs. It runs in a goroutine instead: if it +// does not return within timeout the caller gets (nil, false) and should proceed with +// degraded behavior rather than block. On a gathering error it falls back to base GetInfo. +// +// The buffered channel lets the abandoned goroutine finish and exit once its blocking call +// returns, so it does not leak beyond the duration of that call. +func GetInfoWithChecksTimeout(ctx context.Context, timeout time.Duration, checks []*proto.Checks) (*Info, bool) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + infoCh := make(chan *Info, 1) + go func() { + info, err := GetInfoWithChecks(ctx, checks) + if err != nil { + log.Warnf("failed to get system info with checks: %v", err) + info = GetInfo(ctx) + } + infoCh <- info + }() + + select { + case info := <-infoCh: + return info, true + case <-ctx.Done(): + log.Warnf("gathering system info with checks timed out after %s", timeout) + return nil, false + } +}