mirror of
https://github.com/netbirdio/netbird.git
synced 2026-10-14 01:19:09 +02:00
fix(mgm): report management unhealthy while Sync stream is failing
The health probe (IsHealthy) only checked the gRPC transport and a GetServerKey call. GetServerKey succeeds even when the peer cannot sync (e.g. the server returns "settings not found"), so the probe kept marking management Connected while the Sync stream failed in a tight retry loop — pinning the status to "Connected" forever despite no sync ever succeeding. Track the last Sync stream error and have IsHealthy consult it, so a healthy transport is no longer enough to report the connection healthy.
This commit is contained in:
@@ -55,6 +55,14 @@ type GrpcClient struct {
|
|||||||
connStateCallback ConnStateNotifier
|
connStateCallback ConnStateNotifier
|
||||||
connStateCallbackLock sync.RWMutex
|
connStateCallbackLock sync.RWMutex
|
||||||
serverURL string
|
serverURL string
|
||||||
|
|
||||||
|
// syncStreamErr holds the last Sync stream error, or nil while the stream
|
||||||
|
// is established and healthy. GetServerKey succeeds even when the peer
|
||||||
|
// cannot sync (e.g. the server returns "settings not found"), so the
|
||||||
|
// health probe must consult this to avoid reporting a healthy management
|
||||||
|
// connection while the Sync stream keeps failing.
|
||||||
|
syncStreamMu sync.RWMutex
|
||||||
|
syncStreamErr error
|
||||||
}
|
}
|
||||||
|
|
||||||
type ExposeRequest struct {
|
type ExposeRequest struct {
|
||||||
@@ -372,11 +380,13 @@ func (c *GrpcClient) handleSyncStream(ctx context.Context, serverPubKey wgtypes.
|
|||||||
|
|
||||||
log.Infof("connected to the Management Service stream")
|
log.Infof("connected to the Management Service stream")
|
||||||
c.notifyConnected()
|
c.notifyConnected()
|
||||||
|
c.setSyncStreamConnected()
|
||||||
|
|
||||||
// blocking until error
|
// blocking until error
|
||||||
err = c.receiveUpdatesEvents(stream, serverPubKey, msgHandler)
|
err = c.receiveUpdatesEvents(stream, serverPubKey, msgHandler)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.notifyDisconnected(err)
|
c.notifyDisconnected(err)
|
||||||
|
c.setSyncStreamDisconnected(err)
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
log.Debugf("management connection context has been canceled, this usually indicates shutdown")
|
log.Debugf("management connection context has been canceled, this usually indicates shutdown")
|
||||||
return nil
|
return nil
|
||||||
@@ -530,6 +540,13 @@ func (c *GrpcClient) IsHealthy() bool {
|
|||||||
log.Warnf("health check returned: %s", err)
|
log.Warnf("health check returned: %s", err)
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if syncErr := c.syncStreamError(); syncErr != nil {
|
||||||
|
c.notifyDisconnected(syncErr)
|
||||||
|
log.Warnf("management transport is up but the Sync stream is unhealthy: %s", syncErr)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
c.notifyConnected()
|
c.notifyConnected()
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -771,6 +788,24 @@ func (c *GrpcClient) SyncMeta(sysInfo *system.Info) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *GrpcClient) setSyncStreamConnected() {
|
||||||
|
c.syncStreamMu.Lock()
|
||||||
|
defer c.syncStreamMu.Unlock()
|
||||||
|
c.syncStreamErr = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *GrpcClient) setSyncStreamDisconnected(err error) {
|
||||||
|
c.syncStreamMu.Lock()
|
||||||
|
defer c.syncStreamMu.Unlock()
|
||||||
|
c.syncStreamErr = err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *GrpcClient) syncStreamError() error {
|
||||||
|
c.syncStreamMu.RLock()
|
||||||
|
defer c.syncStreamMu.RUnlock()
|
||||||
|
return c.syncStreamErr
|
||||||
|
}
|
||||||
|
|
||||||
func (c *GrpcClient) notifyDisconnected(err error) {
|
func (c *GrpcClient) notifyDisconnected(err error) {
|
||||||
c.connStateCallbackLock.RLock()
|
c.connStateCallbackLock.RLock()
|
||||||
defer c.connStateCallbackLock.RUnlock()
|
defer c.connStateCallbackLock.RUnlock()
|
||||||
|
|||||||
Reference in New Issue
Block a user