sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-08-24 16:41:30 +02:00
Code Issues Packages Projects Releases Wiki Activity

[client] Reconnect management and signal promptly after a network change

Browse Source 
Three pieces, each verified on Android across WiFi/cellular switches:

- A sweep-aware backoff wrapper: the first retry after a disconnect
  that follows a recent network-change mark comes after 200ms instead
  of the randomized [0..1.6s] interval. Any other failure keeps the
  unchanged spread, so the clients of a restarted server still scatter
  their reconnects.
- The retry sleep wakes on OS network availability transitions
  (nbgrpc.Retry): a disconnect that precedes the offline flag by a few
  milliseconds no longer sleeps blindly through the recovery - the
  loop parks on the netstate gate and resumes the moment the network
  returns.
- The connection state is re-checked after WaitForStateChange: a dial
  settling in Ready proceeds immediately instead of burning another
  backoff round on an already-usable channel.

Measured after a network switch: management and signal recover in
270-470ms deterministically, down from a 312-1593ms lottery.
This commit is contained in:
Zoltán Papp
2026-08-13 15:49:44 +02:00
parent 31e558a64d
commit cfd3a231e2
7 changed files with 291 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
shared/management/client/grpc.go
View File

@@ -235,7 +235,7 @@ func (c *GrpcClient) withMgmtStream(
ctx context.Context,
handler func(ctx context.Context, serverPubKey wgtypes.Key, backOff backoff.BackOff) error,
) error {
backOff := defaultBackoff(ctx)
backOff := c.sweeper.QuickRetryBackoff(ctx, defaultBackoff(ctx), c.netState)
operation := func() error {
// suspend reconnect attempts while the OS reports no usable network.
// Wait only errors on a cancelled context, which means shutdown, so
@@ -247,14 +247,21 @@ func (c *GrpcClient) withMgmtStream(
backOff.Reset()
}
log.Debugf("management connection state %v", c.conn.GetState())
connState := c.conn.GetState()
log.Debugf("management connection state %v", connState)
if connState == connectivity.Shutdown {
return backoff.Permanent(fmt.Errorf("connection to management has been shut down"))
} else if !(connState == connectivity.Ready || connState == connectivity.Idle) {
}
if !(connState == connectivity.Ready || connState == connectivity.Idle) {
// A dial may already be in flight (e.g. the other stream triggered
// it after a network change); wait for it to settle and proceed if
// the channel became usable, instead of burning a backoff round on
// a successful dial. A failed dial errors out as before.
c.conn.WaitForStateChange(ctx, connState)
return fmt.Errorf("connection to management is not ready and in %s state", connState)
connState = c.conn.GetState()
if !(connState == connectivity.Ready || connState == connectivity.Idle) {
return fmt.Errorf("connection to management is not ready and in %s state", connState)
}
}
serverPubKey, err := c.getServerPublicKey()
@@ -266,7 +273,7 @@ func (c *GrpcClient) withMgmtStream(
return handler(ctx, *serverPubKey, backOff)
}
err := backoff.Retry(operation, backOff)
err := nbgrpc.Retry(ctx, operation, backOff, c.netState)
if err != nil {
log.Warnf("exiting the Management service connection retry loop due to the unrecoverable error: %s", err)
}

18
shared/signal/client/grpc.go
View File

@@ -198,7 +198,7 @@ func defaultBackoff(ctx context.Context) backoff.BackOff {
// The connection retry logic will try to reconnect for 30 min and if wasn't successful will propagate the error to the function caller.
func (c *GrpcClient) Receive(ctx context.Context, msgHandler func(msg *proto.Message) error) error {
var backOff = defaultBackoff(ctx)
backOff := c.sweeper.QuickRetryBackoff(ctx, defaultBackoff(ctx), c.netState)
operation := func() error {
// suspend reconnect attempts while the OS reports no usable network.
@@ -213,13 +213,21 @@ func (c *GrpcClient) Receive(ctx context.Context, msgHandler func(msg *proto.Mes
c.notifyStreamDisconnected()
log.Debugf("signal connection state %v", c.signalConn.GetState())
connState := c.signalConn.GetState()
log.Debugf("signal connection state %v", connState)
if connState == connectivity.Shutdown {
return backoff.Permanent(fmt.Errorf("connection to signal has been shut down"))
} else if !(connState == connectivity.Ready || connState == connectivity.Idle) {
}
if !(connState == connectivity.Ready || connState == connectivity.Idle) {
// A dial may already be in flight (e.g. triggered by another RPC
// after a network change); wait for it to settle and proceed if
// the channel became usable, instead of burning a backoff round on
// a successful dial. A failed dial errors out as before.
c.signalConn.WaitForStateChange(ctx, connState)
return fmt.Errorf("connection to signal is not ready and in %s state", connState)
connState = c.signalConn.GetState()
if !(connState == connectivity.Ready || connState == connectivity.Idle) {
return fmt.Errorf("connection to signal is not ready and in %s state", connState)
}
}
// connect to Signal stream identifying ourselves with a public WireGuard key
@@ -273,7 +281,7 @@ func (c *GrpcClient) Receive(ctx context.Context, msgHandler func(msg *proto.Mes
return nil
}
err := backoff.Retry(operation, backOff)
err := nbgrpc.Retry(ctx, operation, backOff, c.netState)
if err != nil {
log.Errorf("exiting the Signal service connection retry loop due to the unrecoverable error: %v", err)
return err