mirror of
https://github.com/netbirdio/netbird.git
synced 2026-10-14 09:29:07 +02:00
Riccardo Manfrin
70ddb2a393
* [client] Move the Signal registration wait into its own helper Pure move, no behavior change. The wait for the registration header and the check on its value leave connect() for awaitRegistration(), so the next commit can bound that wait in one place. * [client] Split the Signal test server helper from its listener setup Pure move, no behavior change. serveTestSignalExchange serves any SignalExchangeServer on a local listener; startTestSignalServer keeps serving the real server through it. Lets a test wrap the real server. * [client] Bound the wait for the Signal registration header After an outage the Signal client could wait forever for the registration header of a new stream. Keepalive cannot end that wait when whatever answers the transport (a proxy in front of Signal, or Signal itself) accepts the stream and never answers it, and the receive watchdog only starts once registration succeeds. Signal then stayed down until the daemon was restarted, while Management and Relay recovered. Cancel the stream when the header does not arrive within 10 seconds, so connect() fails and the existing retry loop opens a new stream. A timer is used instead of a context deadline because the stream context must outlive registration. * [client] Report Signal unhealthy while no stream is registered IsHealthy judged Signal by the gRPC connection state alone and marked it connected whenever the connection was Ready or Idle. With the connection up but no stream registered, the client cannot receive offers or answers, yet netbird status showed Signal: Connected, hiding a stuck registration for as long as it lasted. Require a registered stream before reporting healthy. When there is none, return false without touching the recorder: the disconnect that dropped the stream already reported its reason. * [client] Name the server that reconnected in the SR watcher log Signal and Relay shared one reconnect callback, which logged "reconnected to Signal or Relay server" for either. A Relay reconnect while the Signal stream was down read as Signal having recovered, which misled diagnosis of a stuck Signal registration. Register one callback per server so the log names the one that actually reconnected. Which listeners get notified, and when, is unchanged. * [client] Trim comments in the Signal registration and health code Comment-only change, no behavior change. Drop comments that restate the code and keep only the reason a timer is used instead of a context deadline. * [client] Check Signal stream registration before the connection state IsHealthy returned true for a Connecting or Shutdown channel before the registration check ran, so a client with no registered stream could still be reported healthy while the channel was reconnecting. Check the stream first; the connection-state cases now only apply to a client that has a registered stream. * [client] Log a plain connect in the SR watcher The Signal and Relay callbacks also fire on the first connection, not only after a disconnect, so "reconnected" was wrong at startup. Log "connected to <server> server" for both cases. * [client] Say what the SR watcher does when it logs The line was read as a per-server connect log, which Signal and Relay already emit themselves. What it marks is the peer guards being told to reset their reconnection backoff, which happens only once both are ready. Log that, naming the server whose event triggered it.
87 lines
2.7 KiB
Go
87 lines
2.7 KiB
Go
package client
|
|
|
|
import (
|
|
"context"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
"go.opentelemetry.io/otel"
|
|
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
|
|
|
|
sigProto "github.com/netbirdio/netbird/shared/signal/proto"
|
|
"github.com/netbirdio/netbird/signal/server"
|
|
)
|
|
|
|
// stallingSignalServer holds the first ConnectStream open without ever sending
|
|
// the registration header, while the transport keeps answering keepalive pings.
|
|
// Later streams go to the real server. This is what a responder in front of or
|
|
// inside Signal looks like when it accepts a stream and never answers it.
|
|
type stallingSignalServer struct {
|
|
*server.Server
|
|
stalled atomic.Bool
|
|
held chan struct{}
|
|
}
|
|
|
|
func (s *stallingSignalServer) ConnectStream(stream sigProto.SignalExchange_ConnectStreamServer) error {
|
|
if s.stalled.CompareAndSwap(false, true) {
|
|
close(s.held)
|
|
<-stream.Context().Done()
|
|
return stream.Context().Err()
|
|
}
|
|
return s.Server.ConnectStream(stream)
|
|
}
|
|
|
|
// TestReceiveRetriesWhenRegistrationHeaderNeverArrives reproduces a client stuck
|
|
// forever waiting for the registration header of a stream the server side never
|
|
// answers: the receive watchdog only starts after registration, so nothing ended
|
|
// the wait and Signal never came back until the daemon was restarted. The client
|
|
// must give up on the silent stream and register on a new one.
|
|
func TestReceiveRetriesWhenRegistrationHeaderNeverArrives(t *testing.T) {
|
|
realServer, err := server.NewServer(context.Background(), otel.Meter(""))
|
|
require.NoError(t, err)
|
|
stalling := &stallingSignalServer{Server: realServer, held: make(chan struct{})}
|
|
addr := serveTestSignalExchange(t, stalling)
|
|
|
|
key, err := wgtypes.GenerateKey()
|
|
require.NoError(t, err)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
t.Cleanup(cancel)
|
|
|
|
client, err := NewClient(ctx, addr, key, false)
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() { _ = client.Close() })
|
|
client.registrationTimeout = 200 * time.Millisecond
|
|
|
|
// Stop Receive before Close runs (cleanups are LIFO), so the two never overlap.
|
|
receiveDone := make(chan struct{})
|
|
go func() {
|
|
defer close(receiveDone)
|
|
_ = client.Receive(ctx, func(*sigProto.Message) error { return nil })
|
|
}()
|
|
t.Cleanup(func() {
|
|
cancel()
|
|
<-receiveDone
|
|
})
|
|
|
|
select {
|
|
case <-stalling.held:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("first stream did not reach the stalling server")
|
|
}
|
|
|
|
streamReady := make(chan struct{})
|
|
go func() {
|
|
client.WaitStreamConnected(ctx)
|
|
close(streamReady)
|
|
}()
|
|
select {
|
|
case <-streamReady:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("client never registered after the first stream went unanswered")
|
|
}
|
|
require.True(t, client.StreamConnected(), "client should report a registered stream")
|
|
}
|