[relay] Fix race in Relay server shutdown (#7607)

* [relay] Bind listeners before serving to fix the shutdown race

A shutdown signal that arrives while the relay is still starting races
the listener goroutines. Server.Shutdown read the listener's server
field while Listen was writing it, which the race detector reported,
and when Shutdown won it saw a nil server, returned as if nothing was
running, and the listener then bound and served with nothing left to
stop it. Server.Listen never returned and the process hung on exit.

The listener lifecycle is now split into Bind and Serve. Server.Listen
binds every listener under its mutex before spawning the accept loops,
so the fields Shutdown reads are written before the goroutines exist.
A closed flag on the server makes a Listen that runs after Shutdown
return without binding. A bind failure on one listener shuts down the
ones already bound and surfaces the error at once instead of holding
it in a channel until the surviving listener exits.

* [relay] Mark the server closed before shutting down the relay

Shutdown set the closed flag only after the relay had finished closing
peers, so a Listen that started during that window could bind sockets
and start serving on a server that was already going down. The flag is
now set under listenerMux before the relay shutdown, and Accept closes
connections it receives once the relay is closed instead of leaving
them to the client's timeout.

The tests now cover the public Listen bind failure and a QUIC listener
shut down while blocked in Accept, and they pick ports that are free
for both TCP and UDP, reporting a bind error instead of a timeout.

* Fix tests

* [relay] Test the Listen and Shutdown race and the QUIC bind rollback

The existing tests order Listen and Shutdown deterministically, so the
race the fix targets was never exercised. A new test fires both from a
shared start channel across repeated rounds so either side can take the
lock first, and fails on the hang the old code produced.

The rollback test now drives Server.Listen with the real ws and quic
listeners and a UDP blocker, so the ws socket is bound and released when
the quic bind fails. Both tests pass a real TLS config because a nil one
only yields a quic listener in the devcert build.

* [relay] Bound the Shutdown wait in the concurrent Listen and Shutdown test

* [relay] Keep the default WS port for an empty listen address

net.Listen picks a random port for an empty address, while ListenAndServe
used :http or :https. Apply the same default in Bind so an empty address
keeps listening where it did before.

* [relay] Release the WS socket when Serve fails

ServeTLS can return before it takes ownership of the listener, for example
when no certificate is configured, leaving the socket opened in Bind bound
until Shutdown. Close it in Serve on any error other than a server close.

* [relay] Drain the relay outside the listener lock

Shutdown held listenerMux while the relay closed its peers gracefully, so
ListenerProtocols, and with it the healthcheck, blocked for the whole drain.
Mark the server closed and take the listeners under the lock, then drain and
stop them after releasing it. The closed flag still keeps Listen from
registering new listeners.

relay/server/server.go

@@ -3,6 +3,7 @@ package server
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
 	"net/url"
 	"sync"
 
@@ -35,6 +36,7 @@ type Server struct {
 	relay       *Relay
 	listeners   []Listener
 	listenerMux sync.Mutex
+	closed      bool
 }
 
 // NewServer creates and returns a new relay server instance.
@@ -62,39 +64,30 @@ func NewServer(config Config) (*Server, error) {
 	}, nil
 }
 
-// Listen starts the relay server.
+// Listen binds the relay listeners and serves them until Shutdown is called.
 func (r *Server) Listen(cfg ListenerConfig) error {
-	wSListener := &ws.Listener{
-		Address:        cfg.Address,
-		TLSConfig:      cfg.TLSConfig,
-		TrustedProxies: cfg.TrustedProxies,
-	}
-
 	r.listenerMux.Lock()
-	r.listeners = append(r.listeners, wSListener)
-
-	tlsConfigQUIC, err := quictls.ServerQUICTLSConfig(cfg.TLSConfig)
-	if err != nil {
-		log.Warnf("Not starting QUIC listener: %v", err)
-	} else {
-		quicListener := &quic.Listener{
-			Address:   cfg.Address,
-			TLSConfig: tlsConfigQUIC,
-		}
-
-		r.listeners = append(r.listeners, quicListener)
+	if r.closed {
+		r.listenerMux.Unlock()
+		return nil
 	}
 
-	errChan := make(chan error, len(r.listeners))
+	listeners, err := bindListeners(newListeners(cfg))
+	if err != nil {
+		r.listenerMux.Unlock()
+		return err
+	}
+	r.listeners = append(r.listeners, listeners...)
+
+	errChan := make(chan error, len(listeners))
 	wg := sync.WaitGroup{}
-	for _, l := range r.listeners {
+	for _, l := range listeners {
 		wg.Add(1)
 		go func(listener Listener) {
 			defer wg.Done()
-			errChan <- listener.Listen(r.relay.Accept)
+			errChan <- listener.Serve(r.relay.Accept)
 		}(l)
 	}
-
 	r.listenerMux.Unlock()
 
 	wg.Wait()
@@ -110,18 +103,14 @@ func (r *Server) Listen(cfg ListenerConfig) error {
 // Shutdown stops the relay server. If there are active connections, they will be closed gracefully. In case of a context,
 // the connections will be forcefully closed.
 func (r *Server) Shutdown(ctx context.Context) error {
-	r.relay.Shutdown(ctx)
-
 	r.listenerMux.Lock()
-	var multiErr *multierror.Error
-	for _, l := range r.listeners {
-		if err := l.Shutdown(ctx); err != nil {
-			multiErr = multierror.Append(multiErr, err)
-		}
-	}
-	r.listeners = r.listeners[:0]
+	r.closed = true
+	listeners := r.listeners
+	r.listeners = nil
 	r.listenerMux.Unlock()
-	return nberrors.FormatErrorOrNil(multiErr)
+
+	r.relay.Shutdown(ctx)
+	return shutdownListeners(ctx, listeners)
 }
 
 func (r *Server) ListenerProtocols() []protocol.Protocol {
@@ -145,3 +134,48 @@ func (r *Server) InstanceURL() url.URL {
 func (r *Server) RelayAccept() func(conn listener.Conn) {
 	return r.relay.Accept
 }
+
+func newListeners(cfg ListenerConfig) []Listener {
+	listeners := []Listener{
+		&ws.Listener{
+			Address:        cfg.Address,
+			TLSConfig:      cfg.TLSConfig,
+			TrustedProxies: cfg.TrustedProxies,
+		},
+	}
+
+	tlsConfigQUIC, err := quictls.ServerQUICTLSConfig(cfg.TLSConfig)
+	if err != nil {
+		log.Warnf("Not starting QUIC listener: %v", err)
+		return listeners
+	}
+
+	return append(listeners, &quic.Listener{
+		Address:   cfg.Address,
+		TLSConfig: tlsConfigQUIC,
+	})
+}
+
+func bindListeners(listeners []Listener) ([]Listener, error) {
+	bound := make([]Listener, 0, len(listeners))
+	for _, l := range listeners {
+		if err := l.Bind(); err != nil {
+			if shutdownErr := shutdownListeners(context.Background(), bound); shutdownErr != nil {
+				log.Warnf("failed to close listeners after bind error: %v", shutdownErr)
+			}
+			return nil, fmt.Errorf("%s listener: %w", l.Protocol(), err)
+		}
+		bound = append(bound, l)
+	}
+	return bound, nil
+}
+
+func shutdownListeners(ctx context.Context, listeners []Listener) error {
+	var multiErr *multierror.Error
+	for _, l := range listeners {
+		if err := l.Shutdown(ctx); err != nil {
+			multiErr = multierror.Append(multiErr, err)
+		}
+	}
+	return nberrors.FormatErrorOrNil(multiErr)
+}