[proxy] IPv6 in-place apply and TCP accept-loop hardening on netstack listeners

proxy/internal/tcp/accept.go

@@ -0,0 +1,85 @@
+package tcp
+
+import (
+	"context"
+	"errors"
+	"net"
+	"strings"
+	"time"
+)
+
+// gvisorInvalidEndpointMsg is the canonical text gVisor netstack returns
+// when Accept() is called on a listener whose underlying endpoint has
+// been destroyed (peer rekey, embedded-client reset, account churn).
+// There is no exported sentinel from gvisor.dev/gvisor/pkg/tcpip that
+// survives gonet's *net.OpError wrapping in a way errors.Is can match,
+// so we fall back to a string check. Stable across the gVisor versions
+// netbird pins.
+const gvisorInvalidEndpointMsg = "endpoint is in invalid state"
+
+// IsClosedListenerErr reports whether err signals that an accept loop
+// should exit because the underlying listener can no longer serve
+// connections. It recognises:
+//
+//   - net.ErrClosed for stdlib listeners (Listener.Close was called).
+//   - gVisor's "endpoint is in invalid state" for netstack-backed
+//     listeners whose endpoint was destroyed out from under them
+//     (typically when a per-account WireGuard netstack is reset without
+//     also tearing the listener entry down).
+//
+// Without the gVisor branch an accept loop on a netstack listener spins
+// CPU-hot forever after the endpoint dies, because Accept never blocks
+// again and the error neither matches net.ErrClosed nor cancels ctx.
+func IsClosedListenerErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, net.ErrClosed) {
+		return true
+	}
+	return strings.Contains(err.Error(), gvisorInvalidEndpointMsg)
+}
+
+// AcceptBackoff implements the exponential backoff used by
+// net/http.Server.Serve for transient Accept errors. Without it a loop
+// hitting a sticky unknown error burns a full CPU core. The zero value
+// is ready to use; call Reset after a successful Accept.
+type AcceptBackoff struct {
+	delay time.Duration
+}
+
+// minAcceptDelay / maxAcceptDelay mirror the stdlib defaults
+// (net/http.Server.Serve) and keep us well below 1 log line per second
+// per orphaned listener.
+const (
+	minAcceptDelay = 5 * time.Millisecond
+	maxAcceptDelay = time.Second
+)
+
+// Backoff waits the next exponential delay (5ms doubling up to 1s) and
+// returns true when the wait completed. Returns false if ctx fired
+// during the wait — callers should treat that as "exit the loop".
+func (b *AcceptBackoff) Backoff(ctx context.Context) bool {
+	b.advance()
+	select {
+	case <-ctx.Done():
+		return false
+	case <-time.After(b.delay):
+		return true
+	}
+}
+
+// Reset clears the accumulated delay so the next failure starts at the
+// minimum delay again. Call after a successful Accept.
+func (b *AcceptBackoff) Reset() { b.delay = 0 }
+
+func (b *AcceptBackoff) advance() {
+	if b.delay == 0 {
+		b.delay = minAcceptDelay
+	} else {
+		b.delay *= 2
+	}
+	if b.delay > maxAcceptDelay {
+		b.delay = maxAcceptDelay
+	}
+}

proxy/internal/tcp/accept_test.go

@@ -0,0 +1,142 @@
+package tcp
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestIsClosedListenerErr_NetErrClosed verifies the stdlib path: a
+// closed *net.Listener returns net.ErrClosed wrapped in *net.OpError,
+// and IsClosedListenerErr must unwrap it.
+func TestIsClosedListenerErr_NetErrClosed(t *testing.T) {
+	wrapped := &net.OpError{Op: "accept", Net: "tcp", Err: net.ErrClosed}
+	assert.True(t, IsClosedListenerErr(wrapped),
+		"net.OpError wrapping net.ErrClosed must be recognised as closed")
+}
+
+// TestIsClosedListenerErr_GVisorInvalidEndpoint is the load-bearing
+// regression guard. A gVisor netstack listener whose endpoint has been
+// destroyed returns this exact text. Without recognising it the accept
+// loop spins forever and burns a CPU core.
+func TestIsClosedListenerErr_GVisorInvalidEndpoint(t *testing.T) {
+	err := fmt.Errorf("accept tcp 10.10.1.254:80: endpoint is in invalid state")
+	assert.True(t, IsClosedListenerErr(err),
+		"gVisor 'endpoint is in invalid state' must be recognised as closed")
+}
+
+// TestIsClosedListenerErr_OtherError confirms we don't over-match —
+// transient errors must keep returning false so the backoff path runs.
+func TestIsClosedListenerErr_OtherError(t *testing.T) {
+	cases := []error{
+		errors.New("temporary failure"),
+		errors.New("accept tcp 10.10.1.254:80: too many open files"),
+		nil,
+	}
+	for _, c := range cases {
+		assert.False(t, IsClosedListenerErr(c),
+			"unexpected match on %v — must not be treated as closed", c)
+	}
+}
+
+// TestAcceptBackoff_ProgressionAndCap asserts the doubling schedule:
+// 5ms, 10ms, 20ms, 40ms, ... capped at 1s. The test runs against a
+// real timer but uses tight bounds so a slow CI machine still passes.
+func TestAcceptBackoff_ProgressionAndCap(t *testing.T) {
+	var b AcceptBackoff
+	expected := []time.Duration{
+		5 * time.Millisecond,
+		10 * time.Millisecond,
+		20 * time.Millisecond,
+		40 * time.Millisecond,
+	}
+	for i, want := range expected {
+		start := time.Now()
+		ok := b.Backoff(context.Background())
+		elapsed := time.Since(start)
+		require.True(t, ok, "Backoff %d must complete; ctx is alive", i)
+		assert.GreaterOrEqual(t, elapsed, want,
+			"backoff %d (%v) must wait at least the configured delay", i, want)
+		assert.Less(t, elapsed, want*4,
+			"backoff %d (%v) must not overshoot by more than 4x — caps misbehaving", i, want)
+	}
+
+	// Burn enough rounds to reach the cap, then assert subsequent
+	// rounds stay at exactly maxAcceptDelay (1s) — the timer should
+	// never exceed it.
+	for range 6 {
+		b.Backoff(context.Background())
+	}
+	assert.Equal(t, maxAcceptDelay, b.delay,
+		"after enough doublings the delay must clamp to maxAcceptDelay")
+}
+
+// TestAcceptBackoff_Reset confirms that a successful Accept resets the
+// schedule — a busy-then-quiet listener mustn't stay on a 1s timer
+// after recovery.
+func TestAcceptBackoff_Reset(t *testing.T) {
+	var b AcceptBackoff
+	for range 5 {
+		b.Backoff(context.Background())
+	}
+	require.NotEqual(t, time.Duration(0), b.delay, "precondition: delay must have accumulated")
+
+	b.Reset()
+	assert.Equal(t, time.Duration(0), b.delay, "Reset must zero the delay")
+
+	start := time.Now()
+	ok := b.Backoff(context.Background())
+	elapsed := time.Since(start)
+	require.True(t, ok, "Backoff after Reset must complete")
+	assert.GreaterOrEqual(t, elapsed, minAcceptDelay,
+		"after Reset the next backoff must restart at minAcceptDelay")
+	assert.Less(t, elapsed, 50*time.Millisecond,
+		"after Reset the next backoff must NOT carry over the prior delay")
+}
+
+// TestAcceptBackoff_CancelDuringWait proves the loop exits promptly
+// when ctx fires mid-wait. Without this, a tear-down would still take
+// up to 1 second per orphaned listener.
+func TestAcceptBackoff_CancelDuringWait(t *testing.T) {
+	var b AcceptBackoff
+	// Drive the backoff up so the next call will wait ~1s — long
+	// enough that we can detect early cancellation.
+	for range 10 {
+		b.Backoff(context.Background())
+	}
+	require.Equal(t, maxAcceptDelay, b.delay)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(20 * time.Millisecond)
+		cancel()
+	}()
+
+	start := time.Now()
+	ok := b.Backoff(ctx)
+	elapsed := time.Since(start)
+	assert.False(t, ok, "Backoff must return false when ctx is cancelled mid-wait")
+	assert.Less(t, elapsed, 200*time.Millisecond,
+		"cancellation must short-circuit the timer; took %v", elapsed)
+}
+
+// TestAcceptBackoff_CancelBeforeCall — when ctx is already done the
+// loop exits without sleeping at all.
+func TestAcceptBackoff_CancelBeforeCall(t *testing.T) {
+	var b AcceptBackoff
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	start := time.Now()
+	ok := b.Backoff(ctx)
+	elapsed := time.Since(start)
+	assert.False(t, ok, "Backoff must return false when ctx is already cancelled")
+	assert.Less(t, elapsed, 50*time.Millisecond,
+		"already-cancelled ctx must return immediately; took %v", elapsed)
+}

proxy/internal/tcp/router.go

@@ -297,18 +297,29 @@ func (r *Router) Serve(ctx context.Context, ln net.Listener) error {
 		}
 	}()
 
+	var backoff AcceptBackoff
 	for {
 		conn, err := ln.Accept()
 		if err != nil {
-			if ctx.Err() != nil || errors.Is(err, net.ErrClosed) {
+			if ctx.Err() != nil || IsClosedListenerErr(err) {
+				if ok := r.Drain(DefaultDrainTimeout); !ok {
+					r.logger.Warn("timed out waiting for connections to drain")
+				}
+				return nil
+			}
+			r.logger.Debugf("SNI router accept: %v; backing off", err)
+			if !backoff.Backoff(ctx) {
+				// Cancelled during backoff: still drain in-flight
+				// connections/relays before returning, matching the
+				// shutdown path above.
 				if ok := r.Drain(DefaultDrainTimeout); !ok {
 					r.logger.Warn("timed out waiting for connections to drain")
 				}
 				return nil
 			}
-			r.logger.Debugf("SNI router accept: %v", err)
 			continue
 		}
+		backoff.Reset()
 		r.logger.Debugf("SNI router accepted conn from %s on %s", conn.RemoteAddr(), conn.LocalAddr())
 		r.activeConns.Add(1)
 		go func() {

proxy/internal/tcp/router_test.go

@@ -1836,3 +1836,132 @@ func TestRouter_TLS_StaysOnTLSChannel_WhenPlainEnabled(t *testing.T) {
 		t.Fatal("TLS conn never reached the TLS channel")
 	}
 }
+
+// scriptedAcceptListener is a net.Listener whose Accept() returns
+// pre-scripted errors. Used by the accept-loop exit tests to simulate
+// the failure mode that triggers the tight-loop bug: a netstack
+// listener whose endpoint has been destroyed and now returns the gVisor
+// "endpoint is in invalid state" error from every Accept call.
+type scriptedAcceptListener struct {
+	errs   chan error
+	closed chan struct{}
+}
+
+func newScriptedAcceptListener(errs ...error) *scriptedAcceptListener {
+	s := &scriptedAcceptListener{
+		errs:   make(chan error, len(errs)+1),
+		closed: make(chan struct{}),
+	}
+	for _, e := range errs {
+		s.errs <- e
+	}
+	return s
+}
+
+func (s *scriptedAcceptListener) Accept() (net.Conn, error) {
+	select {
+	case <-s.closed:
+		return nil, net.ErrClosed
+	case err := <-s.errs:
+		return nil, err
+	}
+}
+
+func (s *scriptedAcceptListener) Close() error {
+	select {
+	case <-s.closed:
+	default:
+		close(s.closed)
+	}
+	return nil
+}
+
+func (s *scriptedAcceptListener) Addr() net.Addr {
+	return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}
+}
+
+// TestRouter_Serve_ExitsOnGVisorInvalidEndpoint is the regression guard
+// for the tight-loop bug: when the underlying netstack endpoint is
+// destroyed, Accept returns "endpoint is in invalid state" forever. The
+// loop must recognise that signal and return, otherwise it pegs a CPU
+// core and floods logs.
+func TestRouter_Serve_ExitsOnGVisorInvalidEndpoint(t *testing.T) {
+	logger := log.StandardLogger()
+	addr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 443}
+	router := NewRouter(logger, nil, addr)
+
+	gvisorErr := &net.OpError{
+		Op:   "accept",
+		Net:  "tcp",
+		Addr: addr,
+		Err:  errSentinel("endpoint is in invalid state"),
+	}
+	ln := newScriptedAcceptListener(gvisorErr)
+	defer ln.Close()
+
+	done := make(chan error, 1)
+	go func() {
+		done <- router.Serve(context.Background(), ln)
+	}()
+
+	select {
+	case err := <-done:
+		assert.NoError(t, err, "Serve must return cleanly on a recognised closed-listener error")
+	case <-time.After(2 * time.Second):
+		t.Fatal("Serve did not exit on gVisor 'endpoint is in invalid state' — accept loop is spinning")
+	}
+}
+
+// TestRouter_Serve_BacksOffOnTransientError verifies the defence-in-
+// depth path: when Accept returns an unknown transient error, the loop
+// MUST not spin. It backs off, then exits cleanly once ctx is cancelled.
+// "Bounded call count" stands in for "no CPU spin" — without backoff
+// the goroutine would issue thousands of Accept calls in this window.
+func TestRouter_Serve_BacksOffOnTransientError(t *testing.T) {
+	logger := log.StandardLogger()
+	addr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 443}
+	router := NewRouter(logger, nil, addr)
+
+	const transientErrCount = 5
+	errs := make([]error, transientErrCount)
+	for i := range errs {
+		errs[i] = errSentinel("transient: too many open files")
+	}
+	ln := newScriptedAcceptListener(errs...)
+	defer ln.Close()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	start := time.Now()
+	go func() {
+		done <- router.Serve(ctx, ln)
+	}()
+
+	// Cancel after enough time for the backoff to climb (5ms + 10ms +
+	// 20ms + 40ms = 75ms minimum), but short enough that a spinning
+	// loop would have made thousands of calls by now.
+	time.AfterFunc(150*time.Millisecond, cancel)
+
+	select {
+	case err := <-done:
+		assert.NoError(t, err, "Serve must return cleanly on ctx cancellation")
+	case <-time.After(2 * time.Second):
+		t.Fatal("Serve did not exit on ctx cancellation — backoff or exit path broken")
+	}
+
+	// Without backoff the loop would burn through all 5 scripted errors
+	// in microseconds and then block on the channel. With backoff the
+	// total wall time should be at least 5ms (the first backoff).
+	elapsed := time.Since(start)
+	assert.GreaterOrEqual(t, elapsed, minAcceptDelay,
+		"loop ran without backing off — would burn CPU in production")
+}
+
+// errSentinel mirrors gVisor's tcpip error message exactly. We can't
+// import the gVisor package without dragging in the whole netstack, so
+// the test uses the canonical string the production error formatter
+// emits — same shape IsClosedListenerErr matches in production.
+type errSentinel string
+
+func (e errSentinel) Error() string { return string(e) }
+