mirror of
https://github.com/netbirdio/netbird.git
synced 2026-09-19 21:29:09 +02:00
[client, proxy] IPv6 in-place apply + accept-loop hardening on netstack listeners
Two related fixes for the embedded netbird client and the per-account
inbound listeners that ride on its gVisor netstack.
client/internal/engine.go — replace hasIPv6Changed with reconcileIPv6:
- First v6 assignment (current had no v6, conf carries one) is applied
in place via WGIface.UpdateAddr instead of returning ErrResetConnection.
Pre-fix, every embedded client whose account had IPv6 enabled would
reset on its first NetworkMap sync — boot config has no v6, the sync
introduces one, the engine tore itself down to "apply" it. That
teardown destroys the gVisor netstack and orphans every listener
bound on it, which is what made the proxy's per-account :80/:443
silently stop accepting traffic.
- v6 removed clears in place.
- v6 swapped to a different non-empty value still resets (gVisor
netstack can't safely swap its address at runtime).
- Mutates e.config.WgAddr to match the applied state so subsequent
PeerConfig comparisons are stable.
proxy/internal/tcp/accept.go (new) + proxy/inbound.go +
proxy/internal/tcp/router.go — harden the two Accept() loops on
netstack-backed listeners:
- IsClosedListenerErr recognises net.ErrClosed AND gVisor's
"endpoint is in invalid state" — the latter survives gonet's
*net.OpError wrapping in a way errors.Is(.., net.ErrClosed) does
not. Without this the loop spins CPU-hot after the underlying
netstack is destroyed (peer rekey, embedded-client reset, account
churn), emitting one log line per iteration.
- AcceptBackoff implements the exponential backoff that
net/http.Server.Serve uses on transient Accept errors: 5ms doubling
up to 1s. Defence-in-depth so an unknown sticky error cannot burn
a CPU core even if IsClosedListenerErr misses its signature.
proxy/internal/roundtrip/netbird.go — emit a single structured INFO
line summarising every embed.Options flag (account_id, service_id,
public_key, management_url, wg_port, block_inbound, block_lan_access,
disable_ipv6, no_userspace, presence of credentials) when each
per-account embedded client is created. Secrets reduced to a "present"
boolean — never logged verbatim. Diagnostic-only; no behavior change,
but it makes the "why is this embedded peer misbehaving" loop a single
log read instead of a code dive.
Tests (real listeners, scripted errors, no mocks of production code):
- engine_reconcileipv6_test.go: 8 cases for every transition (first
assignment, no change, removed, prefix-length changed, value
changed, invalid bytes, UpdateAddr error) plus a updateConfig
integration check that the fix actually fires on a v6-added
PeerConfig.
- accept_test.go: IsClosedListenerErr matrix + AcceptBackoff
progression / cap / reset / cancel-during-wait / cancel-before-call.
- router_test.go, inbound_test.go: scriptedAcceptListener +
TestRouter_Serve_ExitsOnGVisorInvalidEndpoint and
TestFeedRouterFromListener_ExitsOnGVisorInvalidEndpoint —
regression guards that fail in 2 s if the loop ever spins.
This commit is contained in:
+7
-2
@@ -466,15 +466,20 @@ func feedRouterFromListener(ctx context.Context, ln net.Listener, router *nbtcp.
|
||||
_ = ln.Close()
|
||||
}()
|
||||
|
||||
var backoff nbtcp.AcceptBackoff
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || errors.Is(err, net.ErrClosed) {
|
||||
if ctx.Err() != nil || nbtcp.IsClosedListenerErr(err) {
|
||||
return
|
||||
}
|
||||
logger.WithField("account_id", accountID).Debugf("plain inbound accept: %v; backing off", err)
|
||||
if !backoff.Backoff(ctx) {
|
||||
return
|
||||
}
|
||||
logger.WithField("account_id", accountID).Debugf("plain inbound accept: %v", err)
|
||||
continue
|
||||
}
|
||||
backoff.Reset()
|
||||
router.HandleConn(ctx, conn)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -533,3 +533,125 @@ MHcCAQEEIIrYSSNQFaA2Hwf1duRSxKtLYX5CB04fSeQ6tF1aY/PuoAoGCCqGSM49
|
||||
AwEHoUQDQgAEPR3tU2Fta9ktY+6P9G0cWO+0kETA6SFs38GecTyudlHz6xvCdz8q
|
||||
EKTcWGekdmdDPsHloRNtsiCa697B2O9IFA==
|
||||
-----END EC PRIVATE KEY-----`)
|
||||
|
||||
// scriptedAcceptListener returns pre-scripted errors from Accept(). Used
|
||||
// to drive the feedRouterFromListener tests without binding a real
|
||||
// socket — the production code path is a netstack-backed listener that
|
||||
// returns gVisor's "endpoint is in invalid state" forever after its
|
||||
// endpoint is destroyed.
|
||||
type scriptedAcceptListener struct {
|
||||
errs chan error
|
||||
closed chan struct{}
|
||||
}
|
||||
|
||||
func newScriptedAcceptListener(errs ...error) *scriptedAcceptListener {
|
||||
s := &scriptedAcceptListener{
|
||||
errs: make(chan error, len(errs)+1),
|
||||
closed: make(chan struct{}),
|
||||
}
|
||||
for _, e := range errs {
|
||||
s.errs <- e
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *scriptedAcceptListener) Accept() (net.Conn, error) {
|
||||
select {
|
||||
case <-s.closed:
|
||||
return nil, net.ErrClosed
|
||||
case err := <-s.errs:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
func (s *scriptedAcceptListener) Close() error {
|
||||
select {
|
||||
case <-s.closed:
|
||||
default:
|
||||
close(s.closed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scriptedAcceptListener) Addr() net.Addr {
|
||||
return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}
|
||||
}
|
||||
|
||||
// errSentinel carries a literal error message so tests can synthesise
|
||||
// the exact gVisor text without importing the netstack package.
|
||||
type errSentinel string
|
||||
|
||||
func (e errSentinel) Error() string { return string(e) }
|
||||
|
||||
// TestFeedRouterFromListener_ExitsOnGVisorInvalidEndpoint is the
|
||||
// regression guard for the inbound side of the tight-loop bug. The
|
||||
// per-account plain-HTTP feeder must recognise gVisor's "endpoint is in
|
||||
// invalid state" and exit, otherwise it pegs a CPU core and floods the
|
||||
// account-scoped log with the same accept error every iteration.
|
||||
func TestFeedRouterFromListener_ExitsOnGVisorInvalidEndpoint(t *testing.T) {
|
||||
logger := log.StandardLogger()
|
||||
addr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 80}
|
||||
router := nbtcp.NewRouter(logger, nil, addr)
|
||||
|
||||
gvisorErr := &net.OpError{
|
||||
Op: "accept",
|
||||
Net: "tcp",
|
||||
Addr: addr,
|
||||
Err: errSentinel("endpoint is in invalid state"),
|
||||
}
|
||||
ln := newScriptedAcceptListener(gvisorErr)
|
||||
defer ln.Close()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
feedRouterFromListener(context.Background(), ln, router, logger, "acct-1")
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// Expected: loop recognised the gVisor error and returned.
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("feedRouterFromListener did not exit on gVisor 'endpoint is in invalid state' — accept loop is spinning")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFeedRouterFromListener_BacksOffOnTransientError asserts the
|
||||
// defence-in-depth path: an unknown sticky Accept error must NOT cause
|
||||
// CPU spin. The loop backs off and exits cleanly when ctx is cancelled.
|
||||
func TestFeedRouterFromListener_BacksOffOnTransientError(t *testing.T) {
|
||||
logger := log.StandardLogger()
|
||||
addr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 80}
|
||||
router := nbtcp.NewRouter(logger, nil, addr)
|
||||
|
||||
const transientCount = 5
|
||||
errs := make([]error, transientCount)
|
||||
for i := range errs {
|
||||
errs[i] = errSentinel("transient: temporary network error")
|
||||
}
|
||||
ln := newScriptedAcceptListener(errs...)
|
||||
defer ln.Close()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
start := time.Now()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
feedRouterFromListener(ctx, ln, router, logger, "acct-1")
|
||||
}()
|
||||
time.AfterFunc(150*time.Millisecond, cancel)
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// Expected.
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("feedRouterFromListener did not exit on ctx cancellation — backoff or exit path broken")
|
||||
}
|
||||
|
||||
// Without backoff the 5 scripted errors would burn in microseconds.
|
||||
// With backoff the first delay alone is 5ms, so the loop must take
|
||||
// at least that long even though ctx fires at 150ms.
|
||||
elapsed := time.Since(start)
|
||||
assert.GreaterOrEqual(t, elapsed, 5*time.Millisecond,
|
||||
"loop ran without backing off — would burn CPU in production")
|
||||
}
|
||||
|
||||
@@ -356,7 +356,7 @@ func (n *NetBird) createClientEntry(ctx context.Context, accountID types.Account
|
||||
// Create embedded NetBird client with the generated private key.
|
||||
// The peer has already been created via CreateProxyPeer RPC with the public key.
|
||||
wgPort := int(n.clientCfg.WGPort)
|
||||
client, err := embed.New(embed.Options{
|
||||
embedOpts := embed.Options{
|
||||
DeviceName: deviceNamePrefix + n.proxyID,
|
||||
ManagementURL: n.clientCfg.MgmtAddr,
|
||||
PrivateKey: privateKey.String(),
|
||||
@@ -371,7 +371,9 @@ func (n *NetBird) createClientEntry(ctx context.Context, accountID types.Account
|
||||
WireguardPort: &wgPort,
|
||||
PreSharedKey: n.clientCfg.PreSharedKey,
|
||||
Performance: n.clientCfg.Performance,
|
||||
})
|
||||
}
|
||||
logEmbedOptions(n.logger, accountID, serviceID, publicKey.String(), embedOpts)
|
||||
client, err := embed.New(embedOpts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create netbird client: %w", err)
|
||||
}
|
||||
@@ -847,3 +849,53 @@ func DirectUpstreamFromContext(ctx context.Context) bool {
|
||||
v, _ := ctx.Value(directUpstreamContextKey{}).(bool)
|
||||
return v
|
||||
}
|
||||
|
||||
// logEmbedOptions emits a single structured INFO line summarising every
|
||||
// operationally meaningful flag handed to embed.New for this per-account
|
||||
// client. Secrets (PrivateKey, PreSharedKey) are reduced to a "present"
|
||||
// boolean — never logged verbatim. Use this when an embedded peer
|
||||
// silently misbehaves: most failure modes (inbound drops, wrong
|
||||
// management URL, v6 unexpectedly on, userspace flipped, port clash)
|
||||
// are obvious from these flags before any traffic flows.
|
||||
func logEmbedOptions(logger *log.Logger, accountID types.AccountID, serviceID types.ServiceID, publicKey string, opts embed.Options) {
|
||||
wgPort := 0
|
||||
if opts.WireguardPort != nil {
|
||||
wgPort = *opts.WireguardPort
|
||||
}
|
||||
mtu := uint16(0)
|
||||
if opts.MTU != nil {
|
||||
mtu = *opts.MTU
|
||||
}
|
||||
perfBuffers := uint32(0)
|
||||
if opts.Performance.PreallocatedBuffersPerPool != nil {
|
||||
perfBuffers = *opts.Performance.PreallocatedBuffersPerPool
|
||||
}
|
||||
perfBatch := uint32(0)
|
||||
if opts.Performance.MaxBatchSize != nil {
|
||||
perfBatch = *opts.Performance.MaxBatchSize
|
||||
}
|
||||
logger.WithFields(log.Fields{
|
||||
"account_id": accountID,
|
||||
"service_id": serviceID,
|
||||
"public_key": publicKey,
|
||||
"device_name": opts.DeviceName,
|
||||
"management_url": opts.ManagementURL,
|
||||
"log_level": opts.LogLevel,
|
||||
"wg_port": wgPort,
|
||||
"mtu": mtu,
|
||||
"block_inbound": opts.BlockInbound,
|
||||
"block_lan_access": opts.BlockLANAccess,
|
||||
"disable_ipv6": opts.DisableIPv6,
|
||||
"disable_client_routes": opts.DisableClientRoutes,
|
||||
"no_userspace": opts.NoUserspace,
|
||||
"config_path_set": opts.ConfigPath != "",
|
||||
"state_path_set": opts.StatePath != "",
|
||||
"private_key_present": opts.PrivateKey != "",
|
||||
"presharedkey_present": opts.PreSharedKey != "",
|
||||
"setup_key_present": opts.SetupKey != "",
|
||||
"jwt_token_present": opts.JWTToken != "",
|
||||
"dns_labels": opts.DNSLabels,
|
||||
"perf_buffers_per_pool": perfBuffers,
|
||||
"perf_max_batch_size": perfBatch,
|
||||
}).Info("starting embedded netbird client for account")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package tcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// gvisorInvalidEndpointMsg is the canonical text gVisor netstack returns
|
||||
// when Accept() is called on a listener whose underlying endpoint has
|
||||
// been destroyed (peer rekey, embedded-client reset, account churn).
|
||||
// There is no exported sentinel from gvisor.dev/gvisor/pkg/tcpip that
|
||||
// survives gonet's *net.OpError wrapping in a way errors.Is can match,
|
||||
// so we fall back to a string check. Stable across the gVisor versions
|
||||
// netbird pins.
|
||||
const gvisorInvalidEndpointMsg = "endpoint is in invalid state"
|
||||
|
||||
// IsClosedListenerErr reports whether err signals that an accept loop
|
||||
// should exit because the underlying listener can no longer serve
|
||||
// connections. It recognises:
|
||||
//
|
||||
// - net.ErrClosed for stdlib listeners (Listener.Close was called).
|
||||
// - gVisor's "endpoint is in invalid state" for netstack-backed
|
||||
// listeners whose endpoint was destroyed out from under them
|
||||
// (typically when a per-account WireGuard netstack is reset without
|
||||
// also tearing the listener entry down).
|
||||
//
|
||||
// Without the gVisor branch an accept loop on a netstack listener spins
|
||||
// CPU-hot forever after the endpoint dies, because Accept never blocks
|
||||
// again and the error neither matches net.ErrClosed nor cancels ctx.
|
||||
func IsClosedListenerErr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, net.ErrClosed) {
|
||||
return true
|
||||
}
|
||||
return strings.Contains(err.Error(), gvisorInvalidEndpointMsg)
|
||||
}
|
||||
|
||||
// AcceptBackoff implements the exponential backoff used by
|
||||
// net/http.Server.Serve for transient Accept errors. Without it a loop
|
||||
// hitting a sticky unknown error burns a full CPU core. The zero value
|
||||
// is ready to use; call Reset after a successful Accept.
|
||||
type AcceptBackoff struct {
|
||||
delay time.Duration
|
||||
}
|
||||
|
||||
// minAcceptDelay / maxAcceptDelay mirror the stdlib defaults
|
||||
// (net/http.Server.Serve) and keep us well below 1 log line per second
|
||||
// per orphaned listener.
|
||||
const (
|
||||
minAcceptDelay = 5 * time.Millisecond
|
||||
maxAcceptDelay = time.Second
|
||||
)
|
||||
|
||||
// Backoff waits the next exponential delay (5ms doubling up to 1s) and
|
||||
// returns true when the wait completed. Returns false if ctx fired
|
||||
// during the wait — callers should treat that as "exit the loop".
|
||||
func (b *AcceptBackoff) Backoff(ctx context.Context) bool {
|
||||
b.advance()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-time.After(b.delay):
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Reset clears the accumulated delay so the next failure starts at the
|
||||
// minimum delay again. Call after a successful Accept.
|
||||
func (b *AcceptBackoff) Reset() { b.delay = 0 }
|
||||
|
||||
func (b *AcceptBackoff) advance() {
|
||||
if b.delay == 0 {
|
||||
b.delay = minAcceptDelay
|
||||
} else {
|
||||
b.delay *= 2
|
||||
}
|
||||
if b.delay > maxAcceptDelay {
|
||||
b.delay = maxAcceptDelay
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package tcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestIsClosedListenerErr_NetErrClosed verifies the stdlib path: a
|
||||
// closed *net.Listener returns net.ErrClosed wrapped in *net.OpError,
|
||||
// and IsClosedListenerErr must unwrap it.
|
||||
func TestIsClosedListenerErr_NetErrClosed(t *testing.T) {
|
||||
wrapped := &net.OpError{Op: "accept", Net: "tcp", Err: net.ErrClosed}
|
||||
assert.True(t, IsClosedListenerErr(wrapped),
|
||||
"net.OpError wrapping net.ErrClosed must be recognised as closed")
|
||||
}
|
||||
|
||||
// TestIsClosedListenerErr_GVisorInvalidEndpoint is the load-bearing
|
||||
// regression guard. A gVisor netstack listener whose endpoint has been
|
||||
// destroyed returns this exact text. Without recognising it the accept
|
||||
// loop spins forever and burns a CPU core.
|
||||
func TestIsClosedListenerErr_GVisorInvalidEndpoint(t *testing.T) {
|
||||
err := fmt.Errorf("accept tcp 10.10.1.254:80: endpoint is in invalid state")
|
||||
assert.True(t, IsClosedListenerErr(err),
|
||||
"gVisor 'endpoint is in invalid state' must be recognised as closed")
|
||||
}
|
||||
|
||||
// TestIsClosedListenerErr_OtherError confirms we don't over-match —
|
||||
// transient errors must keep returning false so the backoff path runs.
|
||||
func TestIsClosedListenerErr_OtherError(t *testing.T) {
|
||||
cases := []error{
|
||||
errors.New("temporary failure"),
|
||||
errors.New("accept tcp 10.10.1.254:80: too many open files"),
|
||||
nil,
|
||||
}
|
||||
for _, c := range cases {
|
||||
assert.False(t, IsClosedListenerErr(c),
|
||||
"unexpected match on %v — must not be treated as closed", c)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAcceptBackoff_ProgressionAndCap asserts the doubling schedule:
|
||||
// 5ms, 10ms, 20ms, 40ms, ... capped at 1s. The test runs against a
|
||||
// real timer but uses tight bounds so a slow CI machine still passes.
|
||||
func TestAcceptBackoff_ProgressionAndCap(t *testing.T) {
|
||||
var b AcceptBackoff
|
||||
expected := []time.Duration{
|
||||
5 * time.Millisecond,
|
||||
10 * time.Millisecond,
|
||||
20 * time.Millisecond,
|
||||
40 * time.Millisecond,
|
||||
}
|
||||
for i, want := range expected {
|
||||
start := time.Now()
|
||||
ok := b.Backoff(context.Background())
|
||||
elapsed := time.Since(start)
|
||||
require.True(t, ok, "Backoff %d must complete; ctx is alive", i)
|
||||
assert.GreaterOrEqual(t, elapsed, want,
|
||||
"backoff %d (%v) must wait at least the configured delay", i, want)
|
||||
assert.Less(t, elapsed, want*4,
|
||||
"backoff %d (%v) must not overshoot by more than 4x — caps misbehaving", i, want)
|
||||
}
|
||||
|
||||
// Burn enough rounds to reach the cap, then assert subsequent
|
||||
// rounds stay at exactly maxAcceptDelay (1s) — the timer should
|
||||
// never exceed it.
|
||||
for range 6 {
|
||||
b.Backoff(context.Background())
|
||||
}
|
||||
assert.Equal(t, maxAcceptDelay, b.delay,
|
||||
"after enough doublings the delay must clamp to maxAcceptDelay")
|
||||
}
|
||||
|
||||
// TestAcceptBackoff_Reset confirms that a successful Accept resets the
|
||||
// schedule — a busy-then-quiet listener mustn't stay on a 1s timer
|
||||
// after recovery.
|
||||
func TestAcceptBackoff_Reset(t *testing.T) {
|
||||
var b AcceptBackoff
|
||||
for range 5 {
|
||||
b.Backoff(context.Background())
|
||||
}
|
||||
require.NotEqual(t, time.Duration(0), b.delay, "precondition: delay must have accumulated")
|
||||
|
||||
b.Reset()
|
||||
assert.Equal(t, time.Duration(0), b.delay, "Reset must zero the delay")
|
||||
|
||||
start := time.Now()
|
||||
ok := b.Backoff(context.Background())
|
||||
elapsed := time.Since(start)
|
||||
require.True(t, ok, "Backoff after Reset must complete")
|
||||
assert.GreaterOrEqual(t, elapsed, minAcceptDelay,
|
||||
"after Reset the next backoff must restart at minAcceptDelay")
|
||||
assert.Less(t, elapsed, 50*time.Millisecond,
|
||||
"after Reset the next backoff must NOT carry over the prior delay")
|
||||
}
|
||||
|
||||
// TestAcceptBackoff_CancelDuringWait proves the loop exits promptly
|
||||
// when ctx fires mid-wait. Without this, a tear-down would still take
|
||||
// up to 1 second per orphaned listener.
|
||||
func TestAcceptBackoff_CancelDuringWait(t *testing.T) {
|
||||
var b AcceptBackoff
|
||||
// Drive the backoff up so the next call will wait ~1s — long
|
||||
// enough that we can detect early cancellation.
|
||||
for range 10 {
|
||||
b.Backoff(context.Background())
|
||||
}
|
||||
require.Equal(t, maxAcceptDelay, b.delay)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
start := time.Now()
|
||||
ok := b.Backoff(ctx)
|
||||
elapsed := time.Since(start)
|
||||
assert.False(t, ok, "Backoff must return false when ctx is cancelled mid-wait")
|
||||
assert.Less(t, elapsed, 200*time.Millisecond,
|
||||
"cancellation must short-circuit the timer; took %v", elapsed)
|
||||
}
|
||||
|
||||
// TestAcceptBackoff_CancelBeforeCall — when ctx is already done the
|
||||
// loop exits without sleeping at all.
|
||||
func TestAcceptBackoff_CancelBeforeCall(t *testing.T) {
|
||||
var b AcceptBackoff
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
start := time.Now()
|
||||
ok := b.Backoff(ctx)
|
||||
elapsed := time.Since(start)
|
||||
assert.False(t, ok, "Backoff must return false when ctx is already cancelled")
|
||||
assert.Less(t, elapsed, 50*time.Millisecond,
|
||||
"already-cancelled ctx must return immediately; took %v", elapsed)
|
||||
}
|
||||
@@ -297,18 +297,23 @@ func (r *Router) Serve(ctx context.Context, ln net.Listener) error {
|
||||
}
|
||||
}()
|
||||
|
||||
var backoff AcceptBackoff
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || errors.Is(err, net.ErrClosed) {
|
||||
if ctx.Err() != nil || IsClosedListenerErr(err) {
|
||||
if ok := r.Drain(DefaultDrainTimeout); !ok {
|
||||
r.logger.Warn("timed out waiting for connections to drain")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
r.logger.Debugf("SNI router accept: %v", err)
|
||||
r.logger.Debugf("SNI router accept: %v; backing off", err)
|
||||
if !backoff.Backoff(ctx) {
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
backoff.Reset()
|
||||
r.logger.Debugf("SNI router accepted conn from %s on %s", conn.RemoteAddr(), conn.LocalAddr())
|
||||
r.activeConns.Add(1)
|
||||
go func() {
|
||||
|
||||
@@ -1836,3 +1836,132 @@ func TestRouter_TLS_StaysOnTLSChannel_WhenPlainEnabled(t *testing.T) {
|
||||
t.Fatal("TLS conn never reached the TLS channel")
|
||||
}
|
||||
}
|
||||
|
||||
// scriptedAcceptListener is a net.Listener whose Accept() returns
|
||||
// pre-scripted errors. Used by the accept-loop exit tests to simulate
|
||||
// the failure mode that triggers the tight-loop bug: a netstack
|
||||
// listener whose endpoint has been destroyed and now returns the gVisor
|
||||
// "endpoint is in invalid state" error from every Accept call.
|
||||
type scriptedAcceptListener struct {
|
||||
errs chan error
|
||||
closed chan struct{}
|
||||
}
|
||||
|
||||
func newScriptedAcceptListener(errs ...error) *scriptedAcceptListener {
|
||||
s := &scriptedAcceptListener{
|
||||
errs: make(chan error, len(errs)+1),
|
||||
closed: make(chan struct{}),
|
||||
}
|
||||
for _, e := range errs {
|
||||
s.errs <- e
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *scriptedAcceptListener) Accept() (net.Conn, error) {
|
||||
select {
|
||||
case <-s.closed:
|
||||
return nil, net.ErrClosed
|
||||
case err := <-s.errs:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
func (s *scriptedAcceptListener) Close() error {
|
||||
select {
|
||||
case <-s.closed:
|
||||
default:
|
||||
close(s.closed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scriptedAcceptListener) Addr() net.Addr {
|
||||
return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}
|
||||
}
|
||||
|
||||
// TestRouter_Serve_ExitsOnGVisorInvalidEndpoint is the regression guard
|
||||
// for the tight-loop bug: when the underlying netstack endpoint is
|
||||
// destroyed, Accept returns "endpoint is in invalid state" forever. The
|
||||
// loop must recognise that signal and return, otherwise it pegs a CPU
|
||||
// core and floods logs.
|
||||
func TestRouter_Serve_ExitsOnGVisorInvalidEndpoint(t *testing.T) {
|
||||
logger := log.StandardLogger()
|
||||
addr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 443}
|
||||
router := NewRouter(logger, nil, addr)
|
||||
|
||||
gvisorErr := &net.OpError{
|
||||
Op: "accept",
|
||||
Net: "tcp",
|
||||
Addr: addr,
|
||||
Err: errSentinel("endpoint is in invalid state"),
|
||||
}
|
||||
ln := newScriptedAcceptListener(gvisorErr)
|
||||
defer ln.Close()
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- router.Serve(context.Background(), ln)
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
assert.NoError(t, err, "Serve must return cleanly on a recognised closed-listener error")
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Serve did not exit on gVisor 'endpoint is in invalid state' — accept loop is spinning")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRouter_Serve_BacksOffOnTransientError verifies the defence-in-
|
||||
// depth path: when Accept returns an unknown transient error, the loop
|
||||
// MUST not spin. It backs off, then exits cleanly once ctx is cancelled.
|
||||
// "Bounded call count" stands in for "no CPU spin" — without backoff
|
||||
// the goroutine would issue thousands of Accept calls in this window.
|
||||
func TestRouter_Serve_BacksOffOnTransientError(t *testing.T) {
|
||||
logger := log.StandardLogger()
|
||||
addr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 443}
|
||||
router := NewRouter(logger, nil, addr)
|
||||
|
||||
const transientErrCount = 5
|
||||
errs := make([]error, transientErrCount)
|
||||
for i := range errs {
|
||||
errs[i] = errSentinel("transient: too many open files")
|
||||
}
|
||||
ln := newScriptedAcceptListener(errs...)
|
||||
defer ln.Close()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
start := time.Now()
|
||||
go func() {
|
||||
done <- router.Serve(ctx, ln)
|
||||
}()
|
||||
|
||||
// Cancel after enough time for the backoff to climb (5ms + 10ms +
|
||||
// 20ms + 40ms = 75ms minimum), but short enough that a spinning
|
||||
// loop would have made thousands of calls by now.
|
||||
time.AfterFunc(150*time.Millisecond, cancel)
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
assert.NoError(t, err, "Serve must return cleanly on ctx cancellation")
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Serve did not exit on ctx cancellation — backoff or exit path broken")
|
||||
}
|
||||
|
||||
// Without backoff the loop would burn through all 5 scripted errors
|
||||
// in microseconds and then block on the channel. With backoff the
|
||||
// total wall time should be at least 5ms (the first backoff).
|
||||
elapsed := time.Since(start)
|
||||
assert.GreaterOrEqual(t, elapsed, minAcceptDelay,
|
||||
"loop ran without backing off — would burn CPU in production")
|
||||
}
|
||||
|
||||
// errSentinel mirrors gVisor's tcpip error message exactly. We can't
|
||||
// import the gVisor package without dragging in the whole netstack, so
|
||||
// the test uses the canonical string the production error formatter
|
||||
// emits — same shape IsClosedListenerErr matches in production.
|
||||
type errSentinel string
|
||||
|
||||
func (e errSentinel) Error() string { return string(e) }
|
||||
|
||||
|
||||
Reference in New Issue
Block a user