[client] Redirect DNS port 53 with UDP and TCP DNAT instead of the eBPF forwarder (#7439)

2026-10-27 06:49:07 +01:00 · 2026-09-09 11:32:11 +02:00
parent d2e62e358a
commit d101f6cc46
14 changed files with 363 additions and 254 deletions
@@ -6,6 +6,7 @@ import (
 	"net"
 	"net/netip"
 	"runtime"
+	"slices"
 	"strconv"
 	"sync"
 	"time"
@@ -17,17 +18,20 @@ import (
 	nberrors "github.com/netbirdio/netbird/client/errors"

 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
-	"github.com/netbirdio/netbird/client/internal/ebpf"
-	ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager"
 )

 const (
 	customPort = 5053
+	// randomPortAttempts bounds the search for a port free on both protocols.
+	randomPortAttempts = 5
 )

 var (
 	defaultIP = netip.MustParseAddr("127.0.0.1")
 	customIP  = netip.MustParseAddr("127.0.0.153")
+
+	// dnatProtocols are the protocols the port 53 redirect covers.
+	dnatProtocols = []firewall.Protocol{firewall.ProtocolUDP, firewall.ProtocolTCP}
 )

 type serviceViaListener struct {
@@ -40,9 +44,20 @@ type serviceViaListener struct {
 	listenPort        uint16
 	listenerIsRunning bool
 	listenerFlagLock  sync.Mutex
-	ebpfService       ebpfMgr.Manager
 	firewall          Firewall
-	tcpDNATConfigured bool
+	// dnatRules holds the port 53 redirects that are installed and not yet
+	// removed, so a removal that fails can be retried.
+	dnatRules []dnatRule
+}
+
+// dnatRule is a port 53 redirect as it was installed. The target is kept with
+// the rule because the listener can come back on a different address or port,
+// and a retried removal has to name the address and port the rule was added
+// with, not the ones in use now.
+type dnatRule struct {
+	protocol firewall.Protocol
+	ip       netip.Addr
+	port     uint16
 }

 func newServiceViaListener(wgIface WGIface, customAddr *netip.AddrPort, fw Firewall) *serviceViaListener {
@@ -112,34 +127,93 @@ func (s *serviceViaListener) Listen() error {
 		}
 	}()

-	// When eBPF redirects UDP port 53 to our listen port, TCP still needs
-	// a DNAT rule because eBPF only handles UDP.
-	if s.ebpfService != nil && s.firewall != nil && s.listenPort != DefaultPort {
-		if err := s.firewall.AddOutputDNAT(s.listenIP, firewall.ProtocolTCP, DefaultPort, s.listenPort); err != nil {
-			log.Warnf("failed to add DNS TCP DNAT rule, TCP DNS on port 53 will not work: %v", err)
-		} else {
-			s.tcpDNATConfigured = true
-			log.Infof("added DNS TCP DNAT rule: %s:%d -> %s:%d", s.listenIP, DefaultPort, s.listenIP, s.listenPort)
-		}
+	if s.listenPort != DefaultPort {
+		s.setupDNAT()
 	}

 	return nil
 }

+// setupDNAT redirects port 53 to the port the DNS server actually listens on.
+// Both protocols must be redirected or none: RuntimePort reports port 53 only
+// while the full redirect is in place, so a half-configured redirect would
+// advertise a resolver that answers over one protocol.
+func (s *serviceViaListener) setupDNAT() {
+	if s.firewall == nil {
+		log.Errorf("no firewall manager available to redirect DNS port %d to %d, "+
+			"clients pointed at %s will not reach the resolver", DefaultPort, s.listenPort, s.listenIP)
+		return
+	}
+
+	// Clear whatever an earlier removal left behind first. Those rules can point
+	// at an address or port this listener no longer uses, and they are matched
+	// before anything added now, so adding a redirect on top of one would keep
+	// sending port 53 traffic to the previous listener while reporting the
+	// redirect as complete. The rules stay recorded for a later attempt.
+	if err := s.removeDNAT(); err != nil {
+		log.Errorf("failed to remove stale DNS DNAT rules, leaving port %d redirected to the previous listener: %v",
+			DefaultPort, err)
+		return
+	}
+
+	for _, proto := range dnatProtocols {
+		if err := s.firewall.AddOutputDNAT(s.listenIP, proto, DefaultPort, s.listenPort); err != nil {
+			log.Errorf("failed to add DNS %s DNAT rule, DNS on port %d will not work: %v",
+				proto, DefaultPort, err)
+			if err := s.removeDNAT(); err != nil {
+				log.Warnf("failed to roll back DNS DNAT rules, retrying on stop: %v", err)
+			}
+			return
+		}
+		s.dnatRules = append(s.dnatRules, dnatRule{protocol: proto, ip: s.listenIP, port: s.listenPort})
+	}
+
+	log.Infof("added DNS DNAT rules: %s:%d -> %s:%d (UDP + TCP)", s.listenIP, DefaultPort, s.listenIP, s.listenPort)
+}
+
+// removeDNAT removes every installed port 53 redirect. A rule whose removal
+// fails stays recorded so a later setup or Stop retries it, rather than leaving
+// port 53 pointing at a resolver that is no longer listening.
+func (s *serviceViaListener) removeDNAT() error {
+	if s.firewall == nil {
+		return nil
+	}
+
+	var merr *multierror.Error
+	var remaining []dnatRule
+	for _, rule := range s.dnatRules {
+		if err := s.firewall.RemoveOutputDNAT(rule.ip, rule.protocol, DefaultPort, rule.port); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("remove DNS %s DNAT rule for %s:%d: %w",
+				rule.protocol, rule.ip, rule.port, err))
+			remaining = append(remaining, rule)
+		}
+	}
+	s.dnatRules = remaining
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
 func (s *serviceViaListener) Stop() error {
 	s.listenerFlagLock.Lock()
 	defer s.listenerFlagLock.Unlock()

+	var merr *multierror.Error
+
+	// Redirects are removed even when the listener is already stopped, so that
+	// a removal which failed earlier is retried instead of leaving port 53
+	// pointing at a resolver that no longer listens.
+	if err := s.removeDNAT(); err != nil {
+		merr = multierror.Append(merr, err)
+	}
+
 	if !s.listenerIsRunning {
-		return nil
+		return nberrors.FormatErrorOrNil(merr)
 	}
 	s.listenerIsRunning = false

 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()

-	var merr *multierror.Error
-
 	if err := s.server.ShutdownContext(ctx); err != nil {
 		merr = multierror.Append(merr, fmt.Errorf("stop DNS UDP server: %w", err))
 	}
@@ -148,19 +222,6 @@ func (s *serviceViaListener) Stop() error {
 		merr = multierror.Append(merr, fmt.Errorf("stop DNS TCP server: %w", err))
 	}

-	if s.tcpDNATConfigured && s.firewall != nil {
-		if err := s.firewall.RemoveOutputDNAT(s.listenIP, firewall.ProtocolTCP, DefaultPort, s.listenPort); err != nil {
-			merr = multierror.Append(merr, fmt.Errorf("remove DNS TCP DNAT rule: %w", err))
-		}
-		s.tcpDNATConfigured = false
-	}
-
-	if s.ebpfService != nil {
-		if err := s.ebpfService.FreeDNSFwd(); err != nil {
-			merr = multierror.Append(merr, fmt.Errorf("stop traffic forwarder: %w", err))
-		}
-	}
-
 	return nberrors.FormatErrorOrNil(merr)
 }

@@ -177,11 +238,23 @@ func (s *serviceViaListener) RuntimePort() int {
 	s.listenerFlagLock.Lock()
 	defer s.listenerFlagLock.Unlock()

-	if s.ebpfService != nil {
+	if s.redirectInstalled() {
 		return DefaultPort
-	} else {
-		return int(s.listenPort)
 	}
+	return int(s.listenPort)
+}
+
+// redirectInstalled reports whether every protocol is redirected from port 53
+// to the address and port the listener currently serves. Rules left over from
+// an earlier listener do not count.
+func (s *serviceViaListener) redirectInstalled() bool {
+	for _, proto := range dnatProtocols {
+		current := dnatRule{protocol: proto, ip: s.listenIP, port: s.listenPort}
+		if !slices.Contains(s.dnatRules, current) {
+			return false
+		}
+	}
+	return true
 }

 func (s *serviceViaListener) RuntimeIP() netip.Addr {
@@ -190,30 +263,29 @@ func (s *serviceViaListener) RuntimeIP() netip.Addr {

 // evalListenAddress figures out the listen address for the DNS server.
 // IPv4-only: all peers have a v4 overlay address, and DNS config points to v4.
-// First checks port 53 on WG interface or lo, then tries eBPF on a random port,
-// then falls back to port 5053.
+// Prefers port 53 on the overlay interface or lo, so no redirect is needed at
+// all; when it is taken it falls back to port 5053 and then to a random free
+// port, both of which need the port 53 redirect set up by setupDNAT.
 func (s *serviceViaListener) evalListenAddress() (netip.Addr, uint16, error) {
 	if s.customAddr != nil {
 		return s.customAddr.Addr(), s.customAddr.Port(), nil
 	}

-	ip, ok := s.testFreePort(DefaultPort)
-	if ok {
+	if ip, ok := s.testFreePort(DefaultPort); ok {
 		return ip, DefaultPort, nil
 	}

-	ebpfSrv, port, ok := s.tryToUseeBPF()
-	if ok {
-		s.ebpfService = ebpfSrv
-		return s.wgInterface.Address().IP, port, nil
-	}
-
-	ip, ok = s.testFreePort(customPort)
-	if ok {
+	if ip, ok := s.testFreePort(customPort); ok {
 		return ip, customPort, nil
 	}

-	return netip.Addr{}, 0, fmt.Errorf("failed to find a free port for DNS server")
+	ip := s.wgInterface.Address().IP
+	port, err := s.randomFreePort(ip)
+	if err != nil {
+		return netip.Addr{}, 0, fmt.Errorf("find a free port for DNS server: %w", err)
+	}
+
+	return ip, port, nil
 }

 func (s *serviceViaListener) testFreePort(port int) (netip.Addr, bool) {
@@ -260,48 +332,25 @@ func (s *serviceViaListener) tryToBind(ip netip.Addr, port int) bool {
 	return true
 }

-// tryToUseeBPF decides whether to apply eBPF program to capture DNS traffic on port 53.
-// This is needed because on some operating systems if we start a DNS server not on a default port 53,
-// the domain name  resolution won't work. So, in case we are running on Linux and picked a free
-// port we should fall back to the eBPF solution that will capture traffic on port 53 and forward
-// it to a local DNS server running on the chosen port.
-func (s *serviceViaListener) tryToUseeBPF() (ebpfMgr.Manager, uint16, bool) {
-	if runtime.GOOS != "linux" {
-		return nil, 0, false
+// randomFreePort returns a port that is free on ip for both UDP and TCP, since
+// the DNS server binds both. The probe listeners are closed again, so the port
+// is only likely, not guaranteed, to still be free when the server binds it.
+func (s *serviceViaListener) randomFreePort(ip netip.Addr) (uint16, error) {
+	for range randomPortAttempts {
+		probeListener, err := net.ListenUDP("udp4", &net.UDPAddr{})
+		if err != nil {
+			return 0, fmt.Errorf("bind random port: %w", err)
+		}
+
+		port := uint16(probeListener.LocalAddr().(*net.UDPAddr).Port)
+		if err := probeListener.Close(); err != nil {
+			return 0, fmt.Errorf("free up probed port: %w", err)
+		}
+
+		if s.tryToBind(ip, int(port)) {
+			return port, nil
+		}
 	}

-	port, err := s.generateFreePort() //nolint:staticcheck,unused
-	if err != nil {
-		log.Warnf("failed to generate a free port for eBPF DNS forwarder server: %s", err)
-		return nil, 0, false
-	}
-
-	ebpfSrv := ebpf.GetEbpfManagerInstance()
-	err = ebpfSrv.LoadDNSFwd(s.wgInterface.Address().IP, int(port))
-	if err != nil {
-		log.Warnf("failed to load DNS forwarder eBPF program, error: %s", err)
-		return nil, 0, false
-	}
-
-	return ebpfSrv, port, true
-}
-
-func (s *serviceViaListener) generateFreePort() (uint16, error) {
-	ok := s.tryToBind(s.wgInterface.Address().IP, customPort)
-	if ok {
-		return customPort, nil
-	}
-
-	probeListener, err := net.ListenUDP("udp4", &net.UDPAddr{})
-	if err != nil {
-		log.Debugf("failed to bind random port for DNS: %s", err)
-		return 0, err
-	}
-
-	port := uint16(probeListener.LocalAddr().(*net.UDPAddr).Port)
-	if err = probeListener.Close(); err != nil {
-		log.Debugf("failed to free up DNS port: %s", err)
-		return 0, err
-	}
-	return port, nil
+	return 0, fmt.Errorf("no port free for UDP and TCP on %s after %d attempts", ip, randomPortAttempts)
 }
@@ -1,6 +1,7 @@
 package dns

 import (
+	"errors"
 	"fmt"
 	"net"
 	"net/netip"
@@ -10,6 +11,8 @@ import (
 	"github.com/miekg/dns"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 )

 func TestServiceViaListener_TCPAndUDP(t *testing.T) {
@@ -84,3 +87,133 @@ func TestServiceViaListener_TCPAndUDP(t *testing.T) {
 	require.NotEmpty(t, tcpResp.Answer)
 	assert.Contains(t, tcpResp.Answer[0].String(), "192.0.2.1", "TCP response should contain expected IP")
 }
+
+type dnatCall struct {
+	rule  dnatRule
+	added bool
+}
+
+// fakeFirewall records DNAT calls and fails the ones named in addErrs/removeErrs.
+type fakeFirewall struct {
+	calls      []dnatCall
+	addErrs    map[firewall.Protocol]error
+	removeErrs map[firewall.Protocol]error
+}
+
+func (f *fakeFirewall) AddOutputDNAT(ip netip.Addr, protocol firewall.Protocol, _, translatedPort uint16) error {
+	if err := f.addErrs[protocol]; err != nil {
+		return err
+	}
+	f.calls = append(f.calls, dnatCall{rule: dnatRule{protocol: protocol, ip: ip, port: translatedPort}, added: true})
+	return nil
+}
+
+func (f *fakeFirewall) RemoveOutputDNAT(ip netip.Addr, protocol firewall.Protocol, _, translatedPort uint16) error {
+	if err := f.removeErrs[protocol]; err != nil {
+		return err
+	}
+	f.calls = append(f.calls, dnatCall{rule: dnatRule{protocol: protocol, ip: ip, port: translatedPort}})
+	return nil
+}
+
+func newDNATTestService(fw Firewall) *serviceViaListener {
+	return &serviceViaListener{
+		listenIP:   netip.MustParseAddr("100.64.0.1"),
+		listenPort: customPort,
+		firewall:   fw,
+	}
+}
+
+func TestSetupDNAT_BothProtocols(t *testing.T) {
+	svc := newDNATTestService(&fakeFirewall{})
+
+	svc.setupDNAT()
+
+	assert.Len(t, svc.dnatRules, len(dnatProtocols))
+	assert.Equal(t, DefaultPort, svc.RuntimePort(), "port 53 is advertised once both redirects are installed")
+}
+
+func TestSetupDNAT_RollsBackPartialRedirect(t *testing.T) {
+	fw := &fakeFirewall{addErrs: map[firewall.Protocol]error{firewall.ProtocolTCP: errors.New("nftables busy")}}
+	svc := newDNATTestService(fw)
+
+	svc.setupDNAT()
+
+	assert.Empty(t, svc.dnatRules, "the UDP redirect installed before the failure must be rolled back")
+	assert.Equal(t, int(svc.listenPort), svc.RuntimePort(), "an incomplete redirect must not advertise port 53")
+	udp := dnatRule{protocol: firewall.ProtocolUDP, ip: svc.listenIP, port: svc.listenPort}
+	assert.Contains(t, fw.calls, dnatCall{rule: udp}, "UDP removal should have been attempted")
+}
+
+// A rollback that fails must keep the rule recorded, so port 53 is not left
+// redirected to a resolver that no longer listens.
+func TestStop_RetriesFailedDNATRemoval(t *testing.T) {
+	fw := &fakeFirewall{
+		addErrs:    map[firewall.Protocol]error{firewall.ProtocolTCP: errors.New("nftables busy")},
+		removeErrs: map[firewall.Protocol]error{firewall.ProtocolUDP: errors.New("nftables busy")},
+	}
+	svc := newDNATTestService(fw)
+
+	svc.setupDNAT()
+	udp := dnatRule{protocol: firewall.ProtocolUDP, ip: svc.listenIP, port: svc.listenPort}
+	require.Equal(t, []dnatRule{udp}, svc.dnatRules, "a failed rollback keeps the rule for a later retry")
+
+	require.Error(t, svc.Stop(), "the failing removal should be reported")
+	require.Equal(t, []dnatRule{udp}, svc.dnatRules)
+
+	delete(fw.removeErrs, firewall.ProtocolUDP)
+	require.NoError(t, svc.Stop(), "a later stop retries the removal")
+	assert.Empty(t, svc.dnatRules)
+}
+
+// A stale rule that cannot be removed is matched before anything added now, so
+// no new redirect may be installed on top of it and port 53 must not be
+// advertised as reaching this listener.
+func TestSetupDNAT_AbortsWhileStaleRuleRemains(t *testing.T) {
+	fw := &fakeFirewall{removeErrs: map[firewall.Protocol]error{firewall.ProtocolUDP: errors.New("nftables busy")}}
+	svc := newDNATTestService(fw)
+	stalePort := svc.listenPort
+
+	svc.setupDNAT()
+	require.Error(t, svc.Stop())
+	staleUDP := dnatRule{protocol: firewall.ProtocolUDP, ip: svc.listenIP, port: stalePort}
+	require.Equal(t, []dnatRule{staleUDP}, svc.dnatRules)
+
+	svc.listenPort = stalePort + 1
+	fw.calls = nil
+
+	svc.setupDNAT()
+
+	assert.Equal(t, []dnatRule{staleUDP}, svc.dnatRules, "the stale rule stays recorded for a later attempt")
+	for _, call := range fw.calls {
+		assert.False(t, call.added, "no redirect may be installed while a stale one is still in place")
+	}
+	assert.Equal(t, int(svc.listenPort), svc.RuntimePort(), "port 53 must not be advertised")
+}
+
+// A rule left behind by a failed removal must be removed with the address and
+// port it was installed with, even when the listener has since moved to another
+// port, and it must not count towards the redirect the new listener advertises.
+func TestSetupDNAT_ClearsStaleRuleAfterPortChange(t *testing.T) {
+	fw := &fakeFirewall{removeErrs: map[firewall.Protocol]error{firewall.ProtocolUDP: errors.New("nftables busy")}}
+	svc := newDNATTestService(fw)
+	stalePort := svc.listenPort
+
+	svc.setupDNAT()
+	require.Error(t, svc.Stop())
+	staleUDP := dnatRule{protocol: firewall.ProtocolUDP, ip: svc.listenIP, port: stalePort}
+	require.Equal(t, []dnatRule{staleUDP}, svc.dnatRules)
+
+	delete(fw.removeErrs, firewall.ProtocolUDP)
+	svc.listenPort = stalePort + 1
+	fw.calls = nil
+
+	svc.setupDNAT()
+
+	assert.Contains(t, fw.calls, dnatCall{rule: staleUDP}, "the stale rule must be removed with its original port")
+	assert.Len(t, svc.dnatRules, len(dnatProtocols))
+	assert.Equal(t, DefaultPort, svc.RuntimePort(), "the new listener is fully redirected")
+	for _, rule := range svc.dnatRules {
+		assert.Equal(t, svc.listenPort, rule.port, "only rules for the current listener remain")
+	}
+}