[client] Redirect DNS port 53 with UDP and TCP DNAT instead of the eBPF forwarder (#7439)

client/internal/dns/service_listener.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"net/netip"
 	"runtime"
+	"slices"
 	"strconv"
 	"sync"
 	"time"
@@ -17,17 +18,20 @@ import (
 	nberrors "github.com/netbirdio/netbird/client/errors"
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
-	"github.com/netbirdio/netbird/client/internal/ebpf"
-	ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager"
 )
 
 const (
 	customPort = 5053
+	// randomPortAttempts bounds the search for a port free on both protocols.
+	randomPortAttempts = 5
 )
 
 var (
 	defaultIP = netip.MustParseAddr("127.0.0.1")
 	customIP  = netip.MustParseAddr("127.0.0.153")
+
+	// dnatProtocols are the protocols the port 53 redirect covers.
+	dnatProtocols = []firewall.Protocol{firewall.ProtocolUDP, firewall.ProtocolTCP}
 )
 
 type serviceViaListener struct {
@@ -40,9 +44,20 @@ type serviceViaListener struct {
 	listenPort        uint16
 	listenerIsRunning bool
 	listenerFlagLock  sync.Mutex
-	ebpfService       ebpfMgr.Manager
 	firewall          Firewall
-	tcpDNATConfigured bool
+	// dnatRules holds the port 53 redirects that are installed and not yet
+	// removed, so a removal that fails can be retried.
+	dnatRules []dnatRule
+}
+
+// dnatRule is a port 53 redirect as it was installed. The target is kept with
+// the rule because the listener can come back on a different address or port,
+// and a retried removal has to name the address and port the rule was added
+// with, not the ones in use now.
+type dnatRule struct {
+	protocol firewall.Protocol
+	ip       netip.Addr
+	port     uint16
 }
 
 func newServiceViaListener(wgIface WGIface, customAddr *netip.AddrPort, fw Firewall) *serviceViaListener {
@@ -112,34 +127,93 @@ func (s *serviceViaListener) Listen() error {
 		}
 	}()
 
-	// When eBPF redirects UDP port 53 to our listen port, TCP still needs
-	// a DNAT rule because eBPF only handles UDP.
-	if s.ebpfService != nil && s.firewall != nil && s.listenPort != DefaultPort {
-		if err := s.firewall.AddOutputDNAT(s.listenIP, firewall.ProtocolTCP, DefaultPort, s.listenPort); err != nil {
-			log.Warnf("failed to add DNS TCP DNAT rule, TCP DNS on port 53 will not work: %v", err)
-		} else {
-			s.tcpDNATConfigured = true
-			log.Infof("added DNS TCP DNAT rule: %s:%d -> %s:%d", s.listenIP, DefaultPort, s.listenIP, s.listenPort)
-		}
+	if s.listenPort != DefaultPort {
+		s.setupDNAT()
 	}
 
 	return nil
 }
 
+// setupDNAT redirects port 53 to the port the DNS server actually listens on.
+// Both protocols must be redirected or none: RuntimePort reports port 53 only
+// while the full redirect is in place, so a half-configured redirect would
+// advertise a resolver that answers over one protocol.
+func (s *serviceViaListener) setupDNAT() {
+	if s.firewall == nil {
+		log.Errorf("no firewall manager available to redirect DNS port %d to %d, "+
+			"clients pointed at %s will not reach the resolver", DefaultPort, s.listenPort, s.listenIP)
+		return
+	}
+
+	// Clear whatever an earlier removal left behind first. Those rules can point
+	// at an address or port this listener no longer uses, and they are matched
+	// before anything added now, so adding a redirect on top of one would keep
+	// sending port 53 traffic to the previous listener while reporting the
+	// redirect as complete. The rules stay recorded for a later attempt.
+	if err := s.removeDNAT(); err != nil {
+		log.Errorf("failed to remove stale DNS DNAT rules, leaving port %d redirected to the previous listener: %v",
+			DefaultPort, err)
+		return
+	}
+
+	for _, proto := range dnatProtocols {
+		if err := s.firewall.AddOutputDNAT(s.listenIP, proto, DefaultPort, s.listenPort); err != nil {
+			log.Errorf("failed to add DNS %s DNAT rule, DNS on port %d will not work: %v",
+				proto, DefaultPort, err)
+			if err := s.removeDNAT(); err != nil {
+				log.Warnf("failed to roll back DNS DNAT rules, retrying on stop: %v", err)
+			}
+			return
+		}
+		s.dnatRules = append(s.dnatRules, dnatRule{protocol: proto, ip: s.listenIP, port: s.listenPort})
+	}
+
+	log.Infof("added DNS DNAT rules: %s:%d -> %s:%d (UDP + TCP)", s.listenIP, DefaultPort, s.listenIP, s.listenPort)
+}
+
+// removeDNAT removes every installed port 53 redirect. A rule whose removal
+// fails stays recorded so a later setup or Stop retries it, rather than leaving
+// port 53 pointing at a resolver that is no longer listening.
+func (s *serviceViaListener) removeDNAT() error {
+	if s.firewall == nil {
+		return nil
+	}
+
+	var merr *multierror.Error
+	var remaining []dnatRule
+	for _, rule := range s.dnatRules {
+		if err := s.firewall.RemoveOutputDNAT(rule.ip, rule.protocol, DefaultPort, rule.port); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("remove DNS %s DNAT rule for %s:%d: %w",
+				rule.protocol, rule.ip, rule.port, err))
+			remaining = append(remaining, rule)
+		}
+	}
+	s.dnatRules = remaining
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
 func (s *serviceViaListener) Stop() error {
 	s.listenerFlagLock.Lock()
 	defer s.listenerFlagLock.Unlock()
 
+	var merr *multierror.Error
+
+	// Redirects are removed even when the listener is already stopped, so that
+	// a removal which failed earlier is retried instead of leaving port 53
+	// pointing at a resolver that no longer listens.
+	if err := s.removeDNAT(); err != nil {
+		merr = multierror.Append(merr, err)
+	}
+
 	if !s.listenerIsRunning {
-		return nil
+		return nberrors.FormatErrorOrNil(merr)
 	}
 	s.listenerIsRunning = false
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
-	var merr *multierror.Error
-
 	if err := s.server.ShutdownContext(ctx); err != nil {
 		merr = multierror.Append(merr, fmt.Errorf("stop DNS UDP server: %w", err))
 	}
@@ -148,19 +222,6 @@ func (s *serviceViaListener) Stop() error {
 		merr = multierror.Append(merr, fmt.Errorf("stop DNS TCP server: %w", err))
 	}
 
-	if s.tcpDNATConfigured && s.firewall != nil {
-		if err := s.firewall.RemoveOutputDNAT(s.listenIP, firewall.ProtocolTCP, DefaultPort, s.listenPort); err != nil {
-			merr = multierror.Append(merr, fmt.Errorf("remove DNS TCP DNAT rule: %w", err))
-		}
-		s.tcpDNATConfigured = false
-	}
-
-	if s.ebpfService != nil {
-		if err := s.ebpfService.FreeDNSFwd(); err != nil {
-			merr = multierror.Append(merr, fmt.Errorf("stop traffic forwarder: %w", err))
-		}
-	}
-
 	return nberrors.FormatErrorOrNil(merr)
 }
 
@@ -177,11 +238,23 @@ func (s *serviceViaListener) RuntimePort() int {
 	s.listenerFlagLock.Lock()
 	defer s.listenerFlagLock.Unlock()
 
-	if s.ebpfService != nil {
+	if s.redirectInstalled() {
 		return DefaultPort
-	} else {
-		return int(s.listenPort)
 	}
+	return int(s.listenPort)
+}
+
+// redirectInstalled reports whether every protocol is redirected from port 53
+// to the address and port the listener currently serves. Rules left over from
+// an earlier listener do not count.
+func (s *serviceViaListener) redirectInstalled() bool {
+	for _, proto := range dnatProtocols {
+		current := dnatRule{protocol: proto, ip: s.listenIP, port: s.listenPort}
+		if !slices.Contains(s.dnatRules, current) {
+			return false
+		}
+	}
+	return true
 }
 
 func (s *serviceViaListener) RuntimeIP() netip.Addr {
@@ -190,30 +263,29 @@ func (s *serviceViaListener) RuntimeIP() netip.Addr {
 
 // evalListenAddress figures out the listen address for the DNS server.
 // IPv4-only: all peers have a v4 overlay address, and DNS config points to v4.
-// First checks port 53 on WG interface or lo, then tries eBPF on a random port,
-// then falls back to port 5053.
+// Prefers port 53 on the overlay interface or lo, so no redirect is needed at
+// all; when it is taken it falls back to port 5053 and then to a random free
+// port, both of which need the port 53 redirect set up by setupDNAT.
 func (s *serviceViaListener) evalListenAddress() (netip.Addr, uint16, error) {
 	if s.customAddr != nil {
 		return s.customAddr.Addr(), s.customAddr.Port(), nil
 	}
 
-	ip, ok := s.testFreePort(DefaultPort)
-	if ok {
+	if ip, ok := s.testFreePort(DefaultPort); ok {
 		return ip, DefaultPort, nil
 	}
 
-	ebpfSrv, port, ok := s.tryToUseeBPF()
-	if ok {
-		s.ebpfService = ebpfSrv
-		return s.wgInterface.Address().IP, port, nil
-	}
-
-	ip, ok = s.testFreePort(customPort)
-	if ok {
+	if ip, ok := s.testFreePort(customPort); ok {
 		return ip, customPort, nil
 	}
 
-	return netip.Addr{}, 0, fmt.Errorf("failed to find a free port for DNS server")
+	ip := s.wgInterface.Address().IP
+	port, err := s.randomFreePort(ip)
+	if err != nil {
+		return netip.Addr{}, 0, fmt.Errorf("find a free port for DNS server: %w", err)
+	}
+
+	return ip, port, nil
 }
 
 func (s *serviceViaListener) testFreePort(port int) (netip.Addr, bool) {
@@ -260,48 +332,25 @@ func (s *serviceViaListener) tryToBind(ip netip.Addr, port int) bool {
 	return true
 }
 
-// tryToUseeBPF decides whether to apply eBPF program to capture DNS traffic on port 53.
-// This is needed because on some operating systems if we start a DNS server not on a default port 53,
-// the domain name  resolution won't work. So, in case we are running on Linux and picked a free
-// port we should fall back to the eBPF solution that will capture traffic on port 53 and forward
-// it to a local DNS server running on the chosen port.
-func (s *serviceViaListener) tryToUseeBPF() (ebpfMgr.Manager, uint16, bool) {
-	if runtime.GOOS != "linux" {
-		return nil, 0, false
+// randomFreePort returns a port that is free on ip for both UDP and TCP, since
+// the DNS server binds both. The probe listeners are closed again, so the port
+// is only likely, not guaranteed, to still be free when the server binds it.
+func (s *serviceViaListener) randomFreePort(ip netip.Addr) (uint16, error) {
+	for range randomPortAttempts {
+		probeListener, err := net.ListenUDP("udp4", &net.UDPAddr{})
+		if err != nil {
+			return 0, fmt.Errorf("bind random port: %w", err)
+		}
+
+		port := uint16(probeListener.LocalAddr().(*net.UDPAddr).Port)
+		if err := probeListener.Close(); err != nil {
+			return 0, fmt.Errorf("free up probed port: %w", err)
+		}
+
+		if s.tryToBind(ip, int(port)) {
+			return port, nil
+		}
 	}
 
-	port, err := s.generateFreePort() //nolint:staticcheck,unused
-	if err != nil {
-		log.Warnf("failed to generate a free port for eBPF DNS forwarder server: %s", err)
-		return nil, 0, false
-	}
-
-	ebpfSrv := ebpf.GetEbpfManagerInstance()
-	err = ebpfSrv.LoadDNSFwd(s.wgInterface.Address().IP, int(port))
-	if err != nil {
-		log.Warnf("failed to load DNS forwarder eBPF program, error: %s", err)
-		return nil, 0, false
-	}
-
-	return ebpfSrv, port, true
-}
-
-func (s *serviceViaListener) generateFreePort() (uint16, error) {
-	ok := s.tryToBind(s.wgInterface.Address().IP, customPort)
-	if ok {
-		return customPort, nil
-	}
-
-	probeListener, err := net.ListenUDP("udp4", &net.UDPAddr{})
-	if err != nil {
-		log.Debugf("failed to bind random port for DNS: %s", err)
-		return 0, err
-	}
-
-	port := uint16(probeListener.LocalAddr().(*net.UDPAddr).Port)
-	if err = probeListener.Close(); err != nil {
-		log.Debugf("failed to free up DNS port: %s", err)
-		return 0, err
-	}
-	return port, nil
+	return 0, fmt.Errorf("no port free for UDP and TCP on %s after %d attempts", ip, randomPortAttempts)
 }

client/internal/dns/service_listener_test.go

@@ -1,6 +1,7 @@
 package dns
 
 import (
+	"errors"
 	"fmt"
 	"net"
 	"net/netip"
@@ -10,6 +11,8 @@ import (
 	"github.com/miekg/dns"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 )
 
 func TestServiceViaListener_TCPAndUDP(t *testing.T) {
@@ -84,3 +87,133 @@ func TestServiceViaListener_TCPAndUDP(t *testing.T) {
 	require.NotEmpty(t, tcpResp.Answer)
 	assert.Contains(t, tcpResp.Answer[0].String(), "192.0.2.1", "TCP response should contain expected IP")
 }
+
+type dnatCall struct {
+	rule  dnatRule
+	added bool
+}
+
+// fakeFirewall records DNAT calls and fails the ones named in addErrs/removeErrs.
+type fakeFirewall struct {
+	calls      []dnatCall
+	addErrs    map[firewall.Protocol]error
+	removeErrs map[firewall.Protocol]error
+}
+
+func (f *fakeFirewall) AddOutputDNAT(ip netip.Addr, protocol firewall.Protocol, _, translatedPort uint16) error {
+	if err := f.addErrs[protocol]; err != nil {
+		return err
+	}
+	f.calls = append(f.calls, dnatCall{rule: dnatRule{protocol: protocol, ip: ip, port: translatedPort}, added: true})
+	return nil
+}
+
+func (f *fakeFirewall) RemoveOutputDNAT(ip netip.Addr, protocol firewall.Protocol, _, translatedPort uint16) error {
+	if err := f.removeErrs[protocol]; err != nil {
+		return err
+	}
+	f.calls = append(f.calls, dnatCall{rule: dnatRule{protocol: protocol, ip: ip, port: translatedPort}})
+	return nil
+}
+
+func newDNATTestService(fw Firewall) *serviceViaListener {
+	return &serviceViaListener{
+		listenIP:   netip.MustParseAddr("100.64.0.1"),
+		listenPort: customPort,
+		firewall:   fw,
+	}
+}
+
+func TestSetupDNAT_BothProtocols(t *testing.T) {
+	svc := newDNATTestService(&fakeFirewall{})
+
+	svc.setupDNAT()
+
+	assert.Len(t, svc.dnatRules, len(dnatProtocols))
+	assert.Equal(t, DefaultPort, svc.RuntimePort(), "port 53 is advertised once both redirects are installed")
+}
+
+func TestSetupDNAT_RollsBackPartialRedirect(t *testing.T) {
+	fw := &fakeFirewall{addErrs: map[firewall.Protocol]error{firewall.ProtocolTCP: errors.New("nftables busy")}}
+	svc := newDNATTestService(fw)
+
+	svc.setupDNAT()
+
+	assert.Empty(t, svc.dnatRules, "the UDP redirect installed before the failure must be rolled back")
+	assert.Equal(t, int(svc.listenPort), svc.RuntimePort(), "an incomplete redirect must not advertise port 53")
+	udp := dnatRule{protocol: firewall.ProtocolUDP, ip: svc.listenIP, port: svc.listenPort}
+	assert.Contains(t, fw.calls, dnatCall{rule: udp}, "UDP removal should have been attempted")
+}
+
+// A rollback that fails must keep the rule recorded, so port 53 is not left
+// redirected to a resolver that no longer listens.
+func TestStop_RetriesFailedDNATRemoval(t *testing.T) {
+	fw := &fakeFirewall{
+		addErrs:    map[firewall.Protocol]error{firewall.ProtocolTCP: errors.New("nftables busy")},
+		removeErrs: map[firewall.Protocol]error{firewall.ProtocolUDP: errors.New("nftables busy")},
+	}
+	svc := newDNATTestService(fw)
+
+	svc.setupDNAT()
+	udp := dnatRule{protocol: firewall.ProtocolUDP, ip: svc.listenIP, port: svc.listenPort}
+	require.Equal(t, []dnatRule{udp}, svc.dnatRules, "a failed rollback keeps the rule for a later retry")
+
+	require.Error(t, svc.Stop(), "the failing removal should be reported")
+	require.Equal(t, []dnatRule{udp}, svc.dnatRules)
+
+	delete(fw.removeErrs, firewall.ProtocolUDP)
+	require.NoError(t, svc.Stop(), "a later stop retries the removal")
+	assert.Empty(t, svc.dnatRules)
+}
+
+// A stale rule that cannot be removed is matched before anything added now, so
+// no new redirect may be installed on top of it and port 53 must not be
+// advertised as reaching this listener.
+func TestSetupDNAT_AbortsWhileStaleRuleRemains(t *testing.T) {
+	fw := &fakeFirewall{removeErrs: map[firewall.Protocol]error{firewall.ProtocolUDP: errors.New("nftables busy")}}
+	svc := newDNATTestService(fw)
+	stalePort := svc.listenPort
+
+	svc.setupDNAT()
+	require.Error(t, svc.Stop())
+	staleUDP := dnatRule{protocol: firewall.ProtocolUDP, ip: svc.listenIP, port: stalePort}
+	require.Equal(t, []dnatRule{staleUDP}, svc.dnatRules)
+
+	svc.listenPort = stalePort + 1
+	fw.calls = nil
+
+	svc.setupDNAT()
+
+	assert.Equal(t, []dnatRule{staleUDP}, svc.dnatRules, "the stale rule stays recorded for a later attempt")
+	for _, call := range fw.calls {
+		assert.False(t, call.added, "no redirect may be installed while a stale one is still in place")
+	}
+	assert.Equal(t, int(svc.listenPort), svc.RuntimePort(), "port 53 must not be advertised")
+}
+
+// A rule left behind by a failed removal must be removed with the address and
+// port it was installed with, even when the listener has since moved to another
+// port, and it must not count towards the redirect the new listener advertises.
+func TestSetupDNAT_ClearsStaleRuleAfterPortChange(t *testing.T) {
+	fw := &fakeFirewall{removeErrs: map[firewall.Protocol]error{firewall.ProtocolUDP: errors.New("nftables busy")}}
+	svc := newDNATTestService(fw)
+	stalePort := svc.listenPort
+
+	svc.setupDNAT()
+	require.Error(t, svc.Stop())
+	staleUDP := dnatRule{protocol: firewall.ProtocolUDP, ip: svc.listenIP, port: stalePort}
+	require.Equal(t, []dnatRule{staleUDP}, svc.dnatRules)
+
+	delete(fw.removeErrs, firewall.ProtocolUDP)
+	svc.listenPort = stalePort + 1
+	fw.calls = nil
+
+	svc.setupDNAT()
+
+	assert.Contains(t, fw.calls, dnatCall{rule: staleUDP}, "the stale rule must be removed with its original port")
+	assert.Len(t, svc.dnatRules, len(dnatProtocols))
+	assert.Equal(t, DefaultPort, svc.RuntimePort(), "the new listener is fully redirected")
+	for _, rule := range svc.dnatRules {
+		assert.Equal(t, svc.listenPort, rule.port, "only rules for the current listener remain")
+	}
+}

client/internal/ebpf/ebpf/bpf_bpfeb.go

@@ -1,5 +1,5 @@
 // Code generated by bpf2go; DO NOT EDIT.
-//go:build arm64be || armbe || mips || mips64 || mips64p32 || ppc64 || s390 || s390x || sparc || sparc64
+//go:build mips || mips64 || ppc64 || s390x
 
 package ebpf
 
@@ -47,9 +47,10 @@ func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
 type bpfSpecs struct {
 	bpfProgramSpecs
 	bpfMapSpecs
+	bpfVariableSpecs
 }
 
-// bpfSpecs contains programs before they are loaded into the kernel.
+// bpfProgramSpecs contains programs before they are loaded into the kernel.
 //
 // It can be passed ebpf.CollectionSpec.Assign.
 type bpfProgramSpecs struct {
@@ -61,17 +62,28 @@ type bpfProgramSpecs struct {
 // It can be passed ebpf.CollectionSpec.Assign.
 type bpfMapSpecs struct {
 	NbFeatures           *ebpf.MapSpec `ebpf:"nb_features"`
-	NbMapDnsIp           *ebpf.MapSpec `ebpf:"nb_map_dns_ip"`
-	NbMapDnsPort         *ebpf.MapSpec `ebpf:"nb_map_dns_port"`
 	NbWgProxySettingsMap *ebpf.MapSpec `ebpf:"nb_wg_proxy_settings_map"`
 }
 
+// bpfVariableSpecs contains global variables before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfVariableSpecs struct {
+	FlagFeatureWgProxy *ebpf.VariableSpec `ebpf:"flag_feature_wg_proxy"`
+	MapKeyFeatures     *ebpf.VariableSpec `ebpf:"map_key_features"`
+	MapKeyProxyPort    *ebpf.VariableSpec `ebpf:"map_key_proxy_port"`
+	MapKeyWgPort       *ebpf.VariableSpec `ebpf:"map_key_wg_port"`
+	ProxyPort          *ebpf.VariableSpec `ebpf:"proxy_port"`
+	WgPort             *ebpf.VariableSpec `ebpf:"wg_port"`
+}
+
 // bpfObjects contains all objects after they have been loaded into the kernel.
 //
 // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
 type bpfObjects struct {
 	bpfPrograms
 	bpfMaps
+	bpfVariables
 }
 
 func (o *bpfObjects) Close() error {
@@ -86,20 +98,28 @@ func (o *bpfObjects) Close() error {
 // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
 type bpfMaps struct {
 	NbFeatures           *ebpf.Map `ebpf:"nb_features"`
-	NbMapDnsIp           *ebpf.Map `ebpf:"nb_map_dns_ip"`
-	NbMapDnsPort         *ebpf.Map `ebpf:"nb_map_dns_port"`
 	NbWgProxySettingsMap *ebpf.Map `ebpf:"nb_wg_proxy_settings_map"`
 }
 
 func (m *bpfMaps) Close() error {
 	return _BpfClose(
 		m.NbFeatures,
-		m.NbMapDnsIp,
-		m.NbMapDnsPort,
 		m.NbWgProxySettingsMap,
 	)
 }
 
+// bpfVariables contains all global variables after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfVariables struct {
+	FlagFeatureWgProxy *ebpf.Variable `ebpf:"flag_feature_wg_proxy"`
+	MapKeyFeatures     *ebpf.Variable `ebpf:"map_key_features"`
+	MapKeyProxyPort    *ebpf.Variable `ebpf:"map_key_proxy_port"`
+	MapKeyWgPort       *ebpf.Variable `ebpf:"map_key_wg_port"`
+	ProxyPort          *ebpf.Variable `ebpf:"proxy_port"`
+	WgPort             *ebpf.Variable `ebpf:"wg_port"`
+}
+
 // bpfPrograms contains all programs after they have been loaded into the kernel.
 //
 // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.

client/internal/ebpf/ebpf/bpf_bpfel.go

@@ -1,5 +1,5 @@
 // Code generated by bpf2go; DO NOT EDIT.
-//go:build 386 || amd64 || amd64p32 || arm || arm64 || loong64 || mips64le || mips64p32le || mipsle || ppc64le || riscv64
+//go:build 386 || amd64 || arm || arm64 || loong64 || mips64le || mipsle || ppc64le || riscv64 || wasm
 
 package ebpf
 
@@ -47,9 +47,10 @@ func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
 type bpfSpecs struct {
 	bpfProgramSpecs
 	bpfMapSpecs
+	bpfVariableSpecs
 }
 
-// bpfSpecs contains programs before they are loaded into the kernel.
+// bpfProgramSpecs contains programs before they are loaded into the kernel.
 //
 // It can be passed ebpf.CollectionSpec.Assign.
 type bpfProgramSpecs struct {
@@ -61,17 +62,28 @@ type bpfProgramSpecs struct {
 // It can be passed ebpf.CollectionSpec.Assign.
 type bpfMapSpecs struct {
 	NbFeatures           *ebpf.MapSpec `ebpf:"nb_features"`
-	NbMapDnsIp           *ebpf.MapSpec `ebpf:"nb_map_dns_ip"`
-	NbMapDnsPort         *ebpf.MapSpec `ebpf:"nb_map_dns_port"`
 	NbWgProxySettingsMap *ebpf.MapSpec `ebpf:"nb_wg_proxy_settings_map"`
 }
 
+// bpfVariableSpecs contains global variables before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfVariableSpecs struct {
+	FlagFeatureWgProxy *ebpf.VariableSpec `ebpf:"flag_feature_wg_proxy"`
+	MapKeyFeatures     *ebpf.VariableSpec `ebpf:"map_key_features"`
+	MapKeyProxyPort    *ebpf.VariableSpec `ebpf:"map_key_proxy_port"`
+	MapKeyWgPort       *ebpf.VariableSpec `ebpf:"map_key_wg_port"`
+	ProxyPort          *ebpf.VariableSpec `ebpf:"proxy_port"`
+	WgPort             *ebpf.VariableSpec `ebpf:"wg_port"`
+}
+
 // bpfObjects contains all objects after they have been loaded into the kernel.
 //
 // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
 type bpfObjects struct {
 	bpfPrograms
 	bpfMaps
+	bpfVariables
 }
 
 func (o *bpfObjects) Close() error {
@@ -86,20 +98,28 @@ func (o *bpfObjects) Close() error {
 // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
 type bpfMaps struct {
 	NbFeatures           *ebpf.Map `ebpf:"nb_features"`
-	NbMapDnsIp           *ebpf.Map `ebpf:"nb_map_dns_ip"`
-	NbMapDnsPort         *ebpf.Map `ebpf:"nb_map_dns_port"`
 	NbWgProxySettingsMap *ebpf.Map `ebpf:"nb_wg_proxy_settings_map"`
 }
 
 func (m *bpfMaps) Close() error {
 	return _BpfClose(
 		m.NbFeatures,
-		m.NbMapDnsIp,
-		m.NbMapDnsPort,
 		m.NbWgProxySettingsMap,
 	)
 }
 
+// bpfVariables contains all global variables after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfVariables struct {
+	FlagFeatureWgProxy *ebpf.Variable `ebpf:"flag_feature_wg_proxy"`
+	MapKeyFeatures     *ebpf.Variable `ebpf:"map_key_features"`
+	MapKeyProxyPort    *ebpf.Variable `ebpf:"map_key_proxy_port"`
+	MapKeyWgPort       *ebpf.Variable `ebpf:"map_key_wg_port"`
+	ProxyPort          *ebpf.Variable `ebpf:"proxy_port"`
+	WgPort             *ebpf.Variable `ebpf:"wg_port"`
+}
+
 // bpfPrograms contains all programs after they have been loaded into the kernel.
 //
 // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.

client/internal/ebpf/ebpf/dns_fwd_linux.go

@@ -1,52 +0,0 @@
-package ebpf
-
-import (
-	"encoding/binary"
-	"fmt"
-	"net/netip"
-
-	log "github.com/sirupsen/logrus"
-)
-
-const (
-	mapKeyDNSIP   uint32 = 0
-	mapKeyDNSPort uint32 = 1
-)
-
-func (tf *GeneralManager) LoadDNSFwd(ip netip.Addr, dnsPort int) error {
-	log.Debugf("load eBPF DNS forwarder, watching addr: %s:53, redirect to port: %d", ip, dnsPort)
-	tf.lock.Lock()
-	defer tf.lock.Unlock()
-
-	err := tf.loadXdp()
-	if err != nil {
-		return err
-	}
-
-	if !ip.Is4() {
-		return fmt.Errorf("eBPF DNS forwarder only supports IPv4, got %s", ip)
-	}
-	ip4 := ip.As4()
-	err = tf.bpfObjs.NbMapDnsIp.Put(mapKeyDNSIP, binary.BigEndian.Uint32(ip4[:]))
-	if err != nil {
-		return err
-	}
-
-	err = tf.bpfObjs.NbMapDnsPort.Put(mapKeyDNSPort, uint16(dnsPort))
-	if err != nil {
-		return err
-	}
-
-	tf.setFeatureFlag(featureFlagDnsForwarder)
-	err = tf.bpfObjs.NbFeatures.Put(mapKeyFeatures, tf.featureFlags)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (tf *GeneralManager) FreeDNSFwd() error {
-	log.Debugf("free ebpf DNS forwarder")
-	return tf.unsetFeatureFlag(featureFlagDnsForwarder)
-}
-

client/internal/ebpf/ebpf/manager_linux.go

@@ -15,8 +15,7 @@ import (
 const (
 	mapKeyFeatures uint32 = 0
 
-	featureFlagWGProxy      = 0b00000001
-	featureFlagDnsForwarder = 0b00000010
+	featureFlagWGProxy = 0b00000001
 )
 
 var (
@@ -28,9 +27,9 @@ var (
 
 // GeneralManager is used to load multiple eBPF programs with a custom check (if then) done in prog.c
 // The manager simply adds a feature (byte) of each program to a map that is shared between the userspace and kernel.
-// When packet arrives, the C code checks for each feature (if it is set) and executes each enabled program (e.g., dns_fwd.c and wg_proxy.c).
+// When packet arrives, the C code checks for each feature (if it is set) and executes each enabled program (e.g., wg_proxy.c).
 //
-//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang-14 bpf src/prog.c -- -I /usr/x86_64-linux-gnu/include
+//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang-14 bpf src/prog.c -- -I /usr/x86_64-linux-gnu/include -include src/bpf_map_def.h
 type GeneralManager struct {
 	lock         sync.Mutex
 	link         link.Link

client/internal/ebpf/ebpf/manager_linux_test.go

@@ -7,33 +7,24 @@ import (
 func TestManager_setFeatureFlag(t *testing.T) {
 	mgr := GeneralManager{}
 	mgr.setFeatureFlag(featureFlagWGProxy)
-	if mgr.featureFlags != 1 {
+	if mgr.featureFlags != featureFlagWGProxy {
 		t.Errorf("invalid feature state")
 	}
 
-	mgr.setFeatureFlag(featureFlagDnsForwarder)
-	if mgr.featureFlags != 3 {
-		t.Errorf("invalid feature state")
+	mgr.setFeatureFlag(featureFlagWGProxy)
+	if mgr.featureFlags != featureFlagWGProxy {
+		t.Errorf("setting a flag twice must be idempotent, got: %d", mgr.featureFlags)
 	}
 }
 
 func TestManager_unsetFeatureFlag(t *testing.T) {
 	mgr := GeneralManager{}
 	mgr.setFeatureFlag(featureFlagWGProxy)
-	mgr.setFeatureFlag(featureFlagDnsForwarder)
 
 	err := mgr.unsetFeatureFlag(featureFlagWGProxy)
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}
-	if mgr.featureFlags != 2 {
-		t.Errorf("invalid feature state, expected: %d, got: %d", 2, mgr.featureFlags)
-	}
-
-	err = mgr.unsetFeatureFlag(featureFlagDnsForwarder)
-	if err != nil {
-		t.Errorf("unexpected error: %s", err)
-	}
 	if mgr.featureFlags != 0 {
 		t.Errorf("invalid feature state, expected: %d, got: %d", 0, mgr.featureFlags)
 	}

client/internal/ebpf/ebpf/src/bpf_map_def.h

@@ -0,0 +1,16 @@
+// libbpf 1.0 removed struct bpf_map_def, but the programs here keep the legacy
+// map definitions: they load on kernels built without BTF, which BTF-style
+// (SEC(".maps")) definitions do not. Define the struct ourselves so the
+// programs compile against current libbpf headers.
+#ifndef NB_BPF_MAP_DEF_H
+#define NB_BPF_MAP_DEF_H
+
+struct bpf_map_def {
+	unsigned int type;
+	unsigned int key_size;
+	unsigned int value_size;
+	unsigned int max_entries;
+	unsigned int map_flags;
+};
+
+#endif

client/internal/ebpf/ebpf/src/dns_fwd.c

@@ -1,67 +0,0 @@
-const __u32 map_key_dns_ip = 0;
-const __u32 map_key_dns_port = 1;
-
-struct bpf_map_def SEC("maps") nb_map_dns_ip = {
-	.type = BPF_MAP_TYPE_ARRAY,
-	.key_size = sizeof(__u32),
-	.value_size = sizeof(__u32),
-	.max_entries = 10,
-};
-
-struct bpf_map_def SEC("maps") nb_map_dns_port = {
-	.type = BPF_MAP_TYPE_ARRAY,
-	.key_size = sizeof(__u32),
-	.value_size = sizeof(__u16),
-	.max_entries = 10,
-};
-
-__be32 dns_ip = 0;
-__be16 dns_port = 0;
-
-// 13568 is 53 in big endian
-__be16 GENERAL_DNS_PORT = 13568;
-
-bool read_settings() {
-    __u16 *port_value;
-    __u32 *ip_value;
-
-    // read dns ip
-    ip_value = bpf_map_lookup_elem(&nb_map_dns_ip, &map_key_dns_ip);
-    if(!ip_value) {
-        return false;
-    }
-    dns_ip = htonl(*ip_value);
-
-    // read dns port
-    port_value = bpf_map_lookup_elem(&nb_map_dns_port, &map_key_dns_port);
-    if (!port_value) {
-        return false;
-    }
-    dns_port = htons(*port_value);
-    return true;
-}
-
-int xdp_dns_fwd(struct iphdr  *ip, struct udphdr *udp) {
-    if (dns_port == 0) {
-        if(!read_settings()){
-            return XDP_PASS;
-        }
-        // bpf_printk("dns port: %d", ntohs(dns_port));
-        // bpf_printk("dns ip: %d", ntohl(dns_ip));
-    }
-
-    if (udp->dest == GENERAL_DNS_PORT && ip->daddr == dns_ip) {
-        udp->dest = dns_port;
-        // Clear the now-stale checksum; zero means "not computed" for IPv4.
-        udp->check = 0;
-        return XDP_PASS;
-    }
-
-    if (udp->source == dns_port && ip->saddr == dns_ip) {
-        udp->source = GENERAL_DNS_PORT;
-        udp->check = 0;
-        return XDP_PASS;
-    }
-
-    return XDP_PASS;
-}

client/internal/ebpf/ebpf/src/prog.c

@@ -5,11 +5,9 @@
 #include <netinet/in.h>
 #include <linux/bpf.h>
 #include <bpf/bpf_helpers.h>
-#include "dns_fwd.c"
 #include "wg_proxy.c"
 
 const __u16 flag_feature_wg_proxy = 0b01;
-const __u16 flag_feature_dns_fwd = 0b10;
 
 const __u32 map_key_features = 0;
 struct bpf_map_def SEC("maps") nb_features = {
@@ -48,10 +46,6 @@ int nb_xdp_prog(struct xdp_md *ctx) {
        return XDP_PASS;
     }
 
-    if (*features & flag_feature_dns_fwd) {
-        xdp_dns_fwd(ip, udp);
-    }
-
     if (*features & flag_feature_wg_proxy) {
         xdp_wg_proxy(ip, udp);
     }

client/internal/ebpf/ebpf/src/readme.md

@@ -1,8 +1,18 @@
-# DNS forwarder
+# XDP programs
 
-The agent attach the XDP program to the lo device. We can not use fake address in eBPF because the
-traffic does not appear in the eBPF program. The program capture the traffic on wg_ip:53 and 
-overwrite in it the destination port to 5053.
+`prog.c` is attached to the `lo` device and dispatches to the features enabled in the
+`nb_features` map. The only feature is the WireGuard proxy (`wg_proxy.c`): it rewrites
+loopback UDP sent from the WireGuard listen port so it reaches the userspace relay proxy
+port instead, and swaps the peer endpoint port into the source so the proxy can tell
+peers apart.
+
+Maps use the legacy `struct bpf_map_def` form, defined in `bpf_map_def.h` because libbpf
+1.0 removed it. They load on kernels built without BTF, which BTF-style (`SEC(".maps")`)
+definitions do not.
+
+Regenerate the objects with `go generate ./client/internal/ebpf/ebpf/`; it needs
+`clang-14`. Loading a regenerated object needs root, attaching it needs `bpf_link`
+(kernel >= 5.7), and only one XDP program can own `lo` at a time.
 
 # Debug
 

client/internal/ebpf/manager/manager.go

@@ -1,11 +1,7 @@
 package manager
 
-import "net/netip"
-
-// Manager is used to load multiple eBPF programs. E.g., current DNS programs and WireGuard proxy
+// Manager is used to load multiple eBPF programs. E.g., the WireGuard proxy
 type Manager interface {
-	LoadDNSFwd(ip netip.Addr, dnsPort int) error
-	FreeDNSFwd() error
 	LoadWgProxy(proxyPort, wgPort int) error
 	FreeWGProxy() error
 }