sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-09-30 18:49:08 +02:00
Code Issues Packages Projects Releases Wiki Activity

[client] Replace the eBPF WireGuard proxy with loopback endpoint addressing (#7316)

Browse Source
This commit is contained in:
Viktor Liu
2026-09-30 10:41:49 +02:00
committed by GitHub
parent 30dd076b36
commit 8edc120370
41 changed files with 775 additions and 1507 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/firewall/iptables/family_linux.go
+2
View File
@@ -24,6 +24,7 @@ const (
tableFilter = "filter"
tableNat = "nat"
tableMangle = "mangle"
tableRaw = "raw"
// chainACLInput is the peer ACL chain that holds installed
// peer-filtering rules.
@@ -34,6 +35,7 @@ const (
mangleForwardKey chainKey = "MANGLE-FORWARD"
chainInput = "INPUT"
chainOutput = "OUTPUT"
chainPostrouting = "POSTROUTING"
chainPrerouting = "PREROUTING"
chainForward = "FORWARD"
client/firewall/iptables/manager_linux.go
+2 -139
View File
@@ -25,9 +25,8 @@ type Manager struct {
wgIface iFaceMapper
ipv4Client *iptables.IPTables
family4 *family
rawSupported bool
ipv4Client *iptables.IPTables
family4 *family
// IPv6 counterparts, nil when no v6 overlay
ipv6Client *iptables.IPTables
@@ -108,10 +107,6 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error {
return err
}
if err := m.initNoTrackChain(); err != nil {
log.Warnf("raw table not available, notrack rules will be disabled: %v", err)
}
// Trust after all fatal init steps so a later failure doesn't leave the
// interface in firewalld's trusted zone without a corresponding Close.
if err := firewalld.TrustInterface(m.wgIface.Name()); err != nil {
@@ -285,10 +280,6 @@ func (m *Manager) Close(stateManager *statemanager.Manager) error {
var merr *multierror.Error
if err := m.cleanupNoTrackChain(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("cleanup notrack chain: %w", err))
}
if m.hasIPv6() {
if err := m.family6.Reset(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("reset v6 family: %w", err))
@@ -440,134 +431,6 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto
return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort)
}
const (
chainNameRaw = "NETBIRD-RAW"
chainOutput = "OUTPUT"
tableRaw = "raw"
)
// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic.
// This prevents conntrack from tracking WireGuard proxy traffic on loopback, which
// can interfere with MASQUERADE rules (e.g., from container runtimes like Podman/netavark).
//
// Traffic flows that need NOTRACK:
//
// 1. Egress: WireGuard -> fake endpoint (before eBPF rewrite)
// src=127.0.0.1:wgPort -> dst=127.0.0.1:fakePort
// Matched by: sport=wgPort
//
// 2. Egress: Proxy -> WireGuard (via raw socket)
// src=127.0.0.1:fakePort -> dst=127.0.0.1:wgPort
// Matched by: dport=wgPort
//
// 3. Ingress: Packets to WireGuard
// dst=127.0.0.1:wgPort
// Matched by: dport=wgPort
//
// 4. Ingress: Packets to proxy (after eBPF rewrite)
// dst=127.0.0.1:proxyPort
// Matched by: dport=proxyPort
//
// Rules are cleaned up when the firewall manager is closed.
func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error {
m.mutex.Lock()
defer m.mutex.Unlock()
if !m.rawSupported {
return fmt.Errorf("raw table not available")
}
wgPortStr := fmt.Sprintf("%d", wgPort)
proxyPortStr := fmt.Sprintf("%d", proxyPort)
// Egress rules: match outgoing loopback UDP packets
outputRuleSport := []string{"-o", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--sport", wgPortStr, "-j", "NOTRACK"}
if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, outputRuleSport...); err != nil {
return fmt.Errorf("add output sport notrack rule: %w", err)
}
outputRuleDport := []string{"-o", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"}
if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, outputRuleDport...); err != nil {
return fmt.Errorf("add output dport notrack rule: %w", err)
}
// Ingress rules: match incoming loopback UDP packets
preroutingRuleWg := []string{"-i", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"}
if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, preroutingRuleWg...); err != nil {
return fmt.Errorf("add prerouting wg notrack rule: %w", err)
}
preroutingRuleProxy := []string{"-i", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", proxyPortStr, "-j", "NOTRACK"}
if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, preroutingRuleProxy...); err != nil {
return fmt.Errorf("add prerouting proxy notrack rule: %w", err)
}
log.Debugf("set up ebpf proxy notrack rules for ports %d,%d", proxyPort, wgPort)
return nil
}
func (m *Manager) initNoTrackChain() error {
if err := m.cleanupNoTrackChain(); err != nil {
log.Debugf("cleanup notrack chain: %v", err)
}
if err := m.ipv4Client.NewChain(tableRaw, chainNameRaw); err != nil {
return fmt.Errorf("create chain: %w", err)
}
jumpRule := []string{"-j", chainNameRaw}
if err := m.ipv4Client.InsertUnique(tableRaw, chainOutput, 1, jumpRule...); err != nil {
if delErr := m.ipv4Client.DeleteChain(tableRaw, chainNameRaw); delErr != nil {
log.Debugf("delete orphan chain: %v", delErr)
}
return fmt.Errorf("add output jump rule: %w", err)
}
if err := m.ipv4Client.InsertUnique(tableRaw, chainPrerouting, 1, jumpRule...); err != nil {
if delErr := m.ipv4Client.DeleteIfExists(tableRaw, chainOutput, jumpRule...); delErr != nil {
log.Debugf("delete output jump rule: %v", delErr)
}
if delErr := m.ipv4Client.DeleteChain(tableRaw, chainNameRaw); delErr != nil {
log.Debugf("delete orphan chain: %v", delErr)
}
return fmt.Errorf("add prerouting jump rule: %w", err)
}
m.rawSupported = true
return nil
}
func (m *Manager) cleanupNoTrackChain() error {
exists, err := m.ipv4Client.ChainExists(tableRaw, chainNameRaw)
if err != nil {
if !m.rawSupported {
return nil
}
return fmt.Errorf("check chain exists: %w", err)
}
if !exists {
return nil
}
jumpRule := []string{"-j", chainNameRaw}
if err := m.ipv4Client.DeleteIfExists(tableRaw, chainOutput, jumpRule...); err != nil {
return fmt.Errorf("remove output jump rule: %w", err)
}
if err := m.ipv4Client.DeleteIfExists(tableRaw, chainPrerouting, jumpRule...); err != nil {
return fmt.Errorf("remove prerouting jump rule: %w", err)
}
if err := m.ipv4Client.ClearAndDeleteChain(tableRaw, chainNameRaw); err != nil {
return fmt.Errorf("clear and delete chain: %w", err)
}
m.rawSupported = false
return nil
}
func getConntrackEstablished() []string {
return []string{"-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}
}
client/firewall/manager/firewall.go
-4
View File
@@ -192,10 +192,6 @@ type Manager interface {
// RemoveOutputDNAT removes an OUTPUT chain DNAT rule.
RemoveOutputDNAT(localAddr netip.Addr, protocol Protocol, originalPort, translatedPort uint16) error
// SetupEBPFProxyNoTrack creates static notrack rules for eBPF proxy loopback traffic.
// This prevents conntrack from interfering with WireGuard proxy communication.
SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error
}
// GenKey builds the rule id for this pair from the given format.
client/firewall/nftables/manager_linux.go
-182
View File
@@ -12,7 +12,6 @@ import (
"github.com/google/nftables/expr"
"github.com/hashicorp/go-multierror"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
nberrors "github.com/netbirdio/netbird/client/errors"
firewall "github.com/netbirdio/netbird/client/firewall/manager"
@@ -55,9 +54,6 @@ type Manager struct {
// IPv6 counterpart, nil when no v6 overlay.
family6 *family
notrackOutputChain *nftables.Chain
notrackPreroutingChain *nftables.Chain
extMonitor *externalChainMonitor
}
@@ -170,10 +166,6 @@ func (m *Manager) initFirewall() (err error) {
}
}
if err := m.initNoTrackChains(workTable); err != nil {
log.Warnf("raw priority chains not available, notrack rules will be disabled: %v", err)
}
return nil
}
@@ -455,10 +447,6 @@ func (m *Manager) Flush() error {
}
}
if err := m.refreshNoTrackChains(); err != nil {
log.Errorf("failed to refresh notrack chains: %v", err)
}
return nil
}
@@ -571,176 +559,6 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto
return m.family4.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort)
}
const (
chainNameRawOutput = "netbird-raw-out"
chainNameRawPrerouting = "netbird-raw-pre"
)
// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic.
// This prevents conntrack from tracking WireGuard proxy traffic on loopback, which
// can interfere with MASQUERADE rules (e.g., from container runtimes like Podman/netavark).
//
// Traffic flows that need NOTRACK:
//
// 1. Egress: WireGuard -> fake endpoint (before eBPF rewrite)
// src=127.0.0.1:wgPort -> dst=127.0.0.1:fakePort
// Matched by: sport=wgPort
//
// 2. Egress: Proxy -> WireGuard (via raw socket)
// src=127.0.0.1:fakePort -> dst=127.0.0.1:wgPort
// Matched by: dport=wgPort
//
// 3. Ingress: Packets to WireGuard
// dst=127.0.0.1:wgPort
// Matched by: dport=wgPort
//
// 4. Ingress: Packets to proxy (after eBPF rewrite)
// dst=127.0.0.1:proxyPort
// Matched by: dport=proxyPort
//
// Rules are cleaned up when the firewall manager is closed.
func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error {
m.mutex.Lock()
defer m.mutex.Unlock()
if m.notrackOutputChain == nil || m.notrackPreroutingChain == nil {
return fmt.Errorf("notrack chains not initialized")
}
proxyPortBytes := binaryutil.BigEndian.PutUint16(proxyPort)
wgPortBytes := binaryutil.BigEndian.PutUint16(wgPort)
loopback := []byte{127, 0, 0, 1}
// Egress rules: match outgoing loopback UDP packets
m.rConn.AddRule(&nftables.Rule{
Table: m.notrackOutputChain.Table,
Chain: m.notrackOutputChain,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // sport=wgPort
&expr.Counter{},
&expr.Notrack{},
},
})
m.rConn.AddRule(&nftables.Rule{
Table: m.notrackOutputChain.Table,
Chain: m.notrackOutputChain,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // dport=wgPort
&expr.Counter{},
&expr.Notrack{},
},
})
// Ingress rules: match incoming loopback UDP packets
m.rConn.AddRule(&nftables.Rule{
Table: m.notrackPreroutingChain.Table,
Chain: m.notrackPreroutingChain,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: wgPortBytes}, // dport=wgPort
&expr.Counter{},
&expr.Notrack{},
},
})
m.rConn.AddRule(&nftables.Rule{
Table: m.notrackPreroutingChain.Table,
Chain: m.notrackPreroutingChain,
Exprs: []expr.Any{
&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: proxyPortBytes}, // dport=proxyPort
&expr.Counter{},
&expr.Notrack{},
},
})
if err := m.rConn.Flush(); err != nil {
return fmt.Errorf("flush notrack rules: %w", err)
}
log.Debugf("set up ebpf proxy notrack rules for ports %d,%d", proxyPort, wgPort)
return nil
}
func (m *Manager) initNoTrackChains(table *nftables.Table) error {
m.notrackOutputChain = m.rConn.AddChain(&nftables.Chain{
Name: chainNameRawOutput,
Table: table,
Type: nftables.ChainTypeFilter,
Hooknum: nftables.ChainHookOutput,
Priority: nftables.ChainPriorityRaw,
})
m.notrackPreroutingChain = m.rConn.AddChain(&nftables.Chain{
Name: chainNameRawPrerouting,
Table: table,
Type: nftables.ChainTypeFilter,
Hooknum: nftables.ChainHookPrerouting,
Priority: nftables.ChainPriorityRaw,
})
if err := m.rConn.Flush(); err != nil {
return fmt.Errorf("flush chain creation: %w", err)
}
return nil
}
func (m *Manager) refreshNoTrackChains() error {
chains, err := m.rConn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
if err != nil {
return fmt.Errorf("list chains: %w", err)
}
tableName := getTableName()
for _, c := range chains {
if c.Table.Name != tableName {
continue
}
switch c.Name {
case chainNameRawOutput:
m.notrackOutputChain = c
case chainNameRawPrerouting:
m.notrackPreroutingChain = c
}
}
return nil
}
func (m *Manager) createWorkTable() (*nftables.Table, error) {
return m.createWorkTableFamily(nftables.TableFamilyIPv4)
}
client/firewall/nftables/routing_linux.go
+1 -1
View File
@@ -192,7 +192,7 @@ func (r *family) addPostroutingRules() {
Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade),
},
// We need to exclude the loopback interface as this changes the ebpf proxy port
// We need to exclude the loopback interface as this changes the wg proxy port
&expr.Meta{
Key: expr.MetaKeyOIFNAME,
Register: 1,
client/firewall/uspfilter/filter.go
-6
View File
@@ -879,12 +879,6 @@ func (m *Manager) resetState() {
}
}
// SetupEBPFProxyNoTrack is not supported by the userspace firewall: eBPF isn't
// used in userspace mode, so this should never be called.
func (m *Manager) SetupEBPFProxyNoTrack(uint16, uint16) error {
return errNotSupported
}
// UpdateSet updates the rule destinations associated with the given set
// by merging the existing prefixes with the new ones, then deduplicating.
func (m *Manager) UpdateSet(set firewall.Set, prefixes []netip.Prefix) error {
client/iface/iface.go
-7
View File
@@ -51,7 +51,6 @@ func ValidateMTU(mtu uint16) error {
type wgProxyFactory interface {
GetProxy() wgproxy.Proxy
GetProxyPort() uint16
Free() error
}
@@ -81,12 +80,6 @@ func (w *WGIface) GetProxy() wgproxy.Proxy {
return w.wgProxyFactory.GetProxy()
}
// GetProxyPort returns the proxy port used by the WireGuard proxy.
// Returns 0 if no proxy port is used (e.g., for userspace WireGuard).
func (w *WGIface) GetProxyPort() uint16 {
return w.wgProxyFactory.GetProxyPort()
}
// GetBind returns the EndpointManager userspace bind mode.
func (w *WGIface) GetBind() device.EndpointManager {
w.mu.Lock()
client/iface/iface_close_test.go
-1
View File
@@ -52,7 +52,6 @@ func (f *fakeTunDevice) Close() error {
type fakeProxyFactory struct{}
func (fakeProxyFactory) GetProxy() wgproxy.Proxy { return nil }
func (fakeProxyFactory) GetProxyPort() uint16 { return 0 }
func (fakeProxyFactory) Free() error { return nil }
// TestWGIface_CloseReleasesMutexBeforeTunClose guards against a deadlock
client/iface/wgproxy/ebpf/portlookup.go
-32
View File
@@ -1,32 +0,0 @@
package ebpf
import (
"fmt"
"net"
)
var (
portRangeStart = 3128
portRangeEnd = portRangeStart + 100
)
type portLookup struct {
}
func (pl portLookup) searchFreePort() (int, error) {
for i := portRangeStart; i <= portRangeEnd; i++ {
if pl.tryToBind(i) == nil {
return i, nil
}
}
return 0, fmt.Errorf("failed to bind free port for eBPF proxy")
}
func (pl portLookup) tryToBind(port int) error {
l, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port))
if err != nil {
return err
}
_ = l.Close()
return nil
}
client/iface/wgproxy/ebpf/portlookup_test.go
-45
View File
@@ -1,45 +0,0 @@
package ebpf
import (
"fmt"
"net"
"testing"
)
func Test_portLookup_searchFreePort(t *testing.T) {
pl := portLookup{}
_, err := pl.searchFreePort()
if err != nil {
t.Fatal(err)
}
}
func Test_portLookup_on_allocated(t *testing.T) {
pl := portLookup{}
portRangeStart = 4128
portRangeEnd = portRangeStart + 100
allocatedPort, err := allocatePort(portRangeStart)
if err != nil {
t.Fatal(err)
}
defer allocatedPort.Close()
fp, err := pl.searchFreePort()
if err != nil {
t.Fatal(err)
}
if fp != (portRangeStart + 1) {
t.Errorf("invalid free port, expected: %d, got: %d", portRangeStart+1, fp)
}
}
func allocatePort(port int) (net.PacketConn, error) {
c, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port))
if err != nil {
return nil, err
}
return c, err
}
client/iface/wgproxy/ebpf/proxy.go
-243
View File
@@ -1,243 +0,0 @@
//go:build linux && !android
package ebpf
import (
"context"
"fmt"
"net"
"sync"
"github.com/hashicorp/go-multierror"
"github.com/pion/transport/v3"
log "github.com/sirupsen/logrus"
nberrors "github.com/netbirdio/netbird/client/errors"
"github.com/netbirdio/netbird/client/iface/bufsize"
"github.com/netbirdio/netbird/client/iface/wgproxy/rawsocket"
"github.com/netbirdio/netbird/client/internal/ebpf"
ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager"
nbnet "github.com/netbirdio/netbird/client/net"
)
const (
loopbackAddr = "127.0.0.1"
)
// WGEBPFProxy definition for proxy with EBPF support
type WGEBPFProxy struct {
localWGListenPort int
proxyPort int
mtu uint16
ebpfManager ebpfMgr.Manager
relayedConnStore map[uint16]net.Conn
relayedConnMutex sync.Mutex
lastUsedPort uint16
rawConnIPv4 net.PacketConn
rawConnIPv6 net.PacketConn
conn transport.UDPConn
ctx context.Context
ctxCancel context.CancelFunc
}
// NewWGEBPFProxy create new WGEBPFProxy instance
func NewWGEBPFProxy(wgPort int, mtu uint16) *WGEBPFProxy {
log.Debugf("instantiate ebpf proxy")
wgProxy := &WGEBPFProxy{
localWGListenPort: wgPort,
mtu: mtu,
ebpfManager: ebpf.GetEbpfManagerInstance(),
relayedConnStore: make(map[uint16]net.Conn),
}
return wgProxy
}
// Listen load ebpf program and listen the proxy
func (p *WGEBPFProxy) Listen() error {
pl := portLookup{}
proxyPort, err := pl.searchFreePort()
if err != nil {
return err
}
p.proxyPort = proxyPort
// Prepare IPv4 raw socket (required)
p.rawConnIPv4, err = rawsocket.PrepareSenderRawSocketIPv4()
if err != nil {
return err
}
// Prepare IPv6 raw socket (optional)
p.rawConnIPv6, err = rawsocket.PrepareSenderRawSocketIPv6()
if err != nil {
log.Warnf("failed to prepare IPv6 raw socket, continuing with IPv4 only: %v", err)
}
err = p.ebpfManager.LoadWgProxy(proxyPort, p.localWGListenPort)
if err != nil {
if closeErr := p.rawConnIPv4.Close(); closeErr != nil {
log.Warnf("failed to close IPv4 raw socket: %v", closeErr)
}
if p.rawConnIPv6 != nil {
if closeErr := p.rawConnIPv6.Close(); closeErr != nil {
log.Warnf("failed to close IPv6 raw socket: %v", closeErr)
}
}
return err
}
addr := net.UDPAddr{
Port: proxyPort,
IP: net.ParseIP(loopbackAddr),
}
p.ctx, p.ctxCancel = context.WithCancel(context.Background())
conn, err := nbnet.ListenUDP("udp", &addr)
if err != nil {
if cErr := p.Free(); cErr != nil {
log.Errorf("Failed to close the wgproxy: %s", cErr)
}
return err
}
p.conn = conn
go p.proxyToRemote()
log.Infof("local wg proxy listening on: %d", proxyPort)
return nil
}
// AddRelayedConn add new relayed connection for the proxy
func (p *WGEBPFProxy) AddRelayedConn(relayedConn net.Conn) (*net.UDPAddr, error) {
wgEndpointPort, err := p.storeRelayedConn(relayedConn)
if err != nil {
return nil, err
}
log.Infof("relayed conn added to wg proxy store: %s, endpoint port: :%d", relayedConn.RemoteAddr(), wgEndpointPort)
wgEndpoint := &net.UDPAddr{
IP: net.ParseIP(loopbackAddr),
Port: int(wgEndpointPort),
}
return wgEndpoint, nil
}
// Free resources except the remoteConns will be keep open.
func (p *WGEBPFProxy) Free() error {
log.Debugf("free up ebpf wg proxy")
if p.ctx != nil && p.ctx.Err() != nil {
//nolint
return nil
}
p.ctxCancel()
var result *multierror.Error
if p.conn != nil {
if err := p.conn.Close(); err != nil {
result = multierror.Append(result, err)
}
}
if err := p.ebpfManager.FreeWGProxy(); err != nil {
result = multierror.Append(result, err)
}
if p.rawConnIPv4 != nil {
if err := p.rawConnIPv4.Close(); err != nil {
result = multierror.Append(result, err)
}
}
if p.rawConnIPv6 != nil {
if err := p.rawConnIPv6.Close(); err != nil {
result = multierror.Append(result, err)
}
}
return nberrors.FormatErrorOrNil(result)
}
// GetProxyPort returns the proxy listening port.
func (p *WGEBPFProxy) GetProxyPort() uint16 {
return uint16(p.proxyPort)
}
// proxyToRemote read messages from local WireGuard interface and forward it to remote conn
// From this go routine has only one instance.
func (p *WGEBPFProxy) proxyToRemote() {
buf := make([]byte, p.mtu+bufsize.WGBufferOverhead)
for p.ctx.Err() == nil {
if err := p.readAndForwardPacket(buf); err != nil {
if p.ctx.Err() != nil {
return
}
log.Errorf("failed to proxy packet to remote conn: %s", err)
}
}
}
func (p *WGEBPFProxy) readAndForwardPacket(buf []byte) error {
n, addr, err := p.conn.ReadFromUDP(buf)
if err != nil {
return fmt.Errorf("failed to read UDP packet from WG: %w", err)
}
p.relayedConnMutex.Lock()
conn, ok := p.relayedConnStore[uint16(addr.Port)]
p.relayedConnMutex.Unlock()
if !ok {
if p.ctx.Err() == nil {
log.Debugf("relayed conn not found by port because conn already has been closed: %d", addr.Port)
}
return nil
}
if _, err := conn.Write(buf[:n]); err != nil {
return fmt.Errorf("forward local WG packet (%d) to remote relayed conn: %w", addr.Port, err)
}
return nil
}
func (p *WGEBPFProxy) storeRelayedConn(relayedConn net.Conn) (uint16, error) {
p.relayedConnMutex.Lock()
defer p.relayedConnMutex.Unlock()
np, err := p.nextFreePort()
if err != nil {
return np, err
}
p.relayedConnStore[np] = relayedConn
return np, nil
}
func (p *WGEBPFProxy) removeRelayedConn(relayedConnID uint16) {
p.relayedConnMutex.Lock()
defer p.relayedConnMutex.Unlock()
_, ok := p.relayedConnStore[relayedConnID]
if ok {
log.Debugf("remove relayed conn from store by port: %d", relayedConnID)
}
delete(p.relayedConnStore, relayedConnID)
}
func (p *WGEBPFProxy) nextFreePort() (uint16, error) {
if len(p.relayedConnStore) == 65535 {
return 0, fmt.Errorf("reached maximum relayed connection numbers")
}
generatePort:
if p.lastUsedPort == 65535 {
p.lastUsedPort = 1
} else {
p.lastUsedPort++
}
if _, ok := p.relayedConnStore[p.lastUsedPort]; ok {
goto generatePort
}
return p.lastUsedPort, nil
}
client/iface/wgproxy/ebpf/proxy_test.go
-56
View File
@@ -1,56 +0,0 @@
//go:build linux && !android
package ebpf
import (
"testing"
)
func TestWGEBPFProxy_connStore(t *testing.T) {
wgProxy := NewWGEBPFProxy(1, 1280)
p, _ := wgProxy.storeRelayedConn(nil)
if p != 1 {
t.Errorf("invalid initial port: %d", wgProxy.lastUsedPort)
}
numOfConns := 10
for i := 0; i < numOfConns; i++ {
p, _ = wgProxy.storeRelayedConn(nil)
}
if p != uint16(numOfConns)+1 {
t.Errorf("invalid last used port: %d, expected: %d", p, numOfConns+1)
}
if len(wgProxy.relayedConnStore) != numOfConns+1 {
t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.relayedConnStore), numOfConns+1)
}
}
func TestWGEBPFProxy_portCalculation_overflow(t *testing.T) {
wgProxy := NewWGEBPFProxy(1, 1280)
_, _ = wgProxy.storeRelayedConn(nil)
wgProxy.lastUsedPort = 65535
p, _ := wgProxy.storeRelayedConn(nil)
if len(wgProxy.relayedConnStore) != 2 {
t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.relayedConnStore), 2)
}
if p != 2 {
t.Errorf("invalid last used port: %d, expected: %d", p, 2)
}
}
func TestWGEBPFProxy_portCalculation_maxConn(t *testing.T) {
wgProxy := NewWGEBPFProxy(1, 1280)
for i := 0; i < 65535; i++ {
_, _ = wgProxy.storeRelayedConn(nil)
}
_, err := wgProxy.storeRelayedConn(nil)
if err == nil {
t.Errorf("invalid relayed conn store calculation")
}
}
client/iface/wgproxy/factory_kernel.go
+27 -24
View File
@@ -8,11 +8,13 @@ import (
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/iface/wgproxy/ebpf"
"github.com/netbirdio/netbird/client/iface/wgproxy/loopback"
udpProxy "github.com/netbirdio/netbird/client/iface/wgproxy/udp"
)
const (
envDisableKernelWGProxy = "NB_DISABLE_KERNEL_WG_PROXY"
// envDisableEBPFWGProxy is a deprecated alias for envDisableKernelWGProxy.
envDisableEBPFWGProxy = "NB_DISABLE_EBPF_WG_PROXY"
)
@@ -20,7 +22,7 @@ type KernelFactory struct {
wgPort int
mtu uint16
ebpfProxy *ebpf.WGEBPFProxy
loopbackProxy *loopback.Proxy
}
func NewKernelFactory(wgPort int, mtu uint16) *KernelFactory {
@@ -29,55 +31,56 @@ func NewKernelFactory(wgPort int, mtu uint16) *KernelFactory {
mtu: mtu,
}
if isEBPFDisabled() {
if isKernelProxyDisabled() {
log.Infof("WireGuard Proxy Factory will produce UDP proxy")
log.Infof("eBPF WireGuard proxy is disabled via %s environment variable", envDisableEBPFWGProxy)
return f
}
ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, mtu)
if err := ebpfProxy.Listen(); err != nil {
loopbackProxy := loopback.NewProxy(wgPort, mtu)
if err := loopbackProxy.Listen(); err != nil {
log.Infof("WireGuard Proxy Factory will produce UDP proxy")
log.Warnf("failed to initialize ebpf proxy, fallback to user space proxy: %s", err)
log.Warnf("failed to initialize loopback proxy, fallback to user space proxy: %s", err)
return f
}
log.Infof("WireGuard Proxy Factory will produce eBPF proxy")
f.ebpfProxy = ebpfProxy
log.Infof("WireGuard Proxy Factory will produce loopback proxy")
f.loopbackProxy = loopbackProxy
return f
}
func (w *KernelFactory) GetProxy() Proxy {
if w.ebpfProxy == nil {
if w.loopbackProxy == nil {
return udpProxy.NewWGUDPProxy(w.wgPort, w.mtu)
}
return ebpf.NewProxyWrapper(w.ebpfProxy)
}
// GetProxyPort returns the eBPF proxy port, or 0 if eBPF is not active.
func (w *KernelFactory) GetProxyPort() uint16 {
if w.ebpfProxy == nil {
return 0
}
return w.ebpfProxy.GetProxyPort()
return loopback.NewProxyWrapper(w.loopbackProxy)
}
func (w *KernelFactory) Free() error {
if w.ebpfProxy == nil {
if w.loopbackProxy == nil {
return nil
}
return w.ebpfProxy.Free()
return w.loopbackProxy.Free()
}
func isEBPFDisabled() bool {
val := os.Getenv(envDisableEBPFWGProxy)
func isKernelProxyDisabled() bool {
env := envDisableKernelWGProxy
val := os.Getenv(env)
if val == "" {
env = envDisableEBPFWGProxy
val = os.Getenv(env)
}
if val == "" {
return false
}
disabled, err := strconv.ParseBool(val)
if err != nil {
log.Warnf("failed to parse %s: %v", envDisableEBPFWGProxy, err)
log.Warnf("failed to parse %s: %v", env, err)
return false
}
if disabled {
log.Infof("kernel WireGuard proxy is disabled via %s", env)
}
return disabled
}
client/iface/wgproxy/factory_usp.go
-5
View File
@@ -24,11 +24,6 @@ func (w *USPFactory) GetProxy() Proxy {
return proxyBind.NewProxyBind(w.bind, w.mtu)
}
// GetProxyPort returns 0 as userspace WireGuard doesn't use a separate proxy port.
func (w *USPFactory) GetProxyPort() uint16 {
return 0
}
func (w *USPFactory) Free() error {
return nil
}
client/iface/wgproxy/loopback/addr.go
+70
View File
@@ -0,0 +1,70 @@
//go:build linux && !android
package loopback
import (
"fmt"
"net/netip"
)
// Peer endpoints live in the upper half of 127.0.0.0/8. Everything in that
// range is delivered to the loopback device without any address or route being
// configured, and staying out of 127.0.0.0/9 keeps well-known squatters such as
// 127.0.0.53 (systemd-resolved) and 127.0.1.1 out of the way.
const (
addrRangeBase uint32 = 0x7f800000 // 127.128.0.0
addrRangeSize uint32 = 1 << 23 // /9
addrRangePrefix = "127.128.0.0/9"
)
// allocator hands out one loopback address per relayed connection. The address
// is the peer's identity: WireGuard sends to it, and the proxy recovers which
// peer a packet belongs to from the destination address.
type allocator struct {
cursor uint32
}
// next returns the first free address at or after the cursor, wrapping once.
// inUse reports whether an address is already handed out.
func (a *allocator) next(inUse func(netip.Addr) bool) (netip.Addr, error) {
for i := uint32(0); i < addrRangeSize; i++ {
a.cursor = (a.cursor + 1) % addrRangeSize
addr := addrFromOffset(a.cursor)
if !addr.IsValid() {
continue
}
if inUse(addr) {
continue
}
return addr, nil
}
return netip.Addr{}, fmt.Errorf("no free endpoint address in %s", addrRangePrefix)
}
// addrFromOffset maps an offset in the range to an address, skipping the .0 and
// .255 hosts. They are unremarkable on loopback, but tools and firewall rules
// tend to treat them as network and broadcast addresses.
func addrFromOffset(offset uint32) netip.Addr {
last := offset & 0xff
if last == 0 || last == 0xff {
return netip.Addr{}
}
v := addrRangeBase + offset
return netip.AddrFrom4([4]byte{
byte(v >> 24),
byte(v >> 16),
byte(v >> 8),
byte(v),
})
}
// inRange reports whether addr is one this proxy could have handed out.
func inRange(addr netip.Addr) bool {
if !addr.Is4() {
return false
}
b := addr.As4()
v := uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
return v >= addrRangeBase && v < addrRangeBase+addrRangeSize && b[3] != 0 && b[3] != 0xff
}
client/iface/wgproxy/loopback/addr_test.go
+114
View File
@@ -0,0 +1,114 @@
//go:build linux && !android
package loopback
import (
"net/netip"
"testing"
)
func TestAllocatorHandsOutDistinctAddresses(t *testing.T) {
var a allocator
taken := make(map[netip.Addr]bool)
for i := 0; i < 1000; i++ {
addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] })
if err != nil {
t.Fatalf("allocate %d: %v", i, err)
}
if taken[addr] {
t.Fatalf("address %s handed out twice", addr)
}
if !inRange(addr) {
t.Fatalf("address %s outside %s", addr, addrRangePrefix)
}
taken[addr] = true
}
}
func TestAllocatorSkipsNetworkAndBroadcastHosts(t *testing.T) {
var a allocator
taken := make(map[netip.Addr]bool)
// enough allocations to walk past a .255/.0 boundary
for i := 0; i < 600; i++ {
addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] })
if err != nil {
t.Fatalf("allocate %d: %v", i, err)
}
last := addr.As4()[3]
if last == 0 || last == 255 {
t.Fatalf("address %s ends in .%d", addr, last)
}
taken[addr] = true
}
}
func TestAllocatorReusesReleasedAddresses(t *testing.T) {
var a allocator
taken := make(map[netip.Addr]bool)
inUse := func(candidate netip.Addr) bool { return taken[candidate] }
alloc := func() netip.Addr {
t.Helper()
addr, err := a.next(inUse)
if err != nil {
t.Fatalf("allocate: %v", err)
}
taken[addr] = true
return addr
}
first := alloc()
second := alloc()
delete(taken, first)
// The cursor only moves forward, so a released address comes back after a
// wrap. Park the cursor near the end of the range instead of allocating
// 2^23 addresses: the next call takes the last usable address, and the one
// after that wraps past the skipped .255 and .0 hosts to the released one.
a.cursor = addrRangeSize - 3
last := alloc()
if want := netip.MustParseAddr("127.255.255.254"); last != want {
t.Fatalf("expected the last usable address %s before the wrap, got %s", want, last)
}
if reused := alloc(); reused != first {
t.Fatalf("expected the released address %s after the wrap, got %s", first, reused)
}
// second is still held, so the allocator must step over it.
if next := alloc(); next == second {
t.Fatalf("allocator handed out %s while it was still in use", second)
}
}
func TestInRange(t *testing.T) {
tests := []struct {
addr string
want bool
}{
{"127.128.0.1", true},
{"127.255.255.254", true},
{"127.128.0.0", false}, // network host, never handed out
{"127.128.5.255", false}, // broadcast host, never handed out
{"127.127.255.255", false}, // below the range, where 127.0.0.53 and friends live
{"127.0.0.1", false},
{"127.0.0.53", false},
{"127.0.1.1", false},
{"128.0.0.1", false},
{"10.0.0.1", false},
}
for _, tc := range tests {
addr := netip.MustParseAddr(tc.addr)
if got := inRange(addr); got != tc.want {
t.Errorf("inRange(%s) = %v, want %v", tc.addr, got, tc.want)
}
}
}
func TestInRangeIgnoresIPv6(t *testing.T) {
if inRange(netip.MustParseAddr("::1")) {
t.Error("inRange(::1) = true, want false")
}
}
client/iface/wgproxy/loopback/proxy.go
+291
View File
@@ -0,0 +1,291 @@
//go:build linux && !android
package loopback
import (
"context"
"fmt"
"net"
"net/netip"
"sync"
"syscall"
"github.com/hashicorp/go-multierror"
log "github.com/sirupsen/logrus"
"golang.org/x/net/ipv4"
"golang.org/x/sys/unix"
nberrors "github.com/netbirdio/netbird/client/errors"
"github.com/netbirdio/netbird/client/iface/bufsize"
"github.com/netbirdio/netbird/client/iface/wgproxy/rawsocket"
)
const (
loopbackDevice = "lo"
portRangeStart = 3128
portRangeEnd = portRangeStart + 100
)
// Proxy forwards packets between relayed connections and a local kernel
// WireGuard instance. Every relayed peer gets its own loopback address as its
// WireGuard endpoint, so a single socket serves all of them: the destination
// address of an incoming packet identifies the peer.
type Proxy struct {
localWGListenPort int
mtu uint16
proxyPort int
conn *net.UDPConn
packetConn *ipv4.PacketConn
loIndex int
rawConnIPv4 net.PacketConn
rawConnIPv6 net.PacketConn
relayedConnMutex sync.Mutex
relayedConnStore map[netip.Addr]net.Conn
addrs allocator
ctx context.Context
ctxCancel context.CancelFunc
}
// NewProxy creates a proxy for the WireGuard instance listening on wgPort.
func NewProxy(wgPort int, mtu uint16) *Proxy {
log.Debugf("instantiate loopback wg proxy")
return &Proxy{
localWGListenPort: wgPort,
mtu: mtu,
relayedConnStore: make(map[netip.Addr]net.Conn),
}
}
// Listen opens the shared socket and starts forwarding WireGuard packets to the
// relayed connections.
func (p *Proxy) Listen() error {
rawConnIPv4, err := rawsocket.PrepareSenderRawSocketIPv4()
if err != nil {
return fmt.Errorf("prepare IPv4 raw socket: %w", err)
}
p.rawConnIPv4 = rawConnIPv4
p.rawConnIPv6, err = rawsocket.PrepareSenderRawSocketIPv6()
if err != nil {
log.Warnf("failed to prepare IPv6 raw socket, continuing with IPv4 only: %v", err)
}
loopback, err := net.InterfaceByName(loopbackDevice)
if err != nil {
if freeErr := p.Free(); freeErr != nil {
log.Errorf("failed to free the wgproxy: %s", freeErr)
}
return fmt.Errorf("look up %s: %w", loopbackDevice, err)
}
p.loIndex = loopback.Index
if err := p.listen(); err != nil {
if freeErr := p.Free(); freeErr != nil {
log.Errorf("failed to free the wgproxy: %s", freeErr)
}
return err
}
p.ctx, p.ctxCancel = context.WithCancel(context.Background())
go p.proxyToRemote()
log.Infof("local wg proxy listening on %s:%d", addrRangePrefix, p.proxyPort)
return nil
}
// listen binds the shared socket on the first free port of the range. The bind
// has to be a wildcard one to receive every peer address in the range, so it is
// restricted to the loopback device: without that the port would be reachable
// on every interface.
func (p *Proxy) listen() error {
var lastErr error
for port := portRangeStart; port <= portRangeEnd; port++ {
err := p.listenOn(port)
if err == nil {
p.proxyPort = port
return nil
}
lastErr = err
}
return fmt.Errorf("bind proxy port in range %d-%d: %w", portRangeStart, portRangeEnd, lastErr)
}
func (p *Proxy) listenOn(proxyPort int) error {
lc := net.ListenConfig{
Control: func(_, _ string, c syscall.RawConn) error {
var sockErr error
if err := c.Control(func(fd uintptr) {
if err := unix.SetsockoptString(int(fd), unix.SOL_SOCKET, unix.SO_BINDTODEVICE, loopbackDevice); err != nil {
sockErr = fmt.Errorf("bind to %s: %w", loopbackDevice, err)
return
}
}); err != nil {
return fmt.Errorf("control socket: %w", err)
}
return sockErr
},
}
conn, err := lc.ListenPacket(context.Background(), "udp4", fmt.Sprintf(":%d", proxyPort))
if err != nil {
return fmt.Errorf("listen on :%d: %w", proxyPort, err)
}
udpConn, ok := conn.(*net.UDPConn)
if !ok {
if closeErr := conn.Close(); closeErr != nil {
log.Errorf("failed to close proxy conn: %s", closeErr)
}
return fmt.Errorf("unexpected conn type %T", conn)
}
packetConn := ipv4.NewPacketConn(udpConn)
// the destination address carries the peer identity, the interface index is
// checked on receive as a second line of defense behind SO_BINDTODEVICE
if err := packetConn.SetControlMessage(ipv4.FlagDst|ipv4.FlagInterface, true); err != nil {
if closeErr := udpConn.Close(); closeErr != nil {
log.Errorf("failed to close proxy conn: %s", closeErr)
}
return fmt.Errorf("request destination address: %w", err)
}
p.conn = udpConn
p.packetConn = packetConn
return nil
}
// AddRelayedConn assigns an endpoint address to the relayed connection and
// returns the address WireGuard should send to, along with the key the
// connection is stored under.
func (p *Proxy) AddRelayedConn(relayedConn net.Conn) (*net.UDPAddr, netip.Addr, error) {
addr, err := p.storeRelayedConn(relayedConn)
if err != nil {
return nil, netip.Addr{}, err
}
log.Infof("relayed conn added to wg proxy store: %s, endpoint address: %s", relayedConn.RemoteAddr(), addr)
return &net.UDPAddr{
IP: addr.AsSlice(),
Port: p.proxyPort,
}, addr, nil
}
// Free releases the proxy resources. The relayed connections are left open.
func (p *Proxy) Free() error {
log.Debugf("free up loopback wg proxy")
if p.ctx != nil && p.ctx.Err() != nil {
//nolint
return nil
}
if p.ctxCancel != nil {
p.ctxCancel()
}
var result *multierror.Error
if p.conn != nil {
if err := p.conn.Close(); err != nil {
result = multierror.Append(result, err)
}
}
if p.rawConnIPv4 != nil {
if err := p.rawConnIPv4.Close(); err != nil {
result = multierror.Append(result, err)
}
}
if p.rawConnIPv6 != nil {
if err := p.rawConnIPv6.Close(); err != nil {
result = multierror.Append(result, err)
}
}
return nberrors.FormatErrorOrNil(result)
}
// proxyToRemote reads packets from the local WireGuard instance and forwards
// them to the relayed connection the destination address belongs to.
func (p *Proxy) proxyToRemote() {
buf := make([]byte, p.mtu+bufsize.WGBufferOverhead)
for p.ctx.Err() == nil {
if err := p.readAndForwardPacket(buf); err != nil {
if p.ctx.Err() != nil {
return
}
log.Errorf("failed to proxy packet to remote conn: %s", err)
}
}
}
func (p *Proxy) readAndForwardPacket(buf []byte) error {
n, cm, _, err := p.packetConn.ReadFrom(buf)
if err != nil {
return fmt.Errorf("read UDP packet from WG: %w", err)
}
if cm == nil {
return fmt.Errorf("no control message on packet")
}
if cm.IfIndex != p.loIndex {
log.Tracef("dropping packet received on interface %d instead of %s", cm.IfIndex, loopbackDevice)
return nil
}
dst, ok := netip.AddrFromSlice(cm.Dst.To4())
if !ok || !inRange(dst) {
log.Tracef("dropping packet for unexpected destination %s", cm.Dst)
return nil
}
p.relayedConnMutex.Lock()
conn, ok := p.relayedConnStore[dst]
p.relayedConnMutex.Unlock()
if !ok {
if p.ctx.Err() == nil {
log.Debugf("relayed conn not found by address because conn already has been closed: %s", dst)
}
return nil
}
if _, err := conn.Write(buf[:n]); err != nil {
return fmt.Errorf("forward local WG packet (%s) to remote relayed conn: %w", dst, err)
}
return nil
}
func (p *Proxy) storeRelayedConn(relayedConn net.Conn) (netip.Addr, error) {
p.relayedConnMutex.Lock()
defer p.relayedConnMutex.Unlock()
addr, err := p.addrs.next(func(a netip.Addr) bool {
_, ok := p.relayedConnStore[a]
return ok
})
if err != nil {
return netip.Addr{}, err
}
p.relayedConnStore[addr] = relayedConn
return addr, nil
}
// removeRelayedConn releases an endpoint address. It only removes the entry
// while it still belongs to relayedConn, so a late release cannot take an
// address away from the peer it was handed to next.
func (p *Proxy) removeRelayedConn(addr netip.Addr, relayedConn net.Conn) {
p.relayedConnMutex.Lock()
defer p.relayedConnMutex.Unlock()
if stored, ok := p.relayedConnStore[addr]; !ok || stored != relayedConn {
return
}
log.Debugf("remove relayed conn from store by address: %s", addr)
delete(p.relayedConnStore, addr)
}
client/iface/wgproxy/loopback/proxy_privileged_test.go
+196
View File
@@ -0,0 +1,196 @@
//go:build linux && !android && privileged
package loopback
import (
"context"
"net"
"strconv"
"testing"
"time"
)
const testWGPort = 51862
// relayEnd stands in for a relayed connection: the proxy writes what it read
// from WireGuard into it, and the test reads it back out here.
func relayEnd(t *testing.T) (proxySide net.Conn, testSide *net.UDPConn) {
t.Helper()
testSide, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")})
if err != nil {
t.Fatalf("relay listener: %v", err)
}
t.Cleanup(func() {
if err := testSide.Close(); err != nil {
t.Logf("close relay listener: %v", err)
}
})
proxySide, err = net.Dial("udp", testSide.LocalAddr().String())
if err != nil {
t.Fatalf("relay conn: %v", err)
}
t.Cleanup(func() {
if err := proxySide.Close(); err != nil {
t.Logf("close relay conn: %v", err)
}
})
return proxySide, testSide
}
// TestProxyDemuxesByDestinationAddress is the core of the design: one socket
// serves every peer, and the destination address decides which relayed
// connection a WireGuard packet belongs to.
func TestProxyDemuxesByDestinationAddress(t *testing.T) {
proxy := NewProxy(testWGPort, 1280)
if err := proxy.Listen(); err != nil {
t.Fatalf("listen: %v", err)
}
defer func() {
if err := proxy.Free(); err != nil {
t.Errorf("free proxy: %v", err)
}
}()
const peers = 3
endpoints := make([]*net.UDPAddr, 0, peers)
readers := make([]*net.UDPConn, 0, peers)
for i := 0; i < peers; i++ {
proxySide, testSide := relayEnd(t)
endpoint, _, err := proxy.AddRelayedConn(proxySide)
if err != nil {
t.Fatalf("add relayed conn %d: %v", i, err)
}
if endpoint.Port != proxy.proxyPort {
t.Errorf("peer %d endpoint port = %d, want the shared proxy port %d", i, endpoint.Port, proxy.proxyPort)
}
endpoints = append(endpoints, endpoint)
readers = append(readers, testSide)
}
// every peer must have its own address, otherwise they are indistinguishable
seen := make(map[string]bool, peers)
for i, endpoint := range endpoints {
if seen[endpoint.IP.String()] {
t.Fatalf("peer %d reuses endpoint address %s", i, endpoint.IP)
}
seen[endpoint.IP.String()] = true
}
wgSock, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")})
if err != nil {
t.Fatalf("wg socket: %v", err)
}
defer func() {
if err := wgSock.Close(); err != nil {
t.Logf("close wg socket: %v", err)
}
}()
for i, endpoint := range endpoints {
payload := []byte{byte(i), 'p', 'k', 't'}
if _, err := wgSock.WriteTo(payload, endpoint); err != nil {
t.Fatalf("write to peer %d endpoint %s: %v", i, endpoint, err)
}
buf := make([]byte, 1500)
if err := readers[i].SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
t.Fatalf("set read deadline: %v", err)
}
n, _, err := readers[i].ReadFrom(buf)
if err != nil {
t.Fatalf("peer %d did not receive its packet: %v", i, err)
}
if string(buf[:n]) != string(payload) {
t.Errorf("peer %d got %q, want %q", i, buf[:n], payload)
}
// no other peer may see it
for j, other := range readers {
if j == i {
continue
}
if err := other.SetReadDeadline(time.Now().Add(100 * time.Millisecond)); err != nil {
t.Fatalf("set read deadline: %v", err)
}
if _, _, err := other.ReadFrom(buf); err == nil {
t.Errorf("packet for peer %d also delivered to peer %d", i, j)
}
}
}
}
// TestProxyDropsPacketsOutsideTheRange guards the wildcard bind: anything that
// is not addressed to a handed-out endpoint must not reach a relayed peer.
func TestProxyDropsPacketsOutsideTheRange(t *testing.T) {
proxy := NewProxy(testWGPort+1, 1280)
if err := proxy.Listen(); err != nil {
t.Fatalf("listen: %v", err)
}
defer func() {
if err := proxy.Free(); err != nil {
t.Errorf("free proxy: %v", err)
}
}()
proxySide, testSide := relayEnd(t)
if _, _, err := proxy.AddRelayedConn(proxySide); err != nil {
t.Fatalf("add relayed conn: %v", err)
}
sender, err := net.Dial("udp", net.JoinHostPort("127.0.0.1", strconv.Itoa(proxy.proxyPort)))
if err != nil {
t.Fatalf("sender: %v", err)
}
defer func() {
if err := sender.Close(); err != nil {
t.Logf("close sender: %v", err)
}
}()
if _, err := sender.Write([]byte("stray")); err != nil {
t.Fatalf("write stray packet: %v", err)
}
buf := make([]byte, 1500)
if err := testSide.SetReadDeadline(time.Now().Add(500 * time.Millisecond)); err != nil {
t.Fatalf("set read deadline: %v", err)
}
if _, _, err := testSide.ReadFrom(buf); err == nil {
t.Error("packet addressed to 127.0.0.1 was forwarded to a relayed peer")
}
}
// A wrapper that is closed before it starts forwarding still has to give its
// endpoint address back, otherwise the range leaks an address per attempt.
func TestClosingBeforeWorkReleasesTheAddress(t *testing.T) {
proxy := NewProxy(testWGPort+2, 1280)
if err := proxy.Listen(); err != nil {
t.Fatalf("listen: %v", err)
}
defer func() {
if err := proxy.Free(); err != nil {
t.Errorf("free proxy: %v", err)
}
}()
proxySide, _ := relayEnd(t)
wrapper := NewProxyWrapper(proxy)
if err := wrapper.AddRelayedConn(context.Background(), nil, proxySide); err != nil {
t.Fatalf("add relayed conn: %v", err)
}
if got := len(proxy.relayedConnStore); got != 1 {
t.Fatalf("store holds %d entries after adding one conn, want 1", got)
}
if err := wrapper.CloseConn(); err != nil {
t.Fatalf("close conn: %v", err)
}
if got := len(proxy.relayedConnStore); got != 0 {
t.Errorf("store holds %d entries after close, want 0", got)
}
}
client/iface/wgproxy/ebpf/wrapper.go → client/iface/wgproxy/loopback/wrapper.go
+30 -16
View File
@@ -1,6 +1,6 @@
//go:build linux && !android
package ebpf
package loopback
import (
"context"
@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"net"
"net/netip"
"sync"
"github.com/google/gopacket"
@@ -95,13 +96,14 @@ func NewPacketHeaders(localWGListenPort int, endpoint *net.UDPAddr) (*PacketHead
// ProxyWrapper help to keep the remoteConn instance for net.Conn.Close function call
type ProxyWrapper struct {
wgeBPFProxy *WGEBPFProxy
proxy *Proxy
remoteConn net.Conn
ctx context.Context
cancel context.CancelFunc
wgRelayedEndpointAddr *net.UDPAddr
peerAddr netip.Addr
headers *PacketHeaders
headerCurrentUsed *PacketHeaders
rawConn net.PacketConn
@@ -113,36 +115,44 @@ type ProxyWrapper struct {
closeListener *listener.CloseListener
}
func NewProxyWrapper(proxy *WGEBPFProxy) *ProxyWrapper {
func NewProxyWrapper(proxy *Proxy) *ProxyWrapper {
return &ProxyWrapper{
wgeBPFProxy: proxy,
proxy: proxy,
pausedCond: sync.NewCond(&sync.Mutex{}),
closeListener: listener.NewCloseListener(),
}
}
func (p *ProxyWrapper) AddRelayedConn(ctx context.Context, _ *net.UDPAddr, remoteConn net.Conn) error {
addr, err := p.wgeBPFProxy.AddRelayedConn(remoteConn)
addr, peerAddr, err := p.proxy.AddRelayedConn(remoteConn)
if err != nil {
return fmt.Errorf("add relayed conn: %w", err)
}
headers, err := NewPacketHeaders(p.wgeBPFProxy.localWGListenPort, addr)
// the endpoint address is otherwise only released by the forwarding
// goroutine, which never starts when the setup below fails
release := func() { p.proxy.removeRelayedConn(peerAddr, remoteConn) }
headers, err := NewPacketHeaders(p.proxy.localWGListenPort, addr)
if err != nil {
release()
return fmt.Errorf("create packet sender: %w", err)
}
// Check if required raw connection is available
if !headers.isIPv4 && p.wgeBPFProxy.rawConnIPv6 == nil {
if !headers.isIPv4 && p.proxy.rawConnIPv6 == nil {
release()
return errIPv6ConnNotAvailable
}
if headers.isIPv4 && p.wgeBPFProxy.rawConnIPv4 == nil {
if headers.isIPv4 && p.proxy.rawConnIPv4 == nil {
release()
return errIPv4ConnNotAvailable
}
p.remoteConn = remoteConn
p.ctx, p.cancel = context.WithCancel(ctx)
p.wgRelayedEndpointAddr = addr
p.peerAddr = peerAddr
p.headers = headers
p.rawConn = p.selectRawConn(headers)
return nil
@@ -193,18 +203,18 @@ func (p *ProxyWrapper) RedirectAs(endpoint *net.UDPAddr) {
return
}
header, err := NewPacketHeaders(p.wgeBPFProxy.localWGListenPort, endpoint)
header, err := NewPacketHeaders(p.proxy.localWGListenPort, endpoint)
if err != nil {
log.Errorf("failed to create packet headers: %s", err)
return
}
// Check if required raw connection is available
if !header.isIPv4 && p.wgeBPFProxy.rawConnIPv6 == nil {
if !header.isIPv4 && p.proxy.rawConnIPv6 == nil {
log.Error(errIPv6ConnNotAvailable)
return
}
if header.isIPv4 && p.wgeBPFProxy.rawConnIPv4 == nil {
if header.isIPv4 && p.proxy.rawConnIPv4 == nil {
log.Error(errIPv4ConnNotAvailable)
return
}
@@ -240,6 +250,10 @@ func (p *ProxyWrapper) CloseConn() error {
p.closeListener.SetCloseListener(nil)
// releases the endpoint address for a wrapper that was never started, and
// is a no-op once the forwarding goroutine has released it
p.proxy.removeRelayedConn(p.peerAddr, p.remoteConn)
p.pausedCond.L.Lock()
p.paused = false
p.pausedCond.Signal()
@@ -252,9 +266,9 @@ func (p *ProxyWrapper) CloseConn() error {
}
func (p *ProxyWrapper) proxyToLocal(ctx context.Context) {
defer p.wgeBPFProxy.removeRelayedConn(uint16(p.wgRelayedEndpointAddr.Port))
defer p.proxy.removeRelayedConn(p.peerAddr, p.remoteConn)
buf := make([]byte, p.wgeBPFProxy.mtu+bufsize.WGBufferOverhead)
buf := make([]byte, p.proxy.mtu+bufsize.WGBufferOverhead)
for {
n, err := p.readFromRemote(ctx, buf)
if err != nil {
@@ -286,7 +300,7 @@ func (p *ProxyWrapper) readFromRemote(ctx context.Context, buf []byte) (int, err
}
p.closeListener.Notify()
if !errors.Is(err, io.EOF) {
log.Errorf("failed to read from relayed conn (endpoint: :%d): %s", p.wgRelayedEndpointAddr.Port, err)
log.Errorf("failed to read from relayed conn (endpoint: %s): %s", p.wgRelayedEndpointAddr, err)
}
return 0, err
}
@@ -314,7 +328,7 @@ func (p *ProxyWrapper) sendPkg(data []byte, header *PacketHeaders) error {
func (p *ProxyWrapper) selectRawConn(header *PacketHeaders) net.PacketConn {
if header.isIPv4 {
return p.wgeBPFProxy.rawConnIPv4
return p.proxy.rawConnIPv4
}
return p.wgeBPFProxy.rawConnIPv6
return p.proxy.rawConnIPv6
}
client/iface/wgproxy/proxy_linux_test.go
+17 -17
View File
@@ -9,25 +9,25 @@ import (
"github.com/netbirdio/netbird/client/iface/bind"
"github.com/netbirdio/netbird/client/iface/wgaddr"
bindproxy "github.com/netbirdio/netbird/client/iface/wgproxy/bind"
"github.com/netbirdio/netbird/client/iface/wgproxy/ebpf"
"github.com/netbirdio/netbird/client/iface/wgproxy/loopback"
"github.com/netbirdio/netbird/client/iface/wgproxy/udp"
)
func seedProxies() ([]proxyInstance, error) {
pl := make([]proxyInstance, 0)
ebpfProxy := ebpf.NewWGEBPFProxy(51831, 1280)
if err := ebpfProxy.Listen(); err != nil {
return nil, fmt.Errorf("failed to initialize ebpf proxy: %s", err)
loopbackProxy := loopback.NewProxy(51831, 1280)
if err := loopbackProxy.Listen(); err != nil {
return nil, fmt.Errorf("failed to initialize loopback proxy: %s", err)
}
pEbpf := proxyInstance{
name: "ebpf kernel proxy",
proxy: ebpf.NewProxyWrapper(ebpfProxy),
pLoopback := proxyInstance{
name: "loopback kernel proxy",
proxy: loopback.NewProxyWrapper(loopbackProxy),
wgPort: 51831,
closeFn: ebpfProxy.Free,
closeFn: loopbackProxy.Free,
}
pl = append(pl, pEbpf)
pl = append(pl, pLoopback)
pUDP := proxyInstance{
name: "udp kernel proxy",
@@ -42,18 +42,18 @@ func seedProxies() ([]proxyInstance, error) {
func seedProxyForProxyCloseByRemoteConn() ([]proxyInstance, error) {
pl := make([]proxyInstance, 0)
ebpfProxy := ebpf.NewWGEBPFProxy(51831, 1280)
if err := ebpfProxy.Listen(); err != nil {
return nil, fmt.Errorf("failed to initialize ebpf proxy: %s", err)
loopbackProxy := loopback.NewProxy(51831, 1280)
if err := loopbackProxy.Listen(); err != nil {
return nil, fmt.Errorf("failed to initialize loopback proxy: %s", err)
}
pEbpf := proxyInstance{
name: "ebpf kernel proxy",
proxy: ebpf.NewProxyWrapper(ebpfProxy),
pLoopback := proxyInstance{
name: "loopback kernel proxy",
proxy: loopback.NewProxyWrapper(loopbackProxy),
wgPort: 51831,
closeFn: ebpfProxy.Free,
closeFn: loopbackProxy.Free,
}
pl = append(pl, pEbpf)
pl = append(pl, pLoopback)
pUDP := proxyInstance{
name: "udp kernel proxy",
client/iface/wgproxy/redirect_test.go
+23 -23
View File
@@ -8,7 +8,7 @@ import (
"testing"
"time"
"github.com/netbirdio/netbird/client/iface/wgproxy/ebpf"
"github.com/netbirdio/netbird/client/iface/wgproxy/loopback"
"github.com/netbirdio/netbird/client/iface/wgproxy/udp"
)
@@ -198,20 +198,20 @@ func testRedirectAs(t *testing.T, proxy Proxy, wgPort int, nbAddr, p2pEndpoint *
}
}
// TestRedirectAs_eBPF_IPv4 tests RedirectAs with eBPF proxy using IPv4 addresses
func TestRedirectAs_eBPF_IPv4(t *testing.T) {
// TestRedirectAs_Loopback_IPv4 tests RedirectAs with the loopback proxy using IPv4 addresses
func TestRedirectAs_Loopback_IPv4(t *testing.T) {
wgPort := 51850
ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280)
if err := ebpfProxy.Listen(); err != nil {
t.Fatalf("failed to initialize ebpf proxy: %v", err)
loopbackProxy := loopback.NewProxy(wgPort, 1280)
if err := loopbackProxy.Listen(); err != nil {
t.Fatalf("failed to initialize loopback proxy: %v", err)
}
defer func() {
if err := ebpfProxy.Free(); err != nil {
t.Errorf("failed to free ebpf proxy: %v", err)
if err := loopbackProxy.Free(); err != nil {
t.Errorf("failed to free loopback proxy: %v", err)
}
}()
proxy := ebpf.NewProxyWrapper(ebpfProxy)
proxy := loopback.NewProxyWrapper(loopbackProxy)
// NetBird UDP address of the remote peer
nbAddr := &net.UDPAddr{
@@ -227,20 +227,20 @@ func TestRedirectAs_eBPF_IPv4(t *testing.T) {
testRedirectAs(t, proxy, wgPort, nbAddr, p2pEndpoint)
}
// TestRedirectAs_eBPF_IPv6 tests RedirectAs with eBPF proxy using IPv6 addresses
func TestRedirectAs_eBPF_IPv6(t *testing.T) {
// TestRedirectAs_Loopback_IPv6 tests RedirectAs with the loopback proxy using IPv6 addresses
func TestRedirectAs_Loopback_IPv6(t *testing.T) {
wgPort := 51851
ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280)
if err := ebpfProxy.Listen(); err != nil {
t.Fatalf("failed to initialize ebpf proxy: %v", err)
loopbackProxy := loopback.NewProxy(wgPort, 1280)
if err := loopbackProxy.Listen(); err != nil {
t.Fatalf("failed to initialize loopback proxy: %v", err)
}
defer func() {
if err := ebpfProxy.Free(); err != nil {
t.Errorf("failed to free ebpf proxy: %v", err)
if err := loopbackProxy.Free(); err != nil {
t.Errorf("failed to free loopback proxy: %v", err)
}
}()
proxy := ebpf.NewProxyWrapper(ebpfProxy)
proxy := loopback.NewProxyWrapper(loopbackProxy)
// NetBird UDP address of the remote peer
nbAddr := &net.UDPAddr{
@@ -259,17 +259,17 @@ func TestRedirectAs_eBPF_IPv6(t *testing.T) {
// TestRedirectAs_Multiple_Switches tests switching between multiple endpoints
func TestRedirectAs_Multiple_Switches(t *testing.T) {
wgPort := 51856
ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280)
if err := ebpfProxy.Listen(); err != nil {
t.Fatalf("failed to initialize ebpf proxy: %v", err)
loopbackProxy := loopback.NewProxy(wgPort, 1280)
if err := loopbackProxy.Listen(); err != nil {
t.Fatalf("failed to initialize loopback proxy: %v", err)
}
defer func() {
if err := ebpfProxy.Free(); err != nil {
t.Errorf("failed to free ebpf proxy: %v", err)
if err := loopbackProxy.Free(); err != nil {
t.Errorf("failed to free loopback proxy: %v", err)
}
}()
proxy := ebpf.NewProxyWrapper(ebpfProxy)
proxy := loopback.NewProxyWrapper(loopbackProxy)
ctx := context.Background()
client/internal/ebpf/ebpf/bpf_bpfeb.go
-148
View File
@@ -1,148 +0,0 @@
// Code generated by bpf2go; DO NOT EDIT.
//go:build mips || mips64 || ppc64 || s390x
package ebpf
import (
"bytes"
_ "embed"
"fmt"
"io"
"github.com/cilium/ebpf"
)
// loadBpf returns the embedded CollectionSpec for bpf.
func loadBpf() (*ebpf.CollectionSpec, error) {
reader := bytes.NewReader(_BpfBytes)
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
if err != nil {
return nil, fmt.Errorf("can't load bpf: %w", err)
}
return spec, err
}
// loadBpfObjects loads bpf and converts it into a struct.
//
// The following types are suitable as obj argument:
//
// *bpfObjects
// *bpfPrograms
// *bpfMaps
//
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
spec, err := loadBpf()
if err != nil {
return err
}
return spec.LoadAndAssign(obj, opts)
}
// bpfSpecs contains maps and programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfSpecs struct {
bpfProgramSpecs
bpfMapSpecs
bpfVariableSpecs
}
// bpfProgramSpecs contains programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfProgramSpecs struct {
NbXdpProg *ebpf.ProgramSpec `ebpf:"nb_xdp_prog"`
}
// bpfMapSpecs contains maps before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfMapSpecs struct {
NbFeatures *ebpf.MapSpec `ebpf:"nb_features"`
NbWgProxySettingsMap *ebpf.MapSpec `ebpf:"nb_wg_proxy_settings_map"`
}
// bpfVariableSpecs contains global variables before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfVariableSpecs struct {
FlagFeatureWgProxy *ebpf.VariableSpec `ebpf:"flag_feature_wg_proxy"`
MapKeyFeatures *ebpf.VariableSpec `ebpf:"map_key_features"`
MapKeyProxyPort *ebpf.VariableSpec `ebpf:"map_key_proxy_port"`
MapKeyWgPort *ebpf.VariableSpec `ebpf:"map_key_wg_port"`
ProxyPort *ebpf.VariableSpec `ebpf:"proxy_port"`
WgPort *ebpf.VariableSpec `ebpf:"wg_port"`
}
// bpfObjects contains all objects after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfObjects struct {
bpfPrograms
bpfMaps
bpfVariables
}
func (o *bpfObjects) Close() error {
return _BpfClose(
&o.bpfPrograms,
&o.bpfMaps,
)
}
// bpfMaps contains all maps after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfMaps struct {
NbFeatures *ebpf.Map `ebpf:"nb_features"`
NbWgProxySettingsMap *ebpf.Map `ebpf:"nb_wg_proxy_settings_map"`
}
func (m *bpfMaps) Close() error {
return _BpfClose(
m.NbFeatures,
m.NbWgProxySettingsMap,
)
}
// bpfVariables contains all global variables after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfVariables struct {
FlagFeatureWgProxy *ebpf.Variable `ebpf:"flag_feature_wg_proxy"`
MapKeyFeatures *ebpf.Variable `ebpf:"map_key_features"`
MapKeyProxyPort *ebpf.Variable `ebpf:"map_key_proxy_port"`
MapKeyWgPort *ebpf.Variable `ebpf:"map_key_wg_port"`
ProxyPort *ebpf.Variable `ebpf:"proxy_port"`
WgPort *ebpf.Variable `ebpf:"wg_port"`
}
// bpfPrograms contains all programs after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfPrograms struct {
NbXdpProg *ebpf.Program `ebpf:"nb_xdp_prog"`
}
func (p *bpfPrograms) Close() error {
return _BpfClose(
p.NbXdpProg,
)
}
func _BpfClose(closers ...io.Closer) error {
for _, closer := range closers {
if err := closer.Close(); err != nil {
return err
}
}
return nil
}
// Do not access this directly.
//
//go:embed bpf_bpfeb.o
var _BpfBytes []byte
client/internal/ebpf/ebpf/bpf_bpfeb.o
BIN
View File
Binary file not shown.
client/internal/ebpf/ebpf/bpf_bpfel.go
-148
View File
@@ -1,148 +0,0 @@
// Code generated by bpf2go; DO NOT EDIT.
//go:build 386 || amd64 || arm || arm64 || loong64 || mips64le || mipsle || ppc64le || riscv64 || wasm
package ebpf
import (
"bytes"
_ "embed"
"fmt"
"io"
"github.com/cilium/ebpf"
)
// loadBpf returns the embedded CollectionSpec for bpf.
func loadBpf() (*ebpf.CollectionSpec, error) {
reader := bytes.NewReader(_BpfBytes)
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
if err != nil {
return nil, fmt.Errorf("can't load bpf: %w", err)
}
return spec, err
}
// loadBpfObjects loads bpf and converts it into a struct.
//
// The following types are suitable as obj argument:
//
// *bpfObjects
// *bpfPrograms
// *bpfMaps
//
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
spec, err := loadBpf()
if err != nil {
return err
}
return spec.LoadAndAssign(obj, opts)
}
// bpfSpecs contains maps and programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfSpecs struct {
bpfProgramSpecs
bpfMapSpecs
bpfVariableSpecs
}
// bpfProgramSpecs contains programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfProgramSpecs struct {
NbXdpProg *ebpf.ProgramSpec `ebpf:"nb_xdp_prog"`
}
// bpfMapSpecs contains maps before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfMapSpecs struct {
NbFeatures *ebpf.MapSpec `ebpf:"nb_features"`
NbWgProxySettingsMap *ebpf.MapSpec `ebpf:"nb_wg_proxy_settings_map"`
}
// bpfVariableSpecs contains global variables before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfVariableSpecs struct {
FlagFeatureWgProxy *ebpf.VariableSpec `ebpf:"flag_feature_wg_proxy"`
MapKeyFeatures *ebpf.VariableSpec `ebpf:"map_key_features"`
MapKeyProxyPort *ebpf.VariableSpec `ebpf:"map_key_proxy_port"`
MapKeyWgPort *ebpf.VariableSpec `ebpf:"map_key_wg_port"`
ProxyPort *ebpf.VariableSpec `ebpf:"proxy_port"`
WgPort *ebpf.VariableSpec `ebpf:"wg_port"`
}
// bpfObjects contains all objects after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfObjects struct {
bpfPrograms
bpfMaps
bpfVariables
}
func (o *bpfObjects) Close() error {
return _BpfClose(
&o.bpfPrograms,
&o.bpfMaps,
)
}
// bpfMaps contains all maps after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfMaps struct {
NbFeatures *ebpf.Map `ebpf:"nb_features"`
NbWgProxySettingsMap *ebpf.Map `ebpf:"nb_wg_proxy_settings_map"`
}
func (m *bpfMaps) Close() error {
return _BpfClose(
m.NbFeatures,
m.NbWgProxySettingsMap,
)
}
// bpfVariables contains all global variables after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfVariables struct {
FlagFeatureWgProxy *ebpf.Variable `ebpf:"flag_feature_wg_proxy"`
MapKeyFeatures *ebpf.Variable `ebpf:"map_key_features"`
MapKeyProxyPort *ebpf.Variable `ebpf:"map_key_proxy_port"`
MapKeyWgPort *ebpf.Variable `ebpf:"map_key_wg_port"`
ProxyPort *ebpf.Variable `ebpf:"proxy_port"`
WgPort *ebpf.Variable `ebpf:"wg_port"`
}
// bpfPrograms contains all programs after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfPrograms struct {
NbXdpProg *ebpf.Program `ebpf:"nb_xdp_prog"`
}
func (p *bpfPrograms) Close() error {
return _BpfClose(
p.NbXdpProg,
)
}
func _BpfClose(closers ...io.Closer) error {
for _, closer := range closers {
if err := closer.Close(); err != nil {
return err
}
}
return nil
}
// Do not access this directly.
//
//go:embed bpf_bpfel.o
var _BpfBytes []byte
client/internal/ebpf/ebpf/bpf_bpfel.o
BIN
View File
Binary file not shown.
client/internal/ebpf/ebpf/manager_linux.go
-115
View File
@@ -1,115 +0,0 @@
package ebpf
import (
_ "embed"
"net"
"sync"
"github.com/cilium/ebpf/link"
"github.com/cilium/ebpf/rlimit"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal/ebpf/manager"
)
const (
mapKeyFeatures uint32 = 0
featureFlagWGProxy = 0b00000001
)
var (
singleton manager.Manager
singletonLock = &sync.Mutex{}
)
// required packages libbpf-dev, libc6-dev-i386-amd64-cross
// GeneralManager is used to load multiple eBPF programs with a custom check (if then) done in prog.c
// The manager simply adds a feature (byte) of each program to a map that is shared between the userspace and kernel.
// When packet arrives, the C code checks for each feature (if it is set) and executes each enabled program (e.g., wg_proxy.c).
//
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang-14 bpf src/prog.c -- -I /usr/x86_64-linux-gnu/include -include src/bpf_map_def.h
type GeneralManager struct {
lock sync.Mutex
link link.Link
featureFlags uint16
bpfObjs bpfObjects
}
// GetEbpfManagerInstance return a static eBpf Manager instance
func GetEbpfManagerInstance() manager.Manager {
singletonLock.Lock()
defer singletonLock.Unlock()
if singleton != nil {
return singleton
}
singleton = &GeneralManager{}
return singleton
}
func (tf *GeneralManager) setFeatureFlag(feature uint16) {
tf.featureFlags |= feature
}
func (tf *GeneralManager) loadXdp() error {
if tf.link != nil {
return nil
}
// it required for Docker
err := rlimit.RemoveMemlock()
if err != nil {
return err
}
iFace, err := net.InterfaceByName("lo")
if err != nil {
return err
}
// load pre-compiled programs into the kernel.
err = loadBpfObjects(&tf.bpfObjs, nil)
if err != nil {
return err
}
tf.link, err = link.AttachXDP(link.XDPOptions{
Program: tf.bpfObjs.NbXdpProg,
Interface: iFace.Index,
})
if err != nil {
_ = tf.bpfObjs.Close()
tf.link = nil
return err
}
return nil
}
func (tf *GeneralManager) unsetFeatureFlag(feature uint16) error {
tf.lock.Lock()
defer tf.lock.Unlock()
tf.featureFlags &^= feature
if tf.link == nil {
return nil
}
if tf.featureFlags == 0 {
return tf.close()
}
return tf.bpfObjs.NbFeatures.Put(mapKeyFeatures, tf.featureFlags)
}
func (tf *GeneralManager) close() error {
log.Debugf("detach ebpf program ")
err := tf.bpfObjs.Close()
if err != nil {
log.Warnf("failed to close eBpf objects: %s", err)
}
err = tf.link.Close()
tf.link = nil
return err
}
client/internal/ebpf/ebpf/manager_linux_test.go
-31
View File
@@ -1,31 +0,0 @@
package ebpf
import (
"testing"
)
func TestManager_setFeatureFlag(t *testing.T) {
mgr := GeneralManager{}
mgr.setFeatureFlag(featureFlagWGProxy)
if mgr.featureFlags != featureFlagWGProxy {
t.Errorf("invalid feature state")
}
mgr.setFeatureFlag(featureFlagWGProxy)
if mgr.featureFlags != featureFlagWGProxy {
t.Errorf("setting a flag twice must be idempotent, got: %d", mgr.featureFlags)
}
}
func TestManager_unsetFeatureFlag(t *testing.T) {
mgr := GeneralManager{}
mgr.setFeatureFlag(featureFlagWGProxy)
err := mgr.unsetFeatureFlag(featureFlagWGProxy)
if err != nil {
t.Errorf("unexpected error: %s", err)
}
if mgr.featureFlags != 0 {
t.Errorf("invalid feature state, expected: %d, got: %d", 0, mgr.featureFlags)
}
}
client/internal/ebpf/ebpf/src/bpf_map_def.h
-16
View File
@@ -1,16 +0,0 @@
// libbpf 1.0 removed struct bpf_map_def, but the programs here keep the legacy
// map definitions: they load on kernels built without BTF, which BTF-style
// (SEC(".maps")) definitions do not. Define the struct ourselves so the
// programs compile against current libbpf headers.
#ifndef NB_BPF_MAP_DEF_H
#define NB_BPF_MAP_DEF_H
struct bpf_map_def {
unsigned int type;
unsigned int key_size;
unsigned int value_size;
unsigned int max_entries;
unsigned int map_flags;
};
#endif
client/internal/ebpf/ebpf/src/prog.c
-54
View File
@@ -1,54 +0,0 @@
#include <stdbool.h>
#include <linux/if_ether.h> // ETH_P_IP
#include <linux/udp.h>
#include <linux/ip.h>
#include <netinet/in.h>
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include "wg_proxy.c"
const __u16 flag_feature_wg_proxy = 0b01;
const __u32 map_key_features = 0;
struct bpf_map_def SEC("maps") nb_features = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(__u32),
.value_size = sizeof(__u16),
.max_entries = 10,
};
SEC("xdp")
int nb_xdp_prog(struct xdp_md *ctx) {
__u16 *features;
features = bpf_map_lookup_elem(&nb_features, &map_key_features);
if (!features) {
return XDP_PASS;
}
void *data = (void *)(long)ctx->data;
void *data_end = (void *)(long)ctx->data_end;
struct ethhdr *eth = data;
struct iphdr *ip = (data + sizeof(struct ethhdr));
struct udphdr *udp = (data + sizeof(struct ethhdr) + sizeof(struct iphdr));
// return early if not enough data
if (data + sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct udphdr) > data_end){
return XDP_PASS;
}
// skip non IPv4 packages
if (eth->h_proto != htons(ETH_P_IP)) {
return XDP_PASS;
}
// skip non UPD packages
if (ip->protocol != IPPROTO_UDP) {
return XDP_PASS;
}
if (*features & flag_feature_wg_proxy) {
xdp_wg_proxy(ip, udp);
}
return XDP_PASS;
}
char _license[] SEC("license") = "GPL";
client/internal/ebpf/ebpf/src/readme.md
-27
View File
@@ -1,27 +0,0 @@
# XDP programs
`prog.c` is attached to the `lo` device and dispatches to the features enabled in the
`nb_features` map. The only feature is the WireGuard proxy (`wg_proxy.c`): it rewrites
loopback UDP sent from the WireGuard listen port so it reaches the userspace relay proxy
port instead, and swaps the peer endpoint port into the source so the proxy can tell
peers apart.
Maps use the legacy `struct bpf_map_def` form, defined in `bpf_map_def.h` because libbpf
1.0 removed it. They load on kernels built without BTF, which BTF-style (`SEC(".maps")`)
definitions do not.
Regenerate the objects with `go generate ./client/internal/ebpf/ebpf/`; it needs
`clang-14`. Loading a regenerated object needs root, attaching it needs `bpf_link`
(kernel >= 5.7), and only one XDP program can own `lo` at a time.
# Debug
The CONFIG_BPF_EVENTS kernel module is required for bpf_printk.
Apply this code to use bpf_printk
```
#define bpf_printk(fmt, ...) \
({ \
char ____fmt[] = fmt; \
bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
})
```
client/internal/ebpf/ebpf/src/wg_proxy.c
-60
View File
@@ -1,60 +0,0 @@
const __u32 map_key_proxy_port = 0;
const __u32 map_key_wg_port = 1;
struct bpf_map_def SEC("maps") nb_wg_proxy_settings_map = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(__u32),
.value_size = sizeof(__u16),
.max_entries = 10,
};
__u16 proxy_port = 0;
__u16 wg_port = 0;
bool read_port_settings() {
__u16 *value;
value = bpf_map_lookup_elem(&nb_wg_proxy_settings_map, &map_key_proxy_port);
if (!value) {
return false;
}
proxy_port = *value;
value = bpf_map_lookup_elem(&nb_wg_proxy_settings_map, &map_key_wg_port);
if (!value) {
return false;
}
wg_port = htons(*value);
return true;
}
int xdp_wg_proxy(struct iphdr *ip, struct udphdr *udp) {
if (proxy_port == 0 || wg_port == 0) {
if (!read_port_settings()){
return XDP_PASS;
}
// bpf_printk("proxy port: %d, wg port: %d", proxy_port, wg_port);
}
// 2130706433 = 127.0.0.1
if (ip->daddr != htonl(2130706433)) {
return XDP_PASS;
}
if (udp->source != wg_port){
return XDP_PASS;
}
__be16 new_src_port = udp->dest;
__be16 new_dst_port = htons(proxy_port);
udp->dest = new_dst_port;
udp->source = new_src_port;
// The ports are covered by the UDP checksum. This is an IPv4 loopback hop
// and the payload is already integrity-protected, so clear the checksum (a
// zero UDP checksum means "not computed" for IPv4) rather than leave a
// stale value the kernel would drop as UDP_CSUM.
udp->check = 0;
return XDP_PASS;
}
client/internal/ebpf/ebpf/wg_proxy_linux.go
-41
View File
@@ -1,41 +0,0 @@
package ebpf
import log "github.com/sirupsen/logrus"
const (
mapKeyProxyPort uint32 = 0
mapKeyWgPort uint32 = 1
)
func (tf *GeneralManager) LoadWgProxy(proxyPort, wgPort int) error {
log.Debugf("load ebpf WG proxy")
tf.lock.Lock()
defer tf.lock.Unlock()
err := tf.loadXdp()
if err != nil {
return err
}
err = tf.bpfObjs.NbWgProxySettingsMap.Put(mapKeyProxyPort, uint16(proxyPort))
if err != nil {
return err
}
err = tf.bpfObjs.NbWgProxySettingsMap.Put(mapKeyWgPort, uint16(wgPort))
if err != nil {
return err
}
tf.setFeatureFlag(featureFlagWGProxy)
err = tf.bpfObjs.NbFeatures.Put(mapKeyFeatures, tf.featureFlags)
if err != nil {
return err
}
return nil
}
func (tf *GeneralManager) FreeWGProxy() error {
log.Debugf("free ebpf WG proxy")
return tf.unsetFeatureFlag(featureFlagWGProxy)
}
client/internal/ebpf/instantiater_linux.go
-15
View File
@@ -1,15 +0,0 @@
//go:build !android
package ebpf
import (
"github.com/netbirdio/netbird/client/internal/ebpf/ebpf"
"github.com/netbirdio/netbird/client/internal/ebpf/manager"
)
// GetEbpfManagerInstance is a wrapper function. This encapsulation is required because if the code import the internal
// ebpf package the Go compiler will include the object files. But it is not supported on Android. It can cause instant
// panic on older Android version.
func GetEbpfManagerInstance() manager.Manager {
return ebpf.GetEbpfManagerInstance()
}
client/internal/ebpf/instantiater_nonlinux.go
-10
View File
@@ -1,10 +0,0 @@
//go:build !linux || android
package ebpf
import "github.com/netbirdio/netbird/client/internal/ebpf/manager"
// GetEbpfManagerInstance return error because ebpf is not supported on all os
func GetEbpfManagerInstance() manager.Manager {
panic("unsupported os")
}
client/internal/ebpf/manager/manager.go
-7
View File
@@ -1,7 +0,0 @@
package manager
// Manager is used to load multiple eBPF programs. E.g., the WireGuard proxy
type Manager interface {
LoadWgProxy(proxyPort, wgPort int) error
FreeWGProxy() error
}
client/internal/engine.go
-21
View File
@@ -661,10 +661,6 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL)
}
e.wgDevice.Store(e.wgInterface.GetWGDevice())
// Set up notrack rules immediately after proxy is listening to prevent
// conntrack entries from being created before the rules are in place
e.setupWGProxyNoTrack()
// Start after interface is up since port may have been resolved from 0 or changed if occupied
e.shutdownWg.Add(1)
go func() {
@@ -802,23 +798,6 @@ func (e *Engine) initFirewall() error {
return nil
}
// setupWGProxyNoTrack configures connection tracking exclusion for WireGuard proxy traffic.
// This prevents conntrack/MASQUERADE from affecting loopback traffic between WireGuard and the eBPF proxy.
func (e *Engine) setupWGProxyNoTrack() {
if e.firewall == nil {
return
}
proxyPort := e.wgInterface.GetProxyPort()
if proxyPort == 0 {
return
}
if err := e.firewall.SetupEBPFProxyNoTrack(proxyPort, uint16(e.config.WgPort)); err != nil {
log.Warnf("failed to setup ebpf proxy notrack: %v", err)
}
}
func (e *Engine) blockLanAccess() {
if e.config.BlockInbound {
// no need to set up extra deny rules if inbound is already blocked in general
client/internal/engine_test.go
-7
View File
@@ -65,7 +65,6 @@ type MockWGIface struct {
GetStatsFunc func() (map[string]configurer.WGStats, error)
GetInterfaceGUIDStringFunc func() (string, error)
GetProxyFunc func() wgproxy.Proxy
GetProxyPortFunc func() uint16
GetNetFunc func() *netstack.Net
LastActivitiesFunc func() map[string]monotime.Time
}
@@ -162,12 +161,6 @@ func (m *MockWGIface) GetProxy() wgproxy.Proxy {
return m.GetProxyFunc()
}
func (m *MockWGIface) GetProxyPort() uint16 {
if m.GetProxyPortFunc != nil {
return m.GetProxyPortFunc()
}
return 0
}
func (m *MockWGIface) GetNet() *netstack.Net {
return m.GetNetFunc()
client/internal/iface_common.go
-1
View File
@@ -28,7 +28,6 @@ type wgIfaceBase interface {
Up() (*udpmux.UniversalUDPMuxDefault, error)
UpdateAddr(newAddr wgaddr.Address) error
GetProxy() wgproxy.Proxy
GetProxyPort() uint16
UpdatePeer(peerKey string, allowedIps []netip.Prefix, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
RemoveEndpointAddress(key string) error
RemovePeer(peerKey string) error
docs/testing-privileged.md
+1 -2
View File
@@ -1,7 +1,7 @@
# Privileged tests
Some tests in this repo need `root` or mutate host network state: they create
TUN/WireGuard interfaces, open netlink/raw sockets, run eBPF programs, or shell
TUN/WireGuard interfaces, open netlink/raw sockets, or shell
out to `ip`/`iptables`/`nft`/`ifconfig`/`route`. Running them on a developer
machine would require `sudo` and could leave stray interfaces or routes behind.
@@ -44,7 +44,6 @@ A test is privileged if it does any of:
- creates a real interface via `iface.NewWGIFace(...).Create()`,
- opens a netlink or raw socket that hard-fails without `CAP_NET_ADMIN`,
- runs an eBPF program (`ebpf.*.Listen()`),
- shells out to `ip`, `iptables`, `nft`, `ifconfig`, or `route` to change state.
Add the tag to the **top** of the file, combined with any existing platform
go.mod
+1 -1
View File
@@ -42,7 +42,6 @@ require (
github.com/c-robinson/iplib v1.0.3
github.com/caarlos0/env/v11 v11.4.1
github.com/caddyserver/certmagic v0.21.3
github.com/cilium/ebpf v0.19.0
github.com/coder/websocket v1.8.14
github.com/coreos/go-iptables v0.7.0
github.com/coreos/go-oidc/v3 v3.18.0
@@ -190,6 +189,7 @@ require (
github.com/caddyserver/zerossl v0.1.3 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cilium/ebpf v0.19.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect
go.sum
-2
View File
@@ -407,8 +407,6 @@ github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbd
github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM=
github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=