mirror of
https://github.com/netbirdio/netbird.git
synced 2026-08-25 17:11:29 +02:00
Replace the eBPF WireGuard proxy with loopback endpoint addressing
This commit is contained in:
@@ -524,30 +524,33 @@ const (
|
||||
tableRaw = "raw"
|
||||
)
|
||||
|
||||
// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic.
|
||||
// SetupWGProxyNoTrack creates notrack rules for WireGuard proxy loopback traffic.
|
||||
// This prevents conntrack from tracking WireGuard proxy traffic on loopback, which
|
||||
// can interfere with MASQUERADE rules (e.g., from container runtimes like Podman/netavark).
|
||||
//
|
||||
// Every relayed peer has its own loopback endpoint address, so the rules match the
|
||||
// whole 127.0.0.0/8 range.
|
||||
//
|
||||
// Traffic flows that need NOTRACK:
|
||||
//
|
||||
// 1. Egress: WireGuard -> fake endpoint (before eBPF rewrite)
|
||||
// src=127.0.0.1:wgPort -> dst=127.0.0.1:fakePort
|
||||
// 1. Egress: WireGuard -> peer endpoint
|
||||
// src=127.0.0.1:wgPort -> dst=127.x.x.x:proxyPort
|
||||
// Matched by: sport=wgPort
|
||||
//
|
||||
// 2. Egress: Proxy -> WireGuard (via raw socket)
|
||||
// src=127.0.0.1:fakePort -> dst=127.0.0.1:wgPort
|
||||
// src=127.x.x.x:proxyPort -> dst=127.0.0.1:wgPort
|
||||
// Matched by: dport=wgPort
|
||||
//
|
||||
// 3. Ingress: Packets to WireGuard
|
||||
// dst=127.0.0.1:wgPort
|
||||
// Matched by: dport=wgPort
|
||||
//
|
||||
// 4. Ingress: Packets to proxy (after eBPF rewrite)
|
||||
// dst=127.0.0.1:proxyPort
|
||||
// 4. Ingress: Packets to the proxy
|
||||
// dst=127.x.x.x:proxyPort
|
||||
// Matched by: dport=proxyPort
|
||||
//
|
||||
// Rules are cleaned up when the firewall manager is closed.
|
||||
func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
func (m *Manager) SetupWGProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
|
||||
@@ -559,28 +562,28 @@ func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
proxyPortStr := fmt.Sprintf("%d", proxyPort)
|
||||
|
||||
// Egress rules: match outgoing loopback UDP packets
|
||||
outputRuleSport := []string{"-o", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--sport", wgPortStr, "-j", "NOTRACK"}
|
||||
outputRuleSport := []string{"-o", "lo", "-s", "127.0.0.0/8", "-d", "127.0.0.0/8", "-p", "udp", "--sport", wgPortStr, "-j", "NOTRACK"}
|
||||
if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, outputRuleSport...); err != nil {
|
||||
return fmt.Errorf("add output sport notrack rule: %w", err)
|
||||
}
|
||||
|
||||
outputRuleDport := []string{"-o", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"}
|
||||
outputRuleDport := []string{"-o", "lo", "-s", "127.0.0.0/8", "-d", "127.0.0.0/8", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"}
|
||||
if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, outputRuleDport...); err != nil {
|
||||
return fmt.Errorf("add output dport notrack rule: %w", err)
|
||||
}
|
||||
|
||||
// Ingress rules: match incoming loopback UDP packets
|
||||
preroutingRuleWg := []string{"-i", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"}
|
||||
preroutingRuleWg := []string{"-i", "lo", "-s", "127.0.0.0/8", "-d", "127.0.0.0/8", "-p", "udp", "--dport", wgPortStr, "-j", "NOTRACK"}
|
||||
if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, preroutingRuleWg...); err != nil {
|
||||
return fmt.Errorf("add prerouting wg notrack rule: %w", err)
|
||||
}
|
||||
|
||||
preroutingRuleProxy := []string{"-i", "lo", "-s", "127.0.0.1", "-d", "127.0.0.1", "-p", "udp", "--dport", proxyPortStr, "-j", "NOTRACK"}
|
||||
preroutingRuleProxy := []string{"-i", "lo", "-s", "127.0.0.0/8", "-d", "127.0.0.0/8", "-p", "udp", "--dport", proxyPortStr, "-j", "NOTRACK"}
|
||||
if err := m.ipv4Client.AppendUnique(tableRaw, chainNameRaw, preroutingRuleProxy...); err != nil {
|
||||
return fmt.Errorf("add prerouting proxy notrack rule: %w", err)
|
||||
}
|
||||
|
||||
log.Debugf("set up ebpf proxy notrack rules for ports %d,%d", proxyPort, wgPort)
|
||||
log.Debugf("set up wg proxy notrack rules for ports %d,%d", proxyPort, wgPort)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -180,9 +180,9 @@ type Manager interface {
|
||||
// RemoveOutputDNAT removes an OUTPUT chain DNAT rule.
|
||||
RemoveOutputDNAT(localAddr netip.Addr, protocol Protocol, originalPort, translatedPort uint16) error
|
||||
|
||||
// SetupEBPFProxyNoTrack creates static notrack rules for eBPF proxy loopback traffic.
|
||||
// SetupWGProxyNoTrack creates static notrack rules for WireGuard proxy loopback traffic.
|
||||
// This prevents conntrack from interfering with WireGuard proxy communication.
|
||||
SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error
|
||||
SetupWGProxyNoTrack(proxyPort, wgPort uint16) error
|
||||
}
|
||||
|
||||
func GenKey(format string, pair RouterPair) string {
|
||||
|
||||
@@ -672,35 +672,46 @@ func (m *Manager) RemoveOutputDNAT(localAddr netip.Addr, protocol firewall.Proto
|
||||
return m.router.RemoveOutputDNAT(localAddr, protocol, originalPort, translatedPort)
|
||||
}
|
||||
|
||||
// The proxy hands every relayed peer its own address out of 127.0.0.0/8, so the
|
||||
// notrack rules match the whole loopback range.
|
||||
var (
|
||||
loopbackNet = []byte{127, 0, 0, 0}
|
||||
loopbackMask = []byte{255, 0, 0, 0}
|
||||
loopbackXor = []byte{0, 0, 0, 0}
|
||||
)
|
||||
|
||||
const (
|
||||
chainNameRawOutput = "netbird-raw-out"
|
||||
chainNameRawPrerouting = "netbird-raw-pre"
|
||||
)
|
||||
|
||||
// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic.
|
||||
// SetupWGProxyNoTrack creates notrack rules for WireGuard proxy loopback traffic.
|
||||
// This prevents conntrack from tracking WireGuard proxy traffic on loopback, which
|
||||
// can interfere with MASQUERADE rules (e.g., from container runtimes like Podman/netavark).
|
||||
//
|
||||
// Every relayed peer has its own loopback endpoint address, so the rules match the
|
||||
// whole 127.0.0.0/8 range.
|
||||
//
|
||||
// Traffic flows that need NOTRACK:
|
||||
//
|
||||
// 1. Egress: WireGuard -> fake endpoint (before eBPF rewrite)
|
||||
// src=127.0.0.1:wgPort -> dst=127.0.0.1:fakePort
|
||||
// 1. Egress: WireGuard -> peer endpoint
|
||||
// src=127.0.0.1:wgPort -> dst=127.x.x.x:proxyPort
|
||||
// Matched by: sport=wgPort
|
||||
//
|
||||
// 2. Egress: Proxy -> WireGuard (via raw socket)
|
||||
// src=127.0.0.1:fakePort -> dst=127.0.0.1:wgPort
|
||||
// src=127.x.x.x:proxyPort -> dst=127.0.0.1:wgPort
|
||||
// Matched by: dport=wgPort
|
||||
//
|
||||
// 3. Ingress: Packets to WireGuard
|
||||
// dst=127.0.0.1:wgPort
|
||||
// Matched by: dport=wgPort
|
||||
//
|
||||
// 4. Ingress: Packets to proxy (after eBPF rewrite)
|
||||
// dst=127.0.0.1:proxyPort
|
||||
// 4. Ingress: Packets to the proxy
|
||||
// dst=127.x.x.x:proxyPort
|
||||
// Matched by: dport=proxyPort
|
||||
//
|
||||
// Rules are cleaned up when the firewall manager is closed.
|
||||
func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
func (m *Manager) SetupWGProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
|
||||
@@ -710,7 +721,6 @@ func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
|
||||
proxyPortBytes := binaryutil.BigEndian.PutUint16(proxyPort)
|
||||
wgPortBytes := binaryutil.BigEndian.PutUint16(wgPort)
|
||||
loopback := []byte{127, 0, 0, 1}
|
||||
|
||||
// Egress rules: match outgoing loopback UDP packets
|
||||
m.rConn.AddRule(&nftables.Rule{
|
||||
@@ -720,9 +730,11 @@ func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")},
|
||||
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
|
||||
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, Mask: loopbackMask, Xor: loopbackXor},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopbackNet},
|
||||
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
|
||||
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, Mask: loopbackMask, Xor: loopbackXor},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopbackNet},
|
||||
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}},
|
||||
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 0, Len: 2},
|
||||
@@ -738,9 +750,11 @@ func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")},
|
||||
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
|
||||
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, Mask: loopbackMask, Xor: loopbackXor},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopbackNet},
|
||||
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
|
||||
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, Mask: loopbackMask, Xor: loopbackXor},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopbackNet},
|
||||
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}},
|
||||
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
|
||||
@@ -758,9 +772,11 @@ func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")},
|
||||
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
|
||||
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, Mask: loopbackMask, Xor: loopbackXor},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopbackNet},
|
||||
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
|
||||
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, Mask: loopbackMask, Xor: loopbackXor},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopbackNet},
|
||||
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}},
|
||||
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
|
||||
@@ -776,9 +792,11 @@ func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: ifname("lo")},
|
||||
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, // saddr
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
|
||||
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, Mask: loopbackMask, Xor: loopbackXor},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopbackNet},
|
||||
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, // daddr
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopback},
|
||||
&expr.Bitwise{SourceRegister: 1, DestRegister: 1, Len: 4, Mask: loopbackMask, Xor: loopbackXor},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: loopbackNet},
|
||||
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
|
||||
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_UDP}},
|
||||
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2},
|
||||
@@ -792,7 +810,7 @@ func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
return fmt.Errorf("flush notrack rules: %w", err)
|
||||
}
|
||||
|
||||
log.Debugf("set up ebpf proxy notrack rules for ports %d,%d", proxyPort, wgPort)
|
||||
log.Debugf("set up wg proxy notrack rules for ports %d,%d", proxyPort, wgPort)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -815,7 +815,7 @@ func (r *router) addPostroutingRules() {
|
||||
Data: binaryutil.NativeEndian.PutUint32(nbnet.PreroutingFwmarkMasquerade),
|
||||
},
|
||||
|
||||
// We need to exclude the loopback interface as this changes the ebpf proxy port
|
||||
// We need to exclude the loopback interface as this changes the wg proxy port
|
||||
&expr.Meta{
|
||||
Key: expr.MetaKeyOIFNAME,
|
||||
Register: 1,
|
||||
|
||||
@@ -751,12 +751,12 @@ func (m *Manager) resetState() {
|
||||
}
|
||||
}
|
||||
|
||||
// SetupEBPFProxyNoTrack creates notrack rules for eBPF proxy loopback traffic.
|
||||
func (m *Manager) SetupEBPFProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
// SetupWGProxyNoTrack creates notrack rules for WireGuard proxy loopback traffic.
|
||||
func (m *Manager) SetupWGProxyNoTrack(proxyPort, wgPort uint16) error {
|
||||
if m.nativeFirewall == nil {
|
||||
return nil
|
||||
}
|
||||
return m.nativeFirewall.SetupEBPFProxyNoTrack(proxyPort, wgPort)
|
||||
return m.nativeFirewall.SetupWGProxyNoTrack(proxyPort, wgPort)
|
||||
}
|
||||
|
||||
// UpdateSet updates the rule destinations associated with the given set
|
||||
|
||||
187
client/iface/wgproxy/bench_linux_test.go
Normal file
187
client/iface/wgproxy/bench_linux_test.go
Normal file
@@ -0,0 +1,187 @@
|
||||
//go:build linux && !android && privileged
|
||||
|
||||
package wgproxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/netbirdio/netbird/client/iface/wgproxy/loopback"
|
||||
"github.com/netbirdio/netbird/client/iface/wgproxy/udp"
|
||||
)
|
||||
|
||||
// Benchmarks the WireGuard -> relay direction: a stand-in WireGuard socket sends
|
||||
// to each peer's endpoint, the proxy picks the packet up and forwards it to the
|
||||
// relayed connection. Peer count matters because the kernel proxy shares one
|
||||
// socket between all peers while the userspace proxy has one per peer.
|
||||
|
||||
const (
|
||||
benchLoopbackWgPort = 51841
|
||||
benchUDPWgPort = 51842
|
||||
benchPktSize = 1200
|
||||
)
|
||||
|
||||
type benchVariant struct {
|
||||
name string
|
||||
wgPort int
|
||||
setup func(peers int) (proxies []Proxy, cleanup func(), err error)
|
||||
}
|
||||
|
||||
func benchVariants() []benchVariant {
|
||||
return []benchVariant{
|
||||
{
|
||||
name: "loopback",
|
||||
wgPort: benchLoopbackWgPort,
|
||||
setup: func(peers int) ([]Proxy, func(), error) {
|
||||
loopbackProxy := loopback.NewProxy(benchLoopbackWgPort, 1280)
|
||||
if err := loopbackProxy.Listen(); err != nil {
|
||||
return nil, nil, fmt.Errorf("listen loopback proxy: %w", err)
|
||||
}
|
||||
proxies := make([]Proxy, 0, peers)
|
||||
for i := 0; i < peers; i++ {
|
||||
proxies = append(proxies, loopback.NewProxyWrapper(loopbackProxy))
|
||||
}
|
||||
return proxies, func() { _ = loopbackProxy.Free() }, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "udp",
|
||||
wgPort: benchUDPWgPort,
|
||||
setup: func(peers int) ([]Proxy, func(), error) {
|
||||
proxies := make([]Proxy, 0, peers)
|
||||
for i := 0; i < peers; i++ {
|
||||
proxies = append(proxies, udp.NewWGUDPProxy(benchUDPWgPort, 1280))
|
||||
}
|
||||
return proxies, func() {}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkProxyForwarding(b *testing.B) {
|
||||
for _, peers := range []int{1, 8, 32} {
|
||||
for _, v := range benchVariants() {
|
||||
b.Run(fmt.Sprintf("%s/peers=%d", v.name, peers), func(b *testing.B) {
|
||||
runForwardingBench(b, v, peers)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runForwardingBench(b *testing.B, v benchVariant, peers int) {
|
||||
b.Helper()
|
||||
|
||||
relayServer, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")})
|
||||
if err != nil {
|
||||
b.Fatalf("relay server: %v", err)
|
||||
}
|
||||
defer relayServer.Close()
|
||||
if err := relayServer.SetReadBuffer(8 << 20); err != nil {
|
||||
b.Logf("set relay read buffer: %v", err)
|
||||
}
|
||||
|
||||
proxies, cleanup, err := v.setup(peers)
|
||||
if err != nil {
|
||||
b.Skipf("setup %s: %v", v.name, err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
endpoints := make([]*net.UDPAddr, 0, peers)
|
||||
for i, p := range proxies {
|
||||
relayConn, err := net.Dial("udp", relayServer.LocalAddr().String())
|
||||
if err != nil {
|
||||
b.Fatalf("relay conn %d: %v", i, err)
|
||||
}
|
||||
defer relayConn.Close()
|
||||
|
||||
nbAddr := &net.UDPAddr{IP: net.IPv4(10, 0, 0, byte(i+1)), Port: 51820}
|
||||
if err := p.AddRelayedConn(ctx, nbAddr, relayConn); err != nil {
|
||||
b.Fatalf("add relayed conn %d: %v", i, err)
|
||||
}
|
||||
p.Work()
|
||||
|
||||
addr := p.EndpointAddr()
|
||||
if addr == nil {
|
||||
b.Fatalf("proxy %d has no endpoint address", i)
|
||||
}
|
||||
endpoints = append(endpoints, addr)
|
||||
}
|
||||
defer func() {
|
||||
for _, p := range proxies {
|
||||
_ = p.CloseConn()
|
||||
}
|
||||
}()
|
||||
|
||||
// stand-in for the WireGuard socket: bound to the wg port the proxy expects
|
||||
wgSock, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: v.wgPort})
|
||||
if err != nil {
|
||||
b.Fatalf("wg socket: %v", err)
|
||||
}
|
||||
defer wgSock.Close()
|
||||
if err := wgSock.SetWriteBuffer(8 << 20); err != nil {
|
||||
b.Logf("set wg write buffer: %v", err)
|
||||
}
|
||||
|
||||
var received atomic.Int64
|
||||
done := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
buf := make([]byte, 65536)
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
default:
|
||||
}
|
||||
if err := relayServer.SetReadDeadline(time.Now().Add(200 * time.Millisecond)); err != nil {
|
||||
return
|
||||
}
|
||||
n, _, err := relayServer.ReadFrom(buf)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if n > 0 {
|
||||
received.Add(1)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
pkt := make([]byte, benchPktSize)
|
||||
// warm up the datapath and the per-peer lookups
|
||||
for i := 0; i < 200; i++ {
|
||||
if _, err := wgSock.WriteTo(pkt, endpoints[i%len(endpoints)]); err != nil {
|
||||
b.Fatalf("warmup write: %v", err)
|
||||
}
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
received.Store(0)
|
||||
|
||||
b.SetBytes(int64(benchPktSize))
|
||||
b.ResetTimer()
|
||||
start := time.Now()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if _, err := wgSock.WriteTo(pkt, endpoints[i%len(endpoints)]); err != nil {
|
||||
b.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
sent := b.N
|
||||
// let the tail drain
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
elapsed := time.Since(start)
|
||||
b.StopTimer()
|
||||
|
||||
close(done)
|
||||
wg.Wait()
|
||||
|
||||
got := received.Load()
|
||||
b.ReportMetric(float64(got)/elapsed.Seconds(), "fwd_pps")
|
||||
b.ReportMetric(float64(got)/float64(sent)*100, "delivered_%")
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package ebpf
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
)
|
||||
|
||||
var (
|
||||
portRangeStart = 3128
|
||||
portRangeEnd = portRangeStart + 100
|
||||
)
|
||||
|
||||
type portLookup struct {
|
||||
}
|
||||
|
||||
func (pl portLookup) searchFreePort() (int, error) {
|
||||
for i := portRangeStart; i <= portRangeEnd; i++ {
|
||||
if pl.tryToBind(i) == nil {
|
||||
return i, nil
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("failed to bind free port for eBPF proxy")
|
||||
}
|
||||
|
||||
func (pl portLookup) tryToBind(port int) error {
|
||||
l, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = l.Close()
|
||||
return nil
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package ebpf
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func Test_portLookup_searchFreePort(t *testing.T) {
|
||||
pl := portLookup{}
|
||||
_, err := pl.searchFreePort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func Test_portLookup_on_allocated(t *testing.T) {
|
||||
pl := portLookup{}
|
||||
|
||||
portRangeStart = 4128
|
||||
portRangeEnd = portRangeStart + 100
|
||||
|
||||
allocatedPort, err := allocatePort(portRangeStart)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer allocatedPort.Close()
|
||||
|
||||
fp, err := pl.searchFreePort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if fp != (portRangeStart + 1) {
|
||||
t.Errorf("invalid free port, expected: %d, got: %d", portRangeStart+1, fp)
|
||||
}
|
||||
}
|
||||
|
||||
func allocatePort(port int) (net.PacketConn, error) {
|
||||
c, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, err
|
||||
}
|
||||
@@ -1,243 +0,0 @@
|
||||
//go:build linux && !android
|
||||
|
||||
package ebpf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/pion/transport/v3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
nberrors "github.com/netbirdio/netbird/client/errors"
|
||||
"github.com/netbirdio/netbird/client/iface/bufsize"
|
||||
"github.com/netbirdio/netbird/client/iface/wgproxy/rawsocket"
|
||||
"github.com/netbirdio/netbird/client/internal/ebpf"
|
||||
ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager"
|
||||
nbnet "github.com/netbirdio/netbird/client/net"
|
||||
)
|
||||
|
||||
const (
|
||||
loopbackAddr = "127.0.0.1"
|
||||
)
|
||||
|
||||
// WGEBPFProxy definition for proxy with EBPF support
|
||||
type WGEBPFProxy struct {
|
||||
localWGListenPort int
|
||||
proxyPort int
|
||||
mtu uint16
|
||||
|
||||
ebpfManager ebpfMgr.Manager
|
||||
relayedConnStore map[uint16]net.Conn
|
||||
relayedConnMutex sync.Mutex
|
||||
|
||||
lastUsedPort uint16
|
||||
rawConnIPv4 net.PacketConn
|
||||
rawConnIPv6 net.PacketConn
|
||||
conn transport.UDPConn
|
||||
|
||||
ctx context.Context
|
||||
ctxCancel context.CancelFunc
|
||||
}
|
||||
|
||||
// NewWGEBPFProxy create new WGEBPFProxy instance
|
||||
func NewWGEBPFProxy(wgPort int, mtu uint16) *WGEBPFProxy {
|
||||
log.Debugf("instantiate ebpf proxy")
|
||||
wgProxy := &WGEBPFProxy{
|
||||
localWGListenPort: wgPort,
|
||||
mtu: mtu,
|
||||
ebpfManager: ebpf.GetEbpfManagerInstance(),
|
||||
relayedConnStore: make(map[uint16]net.Conn),
|
||||
}
|
||||
return wgProxy
|
||||
}
|
||||
|
||||
// Listen load ebpf program and listen the proxy
|
||||
func (p *WGEBPFProxy) Listen() error {
|
||||
pl := portLookup{}
|
||||
proxyPort, err := pl.searchFreePort()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.proxyPort = proxyPort
|
||||
|
||||
// Prepare IPv4 raw socket (required)
|
||||
p.rawConnIPv4, err = rawsocket.PrepareSenderRawSocketIPv4()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Prepare IPv6 raw socket (optional)
|
||||
p.rawConnIPv6, err = rawsocket.PrepareSenderRawSocketIPv6()
|
||||
if err != nil {
|
||||
log.Warnf("failed to prepare IPv6 raw socket, continuing with IPv4 only: %v", err)
|
||||
}
|
||||
|
||||
err = p.ebpfManager.LoadWgProxy(proxyPort, p.localWGListenPort)
|
||||
if err != nil {
|
||||
if closeErr := p.rawConnIPv4.Close(); closeErr != nil {
|
||||
log.Warnf("failed to close IPv4 raw socket: %v", closeErr)
|
||||
}
|
||||
if p.rawConnIPv6 != nil {
|
||||
if closeErr := p.rawConnIPv6.Close(); closeErr != nil {
|
||||
log.Warnf("failed to close IPv6 raw socket: %v", closeErr)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
addr := net.UDPAddr{
|
||||
Port: proxyPort,
|
||||
IP: net.ParseIP(loopbackAddr),
|
||||
}
|
||||
|
||||
p.ctx, p.ctxCancel = context.WithCancel(context.Background())
|
||||
|
||||
conn, err := nbnet.ListenUDP("udp", &addr)
|
||||
if err != nil {
|
||||
if cErr := p.Free(); cErr != nil {
|
||||
log.Errorf("Failed to close the wgproxy: %s", cErr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
p.conn = conn
|
||||
|
||||
go p.proxyToRemote()
|
||||
log.Infof("local wg proxy listening on: %d", proxyPort)
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddRelayedConn add new relayed connection for the proxy
|
||||
func (p *WGEBPFProxy) AddRelayedConn(relayedConn net.Conn) (*net.UDPAddr, error) {
|
||||
wgEndpointPort, err := p.storeRelayedConn(relayedConn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Infof("relayed conn added to wg proxy store: %s, endpoint port: :%d", relayedConn.RemoteAddr(), wgEndpointPort)
|
||||
|
||||
wgEndpoint := &net.UDPAddr{
|
||||
IP: net.ParseIP(loopbackAddr),
|
||||
Port: int(wgEndpointPort),
|
||||
}
|
||||
return wgEndpoint, nil
|
||||
}
|
||||
|
||||
// Free resources except the remoteConns will be keep open.
|
||||
func (p *WGEBPFProxy) Free() error {
|
||||
log.Debugf("free up ebpf wg proxy")
|
||||
if p.ctx != nil && p.ctx.Err() != nil {
|
||||
//nolint
|
||||
return nil
|
||||
}
|
||||
|
||||
p.ctxCancel()
|
||||
|
||||
var result *multierror.Error
|
||||
if p.conn != nil {
|
||||
if err := p.conn.Close(); err != nil {
|
||||
result = multierror.Append(result, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := p.ebpfManager.FreeWGProxy(); err != nil {
|
||||
result = multierror.Append(result, err)
|
||||
}
|
||||
|
||||
if p.rawConnIPv4 != nil {
|
||||
if err := p.rawConnIPv4.Close(); err != nil {
|
||||
result = multierror.Append(result, err)
|
||||
}
|
||||
}
|
||||
|
||||
if p.rawConnIPv6 != nil {
|
||||
if err := p.rawConnIPv6.Close(); err != nil {
|
||||
result = multierror.Append(result, err)
|
||||
}
|
||||
}
|
||||
return nberrors.FormatErrorOrNil(result)
|
||||
}
|
||||
|
||||
// GetProxyPort returns the proxy listening port.
|
||||
func (p *WGEBPFProxy) GetProxyPort() uint16 {
|
||||
return uint16(p.proxyPort)
|
||||
}
|
||||
|
||||
// proxyToRemote read messages from local WireGuard interface and forward it to remote conn
|
||||
// From this go routine has only one instance.
|
||||
func (p *WGEBPFProxy) proxyToRemote() {
|
||||
buf := make([]byte, p.mtu+bufsize.WGBufferOverhead)
|
||||
for p.ctx.Err() == nil {
|
||||
if err := p.readAndForwardPacket(buf); err != nil {
|
||||
if p.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
log.Errorf("failed to proxy packet to remote conn: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *WGEBPFProxy) readAndForwardPacket(buf []byte) error {
|
||||
n, addr, err := p.conn.ReadFromUDP(buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read UDP packet from WG: %w", err)
|
||||
}
|
||||
|
||||
p.relayedConnMutex.Lock()
|
||||
conn, ok := p.relayedConnStore[uint16(addr.Port)]
|
||||
p.relayedConnMutex.Unlock()
|
||||
if !ok {
|
||||
if p.ctx.Err() == nil {
|
||||
log.Debugf("relayed conn not found by port because conn already has been closed: %d", addr.Port)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, err := conn.Write(buf[:n]); err != nil {
|
||||
return fmt.Errorf("forward local WG packet (%d) to remote relayed conn: %w", addr.Port, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *WGEBPFProxy) storeRelayedConn(relayedConn net.Conn) (uint16, error) {
|
||||
p.relayedConnMutex.Lock()
|
||||
defer p.relayedConnMutex.Unlock()
|
||||
|
||||
np, err := p.nextFreePort()
|
||||
if err != nil {
|
||||
return np, err
|
||||
}
|
||||
p.relayedConnStore[np] = relayedConn
|
||||
return np, nil
|
||||
}
|
||||
|
||||
func (p *WGEBPFProxy) removeRelayedConn(relayedConnID uint16) {
|
||||
p.relayedConnMutex.Lock()
|
||||
defer p.relayedConnMutex.Unlock()
|
||||
|
||||
_, ok := p.relayedConnStore[relayedConnID]
|
||||
if ok {
|
||||
log.Debugf("remove relayed conn from store by port: %d", relayedConnID)
|
||||
}
|
||||
delete(p.relayedConnStore, relayedConnID)
|
||||
}
|
||||
|
||||
func (p *WGEBPFProxy) nextFreePort() (uint16, error) {
|
||||
if len(p.relayedConnStore) == 65535 {
|
||||
return 0, fmt.Errorf("reached maximum relayed connection numbers")
|
||||
}
|
||||
generatePort:
|
||||
if p.lastUsedPort == 65535 {
|
||||
p.lastUsedPort = 1
|
||||
} else {
|
||||
p.lastUsedPort++
|
||||
}
|
||||
|
||||
if _, ok := p.relayedConnStore[p.lastUsedPort]; ok {
|
||||
goto generatePort
|
||||
}
|
||||
return p.lastUsedPort, nil
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
//go:build linux && !android
|
||||
|
||||
package ebpf
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWGEBPFProxy_connStore(t *testing.T) {
|
||||
wgProxy := NewWGEBPFProxy(1, 1280)
|
||||
|
||||
p, _ := wgProxy.storeRelayedConn(nil)
|
||||
if p != 1 {
|
||||
t.Errorf("invalid initial port: %d", wgProxy.lastUsedPort)
|
||||
}
|
||||
|
||||
numOfConns := 10
|
||||
for i := 0; i < numOfConns; i++ {
|
||||
p, _ = wgProxy.storeRelayedConn(nil)
|
||||
}
|
||||
if p != uint16(numOfConns)+1 {
|
||||
t.Errorf("invalid last used port: %d, expected: %d", p, numOfConns+1)
|
||||
}
|
||||
if len(wgProxy.relayedConnStore) != numOfConns+1 {
|
||||
t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.relayedConnStore), numOfConns+1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWGEBPFProxy_portCalculation_overflow(t *testing.T) {
|
||||
wgProxy := NewWGEBPFProxy(1, 1280)
|
||||
|
||||
_, _ = wgProxy.storeRelayedConn(nil)
|
||||
wgProxy.lastUsedPort = 65535
|
||||
p, _ := wgProxy.storeRelayedConn(nil)
|
||||
|
||||
if len(wgProxy.relayedConnStore) != 2 {
|
||||
t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.relayedConnStore), 2)
|
||||
}
|
||||
|
||||
if p != 2 {
|
||||
t.Errorf("invalid last used port: %d, expected: %d", p, 2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWGEBPFProxy_portCalculation_maxConn(t *testing.T) {
|
||||
wgProxy := NewWGEBPFProxy(1, 1280)
|
||||
|
||||
for i := 0; i < 65535; i++ {
|
||||
_, _ = wgProxy.storeRelayedConn(nil)
|
||||
}
|
||||
|
||||
_, err := wgProxy.storeRelayedConn(nil)
|
||||
if err == nil {
|
||||
t.Errorf("invalid relayed conn store calculation")
|
||||
}
|
||||
}
|
||||
@@ -8,11 +8,13 @@ import (
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/netbirdio/netbird/client/iface/wgproxy/ebpf"
|
||||
"github.com/netbirdio/netbird/client/iface/wgproxy/loopback"
|
||||
udpProxy "github.com/netbirdio/netbird/client/iface/wgproxy/udp"
|
||||
)
|
||||
|
||||
const (
|
||||
envDisableKernelWGProxy = "NB_DISABLE_KERNEL_WG_PROXY"
|
||||
// envDisableEBPFWGProxy is a deprecated alias for envDisableKernelWGProxy.
|
||||
envDisableEBPFWGProxy = "NB_DISABLE_EBPF_WG_PROXY"
|
||||
)
|
||||
|
||||
@@ -20,7 +22,7 @@ type KernelFactory struct {
|
||||
wgPort int
|
||||
mtu uint16
|
||||
|
||||
ebpfProxy *ebpf.WGEBPFProxy
|
||||
loopbackProxy *loopback.Proxy
|
||||
}
|
||||
|
||||
func NewKernelFactory(wgPort int, mtu uint16) *KernelFactory {
|
||||
@@ -29,55 +31,64 @@ func NewKernelFactory(wgPort int, mtu uint16) *KernelFactory {
|
||||
mtu: mtu,
|
||||
}
|
||||
|
||||
if isEBPFDisabled() {
|
||||
if isKernelProxyDisabled() {
|
||||
log.Infof("WireGuard Proxy Factory will produce UDP proxy")
|
||||
log.Infof("eBPF WireGuard proxy is disabled via %s environment variable", envDisableEBPFWGProxy)
|
||||
return f
|
||||
}
|
||||
|
||||
ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, mtu)
|
||||
if err := ebpfProxy.Listen(); err != nil {
|
||||
loopbackProxy := loopback.NewProxy(wgPort, mtu)
|
||||
if err := loopbackProxy.Listen(); err != nil {
|
||||
log.Infof("WireGuard Proxy Factory will produce UDP proxy")
|
||||
log.Warnf("failed to initialize ebpf proxy, fallback to user space proxy: %s", err)
|
||||
log.Warnf("failed to initialize loopback proxy, fallback to user space proxy: %s", err)
|
||||
return f
|
||||
}
|
||||
log.Infof("WireGuard Proxy Factory will produce eBPF proxy")
|
||||
f.ebpfProxy = ebpfProxy
|
||||
log.Infof("WireGuard Proxy Factory will produce loopback proxy")
|
||||
f.loopbackProxy = loopbackProxy
|
||||
return f
|
||||
}
|
||||
|
||||
func (w *KernelFactory) GetProxy() Proxy {
|
||||
if w.ebpfProxy == nil {
|
||||
if w.loopbackProxy == nil {
|
||||
return udpProxy.NewWGUDPProxy(w.wgPort, w.mtu)
|
||||
}
|
||||
|
||||
return ebpf.NewProxyWrapper(w.ebpfProxy)
|
||||
return loopback.NewProxyWrapper(w.loopbackProxy)
|
||||
}
|
||||
|
||||
// GetProxyPort returns the eBPF proxy port, or 0 if eBPF is not active.
|
||||
// GetProxyPort returns the loopback proxy port, or 0 if the kernel proxy is not active.
|
||||
func (w *KernelFactory) GetProxyPort() uint16 {
|
||||
if w.ebpfProxy == nil {
|
||||
if w.loopbackProxy == nil {
|
||||
return 0
|
||||
}
|
||||
return w.ebpfProxy.GetProxyPort()
|
||||
return w.loopbackProxy.GetProxyPort()
|
||||
}
|
||||
|
||||
func (w *KernelFactory) Free() error {
|
||||
if w.ebpfProxy == nil {
|
||||
if w.loopbackProxy == nil {
|
||||
return nil
|
||||
}
|
||||
return w.ebpfProxy.Free()
|
||||
return w.loopbackProxy.Free()
|
||||
}
|
||||
|
||||
func isEBPFDisabled() bool {
|
||||
val := os.Getenv(envDisableEBPFWGProxy)
|
||||
func isKernelProxyDisabled() bool {
|
||||
env := envDisableKernelWGProxy
|
||||
val := os.Getenv(env)
|
||||
if val == "" {
|
||||
env = envDisableEBPFWGProxy
|
||||
val = os.Getenv(env)
|
||||
}
|
||||
if val == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
disabled, err := strconv.ParseBool(val)
|
||||
if err != nil {
|
||||
log.Warnf("failed to parse %s: %v", envDisableEBPFWGProxy, err)
|
||||
log.Warnf("failed to parse %s: %v", env, err)
|
||||
return false
|
||||
}
|
||||
|
||||
if disabled {
|
||||
log.Infof("kernel WireGuard proxy is disabled via %s", env)
|
||||
}
|
||||
return disabled
|
||||
}
|
||||
|
||||
70
client/iface/wgproxy/loopback/addr.go
Normal file
70
client/iface/wgproxy/loopback/addr.go
Normal file
@@ -0,0 +1,70 @@
|
||||
//go:build linux && !android
|
||||
|
||||
package loopback
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/netip"
|
||||
)
|
||||
|
||||
// Peer endpoints live in the upper half of 127.0.0.0/8. Everything in that
|
||||
// range is delivered to the loopback device without any address or route being
|
||||
// configured, and staying out of 127.0.0.0/9 keeps well-known squatters such as
|
||||
// 127.0.0.53 (systemd-resolved) and 127.0.1.1 out of the way.
|
||||
const (
|
||||
addrRangeBase uint32 = 0x7f800000 // 127.128.0.0
|
||||
addrRangeSize uint32 = 1 << 23 // /9
|
||||
addrRangePrefix = "127.128.0.0/9"
|
||||
)
|
||||
|
||||
// allocator hands out one loopback address per relayed connection. The address
|
||||
// is the peer's identity: WireGuard sends to it, and the proxy recovers which
|
||||
// peer a packet belongs to from the destination address.
|
||||
type allocator struct {
|
||||
cursor uint32
|
||||
}
|
||||
|
||||
// next returns the first free address at or after the cursor, wrapping once.
|
||||
// inUse reports whether an address is already handed out.
|
||||
func (a *allocator) next(inUse func(netip.Addr) bool) (netip.Addr, error) {
|
||||
for i := uint32(0); i < addrRangeSize; i++ {
|
||||
a.cursor = (a.cursor + 1) % addrRangeSize
|
||||
addr := addrFromOffset(a.cursor)
|
||||
if !addr.IsValid() {
|
||||
continue
|
||||
}
|
||||
if inUse(addr) {
|
||||
continue
|
||||
}
|
||||
return addr, nil
|
||||
}
|
||||
return netip.Addr{}, fmt.Errorf("no free endpoint address in %s", addrRangePrefix)
|
||||
}
|
||||
|
||||
// addrFromOffset maps an offset in the range to an address, skipping the .0 and
|
||||
// .255 hosts. They are unremarkable on loopback, but tools and firewall rules
|
||||
// tend to treat them as network and broadcast addresses.
|
||||
func addrFromOffset(offset uint32) netip.Addr {
|
||||
last := offset & 0xff
|
||||
if last == 0 || last == 0xff {
|
||||
return netip.Addr{}
|
||||
}
|
||||
|
||||
v := addrRangeBase + offset
|
||||
return netip.AddrFrom4([4]byte{
|
||||
byte(v >> 24),
|
||||
byte(v >> 16),
|
||||
byte(v >> 8),
|
||||
byte(v),
|
||||
})
|
||||
}
|
||||
|
||||
// inRange reports whether addr is one this proxy could have handed out.
|
||||
func inRange(addr netip.Addr) bool {
|
||||
if !addr.Is4() {
|
||||
return false
|
||||
}
|
||||
b := addr.As4()
|
||||
v := uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
|
||||
return v >= addrRangeBase && v < addrRangeBase+addrRangeSize
|
||||
}
|
||||
103
client/iface/wgproxy/loopback/addr_test.go
Normal file
103
client/iface/wgproxy/loopback/addr_test.go
Normal file
@@ -0,0 +1,103 @@
|
||||
//go:build linux && !android
|
||||
|
||||
package loopback
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAllocatorHandsOutDistinctAddresses(t *testing.T) {
|
||||
var a allocator
|
||||
taken := make(map[netip.Addr]bool)
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] })
|
||||
if err != nil {
|
||||
t.Fatalf("allocate %d: %v", i, err)
|
||||
}
|
||||
if taken[addr] {
|
||||
t.Fatalf("address %s handed out twice", addr)
|
||||
}
|
||||
if !inRange(addr) {
|
||||
t.Fatalf("address %s outside %s", addr, addrRangePrefix)
|
||||
}
|
||||
taken[addr] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocatorSkipsNetworkAndBroadcastHosts(t *testing.T) {
|
||||
var a allocator
|
||||
taken := make(map[netip.Addr]bool)
|
||||
|
||||
// enough allocations to walk past a .255/.0 boundary
|
||||
for i := 0; i < 600; i++ {
|
||||
addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] })
|
||||
if err != nil {
|
||||
t.Fatalf("allocate %d: %v", i, err)
|
||||
}
|
||||
last := addr.As4()[3]
|
||||
if last == 0 || last == 255 {
|
||||
t.Fatalf("address %s ends in .%d", addr, last)
|
||||
}
|
||||
taken[addr] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocatorReusesReleasedAddresses(t *testing.T) {
|
||||
var a allocator
|
||||
taken := make(map[netip.Addr]bool)
|
||||
|
||||
first, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] })
|
||||
if err != nil {
|
||||
t.Fatalf("allocate: %v", err)
|
||||
}
|
||||
taken[first] = true
|
||||
|
||||
// release it and allocate until the cursor wraps back around to it
|
||||
delete(taken, first)
|
||||
for i := 0; i < 10; i++ {
|
||||
addr, err := a.next(func(candidate netip.Addr) bool { return taken[candidate] })
|
||||
if err != nil {
|
||||
t.Fatalf("allocate %d: %v", i, err)
|
||||
}
|
||||
if addr == first {
|
||||
return
|
||||
}
|
||||
taken[addr] = true
|
||||
}
|
||||
// the cursor moves forward, so reuse only happens after a full wrap. Assert
|
||||
// the released address is at least still considered free.
|
||||
if inUse := taken[first]; inUse {
|
||||
t.Fatalf("released address %s still marked in use", first)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInRange(t *testing.T) {
|
||||
tests := []struct {
|
||||
addr string
|
||||
want bool
|
||||
}{
|
||||
{"127.128.0.1", true},
|
||||
{"127.255.255.254", true},
|
||||
{"127.127.255.255", false}, // below the range, where 127.0.0.53 and friends live
|
||||
{"127.0.0.1", false},
|
||||
{"127.0.0.53", false},
|
||||
{"127.0.1.1", false},
|
||||
{"128.0.0.1", false},
|
||||
{"10.0.0.1", false},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
addr := netip.MustParseAddr(tc.addr)
|
||||
if got := inRange(addr); got != tc.want {
|
||||
t.Errorf("inRange(%s) = %v, want %v", tc.addr, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInRangeIgnoresIPv6(t *testing.T) {
|
||||
if inRange(netip.MustParseAddr("::1")) {
|
||||
t.Error("inRange(::1) = true, want false")
|
||||
}
|
||||
}
|
||||
275
client/iface/wgproxy/loopback/proxy.go
Normal file
275
client/iface/wgproxy/loopback/proxy.go
Normal file
@@ -0,0 +1,275 @@
|
||||
//go:build linux && !android
|
||||
|
||||
package loopback
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/net/ipv4"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
nberrors "github.com/netbirdio/netbird/client/errors"
|
||||
"github.com/netbirdio/netbird/client/iface/bufsize"
|
||||
"github.com/netbirdio/netbird/client/iface/wgproxy/rawsocket"
|
||||
)
|
||||
|
||||
const (
|
||||
loopbackDevice = "lo"
|
||||
|
||||
portRangeStart = 3128
|
||||
portRangeEnd = portRangeStart + 100
|
||||
)
|
||||
|
||||
// Proxy forwards packets between relayed connections and a local kernel
|
||||
// WireGuard instance. Every relayed peer gets its own loopback address as its
|
||||
// WireGuard endpoint, so a single socket serves all of them: the destination
|
||||
// address of an incoming packet identifies the peer.
|
||||
type Proxy struct {
|
||||
localWGListenPort int
|
||||
mtu uint16
|
||||
proxyPort int
|
||||
|
||||
conn *net.UDPConn
|
||||
packetConn *ipv4.PacketConn
|
||||
rawConnIPv4 net.PacketConn
|
||||
rawConnIPv6 net.PacketConn
|
||||
|
||||
relayedConnMutex sync.Mutex
|
||||
relayedConnStore map[netip.Addr]net.Conn
|
||||
addrs allocator
|
||||
|
||||
ctx context.Context
|
||||
ctxCancel context.CancelFunc
|
||||
}
|
||||
|
||||
// NewProxy creates a proxy for the WireGuard instance listening on wgPort.
|
||||
func NewProxy(wgPort int, mtu uint16) *Proxy {
|
||||
log.Debugf("instantiate loopback wg proxy")
|
||||
return &Proxy{
|
||||
localWGListenPort: wgPort,
|
||||
mtu: mtu,
|
||||
relayedConnStore: make(map[netip.Addr]net.Conn),
|
||||
}
|
||||
}
|
||||
|
||||
// Listen opens the shared socket and starts forwarding WireGuard packets to the
|
||||
// relayed connections.
|
||||
func (p *Proxy) Listen() error {
|
||||
rawConnIPv4, err := rawsocket.PrepareSenderRawSocketIPv4()
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare IPv4 raw socket: %w", err)
|
||||
}
|
||||
p.rawConnIPv4 = rawConnIPv4
|
||||
|
||||
p.rawConnIPv6, err = rawsocket.PrepareSenderRawSocketIPv6()
|
||||
if err != nil {
|
||||
log.Warnf("failed to prepare IPv6 raw socket, continuing with IPv4 only: %v", err)
|
||||
}
|
||||
|
||||
if err := p.listen(); err != nil {
|
||||
if freeErr := p.Free(); freeErr != nil {
|
||||
log.Errorf("failed to free the wgproxy: %s", freeErr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
p.ctx, p.ctxCancel = context.WithCancel(context.Background())
|
||||
|
||||
go p.proxyToRemote()
|
||||
log.Infof("local wg proxy listening on %s:%d", addrRangePrefix, p.proxyPort)
|
||||
return nil
|
||||
}
|
||||
|
||||
// listen binds the shared socket on the first free port of the range. The bind
|
||||
// has to be a wildcard one to receive every peer address in the range, so it is
|
||||
// restricted to the loopback device: without that the port would be reachable
|
||||
// on every interface.
|
||||
func (p *Proxy) listen() error {
|
||||
var lastErr error
|
||||
for port := portRangeStart; port <= portRangeEnd; port++ {
|
||||
err := p.listenOn(port)
|
||||
if err == nil {
|
||||
p.proxyPort = port
|
||||
return nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
return fmt.Errorf("bind proxy port in range %d-%d: %w", portRangeStart, portRangeEnd, lastErr)
|
||||
}
|
||||
|
||||
func (p *Proxy) listenOn(proxyPort int) error {
|
||||
lc := net.ListenConfig{
|
||||
Control: func(_, _ string, c syscall.RawConn) error {
|
||||
var sockErr error
|
||||
if err := c.Control(func(fd uintptr) {
|
||||
if err := unix.SetsockoptString(int(fd), unix.SOL_SOCKET, unix.SO_BINDTODEVICE, loopbackDevice); err != nil {
|
||||
sockErr = fmt.Errorf("bind to %s: %w", loopbackDevice, err)
|
||||
return
|
||||
}
|
||||
}); err != nil {
|
||||
return fmt.Errorf("control socket: %w", err)
|
||||
}
|
||||
return sockErr
|
||||
},
|
||||
}
|
||||
|
||||
conn, err := lc.ListenPacket(context.Background(), "udp4", fmt.Sprintf(":%d", proxyPort))
|
||||
if err != nil {
|
||||
return fmt.Errorf("listen on :%d: %w", proxyPort, err)
|
||||
}
|
||||
|
||||
udpConn, ok := conn.(*net.UDPConn)
|
||||
if !ok {
|
||||
if closeErr := conn.Close(); closeErr != nil {
|
||||
log.Errorf("failed to close proxy conn: %s", closeErr)
|
||||
}
|
||||
return fmt.Errorf("unexpected conn type %T", conn)
|
||||
}
|
||||
|
||||
packetConn := ipv4.NewPacketConn(udpConn)
|
||||
// the destination address carries the peer identity, the interface index is
|
||||
// checked on receive as a second line of defense behind SO_BINDTODEVICE
|
||||
if err := packetConn.SetControlMessage(ipv4.FlagDst|ipv4.FlagInterface, true); err != nil {
|
||||
if closeErr := udpConn.Close(); closeErr != nil {
|
||||
log.Errorf("failed to close proxy conn: %s", closeErr)
|
||||
}
|
||||
return fmt.Errorf("request destination address: %w", err)
|
||||
}
|
||||
|
||||
p.conn = udpConn
|
||||
p.packetConn = packetConn
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddRelayedConn assigns an endpoint address to the relayed connection and
|
||||
// returns the address WireGuard should send to.
|
||||
func (p *Proxy) AddRelayedConn(relayedConn net.Conn) (*net.UDPAddr, error) {
|
||||
addr, err := p.storeRelayedConn(relayedConn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Infof("relayed conn added to wg proxy store: %s, endpoint address: %s", relayedConn.RemoteAddr(), addr)
|
||||
|
||||
return &net.UDPAddr{
|
||||
IP: addr.AsSlice(),
|
||||
Port: p.proxyPort,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Free releases the proxy resources. The relayed connections are left open.
|
||||
func (p *Proxy) Free() error {
|
||||
log.Debugf("free up loopback wg proxy")
|
||||
if p.ctx != nil && p.ctx.Err() != nil {
|
||||
//nolint
|
||||
return nil
|
||||
}
|
||||
|
||||
if p.ctxCancel != nil {
|
||||
p.ctxCancel()
|
||||
}
|
||||
|
||||
var result *multierror.Error
|
||||
if p.conn != nil {
|
||||
if err := p.conn.Close(); err != nil {
|
||||
result = multierror.Append(result, err)
|
||||
}
|
||||
}
|
||||
|
||||
if p.rawConnIPv4 != nil {
|
||||
if err := p.rawConnIPv4.Close(); err != nil {
|
||||
result = multierror.Append(result, err)
|
||||
}
|
||||
}
|
||||
|
||||
if p.rawConnIPv6 != nil {
|
||||
if err := p.rawConnIPv6.Close(); err != nil {
|
||||
result = multierror.Append(result, err)
|
||||
}
|
||||
}
|
||||
return nberrors.FormatErrorOrNil(result)
|
||||
}
|
||||
|
||||
// GetProxyPort returns the port every peer endpoint address is reached on.
|
||||
func (p *Proxy) GetProxyPort() uint16 {
|
||||
return uint16(p.proxyPort)
|
||||
}
|
||||
|
||||
// proxyToRemote reads packets from the local WireGuard instance and forwards
|
||||
// them to the relayed connection the destination address belongs to.
|
||||
func (p *Proxy) proxyToRemote() {
|
||||
buf := make([]byte, p.mtu+bufsize.WGBufferOverhead)
|
||||
for p.ctx.Err() == nil {
|
||||
if err := p.readAndForwardPacket(buf); err != nil {
|
||||
if p.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
log.Errorf("failed to proxy packet to remote conn: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Proxy) readAndForwardPacket(buf []byte) error {
|
||||
n, cm, _, err := p.packetConn.ReadFrom(buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read UDP packet from WG: %w", err)
|
||||
}
|
||||
|
||||
if cm == nil {
|
||||
return fmt.Errorf("no control message on packet")
|
||||
}
|
||||
|
||||
dst, ok := netip.AddrFromSlice(cm.Dst.To4())
|
||||
if !ok || !inRange(dst) {
|
||||
log.Tracef("dropping packet for unexpected destination %s", cm.Dst)
|
||||
return nil
|
||||
}
|
||||
|
||||
p.relayedConnMutex.Lock()
|
||||
conn, ok := p.relayedConnStore[dst]
|
||||
p.relayedConnMutex.Unlock()
|
||||
if !ok {
|
||||
if p.ctx.Err() == nil {
|
||||
log.Debugf("relayed conn not found by address because conn already has been closed: %s", dst)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, err := conn.Write(buf[:n]); err != nil {
|
||||
return fmt.Errorf("forward local WG packet (%s) to remote relayed conn: %w", dst, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Proxy) storeRelayedConn(relayedConn net.Conn) (netip.Addr, error) {
|
||||
p.relayedConnMutex.Lock()
|
||||
defer p.relayedConnMutex.Unlock()
|
||||
|
||||
addr, err := p.addrs.next(func(a netip.Addr) bool {
|
||||
_, ok := p.relayedConnStore[a]
|
||||
return ok
|
||||
})
|
||||
if err != nil {
|
||||
return netip.Addr{}, err
|
||||
}
|
||||
|
||||
p.relayedConnStore[addr] = relayedConn
|
||||
return addr, nil
|
||||
}
|
||||
|
||||
func (p *Proxy) removeRelayedConn(addr netip.Addr) {
|
||||
p.relayedConnMutex.Lock()
|
||||
defer p.relayedConnMutex.Unlock()
|
||||
|
||||
if _, ok := p.relayedConnStore[addr]; ok {
|
||||
log.Debugf("remove relayed conn from store by address: %s", addr)
|
||||
}
|
||||
delete(p.relayedConnStore, addr)
|
||||
}
|
||||
163
client/iface/wgproxy/loopback/proxy_privileged_test.go
Normal file
163
client/iface/wgproxy/loopback/proxy_privileged_test.go
Normal file
@@ -0,0 +1,163 @@
|
||||
//go:build linux && !android && privileged
|
||||
|
||||
package loopback
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const testWGPort = 51862
|
||||
|
||||
// relayEnd stands in for a relayed connection: the proxy writes what it read
|
||||
// from WireGuard into it, and the test reads it back out here.
|
||||
func relayEnd(t *testing.T) (proxySide net.Conn, testSide *net.UDPConn) {
|
||||
t.Helper()
|
||||
|
||||
testSide, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")})
|
||||
if err != nil {
|
||||
t.Fatalf("relay listener: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := testSide.Close(); err != nil {
|
||||
t.Logf("close relay listener: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
proxySide, err = net.Dial("udp", testSide.LocalAddr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("relay conn: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := proxySide.Close(); err != nil {
|
||||
t.Logf("close relay conn: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
return proxySide, testSide
|
||||
}
|
||||
|
||||
// TestProxyDemuxesByDestinationAddress is the core of the design: one socket
|
||||
// serves every peer, and the destination address decides which relayed
|
||||
// connection a WireGuard packet belongs to.
|
||||
func TestProxyDemuxesByDestinationAddress(t *testing.T) {
|
||||
proxy := NewProxy(testWGPort, 1280)
|
||||
if err := proxy.Listen(); err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := proxy.Free(); err != nil {
|
||||
t.Errorf("free proxy: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
const peers = 3
|
||||
endpoints := make([]*net.UDPAddr, 0, peers)
|
||||
readers := make([]*net.UDPConn, 0, peers)
|
||||
for i := 0; i < peers; i++ {
|
||||
proxySide, testSide := relayEnd(t)
|
||||
endpoint, err := proxy.AddRelayedConn(proxySide)
|
||||
if err != nil {
|
||||
t.Fatalf("add relayed conn %d: %v", i, err)
|
||||
}
|
||||
if endpoint.Port != proxy.proxyPort {
|
||||
t.Errorf("peer %d endpoint port = %d, want the shared proxy port %d", i, endpoint.Port, proxy.proxyPort)
|
||||
}
|
||||
endpoints = append(endpoints, endpoint)
|
||||
readers = append(readers, testSide)
|
||||
}
|
||||
|
||||
// every peer must have its own address, otherwise they are indistinguishable
|
||||
seen := make(map[string]bool, peers)
|
||||
for i, endpoint := range endpoints {
|
||||
if seen[endpoint.IP.String()] {
|
||||
t.Fatalf("peer %d reuses endpoint address %s", i, endpoint.IP)
|
||||
}
|
||||
seen[endpoint.IP.String()] = true
|
||||
}
|
||||
|
||||
wgSock, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: testWGPort})
|
||||
if err != nil {
|
||||
t.Fatalf("wg socket: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := wgSock.Close(); err != nil {
|
||||
t.Logf("close wg socket: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
for i, endpoint := range endpoints {
|
||||
payload := []byte{byte(i), 'p', 'k', 't'}
|
||||
if _, err := wgSock.WriteTo(payload, endpoint); err != nil {
|
||||
t.Fatalf("write to peer %d endpoint %s: %v", i, endpoint, err)
|
||||
}
|
||||
|
||||
buf := make([]byte, 1500)
|
||||
if err := readers[i].SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
|
||||
t.Fatalf("set read deadline: %v", err)
|
||||
}
|
||||
n, _, err := readers[i].ReadFrom(buf)
|
||||
if err != nil {
|
||||
t.Fatalf("peer %d did not receive its packet: %v", i, err)
|
||||
}
|
||||
if string(buf[:n]) != string(payload) {
|
||||
t.Errorf("peer %d got %q, want %q", i, buf[:n], payload)
|
||||
}
|
||||
|
||||
// no other peer may see it
|
||||
for j, other := range readers {
|
||||
if j == i {
|
||||
continue
|
||||
}
|
||||
if err := other.SetReadDeadline(time.Now().Add(100 * time.Millisecond)); err != nil {
|
||||
t.Fatalf("set read deadline: %v", err)
|
||||
}
|
||||
if _, _, err := other.ReadFrom(buf); err == nil {
|
||||
t.Errorf("packet for peer %d also delivered to peer %d", i, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProxyDropsPacketsOutsideTheRange guards the wildcard bind: anything that
|
||||
// is not addressed to a handed-out endpoint must not reach a relayed peer.
|
||||
func TestProxyDropsPacketsOutsideTheRange(t *testing.T) {
|
||||
proxy := NewProxy(testWGPort+1, 1280)
|
||||
if err := proxy.Listen(); err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := proxy.Free(); err != nil {
|
||||
t.Errorf("free proxy: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
proxySide, testSide := relayEnd(t)
|
||||
if _, err := proxy.AddRelayedConn(proxySide); err != nil {
|
||||
t.Fatalf("add relayed conn: %v", err)
|
||||
}
|
||||
|
||||
sender, err := net.Dial("udp", net.JoinHostPort("127.0.0.1", strconv.Itoa(proxy.proxyPort)))
|
||||
if err != nil {
|
||||
t.Fatalf("sender: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := sender.Close(); err != nil {
|
||||
t.Logf("close sender: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := sender.Write([]byte("stray")); err != nil {
|
||||
t.Fatalf("write stray packet: %v", err)
|
||||
}
|
||||
|
||||
buf := make([]byte, 1500)
|
||||
if err := testSide.SetReadDeadline(time.Now().Add(500 * time.Millisecond)); err != nil {
|
||||
t.Fatalf("set read deadline: %v", err)
|
||||
}
|
||||
if _, _, err := testSide.ReadFrom(buf); err == nil {
|
||||
t.Error("packet addressed to 127.0.0.1 was forwarded to a relayed peer")
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
//go:build linux && !android
|
||||
|
||||
package ebpf
|
||||
package loopback
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/netip"
|
||||
"sync"
|
||||
|
||||
"github.com/google/gopacket"
|
||||
@@ -95,13 +96,14 @@ func NewPacketHeaders(localWGListenPort int, endpoint *net.UDPAddr) (*PacketHead
|
||||
|
||||
// ProxyWrapper help to keep the remoteConn instance for net.Conn.Close function call
|
||||
type ProxyWrapper struct {
|
||||
wgeBPFProxy *WGEBPFProxy
|
||||
proxy *Proxy
|
||||
|
||||
remoteConn net.Conn
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
|
||||
wgRelayedEndpointAddr *net.UDPAddr
|
||||
peerAddr netip.Addr
|
||||
headers *PacketHeaders
|
||||
headerCurrentUsed *PacketHeaders
|
||||
rawConn net.PacketConn
|
||||
@@ -113,36 +115,42 @@ type ProxyWrapper struct {
|
||||
closeListener *listener.CloseListener
|
||||
}
|
||||
|
||||
func NewProxyWrapper(proxy *WGEBPFProxy) *ProxyWrapper {
|
||||
func NewProxyWrapper(proxy *Proxy) *ProxyWrapper {
|
||||
return &ProxyWrapper{
|
||||
wgeBPFProxy: proxy,
|
||||
proxy: proxy,
|
||||
pausedCond: sync.NewCond(&sync.Mutex{}),
|
||||
closeListener: listener.NewCloseListener(),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ProxyWrapper) AddRelayedConn(ctx context.Context, _ *net.UDPAddr, remoteConn net.Conn) error {
|
||||
addr, err := p.wgeBPFProxy.AddRelayedConn(remoteConn)
|
||||
addr, err := p.proxy.AddRelayedConn(remoteConn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("add relayed conn: %w", err)
|
||||
}
|
||||
|
||||
headers, err := NewPacketHeaders(p.wgeBPFProxy.localWGListenPort, addr)
|
||||
peerAddr, ok := netip.AddrFromSlice(addr.IP.To4())
|
||||
if !ok {
|
||||
return fmt.Errorf("unexpected endpoint address %s", addr.IP)
|
||||
}
|
||||
|
||||
headers, err := NewPacketHeaders(p.proxy.localWGListenPort, addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create packet sender: %w", err)
|
||||
}
|
||||
|
||||
// Check if required raw connection is available
|
||||
if !headers.isIPv4 && p.wgeBPFProxy.rawConnIPv6 == nil {
|
||||
if !headers.isIPv4 && p.proxy.rawConnIPv6 == nil {
|
||||
return errIPv6ConnNotAvailable
|
||||
}
|
||||
if headers.isIPv4 && p.wgeBPFProxy.rawConnIPv4 == nil {
|
||||
if headers.isIPv4 && p.proxy.rawConnIPv4 == nil {
|
||||
return errIPv4ConnNotAvailable
|
||||
}
|
||||
|
||||
p.remoteConn = remoteConn
|
||||
p.ctx, p.cancel = context.WithCancel(ctx)
|
||||
p.wgRelayedEndpointAddr = addr
|
||||
p.peerAddr = peerAddr
|
||||
p.headers = headers
|
||||
p.rawConn = p.selectRawConn(headers)
|
||||
return nil
|
||||
@@ -193,18 +201,18 @@ func (p *ProxyWrapper) RedirectAs(endpoint *net.UDPAddr) {
|
||||
return
|
||||
}
|
||||
|
||||
header, err := NewPacketHeaders(p.wgeBPFProxy.localWGListenPort, endpoint)
|
||||
header, err := NewPacketHeaders(p.proxy.localWGListenPort, endpoint)
|
||||
if err != nil {
|
||||
log.Errorf("failed to create packet headers: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if required raw connection is available
|
||||
if !header.isIPv4 && p.wgeBPFProxy.rawConnIPv6 == nil {
|
||||
if !header.isIPv4 && p.proxy.rawConnIPv6 == nil {
|
||||
log.Error(errIPv6ConnNotAvailable)
|
||||
return
|
||||
}
|
||||
if header.isIPv4 && p.wgeBPFProxy.rawConnIPv4 == nil {
|
||||
if header.isIPv4 && p.proxy.rawConnIPv4 == nil {
|
||||
log.Error(errIPv4ConnNotAvailable)
|
||||
return
|
||||
}
|
||||
@@ -252,9 +260,9 @@ func (p *ProxyWrapper) CloseConn() error {
|
||||
}
|
||||
|
||||
func (p *ProxyWrapper) proxyToLocal(ctx context.Context) {
|
||||
defer p.wgeBPFProxy.removeRelayedConn(uint16(p.wgRelayedEndpointAddr.Port))
|
||||
defer p.proxy.removeRelayedConn(p.peerAddr)
|
||||
|
||||
buf := make([]byte, p.wgeBPFProxy.mtu+bufsize.WGBufferOverhead)
|
||||
buf := make([]byte, p.proxy.mtu+bufsize.WGBufferOverhead)
|
||||
for {
|
||||
n, err := p.readFromRemote(ctx, buf)
|
||||
if err != nil {
|
||||
@@ -286,7 +294,7 @@ func (p *ProxyWrapper) readFromRemote(ctx context.Context, buf []byte) (int, err
|
||||
}
|
||||
p.closeListener.Notify()
|
||||
if !errors.Is(err, io.EOF) {
|
||||
log.Errorf("failed to read from relayed conn (endpoint: :%d): %s", p.wgRelayedEndpointAddr.Port, err)
|
||||
log.Errorf("failed to read from relayed conn (endpoint: %s): %s", p.wgRelayedEndpointAddr, err)
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
@@ -314,7 +322,7 @@ func (p *ProxyWrapper) sendPkg(data []byte, header *PacketHeaders) error {
|
||||
|
||||
func (p *ProxyWrapper) selectRawConn(header *PacketHeaders) net.PacketConn {
|
||||
if header.isIPv4 {
|
||||
return p.wgeBPFProxy.rawConnIPv4
|
||||
return p.proxy.rawConnIPv4
|
||||
}
|
||||
return p.wgeBPFProxy.rawConnIPv6
|
||||
return p.proxy.rawConnIPv6
|
||||
}
|
||||
@@ -9,25 +9,25 @@ import (
|
||||
"github.com/netbirdio/netbird/client/iface/bind"
|
||||
"github.com/netbirdio/netbird/client/iface/wgaddr"
|
||||
bindproxy "github.com/netbirdio/netbird/client/iface/wgproxy/bind"
|
||||
"github.com/netbirdio/netbird/client/iface/wgproxy/ebpf"
|
||||
"github.com/netbirdio/netbird/client/iface/wgproxy/loopback"
|
||||
"github.com/netbirdio/netbird/client/iface/wgproxy/udp"
|
||||
)
|
||||
|
||||
func seedProxies() ([]proxyInstance, error) {
|
||||
pl := make([]proxyInstance, 0)
|
||||
|
||||
ebpfProxy := ebpf.NewWGEBPFProxy(51831, 1280)
|
||||
if err := ebpfProxy.Listen(); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize ebpf proxy: %s", err)
|
||||
loopbackProxy := loopback.NewProxy(51831, 1280)
|
||||
if err := loopbackProxy.Listen(); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize loopback proxy: %s", err)
|
||||
}
|
||||
|
||||
pEbpf := proxyInstance{
|
||||
name: "ebpf kernel proxy",
|
||||
proxy: ebpf.NewProxyWrapper(ebpfProxy),
|
||||
pLoopback := proxyInstance{
|
||||
name: "loopback kernel proxy",
|
||||
proxy: loopback.NewProxyWrapper(loopbackProxy),
|
||||
wgPort: 51831,
|
||||
closeFn: ebpfProxy.Free,
|
||||
closeFn: loopbackProxy.Free,
|
||||
}
|
||||
pl = append(pl, pEbpf)
|
||||
pl = append(pl, pLoopback)
|
||||
|
||||
pUDP := proxyInstance{
|
||||
name: "udp kernel proxy",
|
||||
@@ -42,18 +42,18 @@ func seedProxies() ([]proxyInstance, error) {
|
||||
func seedProxyForProxyCloseByRemoteConn() ([]proxyInstance, error) {
|
||||
pl := make([]proxyInstance, 0)
|
||||
|
||||
ebpfProxy := ebpf.NewWGEBPFProxy(51831, 1280)
|
||||
if err := ebpfProxy.Listen(); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize ebpf proxy: %s", err)
|
||||
loopbackProxy := loopback.NewProxy(51831, 1280)
|
||||
if err := loopbackProxy.Listen(); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize loopback proxy: %s", err)
|
||||
}
|
||||
|
||||
pEbpf := proxyInstance{
|
||||
name: "ebpf kernel proxy",
|
||||
proxy: ebpf.NewProxyWrapper(ebpfProxy),
|
||||
pLoopback := proxyInstance{
|
||||
name: "loopback kernel proxy",
|
||||
proxy: loopback.NewProxyWrapper(loopbackProxy),
|
||||
wgPort: 51831,
|
||||
closeFn: ebpfProxy.Free,
|
||||
closeFn: loopbackProxy.Free,
|
||||
}
|
||||
pl = append(pl, pEbpf)
|
||||
pl = append(pl, pLoopback)
|
||||
|
||||
pUDP := proxyInstance{
|
||||
name: "udp kernel proxy",
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/netbirdio/netbird/client/iface/wgproxy/ebpf"
|
||||
"github.com/netbirdio/netbird/client/iface/wgproxy/loopback"
|
||||
"github.com/netbirdio/netbird/client/iface/wgproxy/udp"
|
||||
)
|
||||
|
||||
@@ -198,20 +198,20 @@ func testRedirectAs(t *testing.T, proxy Proxy, wgPort int, nbAddr, p2pEndpoint *
|
||||
}
|
||||
}
|
||||
|
||||
// TestRedirectAs_eBPF_IPv4 tests RedirectAs with eBPF proxy using IPv4 addresses
|
||||
func TestRedirectAs_eBPF_IPv4(t *testing.T) {
|
||||
// TestRedirectAs_Loopback_IPv4 tests RedirectAs with the loopback proxy using IPv4 addresses
|
||||
func TestRedirectAs_Loopback_IPv4(t *testing.T) {
|
||||
wgPort := 51850
|
||||
ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280)
|
||||
if err := ebpfProxy.Listen(); err != nil {
|
||||
t.Fatalf("failed to initialize ebpf proxy: %v", err)
|
||||
loopbackProxy := loopback.NewProxy(wgPort, 1280)
|
||||
if err := loopbackProxy.Listen(); err != nil {
|
||||
t.Fatalf("failed to initialize loopback proxy: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := ebpfProxy.Free(); err != nil {
|
||||
t.Errorf("failed to free ebpf proxy: %v", err)
|
||||
if err := loopbackProxy.Free(); err != nil {
|
||||
t.Errorf("failed to free loopback proxy: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
proxy := ebpf.NewProxyWrapper(ebpfProxy)
|
||||
proxy := loopback.NewProxyWrapper(loopbackProxy)
|
||||
|
||||
// NetBird UDP address of the remote peer
|
||||
nbAddr := &net.UDPAddr{
|
||||
@@ -227,20 +227,20 @@ func TestRedirectAs_eBPF_IPv4(t *testing.T) {
|
||||
testRedirectAs(t, proxy, wgPort, nbAddr, p2pEndpoint)
|
||||
}
|
||||
|
||||
// TestRedirectAs_eBPF_IPv6 tests RedirectAs with eBPF proxy using IPv6 addresses
|
||||
func TestRedirectAs_eBPF_IPv6(t *testing.T) {
|
||||
// TestRedirectAs_Loopback_IPv6 tests RedirectAs with the loopback proxy using IPv6 addresses
|
||||
func TestRedirectAs_Loopback_IPv6(t *testing.T) {
|
||||
wgPort := 51851
|
||||
ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280)
|
||||
if err := ebpfProxy.Listen(); err != nil {
|
||||
t.Fatalf("failed to initialize ebpf proxy: %v", err)
|
||||
loopbackProxy := loopback.NewProxy(wgPort, 1280)
|
||||
if err := loopbackProxy.Listen(); err != nil {
|
||||
t.Fatalf("failed to initialize loopback proxy: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := ebpfProxy.Free(); err != nil {
|
||||
t.Errorf("failed to free ebpf proxy: %v", err)
|
||||
if err := loopbackProxy.Free(); err != nil {
|
||||
t.Errorf("failed to free loopback proxy: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
proxy := ebpf.NewProxyWrapper(ebpfProxy)
|
||||
proxy := loopback.NewProxyWrapper(loopbackProxy)
|
||||
|
||||
// NetBird UDP address of the remote peer
|
||||
nbAddr := &net.UDPAddr{
|
||||
@@ -259,17 +259,17 @@ func TestRedirectAs_eBPF_IPv6(t *testing.T) {
|
||||
// TestRedirectAs_Multiple_Switches tests switching between multiple endpoints
|
||||
func TestRedirectAs_Multiple_Switches(t *testing.T) {
|
||||
wgPort := 51856
|
||||
ebpfProxy := ebpf.NewWGEBPFProxy(wgPort, 1280)
|
||||
if err := ebpfProxy.Listen(); err != nil {
|
||||
t.Fatalf("failed to initialize ebpf proxy: %v", err)
|
||||
loopbackProxy := loopback.NewProxy(wgPort, 1280)
|
||||
if err := loopbackProxy.Listen(); err != nil {
|
||||
t.Fatalf("failed to initialize loopback proxy: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := ebpfProxy.Free(); err != nil {
|
||||
t.Errorf("failed to free ebpf proxy: %v", err)
|
||||
if err := loopbackProxy.Free(); err != nil {
|
||||
t.Errorf("failed to free loopback proxy: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
proxy := ebpf.NewProxyWrapper(ebpfProxy)
|
||||
proxy := loopback.NewProxyWrapper(loopbackProxy)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ const (
|
||||
|
||||
mapKeyFeatures uint32 = 0
|
||||
|
||||
featureFlagWGProxy = 0b00000001
|
||||
featureFlagDnsForwarder = 0b00000010
|
||||
)
|
||||
|
||||
|
||||
@@ -4,9 +4,13 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// featureFlagTest stands in for a second feature flag, so the set and unset
|
||||
// paths can be exercised with more than the one flag the manager defines.
|
||||
const featureFlagTest = 0b00000001
|
||||
|
||||
func TestManager_setFeatureFlag(t *testing.T) {
|
||||
mgr := GeneralManager{}
|
||||
mgr.setFeatureFlag(featureFlagWGProxy)
|
||||
mgr.setFeatureFlag(featureFlagTest)
|
||||
if mgr.featureFlags != 1 {
|
||||
t.Errorf("invalid feature state")
|
||||
}
|
||||
@@ -19,10 +23,10 @@ func TestManager_setFeatureFlag(t *testing.T) {
|
||||
|
||||
func TestManager_unsetFeatureFlag(t *testing.T) {
|
||||
mgr := GeneralManager{}
|
||||
mgr.setFeatureFlag(featureFlagWGProxy)
|
||||
mgr.setFeatureFlag(featureFlagTest)
|
||||
mgr.setFeatureFlag(featureFlagDnsForwarder)
|
||||
|
||||
err := mgr.unsetFeatureFlag(featureFlagWGProxy)
|
||||
err := mgr.unsetFeatureFlag(featureFlagTest)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
package ebpf
|
||||
|
||||
import log "github.com/sirupsen/logrus"
|
||||
|
||||
const (
|
||||
mapKeyProxyPort uint32 = 0
|
||||
mapKeyWgPort uint32 = 1
|
||||
)
|
||||
|
||||
func (tf *GeneralManager) LoadWgProxy(proxyPort, wgPort int) error {
|
||||
log.Debugf("load ebpf WG proxy")
|
||||
tf.lock.Lock()
|
||||
defer tf.lock.Unlock()
|
||||
|
||||
err := tf.loadXdp()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = tf.bpfObjs.NbWgProxySettingsMap.Put(mapKeyProxyPort, uint16(proxyPort))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = tf.bpfObjs.NbWgProxySettingsMap.Put(mapKeyWgPort, uint16(wgPort))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tf.setFeatureFlag(featureFlagWGProxy)
|
||||
err = tf.bpfObjs.NbFeatures.Put(mapKeyFeatures, tf.featureFlags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tf *GeneralManager) FreeWGProxy() error {
|
||||
log.Debugf("free ebpf WG proxy")
|
||||
return tf.unsetFeatureFlag(featureFlagWGProxy)
|
||||
}
|
||||
@@ -2,10 +2,8 @@ package manager
|
||||
|
||||
import "net/netip"
|
||||
|
||||
// Manager is used to load multiple eBPF programs. E.g., current DNS programs and WireGuard proxy
|
||||
// Manager is used to load eBPF programs. Currently only the DNS forwarder uses one.
|
||||
type Manager interface {
|
||||
LoadDNSFwd(ip netip.Addr, dnsPort int) error
|
||||
FreeDNSFwd() error
|
||||
LoadWgProxy(proxyPort, wgPort int) error
|
||||
FreeWGProxy() error
|
||||
}
|
||||
|
||||
@@ -775,7 +775,7 @@ func (e *Engine) initFirewall() error {
|
||||
}
|
||||
|
||||
// setupWGProxyNoTrack configures connection tracking exclusion for WireGuard proxy traffic.
|
||||
// This prevents conntrack/MASQUERADE from affecting loopback traffic between WireGuard and the eBPF proxy.
|
||||
// This prevents conntrack/MASQUERADE from affecting loopback traffic between WireGuard and the proxy.
|
||||
func (e *Engine) setupWGProxyNoTrack() {
|
||||
if e.firewall == nil {
|
||||
return
|
||||
@@ -786,7 +786,7 @@ func (e *Engine) setupWGProxyNoTrack() {
|
||||
return
|
||||
}
|
||||
|
||||
if err := e.firewall.SetupEBPFProxyNoTrack(proxyPort, uint16(e.config.WgPort)); err != nil {
|
||||
if err := e.firewall.SetupWGProxyNoTrack(proxyPort, uint16(e.config.WgPort)); err != nil {
|
||||
log.Warnf("failed to setup ebpf proxy notrack: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user