From 0ce7e6a8dd7d9b92dffc2e354e2157100b4e47a3 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 13 Aug 2026 10:49:40 -0400 Subject: [PATCH] Apple source NAT for dual IPs on interface --- device/nat.go | 75 ++++++++++++++++ device/nat_test.go | 220 +++++++++++++++++++++++++++++++++++++++++++++ olm/exitnode.go | 67 ++++++++++++-- 3 files changed, 353 insertions(+), 9 deletions(-) create mode 100644 device/nat.go create mode 100644 device/nat_test.go diff --git a/device/nat.go b/device/nat.go new file mode 100644 index 0000000..918f3ed --- /dev/null +++ b/device/nat.go @@ -0,0 +1,75 @@ +package device + +import "encoding/binary" + +// FixIPv4Source rewrites an IPv4 packet's source address to correctSrc if it +// doesn't already match, incrementally fixing up the IPv4 header checksum +// and (for TCP/UDP) the transport checksum so the packet stays valid. +// +// The common case - source already correct - is a single 4-byte comparison +// and nothing else, so this is safe to call unconditionally on every packet +// matched by a MiddleDevice rule. When a rewrite is needed, checksums are +// updated via the RFC 1624 incremental method (add the delta of the changed +// 16-bit words) rather than a full recompute over the packet, since only the +// address field changed. ICMP has no pseudo-header dependency on the IP +// addresses, so its checksum is left untouched. Non-IPv4 or malformed +// packets are left untouched. +func FixIPv4Source(packet []byte, correctSrc [4]byte) { + if len(packet) < 20 || packet[0]>>4 != 4 { + return + } + + if packet[12] == correctSrc[0] && packet[13] == correctSrc[1] && + packet[14] == correctSrc[2] && packet[15] == correctSrc[3] { + return + } + + ihl := int(packet[0]&0x0f) * 4 + if ihl < 20 || len(packet) < ihl { + return + } + + oldSrc := [4]byte{packet[12], packet[13], packet[14], packet[15]} + + ipChecksum := binary.BigEndian.Uint16(packet[10:12]) + binary.BigEndian.PutUint16(packet[10:12], checksumAdjust(ipChecksum, oldSrc[:], correctSrc[:])) + + switch packet[9] { + case 6: // TCP + if len(packet) >= ihl+20 { + off := ihl + 16 + old := binary.BigEndian.Uint16(packet[off : off+2]) + binary.BigEndian.PutUint16(packet[off:off+2], checksumAdjust(old, oldSrc[:], correctSrc[:])) + } + case 17: // UDP + if len(packet) >= ihl+8 { + off := ihl + 6 + old := binary.BigEndian.Uint16(packet[off : off+2]) + if old != 0 { // zero means checksum not used - must stay zero + binary.BigEndian.PutUint16(packet[off:off+2], checksumAdjust(old, oldSrc[:], correctSrc[:])) + } + } + } + + copy(packet[12:16], correctSrc[:]) +} + +// checksumAdjust incrementally updates a ones-complement checksum after some +// of the bytes it covers changed from old to new (RFC 1624), avoiding a full +// recompute over the packet. old and new must be the same (even) length. +func checksumAdjust(checksum uint16, old, new []byte) uint16 { + sum := uint32(^checksum) + + for i := 0; i+1 < len(old); i += 2 { + sum += uint32(^binary.BigEndian.Uint16(old[i:i+2])) & 0xffff + } + for i := 0; i+1 < len(new); i += 2 { + sum += uint32(binary.BigEndian.Uint16(new[i : i+2])) + } + + for sum>>16 != 0 { + sum = (sum & 0xffff) + (sum >> 16) + } + + return ^uint16(sum) +} diff --git a/device/nat_test.go b/device/nat_test.go new file mode 100644 index 0000000..021dfb6 --- /dev/null +++ b/device/nat_test.go @@ -0,0 +1,220 @@ +package device + +import ( + "bytes" + "encoding/binary" + "testing" +) + +// onesComplementSum computes an RFC 1071 ones-complement checksum from +// scratch, independent of checksumAdjust, so it can be used to verify +// FixIPv4Source's incremental updates rather than tautologically re-deriving +// them with the same formula. +func onesComplementSum(data []byte) uint16 { + var sum uint32 + n := len(data) + for i := 0; i+1 < n; i += 2 { + sum += uint32(data[i])<<8 | uint32(data[i+1]) + } + if n%2 == 1 { + sum += uint32(data[n-1]) << 8 + } + for sum>>16 != 0 { + sum = (sum & 0xffff) + (sum >> 16) + } + return ^uint16(sum) +} + +func buildIPv4Header(src, dst [4]byte, proto byte, payloadLen int) []byte { + h := make([]byte, 20) + h[0] = 0x45 + binary.BigEndian.PutUint16(h[2:4], uint16(20+payloadLen)) + h[6] = 0x40 // DF + h[8] = 64 // TTL + h[9] = proto + copy(h[12:16], src[:]) + copy(h[16:20], dst[:]) + binary.BigEndian.PutUint16(h[10:12], onesComplementSum(h)) + return h +} + +func buildUDPSegment(src, dst [4]byte, payload []byte) []byte { + udpLen := 8 + len(payload) + seg := make([]byte, udpLen) + binary.BigEndian.PutUint16(seg[0:2], 12345) + binary.BigEndian.PutUint16(seg[2:4], 53) + binary.BigEndian.PutUint16(seg[4:6], uint16(udpLen)) + copy(seg[8:], payload) + + pseudo := make([]byte, 12+udpLen) + copy(pseudo[0:4], src[:]) + copy(pseudo[4:8], dst[:]) + pseudo[9] = 17 + binary.BigEndian.PutUint16(pseudo[10:12], uint16(udpLen)) + copy(pseudo[12:], seg) + csum := onesComplementSum(pseudo) + if csum == 0 { + csum = 0xffff + } + binary.BigEndian.PutUint16(seg[6:8], csum) + return seg +} + +func buildTCPSegment(src, dst [4]byte, payload []byte) []byte { + tcpLen := 20 + len(payload) + seg := make([]byte, tcpLen) + binary.BigEndian.PutUint16(seg[0:2], 54321) + binary.BigEndian.PutUint16(seg[2:4], 443) + seg[12] = 0x50 // data offset 5 + copy(seg[20:], payload) + + pseudo := make([]byte, 12+tcpLen) + copy(pseudo[0:4], src[:]) + copy(pseudo[4:8], dst[:]) + pseudo[9] = 6 + binary.BigEndian.PutUint16(pseudo[10:12], uint16(tcpLen)) + copy(pseudo[12:], seg) + binary.BigEndian.PutUint16(seg[16:18], onesComplementSum(pseudo)) + return seg +} + +func verifyIPv4HeaderChecksum(t *testing.T, packet []byte) { + t.Helper() + header := append([]byte(nil), packet[:20]...) + binary.BigEndian.PutUint16(header[10:12], 0) + want := onesComplementSum(header) + got := binary.BigEndian.Uint16(packet[10:12]) + if got != want { + t.Errorf("IPv4 header checksum = %#04x, want %#04x", got, want) + } +} + +func verifyUDPChecksum(t *testing.T, packet []byte, src, dst [4]byte) { + t.Helper() + seg := append([]byte(nil), packet[20:]...) + binary.BigEndian.PutUint16(seg[6:8], 0) + pseudo := make([]byte, 12+len(seg)) + copy(pseudo[0:4], src[:]) + copy(pseudo[4:8], dst[:]) + pseudo[9] = 17 + binary.BigEndian.PutUint16(pseudo[10:12], uint16(len(seg))) + copy(pseudo[12:], seg) + want := onesComplementSum(pseudo) + if want == 0 { + want = 0xffff + } + got := binary.BigEndian.Uint16(packet[26:28]) + if got != want { + t.Errorf("UDP checksum = %#04x, want %#04x", got, want) + } +} + +func verifyTCPChecksum(t *testing.T, packet []byte, src, dst [4]byte) { + t.Helper() + seg := append([]byte(nil), packet[20:]...) + binary.BigEndian.PutUint16(seg[16:18], 0) + pseudo := make([]byte, 12+len(seg)) + copy(pseudo[0:4], src[:]) + copy(pseudo[4:8], dst[:]) + pseudo[9] = 6 + binary.BigEndian.PutUint16(pseudo[10:12], uint16(len(seg))) + copy(pseudo[12:], seg) + want := onesComplementSum(pseudo) + got := binary.BigEndian.Uint16(packet[36:38]) + if got != want { + t.Errorf("TCP checksum = %#04x, want %#04x", got, want) + } +} + +func TestFixIPv4SourceUDP(t *testing.T) { + wrongSrc := [4]byte{10, 0, 0, 1} + correctSrc := [4]byte{10, 0, 0, 2} + dst := [4]byte{192, 168, 1, 1} + payload := []byte("hello world") + + udp := buildUDPSegment(wrongSrc, dst, payload) + ip := buildIPv4Header(wrongSrc, dst, 17, len(udp)) + packet := append(ip, udp...) + + FixIPv4Source(packet, correctSrc) + + if got := [4]byte{packet[12], packet[13], packet[14], packet[15]}; got != correctSrc { + t.Fatalf("source = %v, want %v", got, correctSrc) + } + verifyIPv4HeaderChecksum(t, packet) + verifyUDPChecksum(t, packet, correctSrc, dst) + if !bytes.Equal(packet[28:], payload) { + t.Errorf("UDP payload was mutated: got %q, want %q", packet[28:], payload) + } +} + +func TestFixIPv4SourceTCP(t *testing.T) { + wrongSrc := [4]byte{172, 16, 0, 5} + correctSrc := [4]byte{172, 16, 0, 9} + dst := [4]byte{8, 8, 8, 8} + payload := []byte("GET / HTTP/1.1") + + tcp := buildTCPSegment(wrongSrc, dst, payload) + ip := buildIPv4Header(wrongSrc, dst, 6, len(tcp)) + packet := append(ip, tcp...) + + FixIPv4Source(packet, correctSrc) + + if got := [4]byte{packet[12], packet[13], packet[14], packet[15]}; got != correctSrc { + t.Fatalf("source = %v, want %v", got, correctSrc) + } + verifyIPv4HeaderChecksum(t, packet) + verifyTCPChecksum(t, packet, correctSrc, dst) +} + +func TestFixIPv4SourceAlreadyCorrect(t *testing.T) { + correctSrc := [4]byte{10, 0, 0, 2} + dst := [4]byte{192, 168, 1, 1} + udp := buildUDPSegment(correctSrc, dst, []byte("payload")) + ip := buildIPv4Header(correctSrc, dst, 17, len(udp)) + packet := append(ip, udp...) + + original := append([]byte(nil), packet...) + FixIPv4Source(packet, correctSrc) + + if !bytes.Equal(packet, original) { + t.Errorf("fast path mutated an already-correct packet: got %x, want %x", packet, original) + } +} + +func TestFixIPv4SourceICMPChecksumUntouched(t *testing.T) { + wrongSrc := [4]byte{10, 0, 0, 1} + correctSrc := [4]byte{10, 0, 0, 2} + dst := [4]byte{192, 168, 1, 1} + + // Minimal ICMP echo request: type=8, code=0, checksum, id, seq. + icmp := []byte{8, 0, 0xf7, 0xfd, 0x00, 0x01, 0x00, 0x01} + originalICMP := append([]byte(nil), icmp...) + ip := buildIPv4Header(wrongSrc, dst, 1, len(icmp)) + packet := append(ip, icmp...) + + FixIPv4Source(packet, correctSrc) + + if got := [4]byte{packet[12], packet[13], packet[14], packet[15]}; got != correctSrc { + t.Fatalf("source = %v, want %v", got, correctSrc) + } + verifyIPv4HeaderChecksum(t, packet) + if !bytes.Equal(packet[20:], originalICMP) { + t.Errorf("ICMP body was mutated: got %x, want %x", packet[20:], originalICMP) + } +} + +func TestFixIPv4SourceMalformedPacketNoPanic(t *testing.T) { + correctSrc := [4]byte{10, 0, 0, 2} + + defer func() { + if r := recover(); r != nil { + t.Fatalf("FixIPv4Source panicked: %v", r) + } + }() + + FixIPv4Source(nil, correctSrc) + FixIPv4Source([]byte{}, correctSrc) + FixIPv4Source([]byte{0x45, 0x00, 0x00}, correctSrc) + FixIPv4Source([]byte{0x60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, correctSrc) // IPv6 version nibble +} diff --git a/olm/exitnode.go b/olm/exitnode.go index bf96db2..3542ba1 100644 --- a/olm/exitnode.go +++ b/olm/exitnode.go @@ -4,11 +4,14 @@ import ( "encoding/json" "fmt" "net" + "net/netip" + "runtime" "strings" "github.com/fosrl/newt/logger" "github.com/fosrl/newt/network" "github.com/fosrl/newt/util" + olmDevice "github.com/fosrl/olm/device" "github.com/fosrl/olm/peers" "github.com/fosrl/olm/websocket" ) @@ -74,6 +77,22 @@ func (o *Olm) connectExitNode(cfg ExitNodeConfig) error { return fmt.Errorf("failed to resolve exit node endpoint: %w", err) } + interfaceName := o.tunnelConfig.InterfaceName + tunnelIP := cfg.TunnelIP + if !strings.Contains(tunnelIP, "/") { + tunnelIP += "/32" + } + // Add the secondary address before configuring the peer or route below, and + // fail closed if it doesn't succeed: AddSecondaryAddress (via AddIPv4Address) + // refuses to add when no primary address is configured yet, which would + // otherwise silently make the exit node's address the interface's primary + // one on mobile platforms (array order is what determines primary there). + // Bailing out here before touching the WireGuard device at all means there's + // never a half-configured peer left behind to roll back. + if err := network.AddSecondaryAddress(interfaceName, tunnelIP); err != nil { + return fmt.Errorf("failed to add secondary address %s for exit node: %w", tunnelIP, err) + } + persistentKeepalive := 0 if pm := o.getPeerManager(); pm != nil { persistentKeepalive = pm.PersistentKeepalive @@ -90,15 +109,6 @@ persistent_keepalive_interval=%d`, util.FixKey(cfg.PublicKey), allowedIP, resolv return fmt.Errorf("failed to configure exit node peer: %w", err) } - interfaceName := o.tunnelConfig.InterfaceName - tunnelIP := cfg.TunnelIP - if !strings.Contains(tunnelIP, "/") { - tunnelIP += "/32" - } - if err := network.AddSecondaryAddress(interfaceName, tunnelIP); err != nil { - logger.Warn("Failed to add secondary address %s for exit node: %v", tunnelIP, err) - } - // ServerIP arrives as a bare IP with no CIDR suffix, but AddRouteForServerIP // parses it as a CIDR on darwin (to explicitly route the subnet up the tunnel, // since unlike Linux, adding the address to the interface does not implicitly @@ -115,6 +125,28 @@ persistent_keepalive_interval=%d`, util.FixKey(cfg.PublicKey), allowedIP, resolv logger.Warn("Failed to add route for exit node server IP: %v", err) } + // On macOS/iOS NetworkExtension, the OS can't reliably pin an outbound socket's + // source address to this interface's secondary address the way BSD route(8) + // -ifa does for the CLI path above - unbound sockets still get the primary + // (site tunnel) address stamped as source even for traffic destined to the + // exit node, which the exit node's WireGuard AllowedIPs filtering then + // silently drops. Fix it in-tunnel: intercept outbound packets addressed to + // the exit node and rewrite their source back to tunnelIP before WireGuard + // encrypts them. The fast path (source already correct) is cheap enough to + // also leave this on for a macOS CLI run, where the route above already + // gets it right. + if o.middleDev != nil && (runtime.GOOS == "darwin" || runtime.GOOS == "ios") { + if serverAddr, err := netip.ParseAddr(strings.Split(cfg.ServerIP, "/")[0]); err == nil { + if correctSrc, err := netip.ParseAddr(tunnelIPForRoute); err == nil && correctSrc.Is4() { + src := correctSrc.As4() + o.middleDev.AddRule(serverAddr, func(packet []byte) bool { + olmDevice.FixIPv4Source(packet, src) + return false + }) + } + } + } + cfgCopy := cfg o.exitNode = &cfgCopy @@ -175,6 +207,13 @@ func (o *Olm) removeExitNodePeerLocked() error { } interfaceName := o.tunnelConfig.InterfaceName + + if o.middleDev != nil && (runtime.GOOS == "darwin" || runtime.GOOS == "ios") { + if serverAddr, err := netip.ParseAddr(strings.Split(cfg.ServerIP, "/")[0]); err == nil { + o.middleDev.RemoveRule(serverAddr) + } + } + serverIPForRoute := strings.Split(cfg.ServerIP, "/")[0] + "/32" tunnelIPForRoute := strings.Split(cfg.TunnelIP, "/")[0] if err := network.RemoveRouteForServerIPWithSource(serverIPForRoute, interfaceName, tunnelIPForRoute); err != nil { @@ -268,6 +307,16 @@ func (o *Olm) handleExitNodeConnect(msg websocket.WSMessage) { return } + // The primary tunnel interface must already be configured before an exit + // node's secondary address can safely be added (see the ordering + // enforcement in connectExitNode) - o.registered is only set true after + // that happens in handleConnect. This guards against a stray/early + // message reaching connectExitNode before then. + if !o.registered { + logger.Debug("Not yet registered, ignoring exit node connect message") + return + } + jsonData, err := json.Marshal(msg.Data) if err != nil { logger.Error("Error marshaling exit node connect data: %v", err)