Apple source NAT for dual IPs on interface

device/nat.go

@@ -0,0 +1,75 @@
+package device
+
+import "encoding/binary"
+
+// FixIPv4Source rewrites an IPv4 packet's source address to correctSrc if it
+// doesn't already match, incrementally fixing up the IPv4 header checksum
+// and (for TCP/UDP) the transport checksum so the packet stays valid.
+//
+// The common case - source already correct - is a single 4-byte comparison
+// and nothing else, so this is safe to call unconditionally on every packet
+// matched by a MiddleDevice rule. When a rewrite is needed, checksums are
+// updated via the RFC 1624 incremental method (add the delta of the changed
+// 16-bit words) rather than a full recompute over the packet, since only the
+// address field changed. ICMP has no pseudo-header dependency on the IP
+// addresses, so its checksum is left untouched. Non-IPv4 or malformed
+// packets are left untouched.
+func FixIPv4Source(packet []byte, correctSrc [4]byte) {
+	if len(packet) < 20 || packet[0]>>4 != 4 {
+		return
+	}
+
+	if packet[12] == correctSrc[0] && packet[13] == correctSrc[1] &&
+		packet[14] == correctSrc[2] && packet[15] == correctSrc[3] {
+		return
+	}
+
+	ihl := int(packet[0]&0x0f) * 4
+	if ihl < 20 || len(packet) < ihl {
+		return
+	}
+
+	oldSrc := [4]byte{packet[12], packet[13], packet[14], packet[15]}
+
+	ipChecksum := binary.BigEndian.Uint16(packet[10:12])
+	binary.BigEndian.PutUint16(packet[10:12], checksumAdjust(ipChecksum, oldSrc[:], correctSrc[:]))
+
+	switch packet[9] {
+	case 6: // TCP
+		if len(packet) >= ihl+20 {
+			off := ihl + 16
+			old := binary.BigEndian.Uint16(packet[off : off+2])
+			binary.BigEndian.PutUint16(packet[off:off+2], checksumAdjust(old, oldSrc[:], correctSrc[:]))
+		}
+	case 17: // UDP
+		if len(packet) >= ihl+8 {
+			off := ihl + 6
+			old := binary.BigEndian.Uint16(packet[off : off+2])
+			if old != 0 { // zero means checksum not used - must stay zero
+				binary.BigEndian.PutUint16(packet[off:off+2], checksumAdjust(old, oldSrc[:], correctSrc[:]))
+			}
+		}
+	}
+
+	copy(packet[12:16], correctSrc[:])
+}
+
+// checksumAdjust incrementally updates a ones-complement checksum after some
+// of the bytes it covers changed from old to new (RFC 1624), avoiding a full
+// recompute over the packet. old and new must be the same (even) length.
+func checksumAdjust(checksum uint16, old, new []byte) uint16 {
+	sum := uint32(^checksum)
+
+	for i := 0; i+1 < len(old); i += 2 {
+		sum += uint32(^binary.BigEndian.Uint16(old[i:i+2])) & 0xffff
+	}
+	for i := 0; i+1 < len(new); i += 2 {
+		sum += uint32(binary.BigEndian.Uint16(new[i : i+2]))
+	}
+
+	for sum>>16 != 0 {
+		sum = (sum & 0xffff) + (sum >> 16)
+	}
+
+	return ^uint16(sum)
+}

device/nat_test.go

@@ -0,0 +1,220 @@
+package device
+
+import (
+	"bytes"
+	"encoding/binary"
+	"testing"
+)
+
+// onesComplementSum computes an RFC 1071 ones-complement checksum from
+// scratch, independent of checksumAdjust, so it can be used to verify
+// FixIPv4Source's incremental updates rather than tautologically re-deriving
+// them with the same formula.
+func onesComplementSum(data []byte) uint16 {
+	var sum uint32
+	n := len(data)
+	for i := 0; i+1 < n; i += 2 {
+		sum += uint32(data[i])<<8 | uint32(data[i+1])
+	}
+	if n%2 == 1 {
+		sum += uint32(data[n-1]) << 8
+	}
+	for sum>>16 != 0 {
+		sum = (sum & 0xffff) + (sum >> 16)
+	}
+	return ^uint16(sum)
+}
+
+func buildIPv4Header(src, dst [4]byte, proto byte, payloadLen int) []byte {
+	h := make([]byte, 20)
+	h[0] = 0x45
+	binary.BigEndian.PutUint16(h[2:4], uint16(20+payloadLen))
+	h[6] = 0x40 // DF
+	h[8] = 64   // TTL
+	h[9] = proto
+	copy(h[12:16], src[:])
+	copy(h[16:20], dst[:])
+	binary.BigEndian.PutUint16(h[10:12], onesComplementSum(h))
+	return h
+}
+
+func buildUDPSegment(src, dst [4]byte, payload []byte) []byte {
+	udpLen := 8 + len(payload)
+	seg := make([]byte, udpLen)
+	binary.BigEndian.PutUint16(seg[0:2], 12345)
+	binary.BigEndian.PutUint16(seg[2:4], 53)
+	binary.BigEndian.PutUint16(seg[4:6], uint16(udpLen))
+	copy(seg[8:], payload)
+
+	pseudo := make([]byte, 12+udpLen)
+	copy(pseudo[0:4], src[:])
+	copy(pseudo[4:8], dst[:])
+	pseudo[9] = 17
+	binary.BigEndian.PutUint16(pseudo[10:12], uint16(udpLen))
+	copy(pseudo[12:], seg)
+	csum := onesComplementSum(pseudo)
+	if csum == 0 {
+		csum = 0xffff
+	}
+	binary.BigEndian.PutUint16(seg[6:8], csum)
+	return seg
+}
+
+func buildTCPSegment(src, dst [4]byte, payload []byte) []byte {
+	tcpLen := 20 + len(payload)
+	seg := make([]byte, tcpLen)
+	binary.BigEndian.PutUint16(seg[0:2], 54321)
+	binary.BigEndian.PutUint16(seg[2:4], 443)
+	seg[12] = 0x50 // data offset 5
+	copy(seg[20:], payload)
+
+	pseudo := make([]byte, 12+tcpLen)
+	copy(pseudo[0:4], src[:])
+	copy(pseudo[4:8], dst[:])
+	pseudo[9] = 6
+	binary.BigEndian.PutUint16(pseudo[10:12], uint16(tcpLen))
+	copy(pseudo[12:], seg)
+	binary.BigEndian.PutUint16(seg[16:18], onesComplementSum(pseudo))
+	return seg
+}
+
+func verifyIPv4HeaderChecksum(t *testing.T, packet []byte) {
+	t.Helper()
+	header := append([]byte(nil), packet[:20]...)
+	binary.BigEndian.PutUint16(header[10:12], 0)
+	want := onesComplementSum(header)
+	got := binary.BigEndian.Uint16(packet[10:12])
+	if got != want {
+		t.Errorf("IPv4 header checksum = %#04x, want %#04x", got, want)
+	}
+}
+
+func verifyUDPChecksum(t *testing.T, packet []byte, src, dst [4]byte) {
+	t.Helper()
+	seg := append([]byte(nil), packet[20:]...)
+	binary.BigEndian.PutUint16(seg[6:8], 0)
+	pseudo := make([]byte, 12+len(seg))
+	copy(pseudo[0:4], src[:])
+	copy(pseudo[4:8], dst[:])
+	pseudo[9] = 17
+	binary.BigEndian.PutUint16(pseudo[10:12], uint16(len(seg)))
+	copy(pseudo[12:], seg)
+	want := onesComplementSum(pseudo)
+	if want == 0 {
+		want = 0xffff
+	}
+	got := binary.BigEndian.Uint16(packet[26:28])
+	if got != want {
+		t.Errorf("UDP checksum = %#04x, want %#04x", got, want)
+	}
+}
+
+func verifyTCPChecksum(t *testing.T, packet []byte, src, dst [4]byte) {
+	t.Helper()
+	seg := append([]byte(nil), packet[20:]...)
+	binary.BigEndian.PutUint16(seg[16:18], 0)
+	pseudo := make([]byte, 12+len(seg))
+	copy(pseudo[0:4], src[:])
+	copy(pseudo[4:8], dst[:])
+	pseudo[9] = 6
+	binary.BigEndian.PutUint16(pseudo[10:12], uint16(len(seg)))
+	copy(pseudo[12:], seg)
+	want := onesComplementSum(pseudo)
+	got := binary.BigEndian.Uint16(packet[36:38])
+	if got != want {
+		t.Errorf("TCP checksum = %#04x, want %#04x", got, want)
+	}
+}
+
+func TestFixIPv4SourceUDP(t *testing.T) {
+	wrongSrc := [4]byte{10, 0, 0, 1}
+	correctSrc := [4]byte{10, 0, 0, 2}
+	dst := [4]byte{192, 168, 1, 1}
+	payload := []byte("hello world")
+
+	udp := buildUDPSegment(wrongSrc, dst, payload)
+	ip := buildIPv4Header(wrongSrc, dst, 17, len(udp))
+	packet := append(ip, udp...)
+
+	FixIPv4Source(packet, correctSrc)
+
+	if got := [4]byte{packet[12], packet[13], packet[14], packet[15]}; got != correctSrc {
+		t.Fatalf("source = %v, want %v", got, correctSrc)
+	}
+	verifyIPv4HeaderChecksum(t, packet)
+	verifyUDPChecksum(t, packet, correctSrc, dst)
+	if !bytes.Equal(packet[28:], payload) {
+		t.Errorf("UDP payload was mutated: got %q, want %q", packet[28:], payload)
+	}
+}
+
+func TestFixIPv4SourceTCP(t *testing.T) {
+	wrongSrc := [4]byte{172, 16, 0, 5}
+	correctSrc := [4]byte{172, 16, 0, 9}
+	dst := [4]byte{8, 8, 8, 8}
+	payload := []byte("GET / HTTP/1.1")
+
+	tcp := buildTCPSegment(wrongSrc, dst, payload)
+	ip := buildIPv4Header(wrongSrc, dst, 6, len(tcp))
+	packet := append(ip, tcp...)
+
+	FixIPv4Source(packet, correctSrc)
+
+	if got := [4]byte{packet[12], packet[13], packet[14], packet[15]}; got != correctSrc {
+		t.Fatalf("source = %v, want %v", got, correctSrc)
+	}
+	verifyIPv4HeaderChecksum(t, packet)
+	verifyTCPChecksum(t, packet, correctSrc, dst)
+}
+
+func TestFixIPv4SourceAlreadyCorrect(t *testing.T) {
+	correctSrc := [4]byte{10, 0, 0, 2}
+	dst := [4]byte{192, 168, 1, 1}
+	udp := buildUDPSegment(correctSrc, dst, []byte("payload"))
+	ip := buildIPv4Header(correctSrc, dst, 17, len(udp))
+	packet := append(ip, udp...)
+
+	original := append([]byte(nil), packet...)
+	FixIPv4Source(packet, correctSrc)
+
+	if !bytes.Equal(packet, original) {
+		t.Errorf("fast path mutated an already-correct packet: got %x, want %x", packet, original)
+	}
+}
+
+func TestFixIPv4SourceICMPChecksumUntouched(t *testing.T) {
+	wrongSrc := [4]byte{10, 0, 0, 1}
+	correctSrc := [4]byte{10, 0, 0, 2}
+	dst := [4]byte{192, 168, 1, 1}
+
+	// Minimal ICMP echo request: type=8, code=0, checksum, id, seq.
+	icmp := []byte{8, 0, 0xf7, 0xfd, 0x00, 0x01, 0x00, 0x01}
+	originalICMP := append([]byte(nil), icmp...)
+	ip := buildIPv4Header(wrongSrc, dst, 1, len(icmp))
+	packet := append(ip, icmp...)
+
+	FixIPv4Source(packet, correctSrc)
+
+	if got := [4]byte{packet[12], packet[13], packet[14], packet[15]}; got != correctSrc {
+		t.Fatalf("source = %v, want %v", got, correctSrc)
+	}
+	verifyIPv4HeaderChecksum(t, packet)
+	if !bytes.Equal(packet[20:], originalICMP) {
+		t.Errorf("ICMP body was mutated: got %x, want %x", packet[20:], originalICMP)
+	}
+}
+
+func TestFixIPv4SourceMalformedPacketNoPanic(t *testing.T) {
+	correctSrc := [4]byte{10, 0, 0, 2}
+
+	defer func() {
+		if r := recover(); r != nil {
+			t.Fatalf("FixIPv4Source panicked: %v", r)
+		}
+	}()
+
+	FixIPv4Source(nil, correctSrc)
+	FixIPv4Source([]byte{}, correctSrc)
+	FixIPv4Source([]byte{0x45, 0x00, 0x00}, correctSrc)
+	FixIPv4Source([]byte{0x60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, correctSrc) // IPv6 version nibble
+}

olm/exitnode.go

@@ -4,11 +4,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"net/netip"
+	"runtime"
 	"strings"
 
 	"github.com/fosrl/newt/logger"
 	"github.com/fosrl/newt/network"
 	"github.com/fosrl/newt/util"
+	olmDevice "github.com/fosrl/olm/device"
 	"github.com/fosrl/olm/peers"
 	"github.com/fosrl/olm/websocket"
 )
@@ -74,6 +77,22 @@ func (o *Olm) connectExitNode(cfg ExitNodeConfig) error {
 		return fmt.Errorf("failed to resolve exit node endpoint: %w", err)
 	}
 
+	interfaceName := o.tunnelConfig.InterfaceName
+	tunnelIP := cfg.TunnelIP
+	if !strings.Contains(tunnelIP, "/") {
+		tunnelIP += "/32"
+	}
+	// Add the secondary address before configuring the peer or route below, and
+	// fail closed if it doesn't succeed: AddSecondaryAddress (via AddIPv4Address)
+	// refuses to add when no primary address is configured yet, which would
+	// otherwise silently make the exit node's address the interface's primary
+	// one on mobile platforms (array order is what determines primary there).
+	// Bailing out here before touching the WireGuard device at all means there's
+	// never a half-configured peer left behind to roll back.
+	if err := network.AddSecondaryAddress(interfaceName, tunnelIP); err != nil {
+		return fmt.Errorf("failed to add secondary address %s for exit node: %w", tunnelIP, err)
+	}
+
 	persistentKeepalive := 0
 	if pm := o.getPeerManager(); pm != nil {
 		persistentKeepalive = pm.PersistentKeepalive
@@ -90,15 +109,6 @@ persistent_keepalive_interval=%d`, util.FixKey(cfg.PublicKey), allowedIP, resolv
 		return fmt.Errorf("failed to configure exit node peer: %w", err)
 	}
 
-	interfaceName := o.tunnelConfig.InterfaceName
-	tunnelIP := cfg.TunnelIP
-	if !strings.Contains(tunnelIP, "/") {
-		tunnelIP += "/32"
-	}
-	if err := network.AddSecondaryAddress(interfaceName, tunnelIP); err != nil {
-		logger.Warn("Failed to add secondary address %s for exit node: %v", tunnelIP, err)
-	}
-
 	// ServerIP arrives as a bare IP with no CIDR suffix, but AddRouteForServerIP
 	// parses it as a CIDR on darwin (to explicitly route the subnet up the tunnel,
 	// since unlike Linux, adding the address to the interface does not implicitly
@@ -115,6 +125,28 @@ persistent_keepalive_interval=%d`, util.FixKey(cfg.PublicKey), allowedIP, resolv
 		logger.Warn("Failed to add route for exit node server IP: %v", err)
 	}
 
+	// On macOS/iOS NetworkExtension, the OS can't reliably pin an outbound socket's
+	// source address to this interface's secondary address the way BSD route(8)
+	// -ifa does for the CLI path above - unbound sockets still get the primary
+	// (site tunnel) address stamped as source even for traffic destined to the
+	// exit node, which the exit node's WireGuard AllowedIPs filtering then
+	// silently drops. Fix it in-tunnel: intercept outbound packets addressed to
+	// the exit node and rewrite their source back to tunnelIP before WireGuard
+	// encrypts them. The fast path (source already correct) is cheap enough to
+	// also leave this on for a macOS CLI run, where the route above already
+	// gets it right.
+	if o.middleDev != nil && (runtime.GOOS == "darwin" || runtime.GOOS == "ios") {
+		if serverAddr, err := netip.ParseAddr(strings.Split(cfg.ServerIP, "/")[0]); err == nil {
+			if correctSrc, err := netip.ParseAddr(tunnelIPForRoute); err == nil && correctSrc.Is4() {
+				src := correctSrc.As4()
+				o.middleDev.AddRule(serverAddr, func(packet []byte) bool {
+					olmDevice.FixIPv4Source(packet, src)
+					return false
+				})
+			}
+		}
+	}
+
 	cfgCopy := cfg
 	o.exitNode = &cfgCopy
 
@@ -175,6 +207,13 @@ func (o *Olm) removeExitNodePeerLocked() error {
 	}
 
 	interfaceName := o.tunnelConfig.InterfaceName
+
+	if o.middleDev != nil && (runtime.GOOS == "darwin" || runtime.GOOS == "ios") {
+		if serverAddr, err := netip.ParseAddr(strings.Split(cfg.ServerIP, "/")[0]); err == nil {
+			o.middleDev.RemoveRule(serverAddr)
+		}
+	}
+
 	serverIPForRoute := strings.Split(cfg.ServerIP, "/")[0] + "/32"
 	tunnelIPForRoute := strings.Split(cfg.TunnelIP, "/")[0]
 	if err := network.RemoveRouteForServerIPWithSource(serverIPForRoute, interfaceName, tunnelIPForRoute); err != nil {
@@ -268,6 +307,16 @@ func (o *Olm) handleExitNodeConnect(msg websocket.WSMessage) {
 		return
 	}
 
+	// The primary tunnel interface must already be configured before an exit
+	// node's secondary address can safely be added (see the ordering
+	// enforcement in connectExitNode) - o.registered is only set true after
+	// that happens in handleConnect. This guards against a stray/early
+	// message reaching connectExitNode before then.
+	if !o.registered {
+		logger.Debug("Not yet registered, ignoring exit node connect message")
+		return
+	}
+
 	jsonData, err := json.Marshal(msg.Data)
 	if err != nil {
 		logger.Error("Error marshaling exit node connect data: %v", err)