Implement first draft on native on main tunnel

2026-09-18 11:59:06 +02:00 · 2026-06-26 16:41:10 -04:00
parent 689acff6f8
commit 237587149b
3 changed files with 204 additions and 59 deletions
@@ -15,6 +15,12 @@ import (
 var wgService *clients.WireGuardService
 var ready bool

+// checkNativeMainPermissions returns an error if the process lacks the
+// privileges needed to create a native TUN interface for the main tunnel.
+func checkNativeMainPermissions() error {
+	return permissions.CheckNativeInterfacePermissions()
+}
+
 func setupClients(client *websocket.Client, credStore *nativessh.CredentialStore) {
 	var host = endpoint
 	if strings.HasPrefix(host, "http://") {
@@ -16,6 +16,7 @@ import (
 	"net/netip"
 	"os"
 	"os/signal"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync/atomic"
@@ -24,9 +25,11 @@ import (

 	"github.com/fosrl/newt/authdaemon"
 	"github.com/fosrl/newt/browsergateway"
+	newtDevice "github.com/fosrl/newt/device"
 	"github.com/fosrl/newt/docker"
 	"github.com/fosrl/newt/healthcheck"
 	"github.com/fosrl/newt/logger"
+	"github.com/fosrl/newt/network"
 	"github.com/fosrl/newt/nativessh"
 	"github.com/fosrl/newt/proxy"
 	"github.com/fosrl/newt/updates"
@@ -38,7 +41,7 @@ import (
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"golang.zx2c4.com/wireguard/conn"
 	"golang.zx2c4.com/wireguard/device"
-	"golang.zx2c4.com/wireguard/tun"
+	wtun "golang.zx2c4.com/wireguard/tun"
 	"golang.zx2c4.com/wireguard/tun/netstack"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
@@ -191,6 +194,10 @@ var (

 	// Path to config file (overrides CONFIG_FILE env var and default location)
 	configFile string
+
+	// Native main tunnel flags
+	useNativeMainInterface  bool
+	nativeMainInterfaceName string
 )

 // generateChainId generates a random chain ID for deduplicating round-trip messages.
@@ -273,6 +280,9 @@ func runNewtMain(ctx context.Context) {
 	disableSSH = disableSSHEnv == "true"
 	useNativeInterfaceEnv := os.Getenv("USE_NATIVE_INTERFACE")
 	useNativeInterface = useNativeInterfaceEnv == "true"
+	useNativeMainInterfaceEnv := os.Getenv("USE_NATIVE_MAIN_INTERFACE")
+	useNativeMainInterface = useNativeMainInterfaceEnv == "true"
+	nativeMainInterfaceName = os.Getenv("INTERFACE_MAIN")
 	enforceHealthcheckCertEnv := os.Getenv("ENFORCE_HC_CERT")
 	enforceHealthcheckCert = enforceHealthcheckCertEnv == "true"
 	dockerSocket = os.Getenv("DOCKER_SOCKET")
@@ -338,7 +348,13 @@ func runNewtMain(ctx context.Context) {
 		flag.StringVar(&portStr, "port", "", "Port for client WireGuard interface")
 	}
 	if useNativeInterfaceEnv == "" {
-		flag.BoolVar(&useNativeInterface, "native", false, "Use native WireGuard interface")
+		flag.BoolVar(&useNativeInterface, "native", false, "Use native WireGuard interface for client tunnels")
+	}
+	if useNativeMainInterfaceEnv == "" {
+		flag.BoolVar(&useNativeMainInterface, "native-main", false, "Use native WireGuard interface for the main tunnel (instead of netstack)")
+	}
+	if nativeMainInterfaceName == "" {
+		flag.StringVar(&nativeMainInterfaceName, "interface-main", "newtm", "Name of the native main tunnel WireGuard interface (used with --native-main)")
 	}
 	if disableClientsEnv == "" {
 		flag.BoolVar(&disableClients, "disable-clients", false, "Disable clients on the WireGuard interface")
@@ -528,6 +544,12 @@ func runNewtMain(ctx context.Context) {
 		logger.Info("Newt version %s", newtVersion)
 	}

+	if useNativeMainInterface {
+		if err := checkNativeMainPermissions(); err != nil {
+			logger.Fatal("Insufficient permissions for native main tunnel interface: %v", err)
+		}
+	}
+
 	logger.Init(nil)
 	loggerLevel := util.ParseLogLevel(logLevel)

@@ -748,7 +770,7 @@ func runNewtMain(ctx context.Context) {
 	}

 	// Create TUN device and network stack
-	var tun tun.Device
+	var tun wtun.Device
 	var tnet *netstack.Net
 	var dev *device.Device
 	var pm *proxy.ProxyManager
@@ -897,13 +919,37 @@ func runNewtMain(ctx context.Context) {
 		}

 		logger.Debug(fmtReceivedMsg, msg)
-		tun, tnet, err = netstack.CreateNetTUN(
-			[]netip.Addr{netip.MustParseAddr(wgData.TunnelIP)},
-			[]netip.Addr{netip.MustParseAddr(dns)},
-			mtuInt)
-		if err != nil {
-			logger.Error("Failed to create TUN device: %v", err)
-			regResult = "failure"
+
+		if useNativeMainInterface {
+			mainIfName := nativeMainInterfaceName
+			if runtime.GOOS == "darwin" {
+				mainIfName, err = network.FindUnusedUTUN()
+				if err != nil {
+					logger.Error("Failed to find unused utun for main tunnel: %v", err)
+					regResult = "failure"
+					return
+				}
+			}
+			tun, err = wtun.CreateTUN(mainIfName, mtuInt)
+			if err != nil {
+				logger.Error("Failed to create native main TUN device: %v", err)
+				regResult = "failure"
+				return
+			}
+			if realName, nameErr := tun.Name(); nameErr == nil {
+				mainIfName = realName
+			}
+			tnet = nil
+			nativeMainInterfaceName = mainIfName
+		} else {
+			tun, tnet, err = netstack.CreateNetTUN(
+				[]netip.Addr{netip.MustParseAddr(wgData.TunnelIP)},
+				[]netip.Addr{netip.MustParseAddr(dns)},
+				mtuInt)
+			if err != nil {
+				logger.Error("Failed to create TUN device: %v", err)
+				regResult = "failure"
+			}
 		}

 		setDownstreamTNetstack(tnet)
@@ -957,6 +1003,34 @@ persistent_keepalive_interval=5`, util.FixKey(privateKey.String()), util.FixKey(
 			regResult = "failure"
 		}

+		if useNativeMainInterface {
+			if cfgErr := network.ConfigureInterface(nativeMainInterfaceName, wgData.TunnelIP+"/32", mtuInt); cfgErr != nil {
+				logger.Error("Failed to configure native main tunnel interface: %v", cfgErr)
+			}
+			if runtime.GOOS == "darwin" {
+				if routeErr := network.AddRoutes([]string{wgData.ServerIP + "/32"}, nativeMainInterfaceName); routeErr != nil {
+					logger.Warn("Failed to add route for main tunnel server IP: %v", routeErr)
+				}
+			}
+			// Set up UAPI so wg(8) can inspect the main tunnel interface
+			if fileUAPI, uapiErr := newtDevice.UapiOpen(nativeMainInterfaceName); uapiErr != nil {
+				logger.Warn("Main tunnel UAPI open error: %v", uapiErr)
+			} else if uapiListener, uapiListenErr := newtDevice.UapiListen(nativeMainInterfaceName, fileUAPI); uapiListenErr != nil {
+				logger.Warn("Main tunnel UAPI listen error: %v", uapiListenErr)
+			} else {
+				go func() {
+					for {
+						c, aErr := uapiListener.Accept()
+						if aErr != nil {
+							return
+						}
+						go dev.IpcHandle(c)
+					}
+				}()
+				logger.Debug("Main tunnel UAPI listener started on %s", nativeMainInterfaceName)
+			}
+		}
+
 		logger.Debug("WireGuard device created. Lets ping the server now...")

 		// Even if pingWithRetry returns an error, it will continue trying in the background
@@ -965,30 +1039,44 @@ persistent_keepalive_interval=5`, util.FixKey(privateKey.String()), util.FixKey(
 			close(pingWithRetryStopChan)
 			pingWithRetryStopChan = nil
 		}
-		// Use reliable ping for initial connection test
-		logger.Debug("Testing initial connection with reliable ping...")
-		lat, err := reliablePing(tnet, wgData.ServerIP, pingTimeout, 5)
-		if err == nil && wgData.PublicKey != "" {
-			telemetry.ObserveTunnelLatency(ctx, wgData.PublicKey, "wireguard", lat.Seconds())
-		}
-		if err != nil {
-			logger.Warn("Initial reliable ping failed, but continuing: %v", err)
-			regResult = "failure"
+
+		if !useNativeMainInterface {
+			// Use reliable ping for initial connection test
+			logger.Debug("Testing initial connection with reliable ping...")
+			lat, err := reliablePing(tnet, wgData.ServerIP, pingTimeout, 5)
+			if err == nil && wgData.PublicKey != "" {
+				telemetry.ObserveTunnelLatency(ctx, wgData.PublicKey, "wireguard", lat.Seconds())
+			}
+			if err != nil {
+				logger.Warn("Initial reliable ping failed, but continuing: %v", err)
+				regResult = "failure"
+			} else {
+				logger.Debug("Initial connection test successful")
+			}
+
+			pingWithRetryStopChan, _ = pingWithRetry(tnet, wgData.ServerIP, pingTimeout)
+
+			// Always mark as connected and start the proxy manager regardless of initial ping result
+			// as the pings will continue in the background
+			if !connected {
+				logger.Debug("Starting ping check")
+				pingStopChan = startPingCheck(tnet, wgData.ServerIP, client, wgData.PublicKey)
+			}
 		} else {
-			logger.Debug("Initial connection test successful")
-		}
-
-		pingWithRetryStopChan, _ = pingWithRetry(tnet, wgData.ServerIP, pingTimeout)
-
-		// Always mark as connected and start the proxy manager regardless of initial ping result
-		// as the pings will continue in the background
-		if !connected {
-			logger.Debug("Starting ping check")
-			pingStopChan = startPingCheck(tnet, wgData.ServerIP, client, wgData.PublicKey)
+			// Native main: no netstack-based ping; write health file directly.
+			if healthFile != "" {
+				if writeErr := os.WriteFile(healthFile, []byte("ok"), 0644); writeErr != nil {
+					logger.Warn(msgHealthFileWriteFailed, writeErr)
+				}
+			}
 		}

 		// Create proxy manager
-		pm = proxy.NewProxyManager(tnet)
+		if useNativeMainInterface {
+			pm = proxy.NewProxyManagerNative(wgData.TunnelIP)
+		} else {
+			pm = proxy.NewProxyManager(tnet)
+		}
 		pm.SetAsyncBytes(metricsAsyncBytes)
 		pm.SetUDPIdleTimeout(udpProxyIdleTimeout)
 		// Set tunnel_id for metrics (WireGuard peer public key)
@@ -1015,8 +1103,11 @@ persistent_keepalive_interval=5`, util.FixKey(privateKey.String()), util.FixKey(
 			// }
 		}

-		// Start direct UDP relay from main tunnel to clients' WireGuard (bypasses proxy)
-		clientsStartDirectRelay(wgData.TunnelIP)
+		// Start direct UDP relay from main tunnel to clients' WireGuard (bypasses proxy).
+		// Not needed for native main – the kernel routes UDP packets directly.
+		if !useNativeMainInterface {
+			clientsStartDirectRelay(wgData.TunnelIP)
+		}

 		if err := healthMonitor.AddTargets(wgData.HealthCheckTargets); err != nil {
 			logger.Error("Failed to bulk add health check targets: %v", err)
@@ -1051,7 +1142,13 @@ persistent_keepalive_interval=5`, util.FixKey(privateKey.String()), util.FixKey(
 			browserGateway = browsergateway.New(browsergateway.Config{SSHCredentials: sshCredStore})
 			browserGateway.SetTargets(bgTargets)

-			ln, bgErr := tnet.ListenTCP(&net.TCPAddr{Port: browsergateway.ListenPort})
+			var ln net.Listener
+			var bgErr error
+			if useNativeMainInterface {
+				ln, bgErr = net.Listen("tcp", fmt.Sprintf("%s:%d", wgData.TunnelIP, browsergateway.ListenPort))
+			} else {
+				ln, bgErr = tnet.ListenTCP(&net.TCPAddr{Port: browsergateway.ListenPort})
+			}
 			if bgErr != nil {
 				logger.Error("Failed to start browser gateway listener: %v", bgErr)
 			} else {
@@ -2030,9 +2127,15 @@ persistent_keepalive_interval=5`, util.FixKey(privateKey.String()), util.FixKey(
 		}

 		// If the gateway doesn't exist yet but we have a tunnel, start it
-		if browserGateway == nil && tnet != nil {
+		if browserGateway == nil && (tnet != nil || useNativeMainInterface) {
 			browserGateway = browsergateway.New(browsergateway.Config{SSHCredentials: sshCredStore})
-			ln, bgErr := tnet.ListenTCP(&net.TCPAddr{Port: browsergateway.ListenPort})
+			var ln net.Listener
+			var bgErr error
+			if useNativeMainInterface {
+				ln, bgErr = net.Listen("tcp", fmt.Sprintf("%s:%d", wgData.TunnelIP, browsergateway.ListenPort))
+			} else {
+				ln, bgErr = tnet.ListenTCP(&net.TCPAddr{Port: browsergateway.ListenPort})
+			}
 			if bgErr != nil {
 				logger.Error("Failed to start browser gateway listener: %v", bgErr)
 				browserGateway = nil
@@ -18,7 +18,6 @@ import (
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
 	"golang.zx2c4.com/wireguard/tun/netstack"
-	"gvisor.dev/gvisor/pkg/tcpip/adapters/gonet"
 )

 const (
@@ -56,13 +55,14 @@ type Target struct {

 // ProxyManager handles the creation and management of proxy connections
 type ProxyManager struct {
-	tnet       *netstack.Net
-	tcpTargets map[string]map[int]string // map[listenIP]map[port]targetAddress
-	udpTargets map[string]map[int]string
-	listeners  []*gonet.TCPListener
-	udpConns   []*gonet.UDPConn
-	running    bool
-	mutex      sync.RWMutex
+	tnet           *netstack.Net
+	tcpTargets     map[string]map[int]string // map[listenIP]map[port]targetAddress
+	udpTargets     map[string]map[int]string
+	listeners      []net.Listener
+	udpConns       []net.PacketConn
+	running        bool
+	mutex          sync.RWMutex
+	nativeListenIP string // when non-empty, use native OS listeners instead of netstack

 	// telemetry (multi-tunnel)
 	currentTunnelID string
@@ -149,14 +149,28 @@ func classifyProxyError(err error) string {
 	}
 }

-// NewProxyManager creates a new proxy manager instance
+// NewProxyManager creates a new proxy manager instance backed by a netstack.
 func NewProxyManager(tnet *netstack.Net) *ProxyManager {
 	return &ProxyManager{
 		tnet:           tnet,
 		tcpTargets:     make(map[string]map[int]string),
 		udpTargets:     make(map[string]map[int]string),
-		listeners:      make([]*gonet.TCPListener, 0),
-		udpConns:       make([]*gonet.UDPConn, 0),
+		listeners:      make([]net.Listener, 0),
+		udpConns:       make([]net.PacketConn, 0),
+		tunnels:        make(map[string]*tunnelEntry),
+		udpIdleTimeout: defaultUDPIdleTimeout,
+	}
+}
+
+// NewProxyManagerNative creates a proxy manager that binds listeners directly
+// to the host network stack on the given IP address.
+func NewProxyManagerNative(listenIP string) *ProxyManager {
+	return &ProxyManager{
+		nativeListenIP: listenIP,
+		tcpTargets:     make(map[string]map[int]string),
+		udpTargets:     make(map[string]map[int]string),
+		listeners:      make([]net.Listener, 0),
+		udpConns:       make([]net.PacketConn, 0),
 		tunnels:        make(map[string]*tunnelEntry),
 		udpIdleTimeout: defaultUDPIdleTimeout,
 	}
@@ -229,13 +243,14 @@ func (pm *ProxyManager) ClearTunnelID() {
 	pm.currentTunnelID = ""
 }

-// init function without tnet
+// NewProxyManagerWithoutTNet creates a proxy manager with no backing network.
+// Call SetTNet before starting.
 func NewProxyManagerWithoutTNet() *ProxyManager {
 	return &ProxyManager{
 		tcpTargets:     make(map[string]map[int]string),
 		udpTargets:     make(map[string]map[int]string),
-		listeners:      make([]*gonet.TCPListener, 0),
-		udpConns:       make([]*gonet.UDPConn, 0),
+		listeners:      make([]net.Listener, 0),
+		udpConns:       make([]net.PacketConn, 0),
 		udpIdleTimeout: defaultUDPIdleTimeout,
 	}
 }
@@ -496,21 +511,42 @@ func (pm *ProxyManager) Stop() error {
 func (pm *ProxyManager) startTarget(proto, listenIP string, port int, targetAddr string) error {
 	switch proto {
 	case "tcp":
-		listener, err := pm.tnet.ListenTCP(&net.TCPAddr{Port: port})
-		if err != nil {
-			return fmt.Errorf("failed to create TCP listener: %v", err)
+		var listener net.Listener
+		if pm.tnet != nil {
+			l, err := pm.tnet.ListenTCP(&net.TCPAddr{Port: port})
+			if err != nil {
+				return fmt.Errorf("failed to create TCP listener: %v", err)
+			}
+			listener = l
+		} else if pm.nativeListenIP != "" {
+			l, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP(pm.nativeListenIP), Port: port})
+			if err != nil {
+				return fmt.Errorf("failed to create native TCP listener on %s:%d: %v", pm.nativeListenIP, port, err)
+			}
+			listener = l
+		} else {
+			return fmt.Errorf("proxy manager has no tnet or native IP configured")
 		}
-
 		pm.listeners = append(pm.listeners, listener)
 		go pm.handleTCPProxy(listener, targetAddr)

 	case "udp":
-		addr := &net.UDPAddr{Port: port}
-		conn, err := pm.tnet.ListenUDP(addr)
-		if err != nil {
-			return fmt.Errorf("failed to create UDP listener: %v", err)
+		var conn net.PacketConn
+		if pm.tnet != nil {
+			c, err := pm.tnet.ListenUDP(&net.UDPAddr{Port: port})
+			if err != nil {
+				return fmt.Errorf("failed to create UDP listener: %v", err)
+			}
+			conn = c
+		} else if pm.nativeListenIP != "" {
+			c, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP(pm.nativeListenIP), Port: port})
+			if err != nil {
+				return fmt.Errorf("failed to create native UDP listener on %s:%d: %v", pm.nativeListenIP, port, err)
+			}
+			conn = c
+		} else {
+			return fmt.Errorf("proxy manager has no tnet or native IP configured")
 		}
-
 		pm.udpConns = append(pm.udpConns, conn)
 		go pm.handleUDPProxy(conn, targetAddr)

@@ -611,7 +647,7 @@ func (pm *ProxyManager) handleTCPProxy(listener net.Listener, targetAddr string)
 	}
 }

-func (pm *ProxyManager) handleUDPProxy(conn *gonet.UDPConn, targetAddr string) {
+func (pm *ProxyManager) handleUDPProxy(conn net.PacketConn, targetAddr string) {
 	bufPtr := getUDPBuffer()
 	defer putUDPBuffer(bufPtr)
 	buffer := *bufPtr