Add dns watchdog and override fixer

dns/override/dns_override_android.go

@@ -20,3 +20,9 @@ func CleanupStaleState(interfaceName string) error {
 	_ = interfaceName
 	return nil
 }
+
+// ForceResetDNS is a no-op on Android.
+func ForceResetDNS(interfaceName string) error {
+	_ = interfaceName
+	return nil
+}

dns/override/dns_override_darwin.go

@@ -15,6 +15,13 @@ var configurator platform.DNSConfigurator
 // SetupDNSOverride configures the system DNS to use the DNS proxy on macOS
 // Uses scutil for DNS configuration
 func SetupDNSOverride(interfaceName string, proxyIp netip.Addr) error {
+	// Defensively clear any stale DNS state from a previous unclean shutdown
+	// before installing the new override. This makes a second tunnel start
+	// safe even if the previous client crashed without restoring DNS.
+	if err := CleanupStaleState(interfaceName); err != nil {
+		logger.Warn("Pre-setup stale DNS cleanup failed (continuing): %v", err)
+	}
+
 	var err error
 	configurator, err = platform.NewDarwinDNSConfigurator()
 	if err != nil {
@@ -78,3 +85,31 @@ func CleanupStaleState(interfaceName string) error {
 	logger.Info("Stale DNS state cleanup completed successfully")
 	return nil
 }
+
+// ForceResetDNS forcibly clears any DNS override state, whether or not the
+// current process installed it. This is intended for the "reset-dns" CLI
+// command and for the watchdog process to recover from a stuck override
+// left behind by a crashed client.
+func ForceResetDNS(interfaceName string) error {
+	logger.Info("Forcing DNS reset on Darwin (interface=%s)", interfaceName)
+
+	// First clean up any persisted state from a previous session.
+	cleanupErr := CleanupStaleState(interfaceName)
+
+	// Then, if the current process happens to hold a live configurator,
+	// instruct it to restore DNS as well so in-memory state is consistent.
+	if configurator != nil {
+		if err := configurator.RestoreDNS(); err != nil {
+			logger.Warn("ForceResetDNS: in-memory restore failed: %v", err)
+		}
+		configurator = nil
+	}
+
+	// As a last-resort defense, sweep any scutil keys matching our naming
+	// convention even if no state file exists.
+	if err := platform.SweepOlmScutilKeys(); err != nil {
+		logger.Warn("ForceResetDNS: scutil sweep failed: %v", err)
+	}
+
+	return cleanupErr
+}

dns/override/dns_override_ios.go

@@ -19,3 +19,9 @@ func CleanupStaleState(interfaceName string) error {
 	_ = interfaceName
 	return nil
 }
+
+// ForceResetDNS is a no-op on iOS.
+func ForceResetDNS(interfaceName string) error {
+	_ = interfaceName
+	return nil
+}

dns/override/dns_override_unix.go

@@ -15,6 +15,13 @@ var configurator platform.DNSConfigurator
 // SetupDNSOverride configures the system DNS to use the DNS proxy on Linux/FreeBSD
 // Detects the DNS manager by reading /etc/resolv.conf and verifying runtime availability
 func SetupDNSOverride(interfaceName string, proxyIp netip.Addr) error {
+	// Defensively clear any stale DNS state from a previous unclean shutdown
+	// before installing the new override. This makes a second tunnel start
+	// safe even if the previous client crashed without restoring DNS.
+	if err := CleanupStaleState(interfaceName); err != nil {
+		logger.Warn("Pre-setup stale DNS cleanup failed (continuing): %v", err)
+	}
+
 	var err error
 
 	// Detect which DNS manager is in use by checking /etc/resolv.conf and runtime availability
@@ -144,3 +151,25 @@ func CleanupStaleState(interfaceName string) error {
 	logger.Info("Stale DNS state cleanup completed successfully")
 	return nil
 }
+
+// ForceResetDNS forcibly clears any DNS override state, whether or not the
+// current process installed it. This is intended for the "reset-dns" CLI
+// command and for the watchdog process to recover from a stuck override
+// left behind by a crashed client.
+func ForceResetDNS(interfaceName string) error {
+	logger.Info("Forcing DNS reset on Linux/FreeBSD (interface=%s)", interfaceName)
+
+	// First clean up any persisted state from a previous session.
+	cleanupErr := CleanupStaleState(interfaceName)
+
+	// Then, if the current process happens to hold a live configurator,
+	// instruct it to restore DNS as well so in-memory state is consistent.
+	if configurator != nil {
+		if err := configurator.RestoreDNS(); err != nil {
+			logger.Warn("ForceResetDNS: in-memory restore failed: %v", err)
+		}
+		configurator = nil
+	}
+
+	return cleanupErr
+}

dns/override/dns_override_windows.go

@@ -15,6 +15,12 @@ var configurator platform.DNSConfigurator
 // SetupDNSOverride configures the system DNS to use the DNS proxy on Windows
 // Uses registry-based configuration (automatically extracts interface GUID)
 func SetupDNSOverride(interfaceName string, proxyIp netip.Addr) error {
+	// Defensively clear any stale DNS state from a previous unclean shutdown
+	// before installing the new override.
+	if err := CleanupStaleState(interfaceName); err != nil {
+		logger.Warn("Pre-setup stale DNS cleanup failed (continuing): %v", err)
+	}
+
 	var err error
 	configurator, err = platform.NewWindowsDNSConfigurator(interfaceName)
 	if err != nil {
@@ -77,3 +83,17 @@ func CleanupStaleState(interfaceName string) error {
 	logger.Debug("Windows DNS cleanup: no stale state to clean (interface-specific)")
 	return nil
 }
+
+// ForceResetDNS forcibly clears any DNS override state. On Windows this is
+// largely a no-op because the registry override is tied to the interface
+// GUID and is reclaimed when the interface is torn down.
+func ForceResetDNS(interfaceName string) error {
+	logger.Info("Forcing DNS reset on Windows (interface=%s)", interfaceName)
+	if configurator != nil {
+		if err := configurator.RestoreDNS(); err != nil {
+			logger.Warn("ForceResetDNS: in-memory restore failed: %v", err)
+		}
+		configurator = nil
+	}
+	return CleanupStaleState(interfaceName)
+}

dns/override/watchdog.go

@@ -0,0 +1,136 @@
+package olm
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/fosrl/newt/logger"
+)
+
+// WatchdogConfig configures the DNS override watchdog. The watchdog runs as
+// an external process (spawned via SpawnWatchdog) and monitors a parent olm
+// process. When the parent appears to have died without restoring DNS, the
+// watchdog forcibly resets the system DNS configuration.
+type WatchdogConfig struct {
+	// ParentPID is the PID of the olm process that installed the DNS
+	// override. The watchdog exits when this PID is no longer alive.
+	ParentPID int
+
+	// SocketPath is the path to the olm Unix domain socket (or named pipe
+	// on Windows). The watchdog uses it as a secondary liveness signal.
+	// May be empty if no socket-based API is enabled.
+	SocketPath string
+
+	// InterfaceName is the name of the WireGuard interface whose DNS
+	// override should be reset on parent death.
+	InterfaceName string
+
+	// CheckInterval is how often to poll the parent's liveness.
+	// Defaults to 5 seconds when zero.
+	CheckInterval time.Duration
+
+	// FailureThreshold is the number of consecutive failed liveness checks
+	// before the watchdog declares the parent dead and resets DNS.
+	// Defaults to 3 when zero.
+	FailureThreshold int
+}
+
+// RunWatchdog runs the watchdog loop in the current process until either
+// (a) the parent dies and DNS is reset, or (b) ctx is cancelled.
+func RunWatchdog(ctx context.Context, cfg WatchdogConfig) error {
+	if cfg.ParentPID <= 0 {
+		return fmt.Errorf("watchdog: invalid parent PID %d", cfg.ParentPID)
+	}
+	if cfg.CheckInterval <= 0 {
+		cfg.CheckInterval = 5 * time.Second
+	}
+	if cfg.FailureThreshold <= 0 {
+		cfg.FailureThreshold = 3
+	}
+
+	logger.Info("DNS watchdog started: parent=%d interval=%s threshold=%d socket=%q interface=%q",
+		cfg.ParentPID, cfg.CheckInterval, cfg.FailureThreshold, cfg.SocketPath, cfg.InterfaceName)
+
+	ticker := time.NewTicker(cfg.CheckInterval)
+	defer ticker.Stop()
+
+	consecutiveFailures := 0
+
+	for {
+		select {
+		case <-ctx.Done():
+			logger.Info("DNS watchdog context cancelled, exiting cleanly")
+			return ctx.Err()
+		case <-ticker.C:
+		}
+
+		alive := isParentAlive(cfg.ParentPID, cfg.SocketPath)
+		if alive {
+			if consecutiveFailures > 0 {
+				logger.Debug("DNS watchdog: parent recovered after %d failures", consecutiveFailures)
+			}
+			consecutiveFailures = 0
+			continue
+		}
+
+		consecutiveFailures++
+		logger.Warn("DNS watchdog: parent liveness check failed (%d/%d)",
+			consecutiveFailures, cfg.FailureThreshold)
+
+		if consecutiveFailures >= cfg.FailureThreshold {
+			logger.Warn("DNS watchdog: parent declared dead, forcing DNS reset")
+			if err := ForceResetDNS(cfg.InterfaceName); err != nil {
+				logger.Error("DNS watchdog: ForceResetDNS failed: %v", err)
+				return err
+			}
+			logger.Info("DNS watchdog: DNS reset complete, exiting")
+			return nil
+		}
+	}
+}
+
+// isParentAlive returns true if the parent process appears to be alive. It
+// considers the parent alive if EITHER the PID is still running OR the
+// socket-based health endpoint responds. This dual check avoids false
+// positives where one signal is flaky (e.g., socket blocked but process
+// still recovering).
+func isParentAlive(pid int, socketPath string) bool {
+	if pidAlive(pid) {
+		return true
+	}
+	// Process is gone; double-check via socket to avoid races where PID
+	// recycling or signal-0 quirks lie to us. Socket should already be
+	// gone too.
+	if socketPath != "" && socketHealthy(socketPath) {
+		return true
+	}
+	return false
+}
+
+// socketHealthy attempts a fast /health request over the unix socket.
+func socketHealthy(socketPath string) bool {
+	if _, err := os.Stat(socketPath); err != nil {
+		return false
+	}
+
+	client := &http.Client{
+		Timeout: 2 * time.Second,
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+				d := net.Dialer{Timeout: 2 * time.Second}
+				return d.DialContext(ctx, "unix", socketPath)
+			},
+		},
+	}
+
+	resp, err := client.Get("http://localhost/health")
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+	return resp.StatusCode == http.StatusOK
+}

dns/override/watchdog_spawn_unix.go

@@ -0,0 +1,119 @@
+//go:build !windows
+
+package olm
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"syscall"
+
+	"github.com/fosrl/newt/logger"
+)
+
+// SpawnWatchdogConfig captures the inputs needed to launch the external
+// watchdog subprocess that monitors the calling olm process and forces a
+// DNS reset if the parent dies before restoring DNS.
+type SpawnWatchdogConfig struct {
+	// Executable is the path to the binary that will host the watchdog
+	// (typically os.Executable()). The binary must understand the
+	// watchdog subcommand layout described below.
+	Executable string
+
+	// Subcommand is the argv prefix the binary uses to enter watchdog
+	// mode (e.g., []string{"watchdog"} or []string{"dns", "watchdog"}).
+	Subcommand []string
+
+	// InterfaceName is the WireGuard interface whose DNS override should
+	// be reset if the parent dies.
+	InterfaceName string
+
+	// SocketPath is the parent's olm API socket path (may be empty).
+	SocketPath string
+
+	// LogFile, if non-empty, is the path the watchdog writes its stdout
+	// and stderr to. If empty, /dev/null is used.
+	LogFile string
+}
+
+// SpawnWatchdog launches the watchdog subprocess in a detached process group
+// so that it survives the death of the parent. The returned *exec.Cmd is the
+// handle the parent should call StopWatchdog on during clean shutdown.
+//
+// The spawned process is invoked as:
+//
+//	<Executable> <Subcommand...> --parent-pid=<ppid> \
+//	    --interface=<InterfaceName> [--socket=<SocketPath>]
+//
+// Both pangolin (cli) and olm should map their watchdog subcommand to
+// RunWatchdog.
+func SpawnWatchdog(cfg SpawnWatchdogConfig) (*exec.Cmd, error) {
+	if cfg.Executable == "" {
+		return nil, fmt.Errorf("watchdog: executable is required")
+	}
+	if len(cfg.Subcommand) == 0 {
+		return nil, fmt.Errorf("watchdog: subcommand is required")
+	}
+
+	args := append([]string{}, cfg.Subcommand...)
+	args = append(args,
+		"--parent-pid="+strconv.Itoa(os.Getpid()),
+		"--interface="+cfg.InterfaceName,
+	)
+	if cfg.SocketPath != "" {
+		args = append(args, "--socket="+cfg.SocketPath)
+	}
+
+	cmd := exec.Command(cfg.Executable, args...)
+
+	// Detach: new session so the watchdog is not killed by a signal
+	// delivered to the parent's process group.
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setsid: true,
+	}
+
+	// Direct watchdog output to a log file or /dev/null so it doesn't
+	// share file descriptors with the parent's TTY.
+	logTarget := cfg.LogFile
+	if logTarget == "" {
+		logTarget = os.DevNull
+	}
+	logFile, err := os.OpenFile(logTarget, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
+	if err != nil {
+		return nil, fmt.Errorf("watchdog: open log file: %w", err)
+	}
+	cmd.Stdin = nil
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+
+	if err := cmd.Start(); err != nil {
+		_ = logFile.Close()
+		return nil, fmt.Errorf("watchdog: start: %w", err)
+	}
+
+	// We don't need our handle on the log file after the subprocess
+	// inherits it.
+	_ = logFile.Close()
+
+	logger.Info("DNS watchdog spawned (pid=%d, exe=%s)", cmd.Process.Pid, cfg.Executable)
+	return cmd, nil
+}
+
+// StopWatchdog asks the watchdog to exit cleanly via SIGTERM and reaps it.
+// Safe to call with a nil cmd.
+func StopWatchdog(cmd *exec.Cmd) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+	pid := cmd.Process.Pid
+	if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
+		logger.Debug("DNS watchdog stop signal failed (pid=%d): %v", pid, err)
+	}
+	// Reap in the background; we don't want to block shutdown if the
+	// watchdog is wedged.
+	go func() {
+		_ = cmd.Wait()
+		logger.Debug("DNS watchdog (pid=%d) reaped", pid)
+	}()
+}

dns/override/watchdog_spawn_windows.go

@@ -0,0 +1,29 @@
+//go:build windows
+
+package olm
+
+import (
+	"os/exec"
+)
+
+// SpawnWatchdogConfig is provided on Windows for API symmetry but the
+// watchdog itself is effectively a no-op there (see watchdog_windows.go).
+type SpawnWatchdogConfig struct {
+	Executable    string
+	Subcommand    []string
+	InterfaceName string
+	SocketPath    string
+	LogFile       string
+}
+
+// SpawnWatchdog is a no-op on Windows; DNS overrides are interface-GUID
+// scoped and reclaimed when the interface is removed.
+func SpawnWatchdog(cfg SpawnWatchdogConfig) (*exec.Cmd, error) {
+	_ = cfg
+	return nil, nil
+}
+
+// StopWatchdog is a no-op on Windows.
+func StopWatchdog(cmd *exec.Cmd) {
+	_ = cmd
+}

dns/override/watchdog_unix.go

@@ -0,0 +1,25 @@
+//go:build !windows
+
+package olm
+
+import (
+	"os"
+	"syscall"
+)
+
+// pidAlive returns true if the process with the given PID is still alive.
+// On Unix-like systems we use signal 0, which performs error checking but
+// does not deliver an actual signal.
+func pidAlive(pid int) bool {
+	if pid <= 0 {
+		return false
+	}
+	proc, err := os.FindProcess(pid)
+	if err != nil {
+		return false
+	}
+	if err := proc.Signal(syscall.Signal(0)); err != nil {
+		return false
+	}
+	return true
+}

dns/override/watchdog_windows.go

@@ -0,0 +1,15 @@
+//go:build windows
+
+package olm
+
+// pidAlive on Windows. Reliable PID probing on Windows requires syscall
+// OpenProcess with PROCESS_QUERY_LIMITED_INFORMATION followed by
+// GetExitCodeProcess, which is non-trivial. Since DNS override on Windows
+// is interface-GUID-scoped and is naturally cleaned up when the WireGuard
+// interface goes away, the watchdog is effectively a no-op on Windows.
+// We always report the parent as alive so the watchdog never tears down
+// DNS based on PID checks.
+func pidAlive(pid int) bool {
+	_ = pid
+	return true
+}

dns/platform/darwin.go

@@ -422,6 +422,17 @@ func (d *DarwinDNSConfigurator) clearState() error {
 // configurator from a previous unclean shutdown. This is a static function that can be
 // called without creating a configurator instance, useful for cleanup before network operations.
 func CleanupStaleDarwinDNS() error {
+	// Always sweep orphaned Olm scutil keys regardless of whether a state
+	// file exists. This protects against cases where the state file was
+	// lost (e.g., user home wiped, write failed) but DNS keys are still
+	// installed in the running scutil session.
+	defer func() {
+		_ = SweepOlmScutilKeys()
+		// Flush DNS cache after any sweep so changes take effect.
+		_ = exec.Command(dscacheutilPath, "-flushcache").Run()
+		_ = exec.Command("killall", "-HUP", "mDNSResponder").Run()
+	}()
+
 	stateFilePath := getDNSStateFilePath()
 
 	// Check if state file exists
@@ -473,3 +484,59 @@ func CleanupStaleDarwinDNS() error {
 
 	return nil
 }
+
+// SweepOlmScutilKeys enumerates scutil State:/Network/Service/Olm-* keys and
+// removes any that are present. This is a best-effort safety net used when
+// state files have been lost or never written.
+func SweepOlmScutilKeys() error {
+	// list scutil keys matching our naming convention
+	listOutput, err := runScutilOnce("list State:/Network/Service/Olm-.*/DNS\n")
+	if err != nil {
+		return fmt.Errorf("scutil list: %w", err)
+	}
+
+	var keys []string
+	scanner := bufio.NewScanner(bytes.NewReader(listOutput))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		// scutil output format: subKey [0] = State:/Network/Service/Olm-Override/DNS
+		idx := strings.Index(line, "State:/Network/Service/Olm-")
+		if idx < 0 {
+			continue
+		}
+		key := strings.TrimSpace(line[idx:])
+		if key != "" {
+			keys = append(keys, key)
+		}
+	}
+
+	if len(keys) == 0 {
+		return nil
+	}
+
+	logger.Info("Sweeping %d orphaned Olm scutil DNS keys", len(keys))
+
+	var commands strings.Builder
+	for _, key := range keys {
+		commands.WriteString(fmt.Sprintf("remove %s\n", key))
+	}
+
+	if _, err := runScutilOnce(commands.String()); err != nil {
+		return fmt.Errorf("scutil sweep remove: %w", err)
+	}
+
+	return nil
+}
+
+// runScutilOnce runs a one-shot scutil command sequence wrapped with open/quit
+// without requiring a configurator instance.
+func runScutilOnce(commands string) ([]byte, error) {
+	wrapped := fmt.Sprintf("open\n%squit\n", commands)
+	cmd := exec.Command(scutilPath)
+	cmd.Stdin = strings.NewReader(wrapped)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("scutil command failed: %w, output: %s", err, output)
+	}
+	return output, nil
+}