Monitor network changes and restart engine on detection (#1904)

2026-10-01 19:19:07 +02:00 · 2024-05-07 18:50:34 +02:00
parent 2e0047daea
commit 920877964f
38 changed files with 1027 additions and 190 deletions
@@ -0,0 +1,15 @@
+package networkmonitor
+
+import (
+	"context"
+)
+
+// NetworkWatcher watches for changes in network configuration.
+type NetworkWatcher struct {
+	cancel context.CancelFunc
+}
+
+// New creates a new network monitor.
+func New() *NetworkWatcher {
+	return &NetworkWatcher{}
+}
@@ -0,0 +1,133 @@
+//go:build (darwin && !ios) || dragonfly || freebsd || netbsd || openbsd
+
+package networkmonitor
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"syscall"
+	"unsafe"
+
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/route"
+	"golang.org/x/sys/unix"
+
+	"github.com/netbirdio/netbird/client/internal/routemanager"
+)
+
+func checkChange(ctx context.Context, nexthopv4 netip.Addr, intfv4 *net.Interface, nexthopv6 netip.Addr, intfv6 *net.Interface, callback func()) error {
+	fd, err := unix.Socket(syscall.AF_ROUTE, syscall.SOCK_RAW, syscall.AF_UNSPEC)
+	if err != nil {
+		return fmt.Errorf("failed to open routing socket: %v", err)
+	}
+	defer func() {
+		if err := unix.Close(fd); err != nil {
+			log.Errorf("Network monitor: failed to close routing socket: %v", err)
+		}
+	}()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			buf := make([]byte, 2048)
+			n, err := unix.Read(fd, buf)
+			if err != nil {
+				log.Errorf("Network monitor: failed to read from routing socket: %v", err)
+				continue
+			}
+			if n < unix.SizeofRtMsghdr {
+				log.Errorf("Network monitor: read from routing socket returned less than expected: %d bytes", n)
+				continue
+			}
+
+			msg := (*unix.RtMsghdr)(unsafe.Pointer(&buf[0]))
+
+			switch msg.Type {
+
+			// handle interface state changes
+			case unix.RTM_IFINFO:
+				ifinfo, err := parseInterfaceMessage(buf[:n])
+				if err != nil {
+					log.Errorf("Network monitor: error parsing interface message: %v", err)
+					continue
+				}
+				if msg.Flags&unix.IFF_UP != 0 {
+					continue
+				}
+				if (intfv4 == nil || ifinfo.Index != intfv4.Index) && (intfv6 == nil || ifinfo.Index != intfv6.Index) {
+					continue
+				}
+
+				log.Infof("Network monitor: monitored interface (%s) is down.", ifinfo.Name)
+				callback()
+
+			// handle route changes
+			case unix.RTM_ADD, syscall.RTM_DELETE:
+				route, err := parseRouteMessage(buf[:n])
+				if err != nil {
+					log.Errorf("Network monitor: error parsing routing message: %v", err)
+					continue
+				}
+
+				if !route.Dst.Addr().IsUnspecified() {
+					continue
+				}
+
+				intf := "<nil>"
+				if route.Interface != nil {
+					intf = route.Interface.Name
+				}
+				switch msg.Type {
+				case unix.RTM_ADD:
+					log.Infof("Network monitor: default route changed: via %s, interface %s", route.Gw, intf)
+					callback()
+				case unix.RTM_DELETE:
+					if intfv4 != nil && route.Gw.Compare(nexthopv4) == 0 || intfv6 != nil && route.Gw.Compare(nexthopv6) == 0 {
+						log.Infof("Network monitor: default route removed: via %s, interface %s", route.Gw, intf)
+						callback()
+					}
+				}
+			}
+		}
+	}
+}
+
+func parseInterfaceMessage(buf []byte) (*route.InterfaceMessage, error) {
+	msgs, err := route.ParseRIB(route.RIBTypeInterface, buf)
+	if err != nil {
+		return nil, fmt.Errorf("parse RIB: %v", err)
+	}
+
+	if len(msgs) != 1 {
+		return nil, fmt.Errorf("unexpected RIB message msgs: %v", msgs)
+	}
+
+	msg, ok := msgs[0].(*route.InterfaceMessage)
+	if !ok {
+		return nil, fmt.Errorf("unexpected RIB message type: %T", msgs[0])
+	}
+
+	return msg, nil
+}
+
+func parseRouteMessage(buf []byte) (*routemanager.Route, error) {
+	msgs, err := route.ParseRIB(route.RIBTypeRoute, buf)
+	if err != nil {
+		return nil, fmt.Errorf("parse RIB: %v", err)
+	}
+
+	if len(msgs) != 1 {
+		return nil, fmt.Errorf("unexpected RIB message msgs: %v", msgs)
+	}
+
+	msg, ok := msgs[0].(*route.RouteMessage)
+	if !ok {
+		return nil, fmt.Errorf("unexpected RIB message type: %T", msgs[0])
+	}
+
+	return routemanager.MsgToRoute(msg)
+}
@@ -0,0 +1,82 @@
+//go:build !ios && !android
+
+package networkmonitor
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/netip"
+	"runtime/debug"
+
+	"github.com/cenkalti/backoff/v4"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/routemanager"
+)
+
+// Start begins watching for network changes and calls the callback function and stops when a change is detected.
+func (nw *NetworkWatcher) Start(ctx context.Context, callback func()) {
+	if nw.cancel != nil {
+		log.Warn("Network monitor: already running, stopping previous watcher")
+		nw.Stop()
+	}
+
+	if ctx.Err() != nil {
+		log.Info("Network monitor: not starting, context is already cancelled")
+		return
+	}
+
+	ctx, nw.cancel = context.WithCancel(ctx)
+	defer nw.Stop()
+
+	var nexthop4, nexthop6 netip.Addr
+	var intf4, intf6 *net.Interface
+
+	operation := func() error {
+		var errv4, errv6 error
+		nexthop4, intf4, errv4 = routemanager.GetNextHop(netip.IPv4Unspecified())
+		nexthop6, intf6, errv6 = routemanager.GetNextHop(netip.IPv6Unspecified())
+
+		if errv4 != nil && errv6 != nil {
+			return errors.New("failed to get default next hops")
+		}
+
+		if errv4 == nil {
+			log.Debugf("Network monitor: IPv4 default route: %s, interface: %s", nexthop4, intf4.Name)
+		}
+		if errv6 == nil {
+			log.Debugf("Network monitor: IPv6 default route: %s, interface: %s", nexthop6, intf6.Name)
+		}
+
+		// continue if either route was found
+		return nil
+	}
+
+	expBackOff := backoff.WithContext(backoff.NewExponentialBackOff(), ctx)
+
+	if err := backoff.Retry(operation, expBackOff); err != nil {
+		log.Errorf("Network monitor: failed to get default next hops: %v", err)
+		return
+	}
+
+	// recover in case sys ops panic
+	defer func() {
+		if r := recover(); r != nil {
+			log.Errorf("Network monitor: panic occurred: %v, stack trace: %s", r, string(debug.Stack()))
+		}
+	}()
+
+	if err := checkChange(ctx, nexthop4, intf4, nexthop6, intf6, callback); err != nil && !errors.Is(err, context.Canceled) {
+		log.Errorf("Network monitor: failed to start: %v", err)
+	}
+}
+
+// Stop stops the network monitor.
+func (nw *NetworkWatcher) Stop() {
+	if nw.cancel != nil {
+		nw.cancel()
+		nw.cancel = nil
+		log.Info("Network monitor: stopped")
+	}
+}
@@ -0,0 +1,81 @@
+//go:build !android
+
+package networkmonitor
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"syscall"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/vishvananda/netlink"
+)
+
+func checkChange(ctx context.Context, nexthopv4 netip.Addr, intfv4 *net.Interface, nexthop6 netip.Addr, intfv6 *net.Interface, callback func()) error {
+	if intfv4 == nil && intfv6 == nil {
+		return errors.New("no interfaces available")
+	}
+
+	linkChan := make(chan netlink.LinkUpdate)
+	done := make(chan struct{})
+	defer close(done)
+
+	if err := netlink.LinkSubscribe(linkChan, done); err != nil {
+		return fmt.Errorf("subscribe to link updates: %v", err)
+	}
+
+	routeChan := make(chan netlink.RouteUpdate)
+	if err := netlink.RouteSubscribe(routeChan, done); err != nil {
+		return fmt.Errorf("subscribe to route updates: %v", err)
+	}
+
+	log.Info("Network monitor: started")
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+
+		// handle interface state changes
+		case update := <-linkChan:
+			if (intfv4 == nil || update.Index != int32(intfv4.Index)) && (intfv6 == nil || update.Index != int32(intfv6.Index)) {
+				continue
+			}
+
+			switch update.Header.Type {
+			case syscall.RTM_DELLINK:
+				log.Infof("Network monitor: monitored interface (%s) is gone", update.Link.Attrs().Name)
+				callback()
+				return nil
+			case syscall.RTM_NEWLINK:
+				if (update.IfInfomsg.Flags&syscall.IFF_RUNNING) == 0 && update.Link.Attrs().OperState == netlink.OperDown {
+					log.Infof("Network monitor: monitored interface (%s) is down.", update.Link.Attrs().Name)
+					callback()
+					return nil
+				}
+			}
+
+		// handle route changes
+		case route := <-routeChan:
+			// default route and main table
+			if route.Dst != nil || route.Table != syscall.RT_TABLE_MAIN {
+				continue
+			}
+			switch route.Type {
+			// triggered on added/replaced routes
+			case syscall.RTM_NEWROUTE:
+				log.Infof("Network monitor: default route changed: via %s, interface %d", route.Gw, route.LinkIndex)
+				callback()
+				return nil
+			case syscall.RTM_DELROUTE:
+				if intfv4 != nil && route.Gw.Equal(nexthopv4.AsSlice()) || intfv6 != nil && route.Gw.Equal(nexthop6.AsSlice()) {
+					log.Infof("Network monitor: default route removed: via %s, interface %d", route.Gw, route.LinkIndex)
+					callback()
+					return nil
+				}
+			}
+		}
+	}
+}
@@ -0,0 +1,11 @@
+//go:build ios || android
+
+package networkmonitor
+
+import "context"
+
+func (nw *NetworkWatcher) Start(context.Context, func()) {
+}
+
+func (nw *NetworkWatcher) Stop() {
+}
@@ -0,0 +1,215 @@
+package networkmonitor
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/routemanager"
+)
+
+const (
+	unreachable = 0
+	incomplete  = 1
+	probe       = 2
+	delay       = 3
+	stale       = 4
+	reachable   = 5
+	permanent   = 6
+	tbd         = 7
+)
+
+const interval = 10 * time.Second
+
+func checkChange(ctx context.Context, nexthopv4 netip.Addr, intfv4 *net.Interface, nexthopv6 netip.Addr, intfv6 *net.Interface, callback func()) error {
+	var neighborv4, neighborv6 *routemanager.Neighbor
+	{
+		initialNeighbors, err := getNeighbors()
+		if err != nil {
+			return fmt.Errorf("get neighbors: %w", err)
+		}
+
+		if n, ok := initialNeighbors[nexthopv4]; ok {
+			neighborv4 = &n
+		}
+		if n, ok := initialNeighbors[nexthopv6]; ok {
+			neighborv6 = &n
+		}
+	}
+	log.Debugf("Network monitor: initial IPv4 neighbor: %v, IPv6 neighbor: %v", neighborv4, neighborv6)
+
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+			if changed(nexthopv4, intfv4, neighborv4, nexthopv6, intfv6, neighborv6) {
+				callback()
+				return nil
+			}
+		}
+	}
+}
+
+func changed(
+	nexthopv4 netip.Addr,
+	intfv4 *net.Interface,
+	neighborv4 *routemanager.Neighbor,
+	nexthopv6 netip.Addr,
+	intfv6 *net.Interface,
+	neighborv6 *routemanager.Neighbor,
+) bool {
+	neighbors, err := getNeighbors()
+	if err != nil {
+		log.Errorf("network monitor: error fetching current neighbors: %v", err)
+		return false
+	}
+	if neighborChanged(nexthopv4, neighborv4, neighbors) || neighborChanged(nexthopv6, neighborv6, neighbors) {
+		return true
+	}
+
+	routes, err := getRoutes()
+	if err != nil {
+		log.Errorf("network monitor: error fetching current routes: %v", err)
+		return false
+	}
+
+	if routeChanged(nexthopv4, intfv4, routes) || routeChanged(nexthopv6, intfv6, routes) {
+		return true
+	}
+
+	return false
+}
+
+// routeChanged checks if the default routes still point to our nexthop/interface
+func routeChanged(nexthop netip.Addr, intf *net.Interface, routes map[netip.Prefix]routemanager.Route) bool {
+	if !nexthop.IsValid() {
+		return false
+	}
+
+	var unspec netip.Prefix
+	if nexthop.Is6() {
+		unspec = netip.PrefixFrom(netip.IPv6Unspecified(), 0)
+	} else {
+		unspec = netip.PrefixFrom(netip.IPv4Unspecified(), 0)
+	}
+
+	if r, ok := routes[unspec]; ok {
+		if r.Nexthop != nexthop || compareIntf(r.Interface, intf) != 0 {
+			intf := "<nil>"
+			if r.Interface != nil {
+				intf = r.Interface.Name
+			}
+			log.Infof("network monitor: default route changed: %s via %s (%s)", r.Destination, r.Nexthop, intf)
+			return true
+		}
+	} else {
+		log.Infof("network monitor: default route is gone")
+		return true
+	}
+
+	return false
+
+}
+
+func neighborChanged(nexthop netip.Addr, neighbor *routemanager.Neighbor, neighbors map[netip.Addr]routemanager.Neighbor) bool {
+	if neighbor == nil {
+		return false
+	}
+
+	// TODO: consider non-local nexthops, e.g. on point-to-point interfaces
+	if n, ok := neighbors[nexthop]; ok {
+		if n.State != reachable && n.State != permanent {
+			log.Infof("network monitor: neighbor %s (%s) is not reachable: %s", neighbor.IPAddress, neighbor.LinkLayerAddress, stateFromInt(n.State))
+			return true
+		} else if n.InterfaceIndex != neighbor.InterfaceIndex {
+			log.Infof(
+				"network monitor: neighbor %s (%s) changed interface from '%s' (%d) to '%s' (%d): %s",
+				neighbor.IPAddress,
+				neighbor.LinkLayerAddress,
+				neighbor.InterfaceAlias,
+				neighbor.InterfaceIndex,
+				n.InterfaceAlias,
+				n.InterfaceIndex,
+				stateFromInt(n.State),
+			)
+			return true
+		}
+	} else {
+		log.Infof("network monitor: neighbor %s (%s) is gone", neighbor.IPAddress, neighbor.LinkLayerAddress)
+		return true
+	}
+
+	return false
+}
+
+func getNeighbors() (map[netip.Addr]routemanager.Neighbor, error) {
+	entries, err := routemanager.GetNeighbors()
+	if err != nil {
+		return nil, fmt.Errorf("get neighbors: %w", err)
+	}
+
+	neighbours := make(map[netip.Addr]routemanager.Neighbor, len(entries))
+	for _, entry := range entries {
+		neighbours[entry.IPAddress] = entry
+	}
+
+	return neighbours, nil
+}
+
+func getRoutes() (map[netip.Prefix]routemanager.Route, error) {
+	entries, err := routemanager.GetRoutes()
+	if err != nil {
+		return nil, fmt.Errorf("get routes: %w", err)
+	}
+
+	routes := make(map[netip.Prefix]routemanager.Route, len(entries))
+	for _, entry := range entries {
+		routes[entry.Destination] = entry
+	}
+
+	return routes, nil
+}
+
+func stateFromInt(state uint8) string {
+	switch state {
+	case unreachable:
+		return "unreachable"
+	case incomplete:
+		return "incomplete"
+	case probe:
+		return "probe"
+	case delay:
+		return "delay"
+	case stale:
+		return "stale"
+	case reachable:
+		return "reachable"
+	case permanent:
+		return "permanent"
+	case tbd:
+		return "tbd"
+	default:
+		return "unknown"
+	}
+}
+
+func compareIntf(a, b *net.Interface) int {
+	if a == nil && b == nil {
+		return 0
+	}
+	if a == nil {
+		return -1
+	}
+	if b == nil {
+		return 1
+	}
+	return a.Index - b.Index
+}