Merge branch 'main' into refactor-get-account-by-token

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

client/internal/acl/manager.go

@@ -3,6 +3,7 @@ package acl
 import (
 	"crypto/md5"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"net"
 	"net/netip"
@@ -10,14 +11,18 @@ import (
 	"sync"
 	"time"
 
+	"github.com/hashicorp/go-multierror"
 	log "github.com/sirupsen/logrus"
 
+	nberrors "github.com/netbirdio/netbird/client/errors"
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/internal/acl/id"
 	"github.com/netbirdio/netbird/client/ssh"
 	mgmProto "github.com/netbirdio/netbird/management/proto"
 )
 
+var ErrSourceRangesEmpty = errors.New("sources range is empty")
+
 // Manager is a ACL rules manager
 type Manager interface {
 	ApplyFiltering(networkMap *mgmProto.NetworkMap)
@@ -167,31 +172,40 @@ func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) {
 }
 
 func (d *DefaultManager) applyRouteACLs(rules []*mgmProto.RouteFirewallRule) error {
-	var newRouteRules = make(map[id.RuleID]struct{})
+	newRouteRules := make(map[id.RuleID]struct{}, len(rules))
+	var merr *multierror.Error
+
+	// Apply new rules - firewall manager will return existing rule ID if already present
 	for _, rule := range rules {
 		id, err := d.applyRouteACL(rule)
 		if err != nil {
-			return fmt.Errorf("apply route ACL: %w", err)
+			if errors.Is(err, ErrSourceRangesEmpty) {
+				log.Debugf("skipping empty rule with destination %s: %v", rule.Destination, err)
+			} else {
+				merr = multierror.Append(merr, fmt.Errorf("add route rule: %w", err))
+			}
+			continue
 		}
 		newRouteRules[id] = struct{}{}
 	}
 
+	// Clean up old firewall rules
 	for id := range d.routeRules {
-		if _, ok := newRouteRules[id]; !ok {
+		if _, exists := newRouteRules[id]; !exists {
 			if err := d.firewall.DeleteRouteRule(id); err != nil {
-				log.Errorf("failed to delete route firewall rule: %v", err)
-				continue
+				merr = multierror.Append(merr, fmt.Errorf("delete route rule: %w", err))
 			}
-			delete(d.routeRules, id)
+			// implicitly deleted from the map
 		}
 	}
+
 	d.routeRules = newRouteRules
-	return nil
+	return nberrors.FormatErrorOrNil(merr)
 }
 
 func (d *DefaultManager) applyRouteACL(rule *mgmProto.RouteFirewallRule) (id.RuleID, error) {
 	if len(rule.SourceRanges) == 0 {
-		return "", fmt.Errorf("source ranges is empty")
+		return "", ErrSourceRangesEmpty
 	}
 
 	var sources []netip.Prefix

client/internal/acl/manager_test.go

@@ -1,7 +1,6 @@
 package acl
 
 import (
-	"context"
 	"net"
 	"testing"
 
@@ -52,13 +51,13 @@ func TestDefaultManager(t *testing.T) {
 	}).AnyTimes()
 
 	// we receive one rule from the management so for testing purposes ignore it
-	fw, err := firewall.NewFirewall(context.Background(), ifaceMock)
+	fw, err := firewall.NewFirewall(ifaceMock, nil)
 	if err != nil {
 		t.Errorf("create firewall: %v", err)
 		return
 	}
 	defer func(fw manager.Manager) {
-		_ = fw.Reset()
+		_ = fw.Reset(nil)
 	}(fw)
 	acl := NewDefaultManager(fw)
 
@@ -345,13 +344,13 @@ func TestDefaultManagerEnableSSHRules(t *testing.T) {
 	}).AnyTimes()
 
 	// we receive one rule from the management so for testing purposes ignore it
-	fw, err := firewall.NewFirewall(context.Background(), ifaceMock)
+	fw, err := firewall.NewFirewall(ifaceMock, nil)
 	if err != nil {
 		t.Errorf("create firewall: %v", err)
 		return
 	}
 	defer func(fw manager.Manager) {
-		_ = fw.Reset()
+		_ = fw.Reset(nil)
 	}(fw)
 	acl := NewDefaultManager(fw)
 

client/internal/connect.go

@@ -62,10 +62,7 @@ func (c *ConnectClient) Run() error {
 }
 
 // RunWithProbes runs the client's main logic with probes attached
-func (c *ConnectClient) RunWithProbes(
-	probes *ProbeHolder,
-	runningChan chan error,
-) error {
+func (c *ConnectClient) RunWithProbes(probes *ProbeHolder, runningChan chan error) error {
 	return c.run(MobileDependency{}, probes, runningChan)
 }
 
@@ -104,11 +101,7 @@ func (c *ConnectClient) RunOniOS(
 	return c.run(mobileDependency, nil, nil)
 }
 
-func (c *ConnectClient) run(
-	mobileDependency MobileDependency,
-	probes *ProbeHolder,
-	runningChan chan error,
-) error {
+func (c *ConnectClient) run(mobileDependency MobileDependency, probes *ProbeHolder, runningChan chan error) error {
 	defer func() {
 		if r := recover(); r != nil {
 			log.Panicf("Panic occurred: %v, stack trace: %s", r, string(debug.Stack()))
@@ -117,12 +110,6 @@ func (c *ConnectClient) run(
 
 	log.Infof("starting NetBird client version %s on %s/%s", version.NetbirdVersion(), runtime.GOOS, runtime.GOARCH)
 
-	// Check if client was not shut down in a clean way and restore DNS config if required.
-	// Otherwise, we might not be able to connect to the management server to retrieve new config.
-	if err := dns.CheckUncleanShutdown(c.config.WgIface); err != nil {
-		log.Errorf("checking unclean shutdown error: %s", err)
-	}
-
 	backOff := &backoff.ExponentialBackOff{
 		InitialInterval:     time.Second,
 		RandomizationFactor: 1,
@@ -358,7 +345,11 @@ func (c *ConnectClient) Stop() error {
 	if c.engine == nil {
 		return nil
 	}
-	return c.engine.Stop()
+	if err := c.engine.Stop(); err != nil {
+		return fmt.Errorf("stop engine: %w", err)
+	}
+
+	return nil
 }
 
 func (c *ConnectClient) isContextCancelled() bool {

client/internal/dns/consts_freebsd.go

@@ -1,6 +1,5 @@
 package dns
 
 const (
-	fileUncleanShutdownResolvConfLocation  = "/var/db/netbird/resolv.conf"
-	fileUncleanShutdownManagerTypeLocation = "/var/db/netbird/manager"
+	fileUncleanShutdownResolvConfLocation = "/var/db/netbird/resolv.conf"
 )

client/internal/dns/consts_linux.go

@@ -3,6 +3,5 @@
 package dns
 
 const (
-	fileUncleanShutdownResolvConfLocation  = "/var/lib/netbird/resolv.conf"
-	fileUncleanShutdownManagerTypeLocation = "/var/lib/netbird/manager"
+	fileUncleanShutdownResolvConfLocation = "/var/lib/netbird/resolv.conf"
 )

client/internal/dns/file_repair_unix.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/fsnotify/fsnotify"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 var (
@@ -20,7 +22,7 @@ var (
 	}
 )
 
-type repairConfFn func([]string, string, *resolvConf) error
+type repairConfFn func([]string, string, *resolvConf, *statemanager.Manager) error
 
 type repair struct {
 	operationFile string
@@ -40,7 +42,7 @@ func newRepair(operationFile string, updateFn repairConfFn) *repair {
 	}
 }
 
-func (f *repair) watchFileChanges(nbSearchDomains []string, nbNameserverIP string) {
+func (f *repair) watchFileChanges(nbSearchDomains []string, nbNameserverIP string, stateManager *statemanager.Manager) {
 	if f.inotify != nil {
 		return
 	}
@@ -81,7 +83,7 @@ func (f *repair) watchFileChanges(nbSearchDomains []string, nbNameserverIP strin
 				log.Errorf("failed to rm inotify watch for resolv.conf: %s", err)
 			}
 
-			err = f.updateFn(nbSearchDomains, nbNameserverIP, rConf)
+			err = f.updateFn(nbSearchDomains, nbNameserverIP, rConf, stateManager)
 			if err != nil {
 				log.Errorf("failed to repair resolv.conf: %v", err)
 			}

client/internal/dns/file_repair_unix_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	"github.com/netbirdio/netbird/util"
 )
 
@@ -104,14 +105,14 @@ nameserver 8.8.8.8`,
 
 			var changed bool
 			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-			updateFn := func([]string, string, *resolvConf) error {
+			updateFn := func([]string, string, *resolvConf, *statemanager.Manager) error {
 				changed = true
 				cancel()
 				return nil
 			}
 
 			r := newRepair(operationFile, updateFn)
-			r.watchFileChanges([]string{"netbird.cloud"}, "10.0.0.1")
+			r.watchFileChanges([]string{"netbird.cloud"}, "10.0.0.1", nil)
 
 			err = os.WriteFile(operationFile, []byte(tt.touchedConfContent), 0755)
 			if err != nil {
@@ -151,14 +152,14 @@ searchdomain netbird.cloud something`
 
 	var changed bool
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	updateFn := func([]string, string, *resolvConf) error {
+	updateFn := func([]string, string, *resolvConf, *statemanager.Manager) error {
 		changed = true
 		cancel()
 		return nil
 	}
 
 	r := newRepair(tmpLink, updateFn)
-	r.watchFileChanges([]string{"netbird.cloud"}, "10.0.0.1")
+	r.watchFileChanges([]string{"netbird.cloud"}, "10.0.0.1", nil)
 
 	err = os.WriteFile(tmpLink, []byte(modifyContent), 0755)
 	if err != nil {

client/internal/dns/file_unix.go

@@ -11,6 +11,8 @@ import (
 	"time"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -36,7 +38,7 @@ type fileConfigurator struct {
 	nbNameserverIP string
 }
 
-func newFileConfigurator() (hostManager, error) {
+func newFileConfigurator() (*fileConfigurator, error) {
 	fc := &fileConfigurator{}
 	fc.repair = newRepair(defaultResolvConfPath, fc.updateConfig)
 	return fc, nil
@@ -46,7 +48,7 @@ func (f *fileConfigurator) supportCustomPort() bool {
 	return false
 }
 
-func (f *fileConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (f *fileConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	backupFileExist := f.isBackupFileExist()
 	if !config.RouteAll {
 		if backupFileExist {
@@ -76,15 +78,15 @@ func (f *fileConfigurator) applyDNSConfig(config HostDNSConfig) error {
 
 	f.repair.stopWatchFileChanges()
 
-	err = f.updateConfig(nbSearchDomains, f.nbNameserverIP, resolvConf)
+	err = f.updateConfig(nbSearchDomains, f.nbNameserverIP, resolvConf, stateManager)
 	if err != nil {
 		return err
 	}
-	f.repair.watchFileChanges(nbSearchDomains, f.nbNameserverIP)
+	f.repair.watchFileChanges(nbSearchDomains, f.nbNameserverIP, stateManager)
 	return nil
 }
 
-func (f *fileConfigurator) updateConfig(nbSearchDomains []string, nbNameserverIP string, cfg *resolvConf) error {
+func (f *fileConfigurator) updateConfig(nbSearchDomains []string, nbNameserverIP string, cfg *resolvConf, stateManager *statemanager.Manager) error {
 	searchDomainList := mergeSearchDomains(nbSearchDomains, cfg.searchDomains)
 	nameServers := generateNsList(nbNameserverIP, cfg)
 
@@ -107,7 +109,7 @@ func (f *fileConfigurator) updateConfig(nbSearchDomains []string, nbNameserverIP
 	log.Infof("created a NetBird managed %s file with the DNS settings. Added %d search domains. Search list: %s", defaultResolvConfPath, len(searchDomainList), searchDomainList)
 
 	// create another backup for unclean shutdown detection right after overwriting the original resolv.conf
-	if err := createUncleanShutdownIndicator(fileDefaultResolvConfBackupLocation, fileManager, nbNameserverIP); err != nil {
+	if err := createUncleanShutdownIndicator(fileDefaultResolvConfBackupLocation, nbNameserverIP, stateManager); err != nil {
 		log.Errorf("failed to create unclean shutdown resolv.conf backup: %s", err)
 	}
 
@@ -145,10 +147,6 @@ func (f *fileConfigurator) restore() error {
 		return fmt.Errorf("restoring %s from %s: %w", defaultResolvConfPath, fileDefaultResolvConfBackupLocation, err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf backup: %s", err)
-	}
-
 	return os.RemoveAll(fileDefaultResolvConfBackupLocation)
 }
 
@@ -176,7 +174,7 @@ func (f *fileConfigurator) restoreUncleanShutdownDNS(storedDNSAddress *netip.Add
 		return restoreResolvConfFile()
 	}
 
-	log.Info("restoring unclean shutdown: first current nameserver differs from saved nameserver pre-netbird: not restoring")
+	log.Infof("restoring unclean shutdown: first current nameserver differs from saved nameserver pre-netbird: %s (current) vs %s (stored): not restoring", currentDNSAddress, storedDNSAddress)
 	return nil
 }
 
@@ -192,10 +190,6 @@ func restoreResolvConfFile() error {
 		return fmt.Errorf("restoring %s from %s: %w", defaultResolvConfPath, fileUncleanShutdownResolvConfLocation, err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf file: %s", err)
-	}
-
 	return nil
 }
 

client/internal/dns/host.go

@@ -5,14 +5,14 @@ import (
 	"net/netip"
 	"strings"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbdns "github.com/netbirdio/netbird/dns"
 )
 
 type hostManager interface {
-	applyDNSConfig(config HostDNSConfig) error
+	applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error
 	restoreHostDNS() error
 	supportCustomPort() bool
-	restoreUncleanShutdownDNS(storedDNSAddress *netip.Addr) error
 }
 
 type SystemDNSSettings struct {
@@ -35,15 +35,15 @@ type DomainConfig struct {
 }
 
 type mockHostConfigurator struct {
-	applyDNSConfigFunc            func(config HostDNSConfig) error
+	applyDNSConfigFunc            func(config HostDNSConfig, stateManager *statemanager.Manager) error
 	restoreHostDNSFunc            func() error
 	supportCustomPortFunc         func() bool
 	restoreUncleanShutdownDNSFunc func(*netip.Addr) error
 }
 
-func (m *mockHostConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (m *mockHostConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	if m.applyDNSConfigFunc != nil {
-		return m.applyDNSConfigFunc(config)
+		return m.applyDNSConfigFunc(config, stateManager)
 	}
 	return fmt.Errorf("method applyDNSSettings is not implemented")
 }
@@ -62,16 +62,9 @@ func (m *mockHostConfigurator) supportCustomPort() bool {
 	return false
 }
 
-func (m *mockHostConfigurator) restoreUncleanShutdownDNS(storedDNSAddress *netip.Addr) error {
-	if m.restoreUncleanShutdownDNSFunc != nil {
-		return m.restoreUncleanShutdownDNSFunc(storedDNSAddress)
-	}
-	return fmt.Errorf("method restoreUncleanShutdownDNS is not implemented")
-}
-
 func newNoopHostMocker() hostManager {
 	return &mockHostConfigurator{
-		applyDNSConfigFunc:            func(config HostDNSConfig) error { return nil },
+		applyDNSConfigFunc:            func(config HostDNSConfig, stateManager *statemanager.Manager) error { return nil },
 		restoreHostDNSFunc:            func() error { return nil },
 		supportCustomPortFunc:         func() bool { return true },
 		restoreUncleanShutdownDNSFunc: func(*netip.Addr) error { return nil },

client/internal/dns/host_android.go

@@ -1,15 +1,17 @@
 package dns
 
-import "net/netip"
+import (
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+)
 
 type androidHostManager struct {
 }
 
-func newHostManager() (hostManager, error) {
+func newHostManager() (*androidHostManager, error) {
 	return &androidHostManager{}, nil
 }
 
-func (a androidHostManager) applyDNSConfig(config HostDNSConfig) error {
+func (a androidHostManager) applyDNSConfig(HostDNSConfig, *statemanager.Manager) error {
 	return nil
 }
 
@@ -20,7 +22,3 @@ func (a androidHostManager) restoreHostDNS() error {
 func (a androidHostManager) supportCustomPort() bool {
 	return false
 }
-
-func (a androidHostManager) restoreUncleanShutdownDNS(*netip.Addr) error {
-	return nil
-}

client/internal/dns/host_darwin.go

@@ -8,12 +8,13 @@ import (
 	"fmt"
 	"io"
 	"net"
-	"net/netip"
 	"os/exec"
 	"strconv"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -37,7 +38,7 @@ type systemConfigurator struct {
 	systemDNSSettings SystemDNSSettings
 }
 
-func newHostManager() (hostManager, error) {
+func newHostManager() (*systemConfigurator, error) {
 	return &systemConfigurator{
 		createdKeys: make(map[string]struct{}),
 	}, nil
@@ -47,12 +48,11 @@ func (s *systemConfigurator) supportCustomPort() bool {
 	return true
 }
 
-func (s *systemConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (s *systemConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	var err error
 
-	// create a file for unclean shutdown detection
-	if err := createUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to create unclean shutdown file: %s", err)
+	if err := stateManager.UpdateState(&ShutdownState{}); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	var (
@@ -123,10 +123,6 @@ func (s *systemConfigurator) restoreHostDNS() error {
 		}
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown file: %s", err)
-	}
-
 	return nil
 }
 
@@ -320,7 +316,7 @@ func (s *systemConfigurator) getPrimaryService() (string, string, error) {
 	return primaryService, router, nil
 }
 
-func (s *systemConfigurator) restoreUncleanShutdownDNS(*netip.Addr) error {
+func (s *systemConfigurator) restoreUncleanShutdownDNS() error {
 	if err := s.restoreHostDNS(); err != nil {
 		return fmt.Errorf("restoring dns via scutil: %w", err)
 	}

client/internal/dns/host_ios.go

@@ -3,9 +3,10 @@ package dns
 import (
 	"encoding/json"
 	"fmt"
-	"net/netip"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 type iosHostManager struct {
@@ -13,13 +14,13 @@ type iosHostManager struct {
 	config     HostDNSConfig
 }
 
-func newHostManager(dnsManager IosDnsManager) (hostManager, error) {
+func newHostManager(dnsManager IosDnsManager) (*iosHostManager, error) {
 	return &iosHostManager{
 		dnsManager: dnsManager,
 	}, nil
 }
 
-func (a iosHostManager) applyDNSConfig(config HostDNSConfig) error {
+func (a iosHostManager) applyDNSConfig(config HostDNSConfig, _ *statemanager.Manager) error {
 	jsonData, err := json.Marshal(config)
 	if err != nil {
 		return fmt.Errorf("marshal: %w", err)
@@ -37,7 +38,3 @@ func (a iosHostManager) restoreHostDNS() error {
 func (a iosHostManager) supportCustomPort() bool {
 	return false
 }
-
-func (a iosHostManager) restoreUncleanShutdownDNS(*netip.Addr) error {
-	return nil
-}

client/internal/dns/host_unix.go

@@ -4,9 +4,9 @@ package dns
 
 import (
 	"bufio"
-	"errors"
 	"fmt"
 	"io"
+	"net/netip"
 	"os"
 	"strings"
 
@@ -21,27 +21,8 @@ const (
 	resolvConfManager
 )
 
-var ErrUnknownOsManagerType = errors.New("unknown os manager type")
-
 type osManagerType int
 
-func newOsManagerType(osManager string) (osManagerType, error) {
-	switch osManager {
-	case "netbird":
-		return fileManager, nil
-	case "file":
-		return netbirdManager, nil
-	case "networkManager":
-		return networkManager, nil
-	case "systemd":
-		return systemdManager, nil
-	case "resolvconf":
-		return resolvConfManager, nil
-	default:
-		return 0, ErrUnknownOsManagerType
-	}
-}
-
 func (t osManagerType) String() string {
 	switch t {
 	case netbirdManager:
@@ -59,6 +40,11 @@ func (t osManagerType) String() string {
 	}
 }
 
+type restoreHostManager interface {
+	hostManager
+	restoreUncleanShutdownDNS(*netip.Addr) error
+}
+
 func newHostManager(wgInterface string) (hostManager, error) {
 	osManager, err := getOSDNSManagerType()
 	if err != nil {
@@ -69,7 +55,7 @@ func newHostManager(wgInterface string) (hostManager, error) {
 	return newHostManagerFromType(wgInterface, osManager)
 }
 
-func newHostManagerFromType(wgInterface string, osManager osManagerType) (hostManager, error) {
+func newHostManagerFromType(wgInterface string, osManager osManagerType) (restoreHostManager, error) {
 	switch osManager {
 	case networkManager:
 		return newNetworkManagerDbusConfigurator(wgInterface)

client/internal/dns/host_windows.go

@@ -3,11 +3,12 @@ package dns
 import (
 	"fmt"
 	"io"
-	"net/netip"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows/registry"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const (
@@ -31,7 +32,7 @@ type registryConfigurator struct {
 	routingAll bool
 }
 
-func newHostManager(wgInterface WGIface) (hostManager, error) {
+func newHostManager(wgInterface WGIface) (*registryConfigurator, error) {
 	guid, err := wgInterface.GetInterfaceGUIDString()
 	if err != nil {
 		return nil, err
@@ -39,7 +40,7 @@ func newHostManager(wgInterface WGIface) (hostManager, error) {
 	return newHostManagerWithGuid(guid)
 }
 
-func newHostManagerWithGuid(guid string) (hostManager, error) {
+func newHostManagerWithGuid(guid string) (*registryConfigurator, error) {
 	return &registryConfigurator{
 		guid: guid,
 	}, nil
@@ -49,7 +50,7 @@ func (r *registryConfigurator) supportCustomPort() bool {
 	return false
 }
 
-func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	var err error
 	if config.RouteAll {
 		err = r.addDNSSetupForAll(config.ServerIP)
@@ -65,9 +66,8 @@ func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig) error {
 		log.Infof("removed %s as main DNS forwarder for this peer", config.ServerIP)
 	}
 
-	// create a file for unclean shutdown detection
-	if err := createUncleanShutdownIndicator(r.guid); err != nil {
-		log.Errorf("failed to create unclean shutdown file: %s", err)
+	if err := stateManager.UpdateState(&ShutdownState{Guid: r.guid}); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	var (
@@ -160,10 +160,6 @@ func (r *registryConfigurator) restoreHostDNS() error {
 		return fmt.Errorf("remove interface registry key: %w", err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown file: %s", err)
-	}
-
 	return nil
 }
 
@@ -221,7 +217,7 @@ func (r *registryConfigurator) getInterfaceRegistryKey() (registry.Key, error) {
 	return regKey, nil
 }
 
-func (r *registryConfigurator) restoreUncleanShutdownDNS(*netip.Addr) error {
+func (r *registryConfigurator) restoreUncleanShutdownDNS() error {
 	if err := r.restoreHostDNS(); err != nil {
 		return fmt.Errorf("restoring dns via registry: %w", err)
 	}

client/internal/dns/network_manager_unix.go

@@ -16,6 +16,7 @@ import (
 	"github.com/miekg/dns"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbversion "github.com/netbirdio/netbird/version"
 )
 
@@ -53,6 +54,7 @@ var supportedNetworkManagerVersionConstraints = []string{
 type networkManagerDbusConfigurator struct {
 	dbusLinkObject dbus.ObjectPath
 	routingAll     bool
+	ifaceName      string
 }
 
 // the types below are based on dbus specification, each field is mapped to a dbus type
@@ -77,7 +79,7 @@ func (s networkManagerConnSettings) cleanDeprecatedSettings() {
 	}
 }
 
-func newNetworkManagerDbusConfigurator(wgInterface string) (hostManager, error) {
+func newNetworkManagerDbusConfigurator(wgInterface string) (*networkManagerDbusConfigurator, error) {
 	obj, closeConn, err := getDbusObject(networkManagerDest, networkManagerDbusObjectNode)
 	if err != nil {
 		return nil, fmt.Errorf("get nm dbus: %w", err)
@@ -93,6 +95,7 @@ func newNetworkManagerDbusConfigurator(wgInterface string) (hostManager, error)
 
 	return &networkManagerDbusConfigurator{
 		dbusLinkObject: dbus.ObjectPath(s),
+		ifaceName:      wgInterface,
 	}, nil
 }
 
@@ -100,7 +103,7 @@ func (n *networkManagerDbusConfigurator) supportCustomPort() bool {
 	return false
 }
 
-func (n *networkManagerDbusConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (n *networkManagerDbusConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	connSettings, configVersion, err := n.getAppliedConnectionSettings()
 	if err != nil {
 		return fmt.Errorf("retrieving the applied connection settings, error: %w", err)
@@ -151,10 +154,12 @@ func (n *networkManagerDbusConfigurator) applyDNSConfig(config HostDNSConfig) er
 	connSettings[networkManagerDbusIPv4Key][networkManagerDbusDNSPriorityKey] = dbus.MakeVariant(priority)
 	connSettings[networkManagerDbusIPv4Key][networkManagerDbusDNSSearchKey] = dbus.MakeVariant(newDomainList)
 
-	// create a backup for unclean shutdown detection before adding domains, as these might end up in the resolv.conf file.
-	// The file content itself is not important for network-manager restoration
-	if err := createUncleanShutdownIndicator(defaultResolvConfPath, networkManager, dnsIP.String()); err != nil {
-		log.Errorf("failed to create unclean shutdown resolv.conf backup: %s", err)
+	state := &ShutdownState{
+		ManagerType: networkManager,
+		WgIface:     n.ifaceName,
+	}
+	if err := stateManager.UpdateState(state); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	log.Infof("adding %d search domains and %d match domains. Search list: %s , Match list: %s", len(searchDomains), len(matchDomains), searchDomains, matchDomains)
@@ -171,10 +176,6 @@ func (n *networkManagerDbusConfigurator) restoreHostDNS() error {
 		return fmt.Errorf("delete connection settings: %w", err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf backup: %s", err)
-	}
-
 	return nil
 }
 

client/internal/dns/resolvconf_unix.go

@@ -9,6 +9,8 @@ import (
 	"os/exec"
 
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
 const resolvconfCommand = "resolvconf"
@@ -22,7 +24,7 @@ type resolvconf struct {
 }
 
 // supported "openresolv" only
-func newResolvConfConfigurator(wgInterface string) (hostManager, error) {
+func newResolvConfConfigurator(wgInterface string) (*resolvconf, error) {
 	resolvConfEntries, err := parseDefaultResolvConf()
 	if err != nil {
 		log.Errorf("could not read original search domains from %s: %s", defaultResolvConfPath, err)
@@ -40,7 +42,7 @@ func (r *resolvconf) supportCustomPort() bool {
 	return false
 }
 
-func (r *resolvconf) applyDNSConfig(config HostDNSConfig) error {
+func (r *resolvconf) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	var err error
 	if !config.RouteAll {
 		err = r.restoreHostDNS()
@@ -60,9 +62,12 @@ func (r *resolvconf) applyDNSConfig(config HostDNSConfig) error {
 		append([]string{config.ServerIP}, r.originalNameServers...),
 		options)
 
-	// create a backup for unclean shutdown detection before the resolv.conf is changed
-	if err := createUncleanShutdownIndicator(defaultResolvConfPath, resolvConfManager, config.ServerIP); err != nil {
-		log.Errorf("failed to create unclean shutdown resolv.conf backup: %s", err)
+	state := &ShutdownState{
+		ManagerType: resolvConfManager,
+		WgIface:     r.ifaceName,
+	}
+	if err := stateManager.UpdateState(state); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	err = r.applyConfig(buf)
@@ -79,11 +84,7 @@ func (r *resolvconf) restoreHostDNS() error {
 	cmd := exec.Command(resolvconfCommand, "-f", "-d", r.ifaceName)
 	_, err := cmd.Output()
 	if err != nil {
-		return fmt.Errorf("removing resolvconf configuration for %s interface, error: %w", r.ifaceName, err)
-	}
-
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf backup: %s", err)
+		return fmt.Errorf("removing resolvconf configuration for %s interface: %w", r.ifaceName, err)
 	}
 
 	return nil
@@ -95,7 +96,7 @@ func (r *resolvconf) applyConfig(content bytes.Buffer) error {
 	cmd.Stdin = &content
 	_, err := cmd.Output()
 	if err != nil {
-		return fmt.Errorf("applying resolvconf configuration for %s interface, error: %w", r.ifaceName, err)
+		return fmt.Errorf("applying resolvconf configuration for %s interface: %w", r.ifaceName, err)
 	}
 	return nil
 }

client/internal/dns/server.go

@@ -7,6 +7,7 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/miekg/dns"
 	"github.com/mitchellh/hashstructure/v2"
@@ -14,6 +15,7 @@ import (
 
 	"github.com/netbirdio/netbird/client/internal/listener"
 	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbdns "github.com/netbirdio/netbird/dns"
 )
 
@@ -63,6 +65,7 @@ type DefaultServer struct {
 	iosDnsManager        IosDnsManager
 
 	statusRecorder *peer.Status
+	stateManager   *statemanager.Manager
 }
 
 type handlerWithStop interface {
@@ -77,12 +80,7 @@ type muxUpdate struct {
 }
 
 // NewDefaultServer returns a new dns server
-func NewDefaultServer(
-	ctx context.Context,
-	wgInterface WGIface,
-	customAddress string,
-	statusRecorder *peer.Status,
-) (*DefaultServer, error) {
+func NewDefaultServer(ctx context.Context, wgInterface WGIface, customAddress string, statusRecorder *peer.Status, stateManager *statemanager.Manager) (*DefaultServer, error) {
 	var addrPort *netip.AddrPort
 	if customAddress != "" {
 		parsedAddrPort, err := netip.ParseAddrPort(customAddress)
@@ -99,7 +97,7 @@ func NewDefaultServer(
 		dnsService = newServiceViaListener(wgInterface, addrPort)
 	}
 
-	return newDefaultServer(ctx, wgInterface, dnsService, statusRecorder), nil
+	return newDefaultServer(ctx, wgInterface, dnsService, statusRecorder, stateManager), nil
 }
 
 // NewDefaultServerPermanentUpstream returns a new dns server. It optimized for mobile systems
@@ -112,7 +110,7 @@ func NewDefaultServerPermanentUpstream(
 	statusRecorder *peer.Status,
 ) *DefaultServer {
 	log.Debugf("host dns address list is: %v", hostsDnsList)
-	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder)
+	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder, nil)
 	ds.hostsDNSHolder.set(hostsDnsList)
 	ds.permanent = true
 	ds.addHostRootZone()
@@ -130,12 +128,12 @@ func NewDefaultServerIos(
 	iosDnsManager IosDnsManager,
 	statusRecorder *peer.Status,
 ) *DefaultServer {
-	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder)
+	ds := newDefaultServer(ctx, wgInterface, NewServiceViaMemory(wgInterface), statusRecorder, nil)
 	ds.iosDnsManager = iosDnsManager
 	return ds
 }
 
-func newDefaultServer(ctx context.Context, wgInterface WGIface, dnsService service, statusRecorder *peer.Status) *DefaultServer {
+func newDefaultServer(ctx context.Context, wgInterface WGIface, dnsService service, statusRecorder *peer.Status, stateManager *statemanager.Manager) *DefaultServer {
 	ctx, stop := context.WithCancel(ctx)
 	defaultServer := &DefaultServer{
 		ctx:       ctx,
@@ -147,6 +145,7 @@ func newDefaultServer(ctx context.Context, wgInterface WGIface, dnsService servi
 		},
 		wgInterface:    wgInterface,
 		statusRecorder: statusRecorder,
+		stateManager:   stateManager,
 		hostsDNSHolder: newHostsDNSHolder(),
 	}
 
@@ -169,6 +168,7 @@ func (s *DefaultServer) Initialize() (err error) {
 		}
 	}
 
+	s.stateManager.RegisterState(&ShutdownState{})
 	s.hostManager, err = s.initialize()
 	if err != nil {
 		return fmt.Errorf("initialize: %w", err)
@@ -191,9 +191,10 @@ func (s *DefaultServer) Stop() {
 	s.ctxCancel()
 
 	if s.hostManager != nil {
-		err := s.hostManager.restoreHostDNS()
-		if err != nil {
-			log.Error(err)
+		if err := s.hostManager.restoreHostDNS(); err != nil {
+			log.Error("failed to restore host DNS settings: ", err)
+		} else if err := s.stateManager.DeleteState(&ShutdownState{}); err != nil {
+			log.Errorf("failed to delete shutdown dns state: %v", err)
 		}
 	}
 
@@ -318,10 +319,17 @@ func (s *DefaultServer) applyConfiguration(update nbdns.Config) error {
 		hostUpdate.RouteAll = false
 	}
 
-	if err = s.hostManager.applyDNSConfig(hostUpdate); err != nil {
+	if err = s.hostManager.applyDNSConfig(hostUpdate, s.stateManager); err != nil {
 		log.Error(err)
 	}
 
+	// persist dns state right away
+	ctx, cancel := context.WithTimeout(s.ctx, 3*time.Second)
+	defer cancel()
+	if err := s.stateManager.PersistState(ctx); err != nil {
+		log.Errorf("Failed to persist dns state: %v", err)
+	}
+
 	if s.searchDomainNotifier != nil {
 		s.searchDomainNotifier.onNewSearchDomains(s.SearchDomains())
 	}
@@ -521,10 +529,17 @@ func (s *DefaultServer) upstreamCallbacks(
 			}
 		}
 
-		if err := s.hostManager.applyDNSConfig(s.currentConfig); err != nil {
+		if err := s.hostManager.applyDNSConfig(s.currentConfig, s.stateManager); err != nil {
 			l.Errorf("Failed to apply nameserver deactivation on the host: %v", err)
 		}
 
+		// persist dns state right away
+		ctx, cancel := context.WithTimeout(s.ctx, 3*time.Second)
+		defer cancel()
+		if err := s.stateManager.PersistState(ctx); err != nil {
+			l.Errorf("Failed to persist dns state: %v", err)
+		}
+
 		if runtime.GOOS == "android" && nsGroup.Primary && len(s.hostsDNSHolder.get()) > 0 {
 			s.addHostRootZone()
 		}
@@ -551,7 +566,7 @@ func (s *DefaultServer) upstreamCallbacks(
 			s.currentConfig.RouteAll = true
 			s.service.RegisterMux(nbdns.RootZone, handler)
 		}
-		if err := s.hostManager.applyDNSConfig(s.currentConfig); err != nil {
+		if err := s.hostManager.applyDNSConfig(s.currentConfig, s.stateManager); err != nil {
 			l.WithError(err).Error("reactivate temporary disabled nameserver group, DNS update apply")
 		}
 

client/internal/dns/server_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/netbirdio/netbird/client/iface/device"
 	pfmock "github.com/netbirdio/netbird/client/iface/mocks"
 	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 	nbdns "github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/formatter"
@@ -267,7 +268,17 @@ func TestUpdateDNSServer(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			wgIface, err := iface.NewWGIFace(fmt.Sprintf("utun230%d", n), fmt.Sprintf("100.66.100.%d/32", n+1), 33100, privKey.String(), iface.DefaultMTU, newNet, nil, nil)
+
+			opts := iface.WGIFaceOpts{
+				IFaceName:    fmt.Sprintf("utun230%d", n),
+				Address:      fmt.Sprintf("100.66.100.%d/32", n+1),
+				WGPort:       33100,
+				WGPrivKey:    privKey.String(),
+				MTU:          iface.DefaultMTU,
+				TransportNet: newNet,
+			}
+
+			wgIface, err := iface.NewWGIFace(opts)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -281,7 +292,7 @@ func TestUpdateDNSServer(t *testing.T) {
 					t.Log(err)
 				}
 			}()
-			dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", &peer.Status{})
+			dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", &peer.Status{}, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -345,7 +356,15 @@ func TestDNSFakeResolverHandleUpdates(t *testing.T) {
 	}
 
 	privKey, _ := wgtypes.GeneratePrivateKey()
-	wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.1/32", 33100, privKey.String(), iface.DefaultMTU, newNet, nil, nil)
+	opts := iface.WGIFaceOpts{
+		IFaceName:    "utun2301",
+		Address:      "100.66.100.1/32",
+		WGPort:       33100,
+		WGPrivKey:    privKey.String(),
+		MTU:          iface.DefaultMTU,
+		TransportNet: newNet,
+	}
+	wgIface, err := iface.NewWGIFace(opts)
 	if err != nil {
 		t.Errorf("build interface wireguard: %v", err)
 		return
@@ -382,7 +401,7 @@ func TestDNSFakeResolverHandleUpdates(t *testing.T) {
 		return
 	}
 
-	dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", &peer.Status{})
+	dnsServer, err := NewDefaultServer(context.Background(), wgIface, "", &peer.Status{}, nil)
 	if err != nil {
 		t.Errorf("create DNS server: %v", err)
 		return
@@ -477,7 +496,7 @@ func TestDNSServerStartStop(t *testing.T) {
 
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
-			dnsServer, err := NewDefaultServer(context.Background(), &mocWGIface{}, testCase.addrPort, &peer.Status{})
+			dnsServer, err := NewDefaultServer(context.Background(), &mocWGIface{}, testCase.addrPort, &peer.Status{}, nil)
 			if err != nil {
 				t.Fatalf("%v", err)
 			}
@@ -536,6 +555,7 @@ func TestDNSServerStartStop(t *testing.T) {
 func TestDNSServerUpstreamDeactivateCallback(t *testing.T) {
 	hostManager := &mockHostConfigurator{}
 	server := DefaultServer{
+		ctx:     context.Background(),
 		service: NewServiceViaMemory(&mocWGIface{}),
 		localResolver: &localResolver{
 			registeredMap: make(registrationMap),
@@ -552,7 +572,7 @@ func TestDNSServerUpstreamDeactivateCallback(t *testing.T) {
 	}
 
 	var domainsUpdate string
-	hostManager.applyDNSConfigFunc = func(config HostDNSConfig) error {
+	hostManager.applyDNSConfigFunc = func(config HostDNSConfig, statemanager *statemanager.Manager) error {
 		domains := []string{}
 		for _, item := range config.Domains {
 			if item.Disabled {
@@ -803,7 +823,17 @@ func createWgInterfaceWithBind(t *testing.T) (*iface.WGIface, error) {
 	}
 
 	privKey, _ := wgtypes.GeneratePrivateKey()
-	wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.2/24", 33100, privKey.String(), iface.DefaultMTU, newNet, nil, nil)
+
+	opts := iface.WGIFaceOpts{
+		IFaceName:    "utun2301",
+		Address:      "100.66.100.2/24",
+		WGPort:       33100,
+		WGPrivKey:    privKey.String(),
+		MTU:          iface.DefaultMTU,
+		TransportNet: newNet,
+	}
+
+	wgIface, err := iface.NewWGIFace(opts)
 	if err != nil {
 		t.Fatalf("build interface wireguard: %v", err)
 		return nil, err

client/internal/dns/server_windows.go

@@ -1,5 +1,5 @@
 package dns
 
-func (s *DefaultServer) initialize() (manager hostManager, err error) {
+func (s *DefaultServer) initialize() (hostManager, error) {
 	return newHostManager(s.wgInterface)
 }

client/internal/dns/systemd_freebsd.go

@@ -7,7 +7,7 @@ import (
 
 var errNotImplemented = errors.New("not implemented")
 
-func newSystemdDbusConfigurator(wgInterface string) (hostManager, error) {
+func newSystemdDbusConfigurator(string) (restoreHostManager, error) {
 	return nil, fmt.Errorf("systemd dns management: %w on freebsd", errNotImplemented)
 }
 

client/internal/dns/systemd_linux.go

@@ -15,6 +15,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbdns "github.com/netbirdio/netbird/dns"
 )
 
@@ -38,6 +39,7 @@ const (
 type systemdDbusConfigurator struct {
 	dbusLinkObject dbus.ObjectPath
 	routingAll     bool
+	ifaceName      string
 }
 
 // the types below are based on dbus specification, each field is mapped to a dbus type
@@ -55,7 +57,7 @@ type systemdDbusLinkDomainsInput struct {
 	MatchOnly bool
 }
 
-func newSystemdDbusConfigurator(wgInterface string) (hostManager, error) {
+func newSystemdDbusConfigurator(wgInterface string) (*systemdDbusConfigurator, error) {
 	iface, err := net.InterfaceByName(wgInterface)
 	if err != nil {
 		return nil, fmt.Errorf("get interface: %w", err)
@@ -77,6 +79,7 @@ func newSystemdDbusConfigurator(wgInterface string) (hostManager, error) {
 
 	return &systemdDbusConfigurator{
 		dbusLinkObject: dbus.ObjectPath(s),
+		ifaceName:      wgInterface,
 	}, nil
 }
 
@@ -84,7 +87,7 @@ func (s *systemdDbusConfigurator) supportCustomPort() bool {
 	return true
 }
 
-func (s *systemdDbusConfigurator) applyDNSConfig(config HostDNSConfig) error {
+func (s *systemdDbusConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
 	parsedIP, err := netip.ParseAddr(config.ServerIP)
 	if err != nil {
 		return fmt.Errorf("unable to parse ip address, error: %w", err)
@@ -135,10 +138,12 @@ func (s *systemdDbusConfigurator) applyDNSConfig(config HostDNSConfig) error {
 		log.Infof("removing %s:%d as main DNS forwarder for this peer", config.ServerIP, config.ServerPort)
 	}
 
-	// create a backup for unclean shutdown detection before adding domains, as these might end up in the resolv.conf file.
-	// The file content itself is not important for systemd restoration
-	if err := createUncleanShutdownIndicator(defaultResolvConfPath, systemdManager, parsedIP.String()); err != nil {
-		log.Errorf("failed to create unclean shutdown resolv.conf backup: %s", err)
+	state := &ShutdownState{
+		ManagerType: systemdManager,
+		WgIface:     s.ifaceName,
+	}
+	if err := stateManager.UpdateState(state); err != nil {
+		log.Errorf("failed to update shutdown state: %s", err)
 	}
 
 	log.Infof("adding %d search domains and %d match domains. Search list: %s , Match list: %s", len(searchDomains), len(matchDomains), searchDomains, matchDomains)
@@ -174,10 +179,6 @@ func (s *systemdDbusConfigurator) restoreHostDNS() error {
 		return fmt.Errorf("unable to revert link configuration, got error: %w", err)
 	}
 
-	if err := removeUncleanShutdownIndicator(); err != nil {
-		log.Errorf("failed to remove unclean shutdown resolv.conf backup: %s", err)
-	}
-
 	return s.flushCaches()
 }
 

client/internal/dns/unclean_shutdown_android.go

@@ -1,5 +0,0 @@
-package dns
-
-func CheckUncleanShutdown(string) error {
-	return nil
-}

client/internal/dns/unclean_shutdown_darwin.go

@@ -3,57 +3,25 @@
 package dns
 
 import (
-	"errors"
 	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-
-	log "github.com/sirupsen/logrus"
 )
 
-const fileUncleanShutdownFileLocation = "/var/lib/netbird/unclean_shutdown_dns"
+type ShutdownState struct {
+}
 
-func CheckUncleanShutdown(string) error {
-	if _, err := os.Stat(fileUncleanShutdownFileLocation); err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			// no file -> clean shutdown
-			return nil
-		} else {
-			return fmt.Errorf("state: %w", err)
-		}
-	}
-
-	log.Warnf("detected unclean shutdown, file %s exists. Restoring unclean shutdown dns settings.", fileUncleanShutdownFileLocation)
+func (s *ShutdownState) Name() string {
+	return "dns_state"
+}
 
+func (s *ShutdownState) Cleanup() error {
 	manager, err := newHostManager()
 	if err != nil {
 		return fmt.Errorf("create host manager: %w", err)
 	}
 
-	if err := manager.restoreUncleanShutdownDNS(nil); err != nil {
-		return fmt.Errorf("restore unclean shutdown backup: %w", err)
+	if err := manager.restoreUncleanShutdownDNS(); err != nil {
+		return fmt.Errorf("restore unclean shutdown dns: %w", err)
 	}
 
 	return nil
 }
-
-func createUncleanShutdownIndicator() error {
-	dir := filepath.Dir(fileUncleanShutdownFileLocation)
-	if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil {
-		return fmt.Errorf("create dir %s: %w", dir, err)
-	}
-
-	if err := os.WriteFile(fileUncleanShutdownFileLocation, nil, 0644); err != nil { //nolint:gosec
-		return fmt.Errorf("create %s: %w", fileUncleanShutdownFileLocation, err)
-	}
-
-	return nil
-}
-
-func removeUncleanShutdownIndicator() error {
-	if err := os.Remove(fileUncleanShutdownFileLocation); err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove %s: %w", fileUncleanShutdownFileLocation, err)
-	}
-	return nil
-}

client/internal/dns/unclean_shutdown_ios.go

@@ -1,5 +0,0 @@
-package dns
-
-func CheckUncleanShutdown(string) error {
-	return nil
-}

client/internal/dns/unclean_shutdown_mobile.go

@@ -0,0 +1,14 @@
+//go:build ios || android
+
+package dns
+
+type ShutdownState struct {
+}
+
+func (s *ShutdownState) Name() string {
+	return "dns_state"
+}
+
+func (s *ShutdownState) Cleanup() error {
+	return nil
+}

client/internal/dns/unclean_shutdown_unix.go

@@ -3,66 +3,44 @@
 package dns
 
 import (
-	"errors"
 	"fmt"
-	"io/fs"
 	"net/netip"
 	"os"
 	"path/filepath"
-	"strings"
 
-	log "github.com/sirupsen/logrus"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
 
-func CheckUncleanShutdown(wgIface string) error {
-	if _, err := os.Stat(fileUncleanShutdownResolvConfLocation); err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			// no file -> clean shutdown
-			return nil
-		} else {
-			return fmt.Errorf("state: %w", err)
-		}
-	}
+type ShutdownState struct {
+	ManagerType osManagerType
+	DNSAddress  netip.Addr
+	WgIface     string
+}
 
-	log.Warnf("detected unclean shutdown, file %s exists", fileUncleanShutdownResolvConfLocation)
+func (s *ShutdownState) Name() string {
+	return "dns_state"
+}
 
-	managerData, err := os.ReadFile(fileUncleanShutdownManagerTypeLocation)
-	if err != nil {
-		return fmt.Errorf("read %s: %w", fileUncleanShutdownManagerTypeLocation, err)
-	}
-
-	managerFields := strings.Split(string(managerData), ",")
-	if len(managerFields) < 2 {
-		return errors.New("split manager data: insufficient number of fields")
-	}
-	osManagerTypeStr, dnsAddressStr := managerFields[0], managerFields[1]
-
-	dnsAddress, err := netip.ParseAddr(dnsAddressStr)
-	if err != nil {
-		return fmt.Errorf("parse dns address %s failed: %w", dnsAddressStr, err)
-	}
-
-	log.Warnf("restoring unclean shutdown dns settings via previously detected manager: %s", osManagerTypeStr)
-
-	// determine os manager type, so we can invoke the respective restore action
-	osManagerType, err := newOsManagerType(osManagerTypeStr)
-	if err != nil {
-		return fmt.Errorf("detect previous host manager: %w", err)
-	}
-
-	manager, err := newHostManagerFromType(wgIface, osManagerType)
+func (s *ShutdownState) Cleanup() error {
+	manager, err := newHostManagerFromType(s.WgIface, s.ManagerType)
 	if err != nil {
 		return fmt.Errorf("create previous host manager: %w", err)
 	}
 
-	if err := manager.restoreUncleanShutdownDNS(&dnsAddress); err != nil {
-		return fmt.Errorf("restore unclean shutdown backup: %w", err)
+	if err := manager.restoreUncleanShutdownDNS(&s.DNSAddress); err != nil {
+		return fmt.Errorf("restore unclean shutdown dns: %w", err)
 	}
 
 	return nil
 }
 
-func createUncleanShutdownIndicator(sourcePath string, managerType osManagerType, dnsAddress string) error {
+// TODO: move file contents to state manager
+func createUncleanShutdownIndicator(sourcePath string, dnsAddressStr string, stateManager *statemanager.Manager) error {
+	dnsAddress, err := netip.ParseAddr(dnsAddressStr)
+	if err != nil {
+		return fmt.Errorf("parse dns address %s: %w", dnsAddressStr, err)
+	}
+
 	dir := filepath.Dir(fileUncleanShutdownResolvConfLocation)
 	if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil {
 		return fmt.Errorf("create dir %s: %w", dir, err)
@@ -72,20 +50,13 @@ func createUncleanShutdownIndicator(sourcePath string, managerType osManagerType
 		return fmt.Errorf("create %s: %w", sourcePath, err)
 	}
 
-	managerData := fmt.Sprintf("%s,%s", managerType, dnsAddress)
-
-	if err := os.WriteFile(fileUncleanShutdownManagerTypeLocation, []byte(managerData), 0644); err != nil { //nolint:gosec
-		return fmt.Errorf("create %s: %w", fileUncleanShutdownManagerTypeLocation, err)
-	}
-	return nil
-}
-
-func removeUncleanShutdownIndicator() error {
-	if err := os.Remove(fileUncleanShutdownResolvConfLocation); err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove %s: %w", fileUncleanShutdownResolvConfLocation, err)
-	}
-	if err := os.Remove(fileUncleanShutdownManagerTypeLocation); err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove %s: %w", fileUncleanShutdownManagerTypeLocation, err)
+	state := &ShutdownState{
+		ManagerType: fileManager,
+		DNSAddress:  dnsAddress,
 	}
+	if err := stateManager.UpdateState(state); err != nil {
+		return fmt.Errorf("update state: %w", err)
+	}
+
 	return nil
 }

client/internal/dns/unclean_shutdown_windows.go

@@ -1,75 +1,26 @@
 package dns
 
 import (
-	"errors"
 	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-
-	"github.com/sirupsen/logrus"
 )
 
-const (
-	netbirdProgramDataLocation = "Netbird"
-	fileUncleanShutdownFile    = "unclean_shutdown_dns.txt"
-)
+type ShutdownState struct {
+	Guid string
+}
 
-func CheckUncleanShutdown(string) error {
-	file := getUncleanShutdownFile()
+func (s *ShutdownState) Name() string {
+	return "dns_state"
+}
 
-	if _, err := os.Stat(file); err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			// no file -> clean shutdown
-			return nil
-		} else {
-			return fmt.Errorf("state: %w", err)
-		}
-	}
-
-	logrus.Warnf("detected unclean shutdown, file %s exists. Restoring unclean shutdown dns settings.", file)
-
-	guid, err := os.ReadFile(file)
-	if err != nil {
-		return fmt.Errorf("read %s: %w", file, err)
-	}
-
-	manager, err := newHostManagerWithGuid(string(guid))
+func (s *ShutdownState) Cleanup() error {
+	manager, err := newHostManagerWithGuid(s.Guid)
 	if err != nil {
 		return fmt.Errorf("create host manager: %w", err)
 	}
 
-	if err := manager.restoreUncleanShutdownDNS(nil); err != nil {
-		return fmt.Errorf("restore unclean shutdown backup: %w", err)
+	if err := manager.restoreUncleanShutdownDNS(); err != nil {
+		return fmt.Errorf("restore unclean shutdown dns: %w", err)
 	}
 
 	return nil
 }
-
-func createUncleanShutdownIndicator(guid string) error {
-	file := getUncleanShutdownFile()
-
-	dir := filepath.Dir(file)
-	if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil {
-		return fmt.Errorf("create dir %s: %w", dir, err)
-	}
-
-	if err := os.WriteFile(file, []byte(guid), 0600); err != nil {
-		return fmt.Errorf("create %s: %w", file, err)
-	}
-
-	return nil
-}
-
-func removeUncleanShutdownIndicator() error {
-	file := getUncleanShutdownFile()
-
-	if err := os.Remove(file); err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("remove %s: %w", file, err)
-	}
-	return nil
-}
-
-func getUncleanShutdownFile() string {
-	return filepath.Join(os.Getenv("PROGRAMDATA"), netbirdProgramDataLocation, fileUncleanShutdownFile)
-}

client/internal/engine.go

@@ -23,19 +23,21 @@ import (
 
 	"github.com/netbirdio/netbird/client/firewall"
 	"github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/iface/bind"
 	"github.com/netbirdio/netbird/client/iface/device"
 	"github.com/netbirdio/netbird/client/internal/acl"
 	"github.com/netbirdio/netbird/client/internal/dns"
-
-	"github.com/netbirdio/netbird/client/iface"
-	"github.com/netbirdio/netbird/client/iface/bind"
 	"github.com/netbirdio/netbird/client/internal/networkmonitor"
 	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/internal/peer/guard"
+	icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
 	"github.com/netbirdio/netbird/client/internal/relay"
 	"github.com/netbirdio/netbird/client/internal/rosenpass"
 	"github.com/netbirdio/netbird/client/internal/routemanager"
 	"github.com/netbirdio/netbird/client/internal/routemanager/systemops"
-	"github.com/netbirdio/netbird/client/internal/wgproxy"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
+
 	nbssh "github.com/netbirdio/netbird/client/ssh"
 	"github.com/netbirdio/netbird/client/system"
 	nbdns "github.com/netbirdio/netbird/dns"
@@ -141,8 +143,7 @@ type Engine struct {
 	ctx    context.Context
 	cancel context.CancelFunc
 
-	wgInterface    iface.IWGIface
-	wgProxyFactory *wgproxy.Factory
+	wgInterface iface.IWGIface
 
 	udpMux *bind.UniversalUDPMuxDefault
 
@@ -168,6 +169,8 @@ type Engine struct {
 	checks []*mgmProto.Checks
 
 	relayManager *relayClient.Manager
+	stateManager *statemanager.Manager
+	srWatcher    *guard.SRWatcher
 }
 
 // Peer is an instance of the Connection Peer
@@ -215,7 +218,7 @@ func NewEngineWithProbes(
 	probes *ProbeHolder,
 	checks []*mgmProto.Checks,
 ) *Engine {
-	return &Engine{
+	engine := &Engine{
 		clientCtx:      clientCtx,
 		clientCancel:   clientCancel,
 		signal:         signalClient,
@@ -234,6 +237,11 @@ func NewEngineWithProbes(
 		probes:         probes,
 		checks:         checks,
 	}
+	if path := statemanager.GetDefaultStatePath(); path != "" {
+		engine.stateManager = statemanager.New(path)
+	}
+
+	return engine
 }
 
 func (e *Engine) Stop() error {
@@ -255,7 +263,11 @@ func (e *Engine) Stop() error {
 	e.stopDNSServer()
 
 	if e.routeManager != nil {
-		e.routeManager.Stop()
+		e.routeManager.Stop(e.stateManager)
+	}
+
+	if e.srWatcher != nil {
+		e.srWatcher.Close()
 	}
 
 	err := e.removeAllPeers()
@@ -277,6 +289,17 @@ func (e *Engine) Stop() error {
 
 	e.close()
 	log.Infof("stopped Netbird Engine")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	if err := e.stateManager.Stop(ctx); err != nil {
+		return fmt.Errorf("failed to stop state manager: %w", err)
+	}
+	if err := e.stateManager.PersistState(ctx); err != nil {
+		log.Errorf("failed to persist state: %v", err)
+	}
+
 	return nil
 }
 
@@ -299,9 +322,6 @@ func (e *Engine) Start() error {
 	}
 	e.wgInterface = wgIface
 
-	userspace := e.wgInterface.IsUserspaceBind()
-	e.wgProxyFactory = wgproxy.NewFactory(userspace, e.config.WgPort)
-
 	if e.config.RosenpassEnabled {
 		log.Infof("rosenpass is enabled")
 		if e.config.RosenpassPermissive {
@@ -319,6 +339,8 @@ func (e *Engine) Start() error {
 		}
 	}
 
+	e.stateManager.Start()
+
 	initialRoutes, dnsServer, err := e.newDnsServer()
 	if err != nil {
 		e.close()
@@ -327,7 +349,7 @@ func (e *Engine) Start() error {
 	e.dnsServer = dnsServer
 
 	e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.config.DNSRouteInterval, e.wgInterface, e.statusRecorder, e.relayManager, initialRoutes)
-	beforePeerHook, afterPeerHook, err := e.routeManager.Init()
+	beforePeerHook, afterPeerHook, err := e.routeManager.Init(e.stateManager)
 	if err != nil {
 		log.Errorf("Failed to initialize route manager: %s", err)
 	} else {
@@ -344,7 +366,7 @@ func (e *Engine) Start() error {
 		return fmt.Errorf("create wg interface: %w", err)
 	}
 
-	e.firewall, err = firewall.NewFirewall(e.ctx, e.wgInterface)
+	e.firewall, err = firewall.NewFirewall(e.wgInterface, e.stateManager)
 	if err != nil {
 		log.Errorf("failed creating firewall manager: %s", err)
 	}
@@ -374,6 +396,18 @@ func (e *Engine) Start() error {
 		return fmt.Errorf("initialize dns server: %w", err)
 	}
 
+	iceCfg := icemaker.Config{
+		StunTurn:             &e.stunTurn,
+		InterfaceBlackList:   e.config.IFaceBlackList,
+		DisableIPv6Discovery: e.config.DisableIPv6Discovery,
+		UDPMux:               e.udpMux.UDPMuxDefault,
+		UDPMuxSrflx:          e.udpMux,
+		NATExternalIPs:       e.parseNATExternalIPMappings(),
+	}
+
+	e.srWatcher = guard.NewSRWatcher(e.signal, e.relayManager, e.mobileDep.IFaceDiscover, iceCfg)
+	e.srWatcher.Start()
+
 	e.receiveSignalEvents()
 	e.receiveManagementEvents()
 	e.receiveProbeEvents()
@@ -956,7 +990,7 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e
 		LocalWgPort:     e.config.WgPort,
 		RosenpassPubKey: e.getRosenpassPubKey(),
 		RosenpassAddr:   e.getRosenpassAddr(),
-		ICEConfig: peer.ICEConfig{
+		ICEConfig: icemaker.Config{
 			StunTurn:             &e.stunTurn,
 			InterfaceBlackList:   e.config.IFaceBlackList,
 			DisableIPv6Discovery: e.config.DisableIPv6Discovery,
@@ -966,7 +1000,7 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e
 		},
 	}
 
-	peerConn, err := peer.NewConn(e.ctx, config, e.statusRecorder, e.wgProxyFactory, e.signaler, e.mobileDep.IFaceDiscover, e.relayManager)
+	peerConn, err := peer.NewConn(e.ctx, config, e.statusRecorder, e.signaler, e.mobileDep.IFaceDiscover, e.relayManager, e.srWatcher)
 	if err != nil {
 		return nil, err
 	}
@@ -1117,12 +1151,6 @@ func (e *Engine) parseNATExternalIPMappings() []string {
 }
 
 func (e *Engine) close() {
-	if e.wgProxyFactory != nil {
-		if err := e.wgProxyFactory.Free(); err != nil {
-			log.Errorf("failed closing ebpf proxy: %s", err)
-		}
-	}
-
 	log.Debugf("removing Netbird interface %s", e.config.WgIfaceName)
 	if e.wgInterface != nil {
 		if err := e.wgInterface.Close(); err != nil {
@@ -1139,7 +1167,7 @@ func (e *Engine) close() {
 	}
 
 	if e.firewall != nil {
-		err := e.firewall.Reset()
+		err := e.firewall.Reset(e.stateManager)
 		if err != nil {
 			log.Warnf("failed to reset firewall: %s", err)
 		}
@@ -1167,21 +1195,29 @@ func (e *Engine) newWgIface() (*iface.WGIface, error) {
 		log.Errorf("failed to create pion's stdnet: %s", err)
 	}
 
-	var mArgs *device.MobileIFaceArguments
+	opts := iface.WGIFaceOpts{
+		IFaceName:    e.config.WgIfaceName,
+		Address:      e.config.WgAddr,
+		WGPort:       e.config.WgPort,
+		WGPrivKey:    e.config.WgPrivateKey.String(),
+		MTU:          iface.DefaultMTU,
+		TransportNet: transportNet,
+		FilterFn:     e.addrViaRoutes,
+	}
+
 	switch runtime.GOOS {
 	case "android":
-		mArgs = &device.MobileIFaceArguments{
+		opts.MobileArgs = &device.MobileIFaceArguments{
 			TunAdapter: e.mobileDep.TunAdapter,
 			TunFd:      int(e.mobileDep.FileDescriptor),
 		}
 	case "ios":
-		mArgs = &device.MobileIFaceArguments{
+		opts.MobileArgs = &device.MobileIFaceArguments{
 			TunFd: int(e.mobileDep.FileDescriptor),
 		}
-	default:
 	}
 
-	return iface.NewWGIFace(e.config.WgIfaceName, e.config.WgAddr, e.config.WgPort, e.config.WgPrivateKey.String(), iface.DefaultMTU, transportNet, mArgs, e.addrViaRoutes)
+	return iface.NewWGIFace(opts)
 }
 
 func (e *Engine) wgInterfaceCreate() (err error) {
@@ -1222,10 +1258,11 @@ func (e *Engine) newDnsServer() ([]*route.Route, dns.Server, error) {
 		dnsServer := dns.NewDefaultServerIos(e.ctx, e.wgInterface, e.mobileDep.DnsManager, e.statusRecorder)
 		return nil, dnsServer, nil
 	default:
-		dnsServer, err := dns.NewDefaultServer(e.ctx, e.wgInterface, e.config.CustomDNSAddress, e.statusRecorder)
+		dnsServer, err := dns.NewDefaultServer(e.ctx, e.wgInterface, e.config.CustomDNSAddress, e.statusRecorder, e.stateManager)
 		if err != nil {
 			return nil, nil, err
 		}
+
 		return nil, dnsServer, nil
 	}
 }

client/internal/engine_test.go

@@ -29,6 +29,8 @@ import (
 	"github.com/netbirdio/netbird/client/iface/device"
 	"github.com/netbirdio/netbird/client/internal/dns"
 	"github.com/netbirdio/netbird/client/internal/peer"
+	"github.com/netbirdio/netbird/client/internal/peer/guard"
+	icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
 	"github.com/netbirdio/netbird/client/internal/routemanager"
 	"github.com/netbirdio/netbird/client/ssh"
 	"github.com/netbirdio/netbird/client/system"
@@ -258,6 +260,7 @@ func TestEngine_UpdateNetworkMap(t *testing.T) {
 	}
 	engine.udpMux = bind.NewUniversalUDPMuxDefault(bind.UniversalUDPMuxParams{UDPConn: conn})
 	engine.ctx = ctx
+	engine.srWatcher = guard.NewSRWatcher(nil, nil, nil, icemaker.Config{})
 
 	type testCase struct {
 		name       string
@@ -602,7 +605,16 @@ func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, engine.config.WgPort, key.String(), iface.DefaultMTU, newNet, nil, nil)
+
+			opts := iface.WGIFaceOpts{
+				IFaceName:    wgIfaceName,
+				Address:      wgAddr,
+				WGPort:       engine.config.WgPort,
+				WGPrivKey:    key.String(),
+				MTU:          iface.DefaultMTU,
+				TransportNet: newNet,
+			}
+			engine.wgInterface, err = iface.NewWGIFace(opts)
 			assert.NoError(t, err, "shouldn't return error")
 			input := struct {
 				inputSerial uint64
@@ -774,7 +786,15 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, 33100, key.String(), iface.DefaultMTU, newNet, nil, nil)
+			opts := iface.WGIFaceOpts{
+				IFaceName:    wgIfaceName,
+				Address:      wgAddr,
+				WGPort:       33100,
+				WGPrivKey:    key.String(),
+				MTU:          iface.DefaultMTU,
+				TransportNet: newNet,
+			}
+			engine.wgInterface, err = iface.NewWGIFace(opts)
 			assert.NoError(t, err, "shouldn't return error")
 
 			mockRouteManager := &routemanager.MockManager{

client/internal/peer/conn.go

@@ -10,15 +10,16 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cenkalti/backoff/v4"
 	"github.com/pion/ice/v3"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 
 	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/iface/configurer"
+	"github.com/netbirdio/netbird/client/iface/wgproxy"
+	"github.com/netbirdio/netbird/client/internal/peer/guard"
+	icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
 	"github.com/netbirdio/netbird/client/internal/stdnet"
-	"github.com/netbirdio/netbird/client/internal/wgproxy"
 	relayClient "github.com/netbirdio/netbird/relay/client"
 	"github.com/netbirdio/netbird/route"
 	nbnet "github.com/netbirdio/netbird/util/net"
@@ -32,8 +33,6 @@ const (
 	connPriorityRelay   ConnPriority = 1
 	connPriorityICETurn ConnPriority = 1
 	connPriorityICEP2P  ConnPriority = 2
-
-	reconnectMaxElapsedTime = 30 * time.Minute
 )
 
 type WgConfig struct {
@@ -63,7 +62,7 @@ type ConnConfig struct {
 	RosenpassAddr string
 
 	// ICEConfig ICE protocol configuration
-	ICEConfig ICEConfig
+	ICEConfig icemaker.Config
 }
 
 type WorkerCallbacks struct {
@@ -81,11 +80,10 @@ type Conn struct {
 	ctxCancel      context.CancelFunc
 	config         ConnConfig
 	statusRecorder *Status
-	wgProxyFactory *wgproxy.Factory
 	signaler       *Signaler
-	iFaceDiscover  stdnet.ExternalIFaceDiscover
 	relayManager   *relayClient.Manager
-	allowedIPsIP   string
+	allowedIP      net.IP
+	allowedNet     string
 	handshaker     *Handshaker
 
 	onConnected    func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string)
@@ -107,17 +105,13 @@ type Conn struct {
 	wgProxyICE   wgproxy.Proxy
 	wgProxyRelay wgproxy.Proxy
 
-	// for reconnection operations
-	iCEDisconnected   chan bool
-	relayDisconnected chan bool
-	connMonitor       *ConnMonitor
-	reconnectCh       <-chan struct{}
+	guard *guard.Guard
 }
 
 // NewConn creates a new not opened Conn to the remote peer.
 // To establish a connection run Conn.Open
-func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, signaler *Signaler, iFaceDiscover stdnet.ExternalIFaceDiscover, relayManager *relayClient.Manager) (*Conn, error) {
-	_, allowedIPsIP, err := net.ParseCIDR(config.WgConfig.AllowedIps)
+func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Status, signaler *Signaler, iFaceDiscover stdnet.ExternalIFaceDiscover, relayManager *relayClient.Manager, srWatcher *guard.SRWatcher) (*Conn, error) {
+	allowedIP, allowedNet, err := net.ParseCIDR(config.WgConfig.AllowedIps)
 	if err != nil {
 		log.Errorf("failed to parse allowedIPS: %v", err)
 		return nil, err
@@ -132,26 +126,14 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu
 		ctxCancel:      ctxCancel,
 		config:         config,
 		statusRecorder: statusRecorder,
-		wgProxyFactory: wgProxyFactory,
 		signaler:       signaler,
-		iFaceDiscover:  iFaceDiscover,
 		relayManager:   relayManager,
-		allowedIPsIP:   allowedIPsIP.String(),
+		allowedIP:      allowedIP,
+		allowedNet:     allowedNet.String(),
 		statusRelay:    NewAtomicConnStatus(),
 		statusICE:      NewAtomicConnStatus(),
-
-		iCEDisconnected:   make(chan bool, 1),
-		relayDisconnected: make(chan bool, 1),
 	}
 
-	conn.connMonitor, conn.reconnectCh = NewConnMonitor(
-		signaler,
-		iFaceDiscover,
-		config,
-		conn.relayDisconnected,
-		conn.iCEDisconnected,
-	)
-
 	rFns := WorkerRelayCallbacks{
 		OnConnReady:    conn.relayConnectionIsReady,
 		OnDisconnected: conn.onWorkerRelayStateDisconnected,
@@ -162,7 +144,8 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu
 		OnStatusChanged: conn.onWorkerICEStateDisconnected,
 	}
 
-	conn.workerRelay = NewWorkerRelay(connLog, config, relayManager, rFns)
+	ctrl := isController(config)
+	conn.workerRelay = NewWorkerRelay(connLog, ctrl, config, relayManager, rFns)
 
 	relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally()
 	conn.workerICE, err = NewWorkerICE(ctx, connLog, config, signaler, iFaceDiscover, statusRecorder, relayIsSupportedLocally, wFns)
@@ -177,6 +160,8 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu
 		conn.handshaker.AddOnNewOfferListener(conn.workerICE.OnNewOffer)
 	}
 
+	conn.guard = guard.NewGuard(connLog, ctrl, conn.isConnectedOnAllWay, config.Timeout, srWatcher)
+
 	go conn.handshaker.Listen()
 
 	return conn, nil
@@ -187,6 +172,7 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu
 // be used.
 func (conn *Conn) Open() {
 	conn.log.Debugf("open connection to peer")
+
 	conn.mu.Lock()
 	defer conn.mu.Unlock()
 	conn.opened = true
@@ -203,24 +189,19 @@ func (conn *Conn) Open() {
 		conn.log.Warnf("error while updating the state err: %v", err)
 	}
 
-	go conn.startHandshakeAndReconnect()
+	go conn.startHandshakeAndReconnect(conn.ctx)
 }
 
-func (conn *Conn) startHandshakeAndReconnect() {
-	conn.waitInitialRandomSleepTime()
+func (conn *Conn) startHandshakeAndReconnect(ctx context.Context) {
+	conn.waitInitialRandomSleepTime(ctx)
 
 	err := conn.handshaker.sendOffer()
 	if err != nil {
 		conn.log.Errorf("failed to send initial offer: %v", err)
 	}
 
-	go conn.connMonitor.Start(conn.ctx)
-
-	if conn.workerRelay.IsController() {
-		conn.reconnectLoopWithRetry()
-	} else {
-		conn.reconnectLoopForOnDisconnectedEvent()
-	}
+	go conn.guard.Start(ctx)
+	go conn.listenGuardEvent(ctx)
 }
 
 // Close closes this peer Conn issuing a close event to the Conn closeCh
@@ -319,104 +300,6 @@ func (conn *Conn) GetKey() string {
 	return conn.config.Key
 }
 
-func (conn *Conn) reconnectLoopWithRetry() {
-	// Give chance to the peer to establish the initial connection.
-	// With it, we can decrease to send necessary offer
-	select {
-	case <-conn.ctx.Done():
-		return
-	case <-time.After(3 * time.Second):
-	}
-
-	ticker := conn.prepareExponentTicker()
-	defer ticker.Stop()
-	time.Sleep(1 * time.Second)
-
-	for {
-		select {
-		case t := <-ticker.C:
-			if t.IsZero() {
-				// in case if the ticker has been canceled by context then avoid the temporary loop
-				return
-			}
-
-			if conn.workerRelay.IsRelayConnectionSupportedWithPeer() {
-				if conn.statusRelay.Get() == StatusDisconnected || conn.statusICE.Get() == StatusDisconnected {
-					conn.log.Tracef("connectivity guard timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE)
-				}
-			} else {
-				if conn.statusICE.Get() == StatusDisconnected {
-					conn.log.Tracef("connectivity guard timedout, ice state: %s", conn.statusICE)
-				}
-			}
-
-			// checks if there is peer connection is established via relay or ice
-			if conn.isConnected() {
-				continue
-			}
-
-			err := conn.handshaker.sendOffer()
-			if err != nil {
-				conn.log.Errorf("failed to do handshake: %v", err)
-			}
-
-		case <-conn.reconnectCh:
-			ticker.Stop()
-			ticker = conn.prepareExponentTicker()
-
-		case <-conn.ctx.Done():
-			conn.log.Debugf("context is done, stop reconnect loop")
-			return
-		}
-	}
-}
-
-func (conn *Conn) prepareExponentTicker() *backoff.Ticker {
-	bo := backoff.WithContext(&backoff.ExponentialBackOff{
-		InitialInterval:     800 * time.Millisecond,
-		RandomizationFactor: 0.1,
-		Multiplier:          2,
-		MaxInterval:         conn.config.Timeout,
-		MaxElapsedTime:      reconnectMaxElapsedTime,
-		Stop:                backoff.Stop,
-		Clock:               backoff.SystemClock,
-	}, conn.ctx)
-
-	ticker := backoff.NewTicker(bo)
-	<-ticker.C // consume the initial tick what is happening right after the ticker has been created
-
-	return ticker
-}
-
-// reconnectLoopForOnDisconnectedEvent is used when the peer is not a controller and it should reconnect to the peer
-// when the connection is lost. It will try to establish a connection only once time if before the connection was established
-// It track separately the ice and relay connection status. Just because a lover priority connection reestablished it does not
-// mean that to switch to it. We always force to use the higher priority connection.
-func (conn *Conn) reconnectLoopForOnDisconnectedEvent() {
-	for {
-		select {
-		case changed := <-conn.relayDisconnected:
-			if !changed {
-				continue
-			}
-			conn.log.Debugf("Relay state changed, try to send new offer")
-		case changed := <-conn.iCEDisconnected:
-			if !changed {
-				continue
-			}
-			conn.log.Debugf("ICE state changed, try to send new offer")
-		case <-conn.ctx.Done():
-			conn.log.Debugf("context is done, stop reconnect loop")
-			return
-		}
-
-		err := conn.handshaker.SendOffer()
-		if err != nil {
-			conn.log.Errorf("failed to do handshake: %v", err)
-		}
-	}
-}
-
 // configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected
 func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICEConnInfo) {
 	conn.mu.Lock()
@@ -516,7 +399,7 @@ func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) {
 	changed := conn.statusICE.Get() != newState && newState != StatusConnecting
 	conn.statusICE.Set(newState)
 
-	conn.notifyReconnectLoopICEDisconnected(changed)
+	conn.guard.SetICEConnDisconnected(changed)
 
 	peerState := State{
 		PubKey:           conn.config.Key,
@@ -607,7 +490,7 @@ func (conn *Conn) onWorkerRelayStateDisconnected() {
 
 	changed := conn.statusRelay.Get() != StatusDisconnected
 	conn.statusRelay.Set(StatusDisconnected)
-	conn.notifyReconnectLoopRelayDisconnected(changed)
+	conn.guard.SetRelayedConnDisconnected(changed)
 
 	peerState := State{
 		PubKey:           conn.config.Key,
@@ -620,6 +503,20 @@ func (conn *Conn) onWorkerRelayStateDisconnected() {
 	}
 }
 
+func (conn *Conn) listenGuardEvent(ctx context.Context) {
+	for {
+		select {
+		case <-conn.guard.Reconnect:
+			conn.log.Debugf("send offer to peer")
+			if err := conn.handshaker.SendOffer(); err != nil {
+				conn.log.Errorf("failed to send offer: %v", err)
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
 func (conn *Conn) configureWGEndpoint(addr *net.UDPAddr) error {
 	return conn.config.WgConfig.WgInterface.UpdatePeer(
 		conn.config.WgConfig.RemoteKey,
@@ -692,11 +589,11 @@ func (conn *Conn) doOnConnected(remoteRosenpassPubKey []byte, remoteRosenpassAdd
 	}
 
 	if conn.onConnected != nil {
-		conn.onConnected(conn.config.Key, remoteRosenpassPubKey, conn.allowedIPsIP, remoteRosenpassAddr)
+		conn.onConnected(conn.config.Key, remoteRosenpassPubKey, conn.allowedNet, remoteRosenpassAddr)
 	}
 }
 
-func (conn *Conn) waitInitialRandomSleepTime() {
+func (conn *Conn) waitInitialRandomSleepTime(ctx context.Context) {
 	minWait := 100
 	maxWait := 800
 	duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond
@@ -705,7 +602,7 @@ func (conn *Conn) waitInitialRandomSleepTime() {
 	defer timeout.Stop()
 
 	select {
-	case <-conn.ctx.Done():
+	case <-ctx.Done():
 	case <-timeout.C:
 	}
 }
@@ -734,11 +631,17 @@ func (conn *Conn) evalStatus() ConnStatus {
 	return StatusDisconnected
 }
 
-func (conn *Conn) isConnected() bool {
+func (conn *Conn) isConnectedOnAllWay() (connected bool) {
 	conn.mu.Lock()
 	defer conn.mu.Unlock()
 
-	if conn.statusICE.Get() != StatusConnected && conn.statusICE.Get() != StatusConnecting {
+	defer func() {
+		if !connected {
+			conn.logTraceConnState()
+		}
+	}()
+
+	if conn.statusICE.Get() == StatusDisconnected {
 		return false
 	}
 
@@ -783,8 +686,13 @@ func (conn *Conn) freeUpConnID() {
 
 func (conn *Conn) newProxy(remoteConn net.Conn) (wgproxy.Proxy, error) {
 	conn.log.Debugf("setup proxied WireGuard connection")
-	wgProxy := conn.wgProxyFactory.GetProxy()
-	if err := wgProxy.AddTurnConn(conn.ctx, remoteConn); err != nil {
+	udpAddr := &net.UDPAddr{
+		IP:   conn.allowedIP,
+		Port: conn.config.WgConfig.WgListenPort,
+	}
+
+	wgProxy := conn.config.WgConfig.WgInterface.GetProxy()
+	if err := wgProxy.AddTurnConn(conn.ctx, udpAddr, remoteConn); err != nil {
 		conn.log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
 		return nil, err
 	}
@@ -803,20 +711,6 @@ func (conn *Conn) removeWgPeer() error {
 	return conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey)
 }
 
-func (conn *Conn) notifyReconnectLoopRelayDisconnected(changed bool) {
-	select {
-	case conn.relayDisconnected <- changed:
-	default:
-	}
-}
-
-func (conn *Conn) notifyReconnectLoopICEDisconnected(changed bool) {
-	select {
-	case conn.iCEDisconnected <- changed:
-	default:
-	}
-}
-
 func (conn *Conn) handleConfigurationFailure(err error, wgProxy wgproxy.Proxy) {
 	conn.log.Warnf("Failed to update wg peer configuration: %v", err)
 	if wgProxy != nil {
@@ -829,6 +723,18 @@ func (conn *Conn) handleConfigurationFailure(err error, wgProxy wgproxy.Proxy) {
 	}
 }
 
+func (conn *Conn) logTraceConnState() {
+	if conn.workerRelay.IsRelayConnectionSupportedWithPeer() {
+		conn.log.Tracef("connectivity guard check, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE)
+	} else {
+		conn.log.Tracef("connectivity guard check, ice state: %s", conn.statusICE)
+	}
+}
+
+func isController(config ConnConfig) bool {
+	return config.LocalKey > config.Key
+}
+
 func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool {
 	return remoteRosenpassPubKey != nil
 }

client/internal/peer/conn_monitor.go

@@ -1,212 +0,0 @@
-package peer
-
-import (
-	"context"
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/pion/ice/v3"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/netbirdio/netbird/client/internal/stdnet"
-)
-
-const (
-	signalerMonitorPeriod     = 5 * time.Second
-	candidatesMonitorPeriod   = 5 * time.Minute
-	candidateGatheringTimeout = 5 * time.Second
-)
-
-type ConnMonitor struct {
-	signaler          *Signaler
-	iFaceDiscover     stdnet.ExternalIFaceDiscover
-	config            ConnConfig
-	relayDisconnected chan bool
-	iCEDisconnected   chan bool
-	reconnectCh       chan struct{}
-	currentCandidates []ice.Candidate
-	candidatesMu      sync.Mutex
-}
-
-func NewConnMonitor(signaler *Signaler, iFaceDiscover stdnet.ExternalIFaceDiscover, config ConnConfig, relayDisconnected, iCEDisconnected chan bool) (*ConnMonitor, <-chan struct{}) {
-	reconnectCh := make(chan struct{}, 1)
-	cm := &ConnMonitor{
-		signaler:          signaler,
-		iFaceDiscover:     iFaceDiscover,
-		config:            config,
-		relayDisconnected: relayDisconnected,
-		iCEDisconnected:   iCEDisconnected,
-		reconnectCh:       reconnectCh,
-	}
-	return cm, reconnectCh
-}
-
-func (cm *ConnMonitor) Start(ctx context.Context) {
-	signalerReady := make(chan struct{}, 1)
-	go cm.monitorSignalerReady(ctx, signalerReady)
-
-	localCandidatesChanged := make(chan struct{}, 1)
-	go cm.monitorLocalCandidatesChanged(ctx, localCandidatesChanged)
-
-	for {
-		select {
-		case changed := <-cm.relayDisconnected:
-			if !changed {
-				continue
-			}
-			log.Debugf("Relay state changed, triggering reconnect")
-			cm.triggerReconnect()
-
-		case changed := <-cm.iCEDisconnected:
-			if !changed {
-				continue
-			}
-			log.Debugf("ICE state changed, triggering reconnect")
-			cm.triggerReconnect()
-
-		case <-signalerReady:
-			log.Debugf("Signaler became ready, triggering reconnect")
-			cm.triggerReconnect()
-
-		case <-localCandidatesChanged:
-			log.Debugf("Local candidates changed, triggering reconnect")
-			cm.triggerReconnect()
-
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (cm *ConnMonitor) monitorSignalerReady(ctx context.Context, signalerReady chan<- struct{}) {
-	if cm.signaler == nil {
-		return
-	}
-
-	ticker := time.NewTicker(signalerMonitorPeriod)
-	defer ticker.Stop()
-
-	lastReady := true
-	for {
-		select {
-		case <-ticker.C:
-			currentReady := cm.signaler.Ready()
-			if !lastReady && currentReady {
-				select {
-				case signalerReady <- struct{}{}:
-				default:
-				}
-			}
-			lastReady = currentReady
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (cm *ConnMonitor) monitorLocalCandidatesChanged(ctx context.Context, localCandidatesChanged chan<- struct{}) {
-	ufrag, pwd, err := generateICECredentials()
-	if err != nil {
-		log.Warnf("Failed to generate ICE credentials: %v", err)
-		return
-	}
-
-	ticker := time.NewTicker(candidatesMonitorPeriod)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ticker.C:
-			if err := cm.handleCandidateTick(ctx, localCandidatesChanged, ufrag, pwd); err != nil {
-				log.Warnf("Failed to handle candidate tick: %v", err)
-			}
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (cm *ConnMonitor) handleCandidateTick(ctx context.Context, localCandidatesChanged chan<- struct{}, ufrag string, pwd string) error {
-	log.Debugf("Gathering ICE candidates")
-
-	transportNet, err := newStdNet(cm.iFaceDiscover, cm.config.ICEConfig.InterfaceBlackList)
-	if err != nil {
-		log.Errorf("failed to create pion's stdnet: %s", err)
-	}
-
-	agent, err := newAgent(cm.config, transportNet, candidateTypesP2P(), ufrag, pwd)
-	if err != nil {
-		return fmt.Errorf("create ICE agent: %w", err)
-	}
-	defer func() {
-		if err := agent.Close(); err != nil {
-			log.Warnf("Failed to close ICE agent: %v", err)
-		}
-	}()
-
-	gatherDone := make(chan struct{})
-	err = agent.OnCandidate(func(c ice.Candidate) {
-		log.Tracef("Got candidate: %v", c)
-		if c == nil {
-			close(gatherDone)
-		}
-	})
-	if err != nil {
-		return fmt.Errorf("set ICE candidate handler: %w", err)
-	}
-
-	if err := agent.GatherCandidates(); err != nil {
-		return fmt.Errorf("gather ICE candidates: %w", err)
-	}
-
-	ctx, cancel := context.WithTimeout(ctx, candidateGatheringTimeout)
-	defer cancel()
-
-	select {
-	case <-ctx.Done():
-		return fmt.Errorf("wait for gathering: %w", ctx.Err())
-	case <-gatherDone:
-	}
-
-	candidates, err := agent.GetLocalCandidates()
-	if err != nil {
-		return fmt.Errorf("get local candidates: %w", err)
-	}
-	log.Tracef("Got candidates: %v", candidates)
-
-	if changed := cm.updateCandidates(candidates); changed {
-		select {
-		case localCandidatesChanged <- struct{}{}:
-		default:
-		}
-	}
-
-	return nil
-}
-
-func (cm *ConnMonitor) updateCandidates(newCandidates []ice.Candidate) bool {
-	cm.candidatesMu.Lock()
-	defer cm.candidatesMu.Unlock()
-
-	if len(cm.currentCandidates) != len(newCandidates) {
-		cm.currentCandidates = newCandidates
-		return true
-	}
-
-	for i, candidate := range cm.currentCandidates {
-		if candidate.Address() != newCandidates[i].Address() {
-			cm.currentCandidates = newCandidates
-			return true
-		}
-	}
-
-	return false
-}
-
-func (cm *ConnMonitor) triggerReconnect() {
-	select {
-	case cm.reconnectCh <- struct{}{}:
-	default:
-	}
-}

client/internal/peer/conn_test.go

@@ -10,8 +10,9 @@ import (
 	"github.com/magiconair/properties/assert"
 
 	"github.com/netbirdio/netbird/client/iface"
+	"github.com/netbirdio/netbird/client/internal/peer/guard"
+	"github.com/netbirdio/netbird/client/internal/peer/ice"
 	"github.com/netbirdio/netbird/client/internal/stdnet"
-	"github.com/netbirdio/netbird/client/internal/wgproxy"
 	"github.com/netbirdio/netbird/util"
 )
 
@@ -20,7 +21,7 @@ var connConf = ConnConfig{
 	LocalKey:    "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=",
 	Timeout:     time.Second,
 	LocalWgPort: 51820,
-	ICEConfig: ICEConfig{
+	ICEConfig: ice.Config{
 		InterfaceBlackList: nil,
 	},
 }
@@ -44,11 +45,8 @@ func TestNewConn_interfaceFilter(t *testing.T) {
 }
 
 func TestConn_GetKey(t *testing.T) {
-	wgProxyFactory := wgproxy.NewFactory(false, connConf.LocalWgPort)
-	defer func() {
-		_ = wgProxyFactory.Free()
-	}()
-	conn, err := NewConn(context.Background(), connConf, nil, wgProxyFactory, nil, nil, nil)
+	swWatcher := guard.NewSRWatcher(nil, nil, nil, connConf.ICEConfig)
+	conn, err := NewConn(context.Background(), connConf, nil, nil, nil, nil, swWatcher)
 	if err != nil {
 		return
 	}
@@ -59,11 +57,8 @@ func TestConn_GetKey(t *testing.T) {
 }
 
 func TestConn_OnRemoteOffer(t *testing.T) {
-	wgProxyFactory := wgproxy.NewFactory(false, connConf.LocalWgPort)
-	defer func() {
-		_ = wgProxyFactory.Free()
-	}()
-	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil)
+	swWatcher := guard.NewSRWatcher(nil, nil, nil, connConf.ICEConfig)
+	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), nil, nil, nil, swWatcher)
 	if err != nil {
 		return
 	}
@@ -96,11 +91,8 @@ func TestConn_OnRemoteOffer(t *testing.T) {
 }
 
 func TestConn_OnRemoteAnswer(t *testing.T) {
-	wgProxyFactory := wgproxy.NewFactory(false, connConf.LocalWgPort)
-	defer func() {
-		_ = wgProxyFactory.Free()
-	}()
-	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil)
+	swWatcher := guard.NewSRWatcher(nil, nil, nil, connConf.ICEConfig)
+	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), nil, nil, nil, swWatcher)
 	if err != nil {
 		return
 	}
@@ -132,11 +124,8 @@ func TestConn_OnRemoteAnswer(t *testing.T) {
 	wg.Wait()
 }
 func TestConn_Status(t *testing.T) {
-	wgProxyFactory := wgproxy.NewFactory(false, connConf.LocalWgPort)
-	defer func() {
-		_ = wgProxyFactory.Free()
-	}()
-	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil)
+	swWatcher := guard.NewSRWatcher(nil, nil, nil, connConf.ICEConfig)
+	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), nil, nil, nil, swWatcher)
 	if err != nil {
 		return
 	}

client/internal/peer/guard/guard.go

@@ -0,0 +1,194 @@
+package guard
+
+import (
+	"context"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	reconnectMaxElapsedTime = 30 * time.Minute
+)
+
+type isConnectedFunc func() bool
+
+// Guard is responsible for the reconnection logic.
+// It will trigger to send an offer to the peer then has connection issues.
+// Watch these events:
+// - Relay client reconnected to home server
+// - Signal server connection state changed
+// - ICE connection disconnected
+// - Relayed connection disconnected
+// - ICE candidate changes
+type Guard struct {
+	Reconnect               chan struct{}
+	log                     *log.Entry
+	isController            bool
+	isConnectedOnAllWay     isConnectedFunc
+	timeout                 time.Duration
+	srWatcher               *SRWatcher
+	relayedConnDisconnected chan bool
+	iCEConnDisconnected     chan bool
+}
+
+func NewGuard(log *log.Entry, isController bool, isConnectedFn isConnectedFunc, timeout time.Duration, srWatcher *SRWatcher) *Guard {
+	return &Guard{
+		Reconnect:               make(chan struct{}, 1),
+		log:                     log,
+		isController:            isController,
+		isConnectedOnAllWay:     isConnectedFn,
+		timeout:                 timeout,
+		srWatcher:               srWatcher,
+		relayedConnDisconnected: make(chan bool, 1),
+		iCEConnDisconnected:     make(chan bool, 1),
+	}
+}
+
+func (g *Guard) Start(ctx context.Context) {
+	if g.isController {
+		g.reconnectLoopWithRetry(ctx)
+	} else {
+		g.listenForDisconnectEvents(ctx)
+	}
+}
+
+func (g *Guard) SetRelayedConnDisconnected(changed bool) {
+	select {
+	case g.relayedConnDisconnected <- changed:
+	default:
+	}
+}
+
+func (g *Guard) SetICEConnDisconnected(changed bool) {
+	select {
+	case g.iCEConnDisconnected <- changed:
+	default:
+	}
+}
+
+// reconnectLoopWithRetry periodically check (max 30 min) the connection status.
+// Try to send offer while the P2P is not established or while the Relay is not connected if is it supported
+func (g *Guard) reconnectLoopWithRetry(ctx context.Context) {
+	waitForInitialConnectionTry(ctx)
+
+	srReconnectedChan := g.srWatcher.NewListener()
+	defer g.srWatcher.RemoveListener(srReconnectedChan)
+
+	ticker := g.prepareExponentTicker(ctx)
+	defer ticker.Stop()
+
+	tickerChannel := ticker.C
+
+	g.log.Infof("start reconnect loop...")
+	for {
+		select {
+		case t := <-tickerChannel:
+			if t.IsZero() {
+				g.log.Infof("retry timed out, stop periodic offer sending")
+				// after backoff timeout the ticker.C will be closed. We need to a dummy channel to avoid loop
+				tickerChannel = make(<-chan time.Time)
+				continue
+			}
+
+			if !g.isConnectedOnAllWay() {
+				g.triggerOfferSending()
+			}
+
+		case changed := <-g.relayedConnDisconnected:
+			if !changed {
+				continue
+			}
+			g.log.Debugf("Relay connection changed, reset reconnection ticker")
+			ticker.Stop()
+			ticker = g.prepareExponentTicker(ctx)
+			tickerChannel = ticker.C
+
+		case changed := <-g.iCEConnDisconnected:
+			if !changed {
+				continue
+			}
+			g.log.Debugf("ICE connection changed, reset reconnection ticker")
+			ticker.Stop()
+			ticker = g.prepareExponentTicker(ctx)
+			tickerChannel = ticker.C
+
+		case <-srReconnectedChan:
+			g.log.Debugf("has network changes, reset reconnection ticker")
+			ticker.Stop()
+			ticker = g.prepareExponentTicker(ctx)
+			tickerChannel = ticker.C
+
+		case <-ctx.Done():
+			g.log.Debugf("context is done, stop reconnect loop")
+			return
+		}
+	}
+}
+
+// listenForDisconnectEvents is used when the peer is not a controller and it should reconnect to the peer
+// when the connection is lost. It will try to establish a connection only once time if before the connection was established
+// It track separately the ice and relay connection status. Just because a lower priority connection reestablished it does not
+// mean that to switch to it. We always force to use the higher priority connection.
+func (g *Guard) listenForDisconnectEvents(ctx context.Context) {
+	srReconnectedChan := g.srWatcher.NewListener()
+	defer g.srWatcher.RemoveListener(srReconnectedChan)
+
+	g.log.Infof("start listen for reconnect events...")
+	for {
+		select {
+		case changed := <-g.relayedConnDisconnected:
+			if !changed {
+				continue
+			}
+			g.log.Debugf("Relay connection changed, triggering reconnect")
+			g.triggerOfferSending()
+		case changed := <-g.iCEConnDisconnected:
+			if !changed {
+				continue
+			}
+			g.log.Debugf("ICE state changed, try to send new offer")
+			g.triggerOfferSending()
+		case <-srReconnectedChan:
+			g.triggerOfferSending()
+		case <-ctx.Done():
+			g.log.Debugf("context is done, stop reconnect loop")
+			return
+		}
+	}
+}
+
+func (g *Guard) prepareExponentTicker(ctx context.Context) *backoff.Ticker {
+	bo := backoff.WithContext(&backoff.ExponentialBackOff{
+		InitialInterval:     800 * time.Millisecond,
+		RandomizationFactor: 0.1,
+		Multiplier:          2,
+		MaxInterval:         g.timeout,
+		MaxElapsedTime:      reconnectMaxElapsedTime,
+		Stop:                backoff.Stop,
+		Clock:               backoff.SystemClock,
+	}, ctx)
+
+	ticker := backoff.NewTicker(bo)
+	<-ticker.C // consume the initial tick what is happening right after the ticker has been created
+
+	return ticker
+}
+
+func (g *Guard) triggerOfferSending() {
+	select {
+	case g.Reconnect <- struct{}{}:
+	default:
+	}
+}
+
+// Give chance to the peer to establish the initial connection.
+// With it, we can decrease to send necessary offer
+func waitForInitialConnectionTry(ctx context.Context) {
+	select {
+	case <-ctx.Done():
+		return
+	case <-time.After(3 * time.Second):
+	}
+}

client/internal/peer/guard/ice_monitor.go

@@ -0,0 +1,135 @@
+package guard
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/pion/ice/v3"
+	log "github.com/sirupsen/logrus"
+
+	icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
+	"github.com/netbirdio/netbird/client/internal/stdnet"
+)
+
+const (
+	candidatesMonitorPeriod   = 5 * time.Minute
+	candidateGatheringTimeout = 5 * time.Second
+)
+
+type ICEMonitor struct {
+	ReconnectCh chan struct{}
+
+	iFaceDiscover stdnet.ExternalIFaceDiscover
+	iceConfig     icemaker.Config
+
+	currentCandidates []ice.Candidate
+	candidatesMu      sync.Mutex
+}
+
+func NewICEMonitor(iFaceDiscover stdnet.ExternalIFaceDiscover, config icemaker.Config) *ICEMonitor {
+	cm := &ICEMonitor{
+		ReconnectCh:   make(chan struct{}, 1),
+		iFaceDiscover: iFaceDiscover,
+		iceConfig:     config,
+	}
+	return cm
+}
+
+func (cm *ICEMonitor) Start(ctx context.Context, onChanged func()) {
+	ufrag, pwd, err := icemaker.GenerateICECredentials()
+	if err != nil {
+		log.Warnf("Failed to generate ICE credentials: %v", err)
+		return
+	}
+
+	ticker := time.NewTicker(candidatesMonitorPeriod)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			changed, err := cm.handleCandidateTick(ctx, ufrag, pwd)
+			if err != nil {
+				log.Warnf("Failed to check ICE changes: %v", err)
+				continue
+			}
+
+			if changed {
+				onChanged()
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (cm *ICEMonitor) handleCandidateTick(ctx context.Context, ufrag string, pwd string) (bool, error) {
+	log.Debugf("Gathering ICE candidates")
+
+	agent, err := icemaker.NewAgent(cm.iFaceDiscover, cm.iceConfig, candidateTypesP2P(), ufrag, pwd)
+	if err != nil {
+		return false, fmt.Errorf("create ICE agent: %w", err)
+	}
+	defer func() {
+		if err := agent.Close(); err != nil {
+			log.Warnf("Failed to close ICE agent: %v", err)
+		}
+	}()
+
+	gatherDone := make(chan struct{})
+	err = agent.OnCandidate(func(c ice.Candidate) {
+		log.Tracef("Got candidate: %v", c)
+		if c == nil {
+			close(gatherDone)
+		}
+	})
+	if err != nil {
+		return false, fmt.Errorf("set ICE candidate handler: %w", err)
+	}
+
+	if err := agent.GatherCandidates(); err != nil {
+		return false, fmt.Errorf("gather ICE candidates: %w", err)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, candidateGatheringTimeout)
+	defer cancel()
+
+	select {
+	case <-ctx.Done():
+		return false, fmt.Errorf("wait for gathering timed out")
+	case <-gatherDone:
+	}
+
+	candidates, err := agent.GetLocalCandidates()
+	if err != nil {
+		return false, fmt.Errorf("get local candidates: %w", err)
+	}
+	log.Tracef("Got candidates: %v", candidates)
+
+	return cm.updateCandidates(candidates), nil
+}
+
+func (cm *ICEMonitor) updateCandidates(newCandidates []ice.Candidate) bool {
+	cm.candidatesMu.Lock()
+	defer cm.candidatesMu.Unlock()
+
+	if len(cm.currentCandidates) != len(newCandidates) {
+		cm.currentCandidates = newCandidates
+		return true
+	}
+
+	for i, candidate := range cm.currentCandidates {
+		if candidate.Address() != newCandidates[i].Address() {
+			cm.currentCandidates = newCandidates
+			return true
+		}
+	}
+
+	return false
+}
+
+func candidateTypesP2P() []ice.CandidateType {
+	return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive}
+}

client/internal/peer/guard/sr_watcher.go

@@ -0,0 +1,119 @@
+package guard
+
+import (
+	"context"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/peer/ice"
+	"github.com/netbirdio/netbird/client/internal/stdnet"
+)
+
+type chNotifier interface {
+	SetOnReconnectedListener(func())
+	Ready() bool
+}
+
+type SRWatcher struct {
+	signalClient chNotifier
+	relayManager chNotifier
+
+	listeners     map[chan struct{}]struct{}
+	mu            sync.Mutex
+	iFaceDiscover stdnet.ExternalIFaceDiscover
+	iceConfig     ice.Config
+
+	cancelIceMonitor context.CancelFunc
+}
+
+// NewSRWatcher creates a new SRWatcher. This watcher will notify the listeners when the ICE candidates change or the
+// Relay connection is reconnected or the Signal client reconnected.
+func NewSRWatcher(signalClient chNotifier, relayManager chNotifier, iFaceDiscover stdnet.ExternalIFaceDiscover, iceConfig ice.Config) *SRWatcher {
+	srw := &SRWatcher{
+		signalClient:  signalClient,
+		relayManager:  relayManager,
+		iFaceDiscover: iFaceDiscover,
+		iceConfig:     iceConfig,
+		listeners:     make(map[chan struct{}]struct{}),
+	}
+	return srw
+}
+
+func (w *SRWatcher) Start() {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if w.cancelIceMonitor != nil {
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	w.cancelIceMonitor = cancel
+
+	iceMonitor := NewICEMonitor(w.iFaceDiscover, w.iceConfig)
+	go iceMonitor.Start(ctx, w.onICEChanged)
+	w.signalClient.SetOnReconnectedListener(w.onReconnected)
+	w.relayManager.SetOnReconnectedListener(w.onReconnected)
+
+}
+
+func (w *SRWatcher) Close() {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if w.cancelIceMonitor == nil {
+		return
+	}
+	w.cancelIceMonitor()
+	w.signalClient.SetOnReconnectedListener(nil)
+	w.relayManager.SetOnReconnectedListener(nil)
+}
+
+func (w *SRWatcher) NewListener() chan struct{} {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	listenerChan := make(chan struct{}, 1)
+	w.listeners[listenerChan] = struct{}{}
+	return listenerChan
+}
+
+func (w *SRWatcher) RemoveListener(listenerChan chan struct{}) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	delete(w.listeners, listenerChan)
+	close(listenerChan)
+}
+
+func (w *SRWatcher) onICEChanged() {
+	if !w.signalClient.Ready() {
+		return
+	}
+
+	log.Infof("network changes detected by ICE agent")
+	w.notify()
+}
+
+func (w *SRWatcher) onReconnected() {
+	if !w.signalClient.Ready() {
+		return
+	}
+	if !w.relayManager.Ready() {
+		return
+	}
+
+	log.Infof("reconnected to Signal or Relay server")
+	w.notify()
+}
+
+func (w *SRWatcher) notify() {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	for listener := range w.listeners {
+		select {
+		case listener <- struct{}{}:
+		default:
+		}
+	}
+}

client/internal/peer/ice/agent.go

@@ -0,0 +1,87 @@
+package ice
+
+import (
+	"time"
+
+	"github.com/pion/ice/v3"
+	"github.com/pion/randutil"
+	"github.com/pion/stun/v2"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/stdnet"
+)
+
+const (
+	lenUFrag   = 16
+	lenPwd     = 32
+	runesAlpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+	iceKeepAliveDefault           = 4 * time.Second
+	iceDisconnectedTimeoutDefault = 6 * time.Second
+	// iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package
+	iceRelayAcceptanceMinWaitDefault = 2 * time.Second
+)
+
+var (
+	failedTimeout = 6 * time.Second
+)
+
+func NewAgent(iFaceDiscover stdnet.ExternalIFaceDiscover, config Config, candidateTypes []ice.CandidateType, ufrag string, pwd string) (*ice.Agent, error) {
+	iceKeepAlive := iceKeepAlive()
+	iceDisconnectedTimeout := iceDisconnectedTimeout()
+	iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait()
+
+	transportNet, err := newStdNet(iFaceDiscover, config.InterfaceBlackList)
+	if err != nil {
+		log.Errorf("failed to create pion's stdnet: %s", err)
+	}
+
+	agentConfig := &ice.AgentConfig{
+		MulticastDNSMode:       ice.MulticastDNSModeDisabled,
+		NetworkTypes:           []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6},
+		Urls:                   config.StunTurn.Load().([]*stun.URI),
+		CandidateTypes:         candidateTypes,
+		InterfaceFilter:        stdnet.InterfaceFilter(config.InterfaceBlackList),
+		UDPMux:                 config.UDPMux,
+		UDPMuxSrflx:            config.UDPMuxSrflx,
+		NAT1To1IPs:             config.NATExternalIPs,
+		Net:                    transportNet,
+		FailedTimeout:          &failedTimeout,
+		DisconnectedTimeout:    &iceDisconnectedTimeout,
+		KeepaliveInterval:      &iceKeepAlive,
+		RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait,
+		LocalUfrag:             ufrag,
+		LocalPwd:               pwd,
+	}
+
+	if config.DisableIPv6Discovery {
+		agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4}
+	}
+
+	return ice.NewAgent(agentConfig)
+}
+
+func GenerateICECredentials() (string, string, error) {
+	ufrag, err := randutil.GenerateCryptoRandomString(lenUFrag, runesAlpha)
+	if err != nil {
+		return "", "", err
+	}
+
+	pwd, err := randutil.GenerateCryptoRandomString(lenPwd, runesAlpha)
+	if err != nil {
+		return "", "", err
+	}
+	return ufrag, pwd, nil
+}
+
+func CandidateTypes() []ice.CandidateType {
+	if hasICEForceRelayConn() {
+		return []ice.CandidateType{ice.CandidateTypeRelay}
+	}
+
+	return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay}
+}
+
+func CandidateTypesP2P() []ice.CandidateType {
+	return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive}
+}

client/internal/peer/ice/config.go

@@ -0,0 +1,22 @@
+package ice
+
+import (
+	"sync/atomic"
+
+	"github.com/pion/ice/v3"
+)
+
+type Config struct {
+	// StunTurn is a list of STUN and TURN URLs
+	StunTurn *atomic.Value // []*stun.URI
+
+	// InterfaceBlackList is a list of machine interfaces that should be filtered out by ICE Candidate gathering
+	// (e.g. if eth0 is in the list, host candidate of this interface won't be used)
+	InterfaceBlackList   []string
+	DisableIPv6Discovery bool
+
+	UDPMux      ice.UDPMux
+	UDPMuxSrflx ice.UniversalUDPMux
+
+	NATExternalIPs []string
+}

client/internal/peer/ice/env.go

@@ -1,4 +1,4 @@
-package peer
+package ice
 
 import (
 	"os"
@@ -10,12 +10,19 @@ import (
 )
 
 const (
+	envICEForceRelayConn            = "NB_ICE_FORCE_RELAY_CONN"
 	envICEKeepAliveIntervalSec      = "NB_ICE_KEEP_ALIVE_INTERVAL_SEC"
 	envICEDisconnectedTimeoutSec    = "NB_ICE_DISCONNECTED_TIMEOUT_SEC"
 	envICERelayAcceptanceMinWaitSec = "NB_ICE_RELAY_ACCEPTANCE_MIN_WAIT_SEC"
-	envICEForceRelayConn            = "NB_ICE_FORCE_RELAY_CONN"
+
+	msgWarnInvalidValue = "invalid value %s set for %s, using default %v"
 )
 
+func hasICEForceRelayConn() bool {
+	disconnectedTimeoutEnv := os.Getenv(envICEForceRelayConn)
+	return strings.ToLower(disconnectedTimeoutEnv) == "true"
+}
+
 func iceKeepAlive() time.Duration {
 	keepAliveEnv := os.Getenv(envICEKeepAliveIntervalSec)
 	if keepAliveEnv == "" {
@@ -25,7 +32,7 @@ func iceKeepAlive() time.Duration {
 	log.Infof("setting ICE keep alive interval to %s seconds", keepAliveEnv)
 	keepAliveEnvSec, err := strconv.Atoi(keepAliveEnv)
 	if err != nil {
-		log.Warnf("invalid value %s set for %s, using default %v", keepAliveEnv, envICEKeepAliveIntervalSec, iceKeepAliveDefault)
+		log.Warnf(msgWarnInvalidValue, keepAliveEnv, envICEKeepAliveIntervalSec, iceKeepAliveDefault)
 		return iceKeepAliveDefault
 	}
 
@@ -41,7 +48,7 @@ func iceDisconnectedTimeout() time.Duration {
 	log.Infof("setting ICE disconnected timeout to %s seconds", disconnectedTimeoutEnv)
 	disconnectedTimeoutSec, err := strconv.Atoi(disconnectedTimeoutEnv)
 	if err != nil {
-		log.Warnf("invalid value %s set for %s, using default %v", disconnectedTimeoutEnv, envICEDisconnectedTimeoutSec, iceDisconnectedTimeoutDefault)
+		log.Warnf(msgWarnInvalidValue, disconnectedTimeoutEnv, envICEDisconnectedTimeoutSec, iceDisconnectedTimeoutDefault)
 		return iceDisconnectedTimeoutDefault
 	}
 
@@ -57,14 +64,9 @@ func iceRelayAcceptanceMinWait() time.Duration {
 	log.Infof("setting ICE relay acceptance min wait to %s seconds", iceRelayAcceptanceMinWaitEnv)
 	disconnectedTimeoutSec, err := strconv.Atoi(iceRelayAcceptanceMinWaitEnv)
 	if err != nil {
-		log.Warnf("invalid value %s set for %s, using default %v", iceRelayAcceptanceMinWaitEnv, envICERelayAcceptanceMinWaitSec, iceRelayAcceptanceMinWaitDefault)
+		log.Warnf(msgWarnInvalidValue, iceRelayAcceptanceMinWaitEnv, envICERelayAcceptanceMinWaitSec, iceRelayAcceptanceMinWaitDefault)
 		return iceRelayAcceptanceMinWaitDefault
 	}
 
 	return time.Duration(disconnectedTimeoutSec) * time.Second
 }
-
-func hasICEForceRelayConn() bool {
-	disconnectedTimeoutEnv := os.Getenv(envICEForceRelayConn)
-	return strings.ToLower(disconnectedTimeoutEnv) == "true"
-}

client/internal/peer/ice/stdnet.go

@@ -1,6 +1,6 @@
 //go:build !android
 
-package peer
+package ice
 
 import (
 	"github.com/netbirdio/netbird/client/internal/stdnet"

client/internal/peer/ice/stdnet_android.go

@@ -1,4 +1,4 @@
-package peer
+package ice
 
 import "github.com/netbirdio/netbird/client/internal/stdnet"
 

client/internal/peer/worker_ice.go

@@ -5,52 +5,20 @@ import (
 	"fmt"
 	"net"
 	"net/netip"
-	"runtime"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/pion/ice/v3"
-	"github.com/pion/randutil"
 	"github.com/pion/stun/v2"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/iface/bind"
+	icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 	"github.com/netbirdio/netbird/route"
 )
 
-const (
-	iceKeepAliveDefault           = 4 * time.Second
-	iceDisconnectedTimeoutDefault = 6 * time.Second
-	// iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package
-	iceRelayAcceptanceMinWaitDefault = 2 * time.Second
-
-	lenUFrag   = 16
-	lenPwd     = 32
-	runesAlpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-)
-
-var (
-	failedTimeout = 6 * time.Second
-)
-
-type ICEConfig struct {
-	// StunTurn is a list of STUN and TURN URLs
-	StunTurn *atomic.Value // []*stun.URI
-
-	// InterfaceBlackList is a list of machine interfaces that should be filtered out by ICE Candidate gathering
-	// (e.g. if eth0 is in the list, host candidate of this interface won't be used)
-	InterfaceBlackList   []string
-	DisableIPv6Discovery bool
-
-	UDPMux      ice.UDPMux
-	UDPMuxSrflx ice.UniversalUDPMux
-
-	NATExternalIPs []string
-}
-
 type ICEConnInfo struct {
 	RemoteConn                 net.Conn
 	RosenpassPubKey            []byte
@@ -103,7 +71,7 @@ func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, signal
 		conn:              callBacks,
 	}
 
-	localUfrag, localPwd, err := generateICECredentials()
+	localUfrag, localPwd, err := icemaker.GenerateICECredentials()
 	if err != nil {
 		return nil, err
 	}
@@ -125,10 +93,10 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
 	var preferredCandidateTypes []ice.CandidateType
 	if w.hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" {
 		w.selectedPriority = connPriorityICEP2P
-		preferredCandidateTypes = candidateTypesP2P()
+		preferredCandidateTypes = icemaker.CandidateTypesP2P()
 	} else {
 		w.selectedPriority = connPriorityICETurn
-		preferredCandidateTypes = candidateTypes()
+		preferredCandidateTypes = icemaker.CandidateTypes()
 	}
 
 	w.log.Debugf("recreate ICE agent")
@@ -232,15 +200,10 @@ func (w *WorkerICE) Close() {
 	}
 }
 
-func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) {
-	transportNet, err := newStdNet(w.iFaceDiscover, w.config.ICEConfig.InterfaceBlackList)
-	if err != nil {
-		w.log.Errorf("failed to create pion's stdnet: %s", err)
-	}
-
+func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, candidates []ice.CandidateType) (*ice.Agent, error) {
 	w.sentExtraSrflx = false
 
-	agent, err := newAgent(w.config, transportNet, relaySupport, w.localUfrag, w.localPwd)
+	agent, err := icemaker.NewAgent(w.iFaceDiscover, w.config.ICEConfig, candidates, w.localUfrag, w.localPwd)
 	if err != nil {
 		return nil, fmt.Errorf("create agent: %w", err)
 	}
@@ -365,36 +328,6 @@ func (w *WorkerICE) turnAgentDial(ctx context.Context, remoteOfferAnswer *OfferA
 	}
 }
 
-func newAgent(config ConnConfig, transportNet *stdnet.Net, candidateTypes []ice.CandidateType, ufrag string, pwd string) (*ice.Agent, error) {
-	iceKeepAlive := iceKeepAlive()
-	iceDisconnectedTimeout := iceDisconnectedTimeout()
-	iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait()
-
-	agentConfig := &ice.AgentConfig{
-		MulticastDNSMode:       ice.MulticastDNSModeDisabled,
-		NetworkTypes:           []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6},
-		Urls:                   config.ICEConfig.StunTurn.Load().([]*stun.URI),
-		CandidateTypes:         candidateTypes,
-		InterfaceFilter:        stdnet.InterfaceFilter(config.ICEConfig.InterfaceBlackList),
-		UDPMux:                 config.ICEConfig.UDPMux,
-		UDPMuxSrflx:            config.ICEConfig.UDPMuxSrflx,
-		NAT1To1IPs:             config.ICEConfig.NATExternalIPs,
-		Net:                    transportNet,
-		FailedTimeout:          &failedTimeout,
-		DisconnectedTimeout:    &iceDisconnectedTimeout,
-		KeepaliveInterval:      &iceKeepAlive,
-		RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait,
-		LocalUfrag:             ufrag,
-		LocalPwd:               pwd,
-	}
-
-	if config.ICEConfig.DisableIPv6Discovery {
-		agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4}
-	}
-
-	return ice.NewAgent(agentConfig)
-}
-
 func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) {
 	relatedAdd := candidate.RelatedAddress()
 	return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{
@@ -435,21 +368,6 @@ func candidateViaRoutes(candidate ice.Candidate, clientRoutes route.HAMap) bool
 	return false
 }
 
-func candidateTypes() []ice.CandidateType {
-	if hasICEForceRelayConn() {
-		return []ice.CandidateType{ice.CandidateTypeRelay}
-	}
-	// TODO: remove this once we have refactored userspace proxy into the bind package
-	if runtime.GOOS == "ios" {
-		return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive}
-	}
-	return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay}
-}
-
-func candidateTypesP2P() []ice.CandidateType {
-	return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive}
-}
-
 func isRelayCandidate(candidate ice.Candidate) bool {
 	return candidate.Type() == ice.CandidateTypeRelay
 }
@@ -460,16 +378,3 @@ func isRelayed(pair *ice.CandidatePair) bool {
 	}
 	return false
 }
-
-func generateICECredentials() (string, string, error) {
-	ufrag, err := randutil.GenerateCryptoRandomString(lenUFrag, runesAlpha)
-	if err != nil {
-		return "", "", err
-	}
-
-	pwd, err := randutil.GenerateCryptoRandomString(lenPwd, runesAlpha)
-	if err != nil {
-		return "", "", err
-	}
-	return ufrag, pwd, nil
-}

client/internal/peer/worker_relay.go

@@ -31,6 +31,7 @@ type WorkerRelayCallbacks struct {
 
 type WorkerRelay struct {
 	log          *log.Entry
+	isController bool
 	config       ConnConfig
 	relayManager relayClient.ManagerService
 	callBacks    WorkerRelayCallbacks
@@ -44,9 +45,10 @@ type WorkerRelay struct {
 	relaySupportedOnRemotePeer atomic.Bool
 }
 
-func NewWorkerRelay(log *log.Entry, config ConnConfig, relayManager relayClient.ManagerService, callbacks WorkerRelayCallbacks) *WorkerRelay {
+func NewWorkerRelay(log *log.Entry, ctrl bool, config ConnConfig, relayManager relayClient.ManagerService, callbacks WorkerRelayCallbacks) *WorkerRelay {
 	r := &WorkerRelay{
 		log:          log,
+		isController: ctrl,
 		config:       config,
 		relayManager: relayManager,
 		callBacks:    callbacks,
@@ -80,6 +82,7 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
 		w.log.Errorf("failed to open connection via Relay: %s", err)
 		return
 	}
+
 	w.relayLock.Lock()
 	w.relayedConn = relayedConn
 	w.relayLock.Unlock()
@@ -136,10 +139,6 @@ func (w *WorkerRelay) IsRelayConnectionSupportedWithPeer() bool {
 	return w.relaySupportedOnRemotePeer.Load() && w.RelayIsSupportedLocally()
 }
 
-func (w *WorkerRelay) IsController() bool {
-	return w.config.LocalKey > w.config.Key
-}
-
 func (w *WorkerRelay) RelayIsSupportedLocally() bool {
 	return w.relayManager.HasRelayAddress()
 }
@@ -212,7 +211,7 @@ func (w *WorkerRelay) isRelaySupported(answer *OfferAnswer) bool {
 }
 
 func (w *WorkerRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string {
-	if w.IsController() {
+	if w.isController {
 		return myRelayAddress
 	}
 	return remoteRelayAddress

client/internal/routemanager/manager.go

@@ -23,6 +23,7 @@ import (
 	"github.com/netbirdio/netbird/client/internal/routemanager/systemops"
 	"github.com/netbirdio/netbird/client/internal/routemanager/vars"
 	"github.com/netbirdio/netbird/client/internal/routeselector"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	relayClient "github.com/netbirdio/netbird/relay/client"
 	"github.com/netbirdio/netbird/route"
 	nbnet "github.com/netbirdio/netbird/util/net"
@@ -31,14 +32,14 @@ import (
 
 // Manager is a route manager interface
 type Manager interface {
-	Init() (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error)
+	Init(*statemanager.Manager) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error)
 	UpdateRoutes(updateSerial uint64, newRoutes []*route.Route) (map[route.ID]*route.Route, route.HAMap, error)
 	TriggerSelection(route.HAMap)
 	GetRouteSelector() *routeselector.RouteSelector
 	SetRouteChangeListener(listener listener.NetworkChangeListener)
 	InitialRouteRange() []string
 	EnableServerRouter(firewall firewall.Manager) error
-	Stop()
+	Stop(stateManager *statemanager.Manager)
 }
 
 // DefaultManager is the default instance of a route manager
@@ -120,12 +121,12 @@ func NewManager(
 }
 
 // Init sets up the routing
-func (m *DefaultManager) Init() (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
+func (m *DefaultManager) Init(stateManager *statemanager.Manager) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
 	if nbnet.CustomRoutingDisabled() {
 		return nil, nil, nil
 	}
 
-	if err := m.sysOps.CleanupRouting(); err != nil {
+	if err := m.sysOps.CleanupRouting(nil); err != nil {
 		log.Warnf("Failed cleaning up routing: %v", err)
 	}
 
@@ -136,7 +137,7 @@ func (m *DefaultManager) Init() (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error)
 
 	ips := resolveURLsToIPs(initialAddresses)
 
-	beforePeerHook, afterPeerHook, err := m.sysOps.SetupRouting(ips)
+	beforePeerHook, afterPeerHook, err := m.sysOps.SetupRouting(ips, stateManager)
 	if err != nil {
 		return nil, nil, fmt.Errorf("setup routing: %w", err)
 	}
@@ -154,7 +155,7 @@ func (m *DefaultManager) EnableServerRouter(firewall firewall.Manager) error {
 }
 
 // Stop stops the manager watchers and clean firewall rules
-func (m *DefaultManager) Stop() {
+func (m *DefaultManager) Stop(stateManager *statemanager.Manager) {
 	m.stop()
 	if m.serverRouter != nil {
 		m.serverRouter.cleanUp()
@@ -172,7 +173,7 @@ func (m *DefaultManager) Stop() {
 	}
 
 	if !nbnet.CustomRoutingDisabled() {
-		if err := m.sysOps.CleanupRouting(); err != nil {
+		if err := m.sysOps.CleanupRouting(stateManager); err != nil {
 			log.Errorf("Error cleaning up routing: %v", err)
 		} else {
 			log.Info("Routing cleanup complete")

client/internal/routemanager/manager_test.go

@@ -407,7 +407,15 @@ func TestManagerUpdateRoutes(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun43%d", n), "100.65.65.2/24", 33100, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil, nil)
+			opts := iface.WGIFaceOpts{
+				IFaceName:    fmt.Sprintf("utun43%d", n),
+				Address:      "100.65.65.2/24",
+				WGPort:       33100,
+				WGPrivKey:    peerPrivateKey.String(),
+				MTU:          iface.DefaultMTU,
+				TransportNet: newNet,
+			}
+			wgInterface, err := iface.NewWGIFace(opts)
 			require.NoError(t, err, "should create testing WGIface interface")
 			defer wgInterface.Close()
 
@@ -418,10 +426,10 @@ func TestManagerUpdateRoutes(t *testing.T) {
 			ctx := context.TODO()
 			routeManager := NewManager(ctx, localPeerKey, 0, wgInterface, statusRecorder, nil, nil)
 
-			_, _, err = routeManager.Init()
+			_, _, err = routeManager.Init(nil)
 
 			require.NoError(t, err, "should init route manager")
-			defer routeManager.Stop()
+			defer routeManager.Stop(nil)
 
 			if testCase.removeSrvRouter {
 				routeManager.serverRouter = nil

client/internal/routemanager/mock.go

@@ -8,6 +8,7 @@ import (
 	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/internal/listener"
 	"github.com/netbirdio/netbird/client/internal/routeselector"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	"github.com/netbirdio/netbird/route"
 	"github.com/netbirdio/netbird/util/net"
 )
@@ -17,10 +18,10 @@ type MockManager struct {
 	UpdateRoutesFunc     func(updateSerial uint64, newRoutes []*route.Route) (map[route.ID]*route.Route, route.HAMap, error)
 	TriggerSelectionFunc func(haMap route.HAMap)
 	GetRouteSelectorFunc func() *routeselector.RouteSelector
-	StopFunc             func()
+	StopFunc             func(manager *statemanager.Manager)
 }
 
-func (m *MockManager) Init() (net.AddHookFunc, net.RemoveHookFunc, error) {
+func (m *MockManager) Init(*statemanager.Manager) (net.AddHookFunc, net.RemoveHookFunc, error) {
 	return nil, nil, nil
 }
 
@@ -65,8 +66,8 @@ func (m *MockManager) EnableServerRouter(firewall firewall.Manager) error {
 }
 
 // Stop mock implementation of Stop from Manager interface
-func (m *MockManager) Stop() {
+func (m *MockManager) Stop(stateManager *statemanager.Manager) {
 	if m.StopFunc != nil {
-		m.StopFunc()
+		m.StopFunc(stateManager)
 	}
 }

client/internal/routemanager/refcounter/refcounter.go

@@ -1,6 +1,7 @@
 package refcounter
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"runtime"
@@ -70,6 +71,19 @@ func New[Key comparable, I, O any](add AddFunc[Key, I, O], remove RemoveFunc[Key
 	}
 }
 
+// LoadData loads the data from the existing counter
+func (rm *Counter[Key, I, O]) LoadData(
+	existingCounter *Counter[Key, I, O],
+) {
+	rm.refCountMu.Lock()
+	defer rm.refCountMu.Unlock()
+	rm.idMu.Lock()
+	defer rm.idMu.Unlock()
+
+	rm.refCountMap = existingCounter.refCountMap
+	rm.idMap = existingCounter.idMap
+}
+
 // Get retrieves the current reference count and associated data for a key.
 // If the key doesn't exist, it returns a zero value Ref and false.
 func (rm *Counter[Key, I, O]) Get(key Key) (Ref[O], bool) {
@@ -201,6 +215,32 @@ func (rm *Counter[Key, I, O]) Clear() {
 	clear(rm.idMap)
 }
 
+// MarshalJSON implements the json.Marshaler interface for Counter.
+func (rm *Counter[Key, I, O]) MarshalJSON() ([]byte, error) {
+	return json.Marshal(struct {
+		RefCountMap map[Key]Ref[O]   `json:"refCountMap"`
+		IDMap       map[string][]Key `json:"idMap"`
+	}{
+		RefCountMap: rm.refCountMap,
+		IDMap:       rm.idMap,
+	})
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface for Counter.
+func (rm *Counter[Key, I, O]) UnmarshalJSON(data []byte) error {
+	var temp struct {
+		RefCountMap map[Key]Ref[O]   `json:"refCountMap"`
+		IDMap       map[string][]Key `json:"idMap"`
+	}
+	if err := json.Unmarshal(data, &temp); err != nil {
+		return err
+	}
+	rm.refCountMap = temp.RefCountMap
+	rm.idMap = temp.IDMap
+
+	return nil
+}
+
 func getCallerInfo(depth int, maxDepth int) (string, bool) {
 	if depth >= maxDepth {
 		return "", false

client/internal/routemanager/systemops/state.go

@@ -0,0 +1,32 @@
+package systemops
+
+import (
+	"net/netip"
+	"sync"
+
+	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
+)
+
+type ShutdownState struct {
+	Counter *ExclusionCounter `json:"counter,omitempty"`
+	mu      sync.RWMutex
+}
+
+func (s *ShutdownState) Name() string {
+	return "route_state"
+}
+
+func (s *ShutdownState) Cleanup() error {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	if s.Counter == nil {
+		return nil
+	}
+
+	sysops := NewSysOps(nil, nil)
+	sysops.refCounter = refcounter.New[netip.Prefix, struct{}, Nexthop](nil, sysops.removeFromRouteTable)
+	sysops.refCounter.LoadData(s.Counter)
+
+	return sysops.refCounter.Flush()
+}

client/internal/routemanager/systemops/systemops_android.go

@@ -9,14 +9,15 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
-func (r *SysOps) SetupRouting([]net.IP) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
+func (r *SysOps) SetupRouting([]net.IP, *statemanager.Manager) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
 	return nil, nil, nil
 }
 
-func (r *SysOps) CleanupRouting() error {
+func (r *SysOps) CleanupRouting(*statemanager.Manager) error {
 	return nil
 }
 
@@ -28,6 +29,10 @@ func (r *SysOps) RemoveVPNRoute(netip.Prefix, *net.Interface) error {
 	return nil
 }
 
+func (r *SysOps) removeFromRouteTable(netip.Prefix, Nexthop) error {
+	return nil
+}
+
 func EnableIPForwarding() error {
 	log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS)
 	return nil

client/internal/routemanager/systemops/systemops_generic.go

@@ -20,6 +20,7 @@ import (
 	"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
 	"github.com/netbirdio/netbird/client/internal/routemanager/util"
 	"github.com/netbirdio/netbird/client/internal/routemanager/vars"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
@@ -30,7 +31,9 @@ var splitDefaultv6_2 = netip.PrefixFrom(netip.AddrFrom16([16]byte{0x80}), 1)
 
 var ErrRoutingIsSeparate = errors.New("routing is separate")
 
-func (r *SysOps) setupRefCounter(initAddresses []net.IP) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
+func (r *SysOps) setupRefCounter(initAddresses []net.IP, stateManager *statemanager.Manager) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
+	stateManager.RegisterState(&ShutdownState{})
+
 	initialNextHopV4, err := GetNextHop(netip.IPv4Unspecified())
 	if err != nil && !errors.Is(err, vars.ErrRouteNotFound) {
 		log.Errorf("Unable to get initial v4 default next hop: %v", err)
@@ -53,9 +56,18 @@ func (r *SysOps) setupRefCounter(initAddresses []net.IP) (nbnet.AddHookFunc, nbn
 				// These errors are not critical, but also we should not track and try to remove the routes either.
 				return nexthop, refcounter.ErrIgnore
 			}
+
+			r.updateState(stateManager)
+
 			return nexthop, err
 		},
-		r.removeFromRouteTable,
+		func(prefix netip.Prefix, nexthop Nexthop) error {
+			// remove from state even if we have trouble removing it from the route table
+			// it could be already gone
+			r.updateState(stateManager)
+
+			return r.removeFromRouteTable(prefix, nexthop)
+		},
 	)
 
 	r.refCounter = refCounter
@@ -63,7 +75,17 @@ func (r *SysOps) setupRefCounter(initAddresses []net.IP) (nbnet.AddHookFunc, nbn
 	return r.setupHooks(initAddresses)
 }
 
-func (r *SysOps) cleanupRefCounter() error {
+func (r *SysOps) updateState(stateManager *statemanager.Manager) {
+	state := getState(stateManager)
+
+	state.Counter = r.refCounter
+
+	if err := stateManager.UpdateState(state); err != nil {
+		log.Errorf("failed to update state: %v", err)
+	}
+}
+
+func (r *SysOps) cleanupRefCounter(stateManager *statemanager.Manager) error {
 	if r.refCounter == nil {
 		return nil
 	}
@@ -76,6 +98,10 @@ func (r *SysOps) cleanupRefCounter() error {
 		return fmt.Errorf("flush route manager: %w", err)
 	}
 
+	if err := stateManager.DeleteState(&ShutdownState{}); err != nil {
+		return fmt.Errorf("delete state: %w", err)
+	}
+
 	return nil
 }
 
@@ -506,3 +532,14 @@ func isVpnRoute(addr netip.Addr, vpnRoutes []netip.Prefix, localRoutes []netip.P
 	// Return true if the longest matching prefix is from vpnRoutes
 	return isVpn, longestPrefix
 }
+
+func getState(stateManager *statemanager.Manager) *ShutdownState {
+	var shutdownState *ShutdownState
+	if state := stateManager.GetState(shutdownState); state != nil {
+		shutdownState = state.(*ShutdownState)
+	} else {
+		shutdownState = &ShutdownState{}
+	}
+
+	return shutdownState
+}

client/internal/routemanager/systemops/systemops_generic_test.go

@@ -61,7 +61,14 @@ func TestAddRemoveRoutes(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun53%d", n), "100.65.75.2/24", 33100, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil, nil)
+			opts := iface.WGIFaceOpts{
+				IFaceName:    fmt.Sprintf("utun53%d", n),
+				Address:      "100.65.75.2/24",
+				WGPrivKey:    peerPrivateKey.String(),
+				MTU:          iface.DefaultMTU,
+				TransportNet: newNet,
+			}
+			wgInterface, err := iface.NewWGIFace(opts)
 			require.NoError(t, err, "should create testing WGIface interface")
 			defer wgInterface.Close()
 
@@ -70,10 +77,10 @@ func TestAddRemoveRoutes(t *testing.T) {
 
 			r := NewSysOps(wgInterface, nil)
 
-			_, _, err = r.SetupRouting(nil)
+			_, _, err = r.SetupRouting(nil, nil)
 			require.NoError(t, err)
 			t.Cleanup(func() {
-				assert.NoError(t, r.CleanupRouting())
+				assert.NoError(t, r.CleanupRouting(nil))
 			})
 
 			index, err := net.InterfaceByName(wgInterface.Name())
@@ -213,7 +220,15 @@ func TestAddExistAndRemoveRoute(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun53%d", n), "100.65.75.2/24", 33100, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil, nil)
+			opts := iface.WGIFaceOpts{
+				IFaceName:    fmt.Sprintf("utun53%d", n),
+				Address:      "100.65.75.2/24",
+				WGPort:       33100,
+				WGPrivKey:    peerPrivateKey.String(),
+				MTU:          iface.DefaultMTU,
+				TransportNet: newNet,
+			}
+			wgInterface, err := iface.NewWGIFace(opts)
 			require.NoError(t, err, "should create testing WGIface interface")
 			defer wgInterface.Close()
 
@@ -345,7 +360,15 @@ func createWGInterface(t *testing.T, interfaceName, ipAddressCIDR string, listen
 	newNet, err := stdnet.NewNet()
 	require.NoError(t, err)
 
-	wgInterface, err := iface.NewWGIFace(interfaceName, ipAddressCIDR, listenPort, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil, nil)
+	opts := iface.WGIFaceOpts{
+		IFaceName:    interfaceName,
+		Address:      ipAddressCIDR,
+		WGPrivKey:    peerPrivateKey.String(),
+		WGPort:       listenPort,
+		MTU:          iface.DefaultMTU,
+		TransportNet: newNet,
+	}
+	wgInterface, err := iface.NewWGIFace(opts)
 	require.NoError(t, err, "should create testing WireGuard interface")
 
 	err = wgInterface.Create()
@@ -380,10 +403,10 @@ func setupTestEnv(t *testing.T) {
 	})
 
 	r := NewSysOps(wgInterface, nil)
-	_, _, err := r.SetupRouting(nil)
+	_, _, err := r.SetupRouting(nil, nil)
 	require.NoError(t, err, "setupRouting should not return err")
 	t.Cleanup(func() {
-		assert.NoError(t, r.CleanupRouting())
+		assert.NoError(t, r.CleanupRouting(nil))
 	})
 
 	index, err := net.InterfaceByName(wgInterface.Name())

client/internal/routemanager/systemops/systemops_ios.go

@@ -9,17 +9,18 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
-func (r *SysOps) SetupRouting([]net.IP) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
+func (r *SysOps) SetupRouting([]net.IP, *statemanager.Manager) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	r.prefixes = make(map[netip.Prefix]struct{})
 	return nil, nil, nil
 }
 
-func (r *SysOps) CleanupRouting() error {
+func (r *SysOps) CleanupRouting(*statemanager.Manager) error {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
@@ -46,6 +47,18 @@ func (r *SysOps) RemoveVPNRoute(prefix netip.Prefix, _ *net.Interface) error {
 	return nil
 }
 
+func (r *SysOps) notify() {
+	prefixes := make([]netip.Prefix, 0, len(r.prefixes))
+	for prefix := range r.prefixes {
+		prefixes = append(prefixes, prefix)
+	}
+	r.notifier.OnNewPrefixes(prefixes)
+}
+
+func (r *SysOps) removeFromRouteTable(netip.Prefix, Nexthop) error {
+	return nil
+}
+
 func EnableIPForwarding() error {
 	log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS)
 	return nil
@@ -54,11 +67,3 @@ func EnableIPForwarding() error {
 func IsAddrRouted(netip.Addr, []netip.Prefix) (bool, netip.Prefix) {
 	return false, netip.Prefix{}
 }
-
-func (r *SysOps) notify() {
-	prefixes := make([]netip.Prefix, 0, len(r.prefixes))
-	for prefix := range r.prefixes {
-		prefixes = append(prefixes, prefix)
-	}
-	r.notifier.OnNewPrefixes(prefixes)
-}

client/internal/routemanager/systemops/systemops_linux.go

@@ -18,6 +18,7 @@ import (
 	nberrors "github.com/netbirdio/netbird/client/errors"
 	"github.com/netbirdio/netbird/client/internal/routemanager/sysctl"
 	"github.com/netbirdio/netbird/client/internal/routemanager/vars"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
@@ -85,10 +86,10 @@ func getSetupRules() []ruleParams {
 // Rule 2 (VPN Traffic Routing): Directs all remaining traffic to the 'NetbirdVPNTableID' custom routing table.
 // This table is where a default route or other specific routes received from the management server are configured,
 // enabling VPN connectivity.
-func (r *SysOps) SetupRouting(initAddresses []net.IP) (_ nbnet.AddHookFunc, _ nbnet.RemoveHookFunc, err error) {
+func (r *SysOps) SetupRouting(initAddresses []net.IP, stateManager *statemanager.Manager) (_ nbnet.AddHookFunc, _ nbnet.RemoveHookFunc, err error) {
 	if isLegacy() {
 		log.Infof("Using legacy routing setup")
-		return r.setupRefCounter(initAddresses)
+		return r.setupRefCounter(initAddresses, stateManager)
 	}
 
 	if err = addRoutingTableName(); err != nil {
@@ -104,7 +105,7 @@ func (r *SysOps) SetupRouting(initAddresses []net.IP) (_ nbnet.AddHookFunc, _ nb
 
 	defer func() {
 		if err != nil {
-			if cleanErr := r.CleanupRouting(); cleanErr != nil {
+			if cleanErr := r.CleanupRouting(stateManager); cleanErr != nil {
 				log.Errorf("Error cleaning up routing: %v", cleanErr)
 			}
 		}
@@ -116,7 +117,7 @@ func (r *SysOps) SetupRouting(initAddresses []net.IP) (_ nbnet.AddHookFunc, _ nb
 			if errors.Is(err, syscall.EOPNOTSUPP) {
 				log.Warnf("Rule operations are not supported, falling back to the legacy routing setup")
 				setIsLegacy(true)
-				return r.setupRefCounter(initAddresses)
+				return r.setupRefCounter(initAddresses, stateManager)
 			}
 			return nil, nil, fmt.Errorf("%s: %w", rule.description, err)
 		}
@@ -128,9 +129,9 @@ func (r *SysOps) SetupRouting(initAddresses []net.IP) (_ nbnet.AddHookFunc, _ nb
 // CleanupRouting performs a thorough cleanup of the routing configuration established by 'setupRouting'.
 // It systematically removes the three rules and any associated routing table entries to ensure a clean state.
 // The function uses error aggregation to report any errors encountered during the cleanup process.
-func (r *SysOps) CleanupRouting() error {
+func (r *SysOps) CleanupRouting(stateManager *statemanager.Manager) error {
 	if isLegacy() {
-		return r.cleanupRefCounter()
+		return r.cleanupRefCounter(stateManager)
 	}
 
 	var result *multierror.Error

client/internal/routemanager/systemops/systemops_unix.go

@@ -13,15 +13,16 @@ import (
 	"github.com/cenkalti/backoff/v4"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
-func (r *SysOps) SetupRouting(initAddresses []net.IP) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
-	return r.setupRefCounter(initAddresses)
+func (r *SysOps) SetupRouting(initAddresses []net.IP, stateManager *statemanager.Manager) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
+	return r.setupRefCounter(initAddresses, stateManager)
 }
 
-func (r *SysOps) CleanupRouting() error {
-	return r.cleanupRefCounter()
+func (r *SysOps) CleanupRouting(stateManager *statemanager.Manager) error {
+	return r.cleanupRefCounter(stateManager)
 }
 
 func (r *SysOps) addToRouteTable(prefix netip.Prefix, nexthop Nexthop) error {

client/internal/routemanager/systemops/systemops_windows.go

@@ -22,6 +22,7 @@ import (
 	"golang.org/x/sys/windows"
 
 	"github.com/netbirdio/netbird/client/firewall/uspfilter"
+	"github.com/netbirdio/netbird/client/internal/statemanager"
 	nbnet "github.com/netbirdio/netbird/util/net"
 )
 
@@ -130,12 +131,12 @@ const (
 	RouteDeleted
 )
 
-func (r *SysOps) SetupRouting(initAddresses []net.IP) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
-	return r.setupRefCounter(initAddresses)
+func (r *SysOps) SetupRouting(initAddresses []net.IP, stateManager *statemanager.Manager) (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) {
+	return r.setupRefCounter(initAddresses, stateManager)
 }
 
-func (r *SysOps) CleanupRouting() error {
-	return r.cleanupRefCounter()
+func (r *SysOps) CleanupRouting(stateManager *statemanager.Manager) error {
+	return r.cleanupRefCounter(stateManager)
 }
 
 func (r *SysOps) addToRouteTable(prefix netip.Prefix, nexthop Nexthop) error {

client/internal/statemanager/manager.go

@@ -0,0 +1,298 @@
+package statemanager
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"reflect"
+	"sync"
+	"time"
+
+	"github.com/hashicorp/go-multierror"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/exp/maps"
+
+	nberrors "github.com/netbirdio/netbird/client/errors"
+)
+
+// State interface defines the methods that all state types must implement
+type State interface {
+	Name() string
+	Cleanup() error
+}
+
+// Manager handles the persistence and management of various states
+type Manager struct {
+	mu     sync.Mutex
+	cancel context.CancelFunc
+	done   chan struct{}
+
+	filePath string
+	// holds the states that are registered with the manager and that are to be persisted
+	states map[string]State
+	// holds the state names that have been updated and need to be persisted with the next save
+	dirty map[string]struct{}
+	// holds the type information for each registered state
+	stateTypes map[string]reflect.Type
+}
+
+// New creates a new Manager instance
+func New(filePath string) *Manager {
+	return &Manager{
+		filePath:   filePath,
+		states:     make(map[string]State),
+		dirty:      make(map[string]struct{}),
+		stateTypes: make(map[string]reflect.Type),
+	}
+}
+
+// Start starts the state manager periodic save routine
+func (m *Manager) Start() {
+	if m == nil {
+		return
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	var ctx context.Context
+	ctx, m.cancel = context.WithCancel(context.Background())
+	m.done = make(chan struct{})
+
+	go m.periodicStateSave(ctx)
+}
+
+func (m *Manager) Stop(ctx context.Context) error {
+	if m == nil {
+		return nil
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if m.cancel != nil {
+		m.cancel()
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-m.done:
+			return nil
+		}
+	}
+
+	return nil
+}
+
+// RegisterState registers a state with the manager but doesn't attempt to persist it.
+// Pass an uninitialized state to register it.
+func (m *Manager) RegisterState(state State) {
+	if m == nil {
+		return
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	name := state.Name()
+	m.states[name] = nil
+	m.stateTypes[name] = reflect.TypeOf(state).Elem()
+}
+
+// GetState returns the state for the given type
+func (m *Manager) GetState(state State) State {
+	if m == nil {
+		return nil
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	return m.states[state.Name()]
+}
+
+// UpdateState updates the state in the manager and marks it as dirty for the next save.
+// The state will be replaced with the new one.
+func (m *Manager) UpdateState(state State) error {
+	if m == nil {
+		return nil
+	}
+
+	return m.setState(state.Name(), state)
+}
+
+// DeleteState removes the state from the manager and marks it as dirty for the next save.
+// Pass an uninitialized state to delete it.
+func (m *Manager) DeleteState(state State) error {
+	if m == nil {
+		return nil
+	}
+
+	return m.setState(state.Name(), nil)
+}
+
+func (m *Manager) setState(name string, state State) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if _, exists := m.states[name]; !exists {
+		return fmt.Errorf("state %s not registered", name)
+	}
+
+	m.states[name] = state
+	m.dirty[name] = struct{}{}
+
+	return nil
+}
+
+func (m *Manager) periodicStateSave(ctx context.Context) {
+	ticker := time.NewTicker(10 * time.Second)
+	defer ticker.Stop()
+	defer close(m.done)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if err := m.PersistState(ctx); err != nil {
+				log.Errorf("failed to persist state: %v", err)
+			}
+		}
+	}
+}
+
+// PersistState persists the states that have been updated since the last save.
+func (m *Manager) PersistState(ctx context.Context) error {
+	if m == nil {
+		return nil
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if len(m.dirty) == 0 {
+		return nil
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+
+	done := make(chan error, 1)
+
+	go func() {
+		data, err := json.MarshalIndent(m.states, "", "  ")
+		if err != nil {
+			done <- fmt.Errorf("marshal states: %w", err)
+			return
+		}
+
+		// nolint:gosec
+		if err := os.WriteFile(m.filePath, data, 0640); err != nil {
+			done <- fmt.Errorf("write state file: %w", err)
+			return
+		}
+
+		done <- nil
+	}()
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case err := <-done:
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Debugf("persisted shutdown states: %v", maps.Keys(m.dirty))
+
+	clear(m.dirty)
+
+	return nil
+}
+
+// loadState loads the existing state from the state file
+func (m *Manager) loadState() error {
+	data, err := os.ReadFile(m.filePath)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			log.Debug("state file does not exist")
+			return nil
+		}
+		return fmt.Errorf("read state file: %w", err)
+	}
+
+	var rawStates map[string]json.RawMessage
+	if err := json.Unmarshal(data, &rawStates); err != nil {
+		log.Warn("State file appears to be corrupted, attempting to delete it")
+		if err := os.Remove(m.filePath); err != nil {
+			log.Errorf("Failed to delete corrupted state file: %v", err)
+		} else {
+			log.Info("State file deleted")
+		}
+		return fmt.Errorf("unmarshal states: %w", err)
+	}
+
+	var merr *multierror.Error
+
+	for name, rawState := range rawStates {
+		stateType, ok := m.stateTypes[name]
+		if !ok {
+			merr = multierror.Append(merr, fmt.Errorf("unknown state type: %s", name))
+			continue
+		}
+
+		if string(rawState) == "null" {
+			continue
+		}
+
+		statePtr := reflect.New(stateType).Interface().(State)
+		if err := json.Unmarshal(rawState, statePtr); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("unmarshal state %s: %w", name, err))
+			continue
+		}
+
+		m.states[name] = statePtr
+		log.Debugf("loaded state: %s", name)
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
+}
+
+// PerformCleanup retrieves all states from the state file for the registered states and calls Cleanup on them.
+// If the cleanup is successful, the state is marked for deletion.
+func (m *Manager) PerformCleanup() error {
+	if m == nil {
+		return nil
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if err := m.loadState(); err != nil {
+		log.Warnf("Failed to load state during cleanup: %v", err)
+	}
+
+	var merr *multierror.Error
+	for name, state := range m.states {
+		if state == nil {
+			// If no state was found in the state file, we don't mark the state dirty nor return an error
+			continue
+		}
+
+		log.Infof("client was not shut down properly, cleaning up %s", name)
+		if err := state.Cleanup(); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("cleanup state for %s: %w", name, err))
+		} else {
+			// mark for deletion on cleanup success
+			m.states[name] = nil
+			m.dirty[name] = struct{}{}
+		}
+	}
+
+	return nberrors.FormatErrorOrNil(merr)
+}

client/internal/statemanager/path.go

@@ -0,0 +1,35 @@
+package statemanager
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// GetDefaultStatePath returns the path to the state file based on the operating system
+// It returns an empty string if the path cannot be determined. It also creates the directory if it does not exist.
+func GetDefaultStatePath() string {
+	var path string
+
+	switch runtime.GOOS {
+	case "windows":
+		path = filepath.Join(os.Getenv("PROGRAMDATA"), "Netbird", "state.json")
+	case "darwin", "linux":
+		path = "/var/lib/netbird/state.json"
+	case "freebsd", "openbsd", "netbsd", "dragonfly":
+		path = "/var/db/netbird/state.json"
+	// ios/android don't need state
+	default:
+		return ""
+	}
+
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		log.Errorf("Error creating directory %s: %v. Continuing without state support.", dir, err)
+		return ""
+	}
+
+	return path
+}

client/internal/wgproxy/ebpf/portlookup.go

@@ -1,32 +0,0 @@
-package ebpf
-
-import (
-	"fmt"
-	"net"
-)
-
-const (
-	portRangeStart = 3128
-	portRangeEnd   = 3228
-)
-
-type portLookup struct {
-}
-
-func (pl portLookup) searchFreePort() (int, error) {
-	for i := portRangeStart; i <= portRangeEnd; i++ {
-		if pl.tryToBind(i) == nil {
-			return i, nil
-		}
-	}
-	return 0, fmt.Errorf("failed to bind free port for eBPF proxy")
-}
-
-func (pl portLookup) tryToBind(port int) error {
-	l, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port))
-	if err != nil {
-		return err
-	}
-	_ = l.Close()
-	return nil
-}

client/internal/wgproxy/ebpf/portlookup_test.go

@@ -1,42 +0,0 @@
-package ebpf
-
-import (
-	"fmt"
-	"net"
-	"testing"
-)
-
-func Test_portLookup_searchFreePort(t *testing.T) {
-	pl := portLookup{}
-	_, err := pl.searchFreePort()
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func Test_portLookup_on_allocated(t *testing.T) {
-	pl := portLookup{}
-
-	allocatedPort, err := allocatePort(portRangeStart)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer allocatedPort.Close()
-
-	fp, err := pl.searchFreePort()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if fp != (portRangeStart + 1) {
-		t.Errorf("invalid free port, expected: %d, got: %d", portRangeStart+1, fp)
-	}
-}
-
-func allocatePort(port int) (net.PacketConn, error) {
-	c, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port))
-	if err != nil {
-		return nil, err
-	}
-	return c, err
-}

client/internal/wgproxy/ebpf/proxy.go

@@ -1,283 +0,0 @@
-//go:build linux && !android
-
-package ebpf
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"os"
-	"sync"
-	"syscall"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-	"github.com/hashicorp/go-multierror"
-	"github.com/pion/transport/v3"
-	log "github.com/sirupsen/logrus"
-
-	nberrors "github.com/netbirdio/netbird/client/errors"
-	"github.com/netbirdio/netbird/client/internal/ebpf"
-	ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager"
-	nbnet "github.com/netbirdio/netbird/util/net"
-)
-
-const (
-	loopbackAddr = "127.0.0.1"
-)
-
-// WGEBPFProxy definition for proxy with EBPF support
-type WGEBPFProxy struct {
-	localWGListenPort int
-
-	ebpfManager   ebpfMgr.Manager
-	turnConnStore map[uint16]net.Conn
-	turnConnMutex sync.Mutex
-
-	lastUsedPort uint16
-	rawConn      net.PacketConn
-	conn         transport.UDPConn
-
-	ctx       context.Context
-	ctxCancel context.CancelFunc
-}
-
-// NewWGEBPFProxy create new WGEBPFProxy instance
-func NewWGEBPFProxy(wgPort int) *WGEBPFProxy {
-	log.Debugf("instantiate ebpf proxy")
-	wgProxy := &WGEBPFProxy{
-		localWGListenPort: wgPort,
-		ebpfManager:       ebpf.GetEbpfManagerInstance(),
-		turnConnStore:     make(map[uint16]net.Conn),
-	}
-	return wgProxy
-}
-
-// Listen load ebpf program and listen the proxy
-func (p *WGEBPFProxy) Listen() error {
-	pl := portLookup{}
-	wgPorxyPort, err := pl.searchFreePort()
-	if err != nil {
-		return err
-	}
-
-	p.rawConn, err = p.prepareSenderRawSocket()
-	if err != nil {
-		return err
-	}
-
-	err = p.ebpfManager.LoadWgProxy(wgPorxyPort, p.localWGListenPort)
-	if err != nil {
-		return err
-	}
-
-	addr := net.UDPAddr{
-		Port: wgPorxyPort,
-		IP:   net.ParseIP(loopbackAddr),
-	}
-
-	p.ctx, p.ctxCancel = context.WithCancel(context.Background())
-
-	conn, err := nbnet.ListenUDP("udp", &addr)
-	if err != nil {
-		if cErr := p.Free(); cErr != nil {
-			log.Errorf("Failed to close the wgproxy: %s", cErr)
-		}
-		return err
-	}
-	p.conn = conn
-
-	go p.proxyToRemote()
-	log.Infof("local wg proxy listening on: %d", wgPorxyPort)
-	return nil
-}
-
-// AddTurnConn add new turn connection for the proxy
-func (p *WGEBPFProxy) AddTurnConn(turnConn net.Conn) (*net.UDPAddr, error) {
-	wgEndpointPort, err := p.storeTurnConn(turnConn)
-	if err != nil {
-		return nil, err
-	}
-
-	log.Infof("turn conn added to wg proxy store: %s, endpoint port: :%d", turnConn.RemoteAddr(), wgEndpointPort)
-
-	wgEndpoint := &net.UDPAddr{
-		IP:   net.ParseIP(loopbackAddr),
-		Port: int(wgEndpointPort),
-	}
-	return wgEndpoint, nil
-}
-
-// Free resources except the remoteConns will be keep open.
-func (p *WGEBPFProxy) Free() error {
-	log.Debugf("free up ebpf wg proxy")
-	if p.ctx != nil && p.ctx.Err() != nil {
-		//nolint
-		return nil
-	}
-
-	p.ctxCancel()
-
-	var result *multierror.Error
-	if p.conn != nil { // p.conn will be nil if we have failed to listen
-		if err := p.conn.Close(); err != nil {
-			result = multierror.Append(result, err)
-		}
-	}
-
-	if err := p.ebpfManager.FreeWGProxy(); err != nil {
-		result = multierror.Append(result, err)
-	}
-
-	if err := p.rawConn.Close(); err != nil {
-		result = multierror.Append(result, err)
-	}
-	return nberrors.FormatErrorOrNil(result)
-}
-
-// proxyToRemote read messages from local WireGuard interface and forward it to remote conn
-// From this go routine has only one instance.
-func (p *WGEBPFProxy) proxyToRemote() {
-	buf := make([]byte, 1500)
-	for p.ctx.Err() == nil {
-		if err := p.readAndForwardPacket(buf); err != nil {
-			if p.ctx.Err() != nil {
-				return
-			}
-			log.Errorf("failed to proxy packet to remote conn: %s", err)
-		}
-	}
-}
-
-func (p *WGEBPFProxy) readAndForwardPacket(buf []byte) error {
-	n, addr, err := p.conn.ReadFromUDP(buf)
-	if err != nil {
-		return fmt.Errorf("failed to read UDP packet from WG: %w", err)
-	}
-
-	p.turnConnMutex.Lock()
-	conn, ok := p.turnConnStore[uint16(addr.Port)]
-	p.turnConnMutex.Unlock()
-	if !ok {
-		if p.ctx.Err() == nil {
-			log.Debugf("turn conn not found by port because conn already has been closed: %d", addr.Port)
-		}
-		return nil
-	}
-
-	if _, err := conn.Write(buf[:n]); err != nil {
-		return fmt.Errorf("failed to forward local WG packet (%d) to remote turn conn: %w", addr.Port, err)
-	}
-	return nil
-}
-
-func (p *WGEBPFProxy) storeTurnConn(turnConn net.Conn) (uint16, error) {
-	p.turnConnMutex.Lock()
-	defer p.turnConnMutex.Unlock()
-
-	np, err := p.nextFreePort()
-	if err != nil {
-		return np, err
-	}
-	p.turnConnStore[np] = turnConn
-	return np, nil
-}
-
-func (p *WGEBPFProxy) removeTurnConn(turnConnID uint16) {
-	p.turnConnMutex.Lock()
-	defer p.turnConnMutex.Unlock()
-
-	_, ok := p.turnConnStore[turnConnID]
-	if ok {
-		log.Debugf("remove turn conn from store by port: %d", turnConnID)
-	}
-	delete(p.turnConnStore, turnConnID)
-}
-
-func (p *WGEBPFProxy) nextFreePort() (uint16, error) {
-	if len(p.turnConnStore) == 65535 {
-		return 0, fmt.Errorf("reached maximum turn connection numbers")
-	}
-generatePort:
-	if p.lastUsedPort == 65535 {
-		p.lastUsedPort = 1
-	} else {
-		p.lastUsedPort++
-	}
-
-	if _, ok := p.turnConnStore[p.lastUsedPort]; ok {
-		goto generatePort
-	}
-	return p.lastUsedPort, nil
-}
-
-func (p *WGEBPFProxy) prepareSenderRawSocket() (net.PacketConn, error) {
-	// Create a raw socket.
-	fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_RAW)
-	if err != nil {
-		return nil, fmt.Errorf("creating raw socket failed: %w", err)
-	}
-
-	// Set the IP_HDRINCL option on the socket to tell the kernel that headers are included in the packet.
-	err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_HDRINCL, 1)
-	if err != nil {
-		return nil, fmt.Errorf("setting IP_HDRINCL failed: %w", err)
-	}
-
-	// Bind the socket to the "lo" interface.
-	err = syscall.SetsockoptString(fd, syscall.SOL_SOCKET, syscall.SO_BINDTODEVICE, "lo")
-	if err != nil {
-		return nil, fmt.Errorf("binding to lo interface failed: %w", err)
-	}
-
-	// Set the fwmark on the socket.
-	err = nbnet.SetSocketOpt(fd)
-	if err != nil {
-		return nil, fmt.Errorf("setting fwmark failed: %w", err)
-	}
-
-	// Convert the file descriptor to a PacketConn.
-	file := os.NewFile(uintptr(fd), fmt.Sprintf("fd %d", fd))
-	if file == nil {
-		return nil, fmt.Errorf("converting fd to file failed")
-	}
-	packetConn, err := net.FilePacketConn(file)
-	if err != nil {
-		return nil, fmt.Errorf("converting file to packet conn failed: %w", err)
-	}
-
-	return packetConn, nil
-}
-
-func (p *WGEBPFProxy) sendPkg(data []byte, port int) error {
-	localhost := net.ParseIP("127.0.0.1")
-
-	payload := gopacket.Payload(data)
-	ipH := &layers.IPv4{
-		DstIP:    localhost,
-		SrcIP:    localhost,
-		Version:  4,
-		TTL:      64,
-		Protocol: layers.IPProtocolUDP,
-	}
-	udpH := &layers.UDP{
-		SrcPort: layers.UDPPort(port),
-		DstPort: layers.UDPPort(p.localWGListenPort),
-	}
-
-	err := udpH.SetNetworkLayerForChecksum(ipH)
-	if err != nil {
-		return fmt.Errorf("set network layer for checksum: %w", err)
-	}
-
-	layerBuffer := gopacket.NewSerializeBuffer()
-
-	err = gopacket.SerializeLayers(layerBuffer, gopacket.SerializeOptions{ComputeChecksums: true, FixLengths: true}, ipH, udpH, payload)
-	if err != nil {
-		return fmt.Errorf("serialize layers: %w", err)
-	}
-	if _, err = p.rawConn.WriteTo(layerBuffer.Bytes(), &net.IPAddr{IP: localhost}); err != nil {
-		return fmt.Errorf("write to raw conn: %w", err)
-	}
-	return nil
-}

client/internal/wgproxy/ebpf/proxy_test.go

@@ -1,56 +0,0 @@
-//go:build linux && !android
-
-package ebpf
-
-import (
-	"testing"
-)
-
-func TestWGEBPFProxy_connStore(t *testing.T) {
-	wgProxy := NewWGEBPFProxy(1)
-
-	p, _ := wgProxy.storeTurnConn(nil)
-	if p != 1 {
-		t.Errorf("invalid initial port: %d", wgProxy.lastUsedPort)
-	}
-
-	numOfConns := 10
-	for i := 0; i < numOfConns; i++ {
-		p, _ = wgProxy.storeTurnConn(nil)
-	}
-	if p != uint16(numOfConns)+1 {
-		t.Errorf("invalid last used port: %d, expected: %d", p, numOfConns+1)
-	}
-	if len(wgProxy.turnConnStore) != numOfConns+1 {
-		t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.turnConnStore), numOfConns+1)
-	}
-}
-
-func TestWGEBPFProxy_portCalculation_overflow(t *testing.T) {
-	wgProxy := NewWGEBPFProxy(1)
-
-	_, _ = wgProxy.storeTurnConn(nil)
-	wgProxy.lastUsedPort = 65535
-	p, _ := wgProxy.storeTurnConn(nil)
-
-	if len(wgProxy.turnConnStore) != 2 {
-		t.Errorf("invalid store size: %d, expected: %d", len(wgProxy.turnConnStore), 2)
-	}
-
-	if p != 2 {
-		t.Errorf("invalid last used port: %d, expected: %d", p, 2)
-	}
-}
-
-func TestWGEBPFProxy_portCalculation_maxConn(t *testing.T) {
-	wgProxy := NewWGEBPFProxy(1)
-
-	for i := 0; i < 65535; i++ {
-		_, _ = wgProxy.storeTurnConn(nil)
-	}
-
-	_, err := wgProxy.storeTurnConn(nil)
-	if err == nil {
-		t.Errorf("invalid turn conn store calculation")
-	}
-}

client/internal/wgproxy/ebpf/wrapper.go

@@ -1,126 +0,0 @@
-//go:build linux && !android
-
-package ebpf
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net"
-	"sync"
-
-	log "github.com/sirupsen/logrus"
-)
-
-// ProxyWrapper help to keep the remoteConn instance for net.Conn.Close function call
-type ProxyWrapper struct {
-	WgeBPFProxy *WGEBPFProxy
-
-	remoteConn net.Conn
-	ctx        context.Context
-	cancel     context.CancelFunc
-
-	wgEndpointAddr *net.UDPAddr
-
-	pausedMu  sync.Mutex
-	paused    bool
-	isStarted bool
-}
-
-func (p *ProxyWrapper) AddTurnConn(ctx context.Context, remoteConn net.Conn) error {
-	addr, err := p.WgeBPFProxy.AddTurnConn(remoteConn)
-	if err != nil {
-		return fmt.Errorf("add turn conn: %w", err)
-	}
-	p.remoteConn = remoteConn
-	p.ctx, p.cancel = context.WithCancel(ctx)
-	p.wgEndpointAddr = addr
-	return err
-}
-
-func (p *ProxyWrapper) EndpointAddr() *net.UDPAddr {
-	return p.wgEndpointAddr
-}
-
-func (p *ProxyWrapper) Work() {
-	if p.remoteConn == nil {
-		return
-	}
-
-	p.pausedMu.Lock()
-	p.paused = false
-	p.pausedMu.Unlock()
-
-	if !p.isStarted {
-		p.isStarted = true
-		go p.proxyToLocal(p.ctx)
-	}
-}
-
-func (p *ProxyWrapper) Pause() {
-	if p.remoteConn == nil {
-		return
-	}
-
-	log.Tracef("pause proxy reading from: %s", p.remoteConn.RemoteAddr())
-	p.pausedMu.Lock()
-	p.paused = true
-	p.pausedMu.Unlock()
-}
-
-// CloseConn close the remoteConn and automatically remove the conn instance from the map
-func (e *ProxyWrapper) CloseConn() error {
-	if e.cancel == nil {
-		return fmt.Errorf("proxy not started")
-	}
-
-	e.cancel()
-
-	if err := e.remoteConn.Close(); err != nil {
-		return fmt.Errorf("failed to close remote conn: %w", err)
-	}
-	return nil
-}
-
-func (p *ProxyWrapper) proxyToLocal(ctx context.Context) {
-	defer p.WgeBPFProxy.removeTurnConn(uint16(p.wgEndpointAddr.Port))
-
-	buf := make([]byte, 1500)
-	for {
-		n, err := p.readFromRemote(ctx, buf)
-		if err != nil {
-			return
-		}
-
-		p.pausedMu.Lock()
-		if p.paused {
-			p.pausedMu.Unlock()
-			continue
-		}
-
-		err = p.WgeBPFProxy.sendPkg(buf[:n], p.wgEndpointAddr.Port)
-		p.pausedMu.Unlock()
-
-		if err != nil {
-			if ctx.Err() != nil {
-				return
-			}
-			log.Errorf("failed to write out turn pkg to local conn: %v", err)
-		}
-	}
-}
-
-func (p *ProxyWrapper) readFromRemote(ctx context.Context, buf []byte) (int, error) {
-	n, err := p.remoteConn.Read(buf)
-	if err != nil {
-		if ctx.Err() != nil {
-			return 0, ctx.Err()
-		}
-		if !errors.Is(err, io.EOF) {
-			log.Errorf("failed to read from turn conn (endpoint: :%d): %s", p.wgEndpointAddr.Port, err)
-		}
-		return 0, err
-	}
-	return n, nil
-}

client/internal/wgproxy/factory_linux.go

@@ -1,50 +0,0 @@
-//go:build !android
-
-package wgproxy
-
-import (
-	log "github.com/sirupsen/logrus"
-
-	"github.com/netbirdio/netbird/client/internal/wgproxy/ebpf"
-	"github.com/netbirdio/netbird/client/internal/wgproxy/usp"
-)
-
-type Factory struct {
-	wgPort    int
-	ebpfProxy *ebpf.WGEBPFProxy
-}
-
-func NewFactory(userspace bool, wgPort int) *Factory {
-	f := &Factory{wgPort: wgPort}
-
-	if userspace {
-		return f
-	}
-
-	ebpfProxy := ebpf.NewWGEBPFProxy(wgPort)
-	err := ebpfProxy.Listen()
-	if err != nil {
-		log.Warnf("failed to initialize ebpf proxy, fallback to user space proxy: %s", err)
-		return f
-	}
-
-	f.ebpfProxy = ebpfProxy
-	return f
-}
-
-func (w *Factory) GetProxy() Proxy {
-	if w.ebpfProxy != nil {
-		p := &ebpf.ProxyWrapper{
-			WgeBPFProxy: w.ebpfProxy,
-		}
-		return p
-	}
-	return usp.NewWGUserSpaceProxy(w.wgPort)
-}
-
-func (w *Factory) Free() error {
-	if w.ebpfProxy == nil {
-		return nil
-	}
-	return w.ebpfProxy.Free()
-}

client/internal/wgproxy/factory_nonlinux.go

@@ -1,21 +0,0 @@
-//go:build !linux || android
-
-package wgproxy
-
-import "github.com/netbirdio/netbird/client/internal/wgproxy/usp"
-
-type Factory struct {
-	wgPort int
-}
-
-func NewFactory(_ bool, wgPort int) *Factory {
-	return &Factory{wgPort: wgPort}
-}
-
-func (w *Factory) GetProxy() Proxy {
-	return usp.NewWGUserSpaceProxy(w.wgPort)
-}
-
-func (w *Factory) Free() error {
-	return nil
-}

client/internal/wgproxy/proxy.go

@@ -1,15 +0,0 @@
-package wgproxy
-
-import (
-	"context"
-	"net"
-)
-
-// Proxy is a transfer layer between the relayed connection and the WireGuard
-type Proxy interface {
-	AddTurnConn(ctx context.Context, turnConn net.Conn) error
-	EndpointAddr() *net.UDPAddr
-	Work()
-	Pause()
-	CloseConn() error
-}

client/internal/wgproxy/proxy_test.go

@@ -1,128 +0,0 @@
-//go:build linux
-
-package wgproxy
-
-import (
-	"context"
-	"io"
-	"net"
-	"os"
-	"runtime"
-	"testing"
-	"time"
-
-	"github.com/netbirdio/netbird/client/internal/wgproxy/ebpf"
-	"github.com/netbirdio/netbird/client/internal/wgproxy/usp"
-	"github.com/netbirdio/netbird/util"
-)
-
-func TestMain(m *testing.M) {
-	_ = util.InitLog("trace", "console")
-	code := m.Run()
-	os.Exit(code)
-}
-
-type mocConn struct {
-	closeChan chan struct{}
-	closed    bool
-}
-
-func newMockConn() *mocConn {
-	return &mocConn{
-		closeChan: make(chan struct{}),
-	}
-}
-
-func (m *mocConn) Read(b []byte) (n int, err error) {
-	<-m.closeChan
-	return 0, io.EOF
-}
-
-func (m *mocConn) Write(b []byte) (n int, err error) {
-	<-m.closeChan
-	return 0, io.EOF
-}
-
-func (m *mocConn) Close() error {
-	if m.closed == true {
-		return nil
-	}
-
-	m.closed = true
-	close(m.closeChan)
-	return nil
-}
-
-func (m *mocConn) LocalAddr() net.Addr {
-	panic("implement me")
-}
-
-func (m *mocConn) RemoteAddr() net.Addr {
-	return &net.UDPAddr{
-		IP: net.ParseIP("172.16.254.1"),
-	}
-}
-
-func (m *mocConn) SetDeadline(t time.Time) error {
-	panic("implement me")
-}
-
-func (m *mocConn) SetReadDeadline(t time.Time) error {
-	panic("implement me")
-}
-
-func (m *mocConn) SetWriteDeadline(t time.Time) error {
-	panic("implement me")
-}
-
-func TestProxyCloseByRemoteConn(t *testing.T) {
-	ctx := context.Background()
-
-	tests := []struct {
-		name  string
-		proxy Proxy
-	}{
-		{
-			name:  "userspace proxy",
-			proxy: usp.NewWGUserSpaceProxy(51830),
-		},
-	}
-
-	if runtime.GOOS == "linux" && os.Getenv("GITHUB_ACTIONS") != "true" {
-		ebpfProxy := ebpf.NewWGEBPFProxy(51831)
-		if err := ebpfProxy.Listen(); err != nil {
-			t.Fatalf("failed to initialize ebpf proxy: %s", err)
-		}
-		defer func() {
-			if err := ebpfProxy.Free(); err != nil {
-				t.Errorf("failed to free ebpf proxy: %s", err)
-			}
-		}()
-		proxyWrapper := &ebpf.ProxyWrapper{
-			WgeBPFProxy: ebpfProxy,
-		}
-
-		tests = append(tests, struct {
-			name  string
-			proxy Proxy
-		}{
-			name:  "ebpf proxy",
-			proxy: proxyWrapper,
-		})
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			relayedConn := newMockConn()
-			err := tt.proxy.AddTurnConn(ctx, relayedConn)
-			if err != nil {
-				t.Errorf("error: %v", err)
-			}
-
-			_ = relayedConn.Close()
-			if err := tt.proxy.CloseConn(); err != nil {
-				t.Errorf("error: %v", err)
-			}
-		})
-	}
-}

client/internal/wgproxy/usp/proxy.go

@@ -1,195 +0,0 @@
-package usp
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"sync"
-
-	"github.com/hashicorp/go-multierror"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/netbirdio/netbird/client/errors"
-)
-
-// WGUserSpaceProxy proxies
-type WGUserSpaceProxy struct {
-	localWGListenPort int
-
-	remoteConn net.Conn
-	localConn  net.Conn
-	ctx        context.Context
-	cancel     context.CancelFunc
-	closeMu    sync.Mutex
-	closed     bool
-
-	pausedMu  sync.Mutex
-	paused    bool
-	isStarted bool
-}
-
-// NewWGUserSpaceProxy instantiate a user space WireGuard proxy. This is not a thread safe implementation
-func NewWGUserSpaceProxy(wgPort int) *WGUserSpaceProxy {
-	log.Debugf("Initializing new user space proxy with port %d", wgPort)
-	p := &WGUserSpaceProxy{
-		localWGListenPort: wgPort,
-	}
-	return p
-}
-
-// AddTurnConn
-// The provided Context must be non-nil. If the context expires before
-// the connection is complete, an error is returned. Once successfully
-// connected, any expiration of the context will not affect the
-// connection.
-func (p *WGUserSpaceProxy) AddTurnConn(ctx context.Context, remoteConn net.Conn) error {
-	dialer := net.Dialer{}
-	localConn, err := dialer.DialContext(ctx, "udp", fmt.Sprintf(":%d", p.localWGListenPort))
-	if err != nil {
-		log.Errorf("failed dialing to local Wireguard port %s", err)
-		return err
-	}
-
-	p.ctx, p.cancel = context.WithCancel(ctx)
-	p.localConn = localConn
-	p.remoteConn = remoteConn
-
-	return err
-}
-
-func (p *WGUserSpaceProxy) EndpointAddr() *net.UDPAddr {
-	if p.localConn == nil {
-		return nil
-	}
-	endpointUdpAddr, _ := net.ResolveUDPAddr(p.localConn.LocalAddr().Network(), p.localConn.LocalAddr().String())
-	return endpointUdpAddr
-}
-
-// Work starts the proxy or resumes it if it was paused
-func (p *WGUserSpaceProxy) Work() {
-	if p.remoteConn == nil {
-		return
-	}
-
-	p.pausedMu.Lock()
-	p.paused = false
-	p.pausedMu.Unlock()
-
-	if !p.isStarted {
-		p.isStarted = true
-		go p.proxyToRemote(p.ctx)
-		go p.proxyToLocal(p.ctx)
-	}
-}
-
-// Pause pauses the proxy from receiving data from the remote peer
-func (p *WGUserSpaceProxy) Pause() {
-	if p.remoteConn == nil {
-		return
-	}
-
-	p.pausedMu.Lock()
-	p.paused = true
-	p.pausedMu.Unlock()
-}
-
-// CloseConn close the localConn
-func (p *WGUserSpaceProxy) CloseConn() error {
-	if p.cancel == nil {
-		return fmt.Errorf("proxy not started")
-	}
-	return p.close()
-}
-
-func (p *WGUserSpaceProxy) close() error {
-	p.closeMu.Lock()
-	defer p.closeMu.Unlock()
-
-	// prevent double close
-	if p.closed {
-		return nil
-	}
-	p.closed = true
-
-	p.cancel()
-
-	var result *multierror.Error
-	if err := p.remoteConn.Close(); err != nil {
-		result = multierror.Append(result, fmt.Errorf("remote conn: %s", err))
-	}
-
-	if err := p.localConn.Close(); err != nil {
-		result = multierror.Append(result, fmt.Errorf("local conn: %s", err))
-	}
-	return errors.FormatErrorOrNil(result)
-}
-
-// proxyToRemote proxies from Wireguard to the RemoteKey
-func (p *WGUserSpaceProxy) proxyToRemote(ctx context.Context) {
-	defer func() {
-		if err := p.close(); err != nil {
-			log.Warnf("error in proxy to remote loop: %s", err)
-		}
-	}()
-
-	buf := make([]byte, 1500)
-	for ctx.Err() == nil {
-		n, err := p.localConn.Read(buf)
-		if err != nil {
-			if ctx.Err() != nil {
-				return
-			}
-			log.Debugf("failed to read from wg interface conn: %s", err)
-			return
-		}
-
-		_, err = p.remoteConn.Write(buf[:n])
-		if err != nil {
-			if ctx.Err() != nil {
-				return
-			}
-
-			log.Debugf("failed to write to remote conn: %s", err)
-			return
-		}
-	}
-}
-
-// proxyToLocal proxies from the Remote peer to local WireGuard
-// if the proxy is paused it will drain the remote conn and drop the packets
-func (p *WGUserSpaceProxy) proxyToLocal(ctx context.Context) {
-	defer func() {
-		if err := p.close(); err != nil {
-			log.Warnf("error in proxy to local loop: %s", err)
-		}
-	}()
-
-	buf := make([]byte, 1500)
-	for {
-		n, err := p.remoteConn.Read(buf)
-		if err != nil {
-			if ctx.Err() != nil {
-				return
-			}
-			log.Errorf("failed to read from remote conn: %s, %s", p.remoteConn.RemoteAddr(), err)
-			return
-		}
-
-		p.pausedMu.Lock()
-		if p.paused {
-			p.pausedMu.Unlock()
-			continue
-		}
-
-		_, err = p.localConn.Write(buf[:n])
-		p.pausedMu.Unlock()
-
-		if err != nil {
-			if ctx.Err() != nil {
-				return
-			}
-			log.Debugf("failed to write to wg interface conn: %s", err)
-			continue
-		}
-	}
-}