Add retry to sending signal message (#906)

Increased the default send timeout from 2 to 5

Added a max of 4 retries
 with an increased timeout after the second attempt

using the grpc client context and
checking the error value for canceled context

.github/workflows/release.yml

@@ -9,7 +9,7 @@ on:
   pull_request:
 
 env:
-  SIGN_PIPE_VER: "v0.0.6"
+  SIGN_PIPE_VER: "v0.0.7"
   GORELEASER_VER: "v1.14.1"
 
 concurrency:

.goreleaser.yaml

@@ -12,11 +12,7 @@ builds:
       - arm
       - amd64
       - arm64
-      - mips
       - 386
-    gomips:
-      - hardfloat
-      - softfloat
     ignore:
       - goos: windows
         goarch: arm64
@@ -30,6 +26,26 @@ builds:
     tags:
       - load_wgnt_from_rsrc
 
+  - id: netbird-static
+    dir: client
+    binary: netbird
+    env: [CGO_ENABLED=0]
+    goos:
+      - linux
+    goarch:
+      - mips
+      - mipsle
+      - mips64
+      - mips64le
+    gomips:
+      - hardfloat
+      - softfloat
+    ldflags:
+      - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
+    mod_timestamp: '{{ .CommitTimestamp }}'
+    tags:
+      - load_wgnt_from_rsrc
+
   - id: netbird-mgmt
     dir: management
     env:
@@ -67,6 +83,7 @@ builds:
 archives:
   - builds:
       - netbird
+      - netbird-static
 
 nfpms:
 
@@ -359,4 +376,4 @@ uploads:
     mode: archive
     target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }}
     username: dev@wiretrustee.com
-    method: PUT
+    method: PUT

base62/base62.go

@@ -0,0 +1,59 @@
+package base62
+
+import (
+	"fmt"
+	"math"
+	"strings"
+)
+
+const (
+	alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+	base     = uint32(len(alphabet))
+)
+
+// Encode encodes a uint32 value to a base62 string.
+func Encode(num uint32) string {
+	if num == 0 {
+		return string(alphabet[0])
+	}
+
+	var encoded strings.Builder
+	remainder := uint32(0)
+
+	for num > 0 {
+		remainder = num % base
+		encoded.WriteByte(alphabet[remainder])
+		num /= base
+	}
+
+	// Reverse the encoded string
+	encodedString := encoded.String()
+	reversed := reverse(encodedString)
+	return reversed
+}
+
+// Decode decodes a base62 string to a uint32 value.
+func Decode(encoded string) (uint32, error) {
+	var decoded uint32
+	strLen := len(encoded)
+
+	for i, char := range encoded {
+		index := strings.IndexRune(alphabet, char)
+		if index < 0 {
+			return 0, fmt.Errorf("invalid character: %c", char)
+		}
+
+		decoded += uint32(index) * uint32(math.Pow(float64(base), float64(strLen-i-1)))
+	}
+
+	return decoded, nil
+}
+
+// Reverse a string.
+func reverse(s string) string {
+	runes := []rune(s)
+	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
+		runes[i], runes[j] = runes[j], runes[i]
+	}
+	return string(runes)
+}

base62/base62_test.go

@@ -0,0 +1,31 @@
+package base62
+
+import (
+	"testing"
+)
+
+func TestEncodeDecode(t *testing.T) {
+	tests := []struct {
+		num uint32
+	}{
+		{0},
+		{1},
+		{42},
+		{12345},
+		{99999},
+		{123456789},
+	}
+
+	for _, tt := range tests {
+		encoded := Encode(tt.num)
+		decoded, err := Decode(encoded)
+
+		if err != nil {
+			t.Errorf("Decode error: %v", err)
+		}
+
+		if decoded != tt.num {
+			t.Errorf("Decode(%v) = %v, want %v", encoded, decoded, tt.num)
+		}
+	}
+}

client/cmd/testutil.go

@@ -2,21 +2,23 @@ package cmd
 
 import (
 	"context"
-	"github.com/netbirdio/netbird/management/server/activity"
 	"net"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/netbirdio/netbird/management/server/activity"
+
 	"github.com/netbirdio/netbird/util"
 
+	"google.golang.org/grpc"
+
 	clientProto "github.com/netbirdio/netbird/client/proto"
 	client "github.com/netbirdio/netbird/client/server"
 	mgmtProto "github.com/netbirdio/netbird/management/proto"
 	mgmt "github.com/netbirdio/netbird/management/server"
 	sigProto "github.com/netbirdio/netbird/signal/proto"
 	sig "github.com/netbirdio/netbird/signal/server"
-	"google.golang.org/grpc"
 )
 
 func startTestingServices(t *testing.T) string {
@@ -63,7 +65,7 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	store, err := mgmt.NewFileStore(config.Datadir)
+	store, err := mgmt.NewFileStore(config.Datadir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

client/internal/dns/file_linux.go

@@ -32,6 +32,10 @@ func newFileConfigurator() (hostManager, error) {
 	return &fileConfigurator{}, nil
 }
 
+func (f *fileConfigurator) supportCustomPort() bool {
+	return false
+}
+
 func (f *fileConfigurator) applyDNSConfig(config hostDNSConfig) error {
 	backupFileExist := false
 	_, err := os.Stat(fileDefaultResolvConfBackupLocation)

client/internal/dns/host.go

@@ -10,6 +10,7 @@ import (
 type hostManager interface {
 	applyDNSConfig(config hostDNSConfig) error
 	restoreHostDNS() error
+	supportCustomPort() bool
 }
 
 type hostDNSConfig struct {
@@ -26,8 +27,9 @@ type domainConfig struct {
 }
 
 type mockHostConfigurator struct {
-	applyDNSConfigFunc func(config hostDNSConfig) error
-	restoreHostDNSFunc func() error
+	applyDNSConfigFunc    func(config hostDNSConfig) error
+	restoreHostDNSFunc    func() error
+	supportCustomPortFunc func() bool
 }
 
 func (m *mockHostConfigurator) applyDNSConfig(config hostDNSConfig) error {
@@ -44,10 +46,18 @@ func (m *mockHostConfigurator) restoreHostDNS() error {
 	return fmt.Errorf("method restoreHostDNS is not implemented")
 }
 
+func (m *mockHostConfigurator) supportCustomPort() bool {
+	if m.supportCustomPortFunc != nil {
+		return m.supportCustomPortFunc()
+	}
+	return false
+}
+
 func newNoopHostMocker() hostManager {
 	return &mockHostConfigurator{
-		applyDNSConfigFunc: func(config hostDNSConfig) error { return nil },
-		restoreHostDNSFunc: func() error { return nil },
+		applyDNSConfigFunc:    func(config hostDNSConfig) error { return nil },
+		restoreHostDNSFunc:    func() error { return nil },
+		supportCustomPortFunc: func() bool { return true },
 	}
 }
 

client/internal/dns/host_darwin.go

@@ -8,8 +8,9 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/netbirdio/netbird/iface"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/iface"
 )
 
 const (
@@ -39,6 +40,10 @@ func newHostManager(_ *iface.WGIface) (hostManager, error) {
 	}, nil
 }
 
+func (s *systemConfigurator) supportCustomPort() bool {
+	return true
+}
+
 func (s *systemConfigurator) applyDNSConfig(config hostDNSConfig) error {
 	var err error
 

client/internal/dns/host_windows.go

@@ -4,9 +4,10 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/netbirdio/netbird/iface"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows/registry"
+
+	"github.com/netbirdio/netbird/iface"
 )
 
 const (
@@ -42,6 +43,10 @@ func newHostManager(wgInterface *iface.WGIface) (hostManager, error) {
 	}, nil
 }
 
+func (s *registryConfigurator) supportCustomPort() bool {
+	return false
+}
+
 func (r *registryConfigurator) applyDNSConfig(config hostDNSConfig) error {
 	var err error
 	if config.routeAll {

client/internal/dns/local.go

@@ -2,10 +2,12 @@ package dns
 
 import (
 	"fmt"
-	"github.com/miekg/dns"
-	nbdns "github.com/netbirdio/netbird/dns"
-	log "github.com/sirupsen/logrus"
 	"sync"
+
+	"github.com/miekg/dns"
+	log "github.com/sirupsen/logrus"
+
+	nbdns "github.com/netbirdio/netbird/dns"
 )
 
 type registrationMap map[string]struct{}
@@ -15,6 +17,9 @@ type localResolver struct {
 	records       sync.Map
 }
 
+func (d *localResolver) stop() {
+}
+
 // ServeDNS handles a DNS request
 func (d *localResolver) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 	log.Tracef("received question: %#v\n", r.Question[0])

client/internal/dns/network_manager_linux.go

@@ -11,8 +11,9 @@ import (
 	"github.com/godbus/dbus/v5"
 	"github.com/hashicorp/go-version"
 	"github.com/miekg/dns"
-	"github.com/netbirdio/netbird/iface"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/iface"
 )
 
 const (
@@ -88,6 +89,10 @@ func newNetworkManagerDbusConfigurator(wgInterface *iface.WGIface) (hostManager,
 	}, nil
 }
 
+func (n *networkManagerDbusConfigurator) supportCustomPort() bool {
+	return false
+}
+
 func (n *networkManagerDbusConfigurator) applyDNSConfig(config hostDNSConfig) error {
 	connSettings, configVersion, err := n.getAppliedConnectionSettings()
 	if err != nil {

client/internal/dns/resolvconf_linux.go

@@ -5,8 +5,9 @@ import (
 	"os/exec"
 	"strings"
 
-	"github.com/netbirdio/netbird/iface"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/iface"
 )
 
 const resolvconfCommand = "resolvconf"
@@ -21,6 +22,10 @@ func newResolvConfConfigurator(wgInterface *iface.WGIface) (hostManager, error)
 	}, nil
 }
 
+func (r *resolvconf) supportCustomPort() bool {
+	return false
+}
+
 func (r *resolvconf) applyDNSConfig(config hostDNSConfig) error {
 	var err error
 	if !config.routeAll {

client/internal/dns/server_nonandroid.go

@@ -13,9 +13,10 @@ import (
 
 	"github.com/miekg/dns"
 	"github.com/mitchellh/hashstructure/v2"
+	log "github.com/sirupsen/logrus"
+
 	nbdns "github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/iface"
-	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -25,15 +26,16 @@ const (
 	customIP    = "127.0.0.153"
 )
 
+type registeredHandlerMap map[string]handlerWithStop
+
 // DefaultServer dns server object
 type DefaultServer struct {
 	ctx                context.Context
 	ctxCancel          context.CancelFunc
-	upstreamCtxCancel  context.CancelFunc
 	mux                sync.Mutex
 	server             *dns.Server
 	dnsMux             *dns.ServeMux
-	dnsMuxMap          registrationMap
+	dnsMuxMap          registeredHandlerMap
 	localResolver      *localResolver
 	wgInterface        *iface.WGIface
 	hostManager        hostManager
@@ -46,9 +48,14 @@ type DefaultServer struct {
 	customAddress      *netip.AddrPort
 }
 
+type handlerWithStop interface {
+	dns.Handler
+	stop()
+}
+
 type muxUpdate struct {
 	domain  string
-	handler dns.Handler
+	handler handlerWithStop
 }
 
 // NewDefaultServer returns a new dns server
@@ -78,7 +85,7 @@ func NewDefaultServer(ctx context.Context, wgInterface *iface.WGIface, customAdd
 		ctxCancel: stop,
 		server:    dnsServer,
 		dnsMux:    mux,
-		dnsMuxMap: make(registrationMap),
+		dnsMuxMap: make(registeredHandlerMap),
 		localResolver: &localResolver{
 			registeredMap: make(registrationMap),
 		},
@@ -254,7 +261,14 @@ func (s *DefaultServer) applyConfiguration(update nbdns.Config) error {
 	s.updateLocalResolver(localRecords)
 	s.currentConfig = dnsConfigToHostDNSConfig(update, s.runtimeIP, s.runtimePort)
 
-	if err = s.hostManager.applyDNSConfig(s.currentConfig); err != nil {
+	hostUpdate := s.currentConfig
+	if s.runtimePort != defaultPort && !s.hostManager.supportCustomPort() {
+		log.Warnf("the DNS manager of this peer doesn't support custom port. Disabling primary DNS setup. " +
+			"Learn more at: https://netbird.io/docs/how-to-guides/nameservers#local-resolver")
+		hostUpdate.routeAll = false
+	}
+
+	if err = s.hostManager.applyDNSConfig(hostUpdate); err != nil {
 		log.Error(err)
 	}
 
@@ -289,10 +303,6 @@ func (s *DefaultServer) buildLocalHandlerUpdate(customZones []nbdns.CustomZone)
 }
 
 func (s *DefaultServer) buildUpstreamHandlerUpdate(nameServerGroups []*nbdns.NameServerGroup) ([]muxUpdate, error) {
-	// clean up the previous upstream resolver
-	if s.upstreamCtxCancel != nil {
-		s.upstreamCtxCancel()
-	}
 
 	var muxUpdates []muxUpdate
 	for _, nsGroup := range nameServerGroups {
@@ -301,10 +311,7 @@ func (s *DefaultServer) buildUpstreamHandlerUpdate(nameServerGroups []*nbdns.Nam
 			continue
 		}
 
-		var ctx context.Context
-		ctx, s.upstreamCtxCancel = context.WithCancel(s.ctx)
-
-		handler := newUpstreamResolver(ctx)
+		handler := newUpstreamResolver(s.ctx)
 		for _, ns := range nsGroup.NameServers {
 			if ns.NSType != nbdns.UDPNameServerType {
 				log.Warnf("skiping nameserver %s with type %s, this peer supports only %s",
@@ -315,6 +322,7 @@ func (s *DefaultServer) buildUpstreamHandlerUpdate(nameServerGroups []*nbdns.Nam
 		}
 
 		if len(handler.upstreamServers) == 0 {
+			handler.stop()
 			log.Errorf("received a nameserver group with an invalid nameserver list")
 			continue
 		}
@@ -338,11 +346,13 @@ func (s *DefaultServer) buildUpstreamHandlerUpdate(nameServerGroups []*nbdns.Nam
 		}
 
 		if len(nsGroup.Domains) == 0 {
+			handler.stop()
 			return nil, fmt.Errorf("received a non primary nameserver group with an empty domain list")
 		}
 
 		for _, domain := range nsGroup.Domains {
 			if domain == "" {
+				handler.stop()
 				return nil, fmt.Errorf("received a nameserver group with an empty domain element")
 			}
 			muxUpdates = append(muxUpdates, muxUpdate{
@@ -355,16 +365,20 @@ func (s *DefaultServer) buildUpstreamHandlerUpdate(nameServerGroups []*nbdns.Nam
 }
 
 func (s *DefaultServer) updateMux(muxUpdates []muxUpdate) {
-	muxUpdateMap := make(registrationMap)
+	muxUpdateMap := make(registeredHandlerMap)
 
 	for _, update := range muxUpdates {
 		s.registerMux(update.domain, update.handler)
-		muxUpdateMap[update.domain] = struct{}{}
+		muxUpdateMap[update.domain] = update.handler
+		if existingHandler, ok := s.dnsMuxMap[update.domain]; ok {
+			existingHandler.stop()
+		}
 	}
 
-	for key := range s.dnsMuxMap {
+	for key, existingHandler := range s.dnsMuxMap {
 		_, found := muxUpdateMap[key]
 		if !found {
+			existingHandler.stop()
 			s.deregisterMux(key)
 		}
 	}

client/internal/dns/server_test.go

@@ -41,21 +41,23 @@ func TestUpdateDNSServer(t *testing.T) {
 		},
 	}
 
+	dummyHandler := &localResolver{}
+
 	testCases := []struct {
 		name                string
-		initUpstreamMap     registrationMap
+		initUpstreamMap     registeredHandlerMap
 		initLocalMap        registrationMap
 		initSerial          uint64
 		inputSerial         uint64
 		inputUpdate         nbdns.Config
 		shouldFail          bool
-		expectedUpstreamMap registrationMap
+		expectedUpstreamMap registeredHandlerMap
 		expectedLocalMap    registrationMap
 	}{
 		{
 			name:            "Initial Config Should Succeed",
 			initLocalMap:    make(registrationMap),
-			initUpstreamMap: make(registrationMap),
+			initUpstreamMap: make(registeredHandlerMap),
 			initSerial:      0,
 			inputSerial:     1,
 			inputUpdate: nbdns.Config{
@@ -77,13 +79,13 @@ func TestUpdateDNSServer(t *testing.T) {
 					},
 				},
 			},
-			expectedUpstreamMap: registrationMap{"netbird.io": struct{}{}, "netbird.cloud": struct{}{}, nbdns.RootZone: struct{}{}},
+			expectedUpstreamMap: registeredHandlerMap{"netbird.io": dummyHandler, "netbird.cloud": dummyHandler, nbdns.RootZone: dummyHandler},
 			expectedLocalMap:    registrationMap{buildRecordKey(zoneRecords[0].Name, 1, 1): struct{}{}},
 		},
 		{
 			name:            "New Config Should Succeed",
 			initLocalMap:    registrationMap{"netbird.cloud": struct{}{}},
-			initUpstreamMap: registrationMap{buildRecordKey(zoneRecords[0].Name, 1, 1): struct{}{}},
+			initUpstreamMap: registeredHandlerMap{buildRecordKey(zoneRecords[0].Name, 1, 1): dummyHandler},
 			initSerial:      0,
 			inputSerial:     1,
 			inputUpdate: nbdns.Config{
@@ -101,13 +103,13 @@ func TestUpdateDNSServer(t *testing.T) {
 					},
 				},
 			},
-			expectedUpstreamMap: registrationMap{"netbird.io": struct{}{}, "netbird.cloud": struct{}{}},
+			expectedUpstreamMap: registeredHandlerMap{"netbird.io": dummyHandler, "netbird.cloud": dummyHandler},
 			expectedLocalMap:    registrationMap{buildRecordKey(zoneRecords[0].Name, 1, 1): struct{}{}},
 		},
 		{
 			name:            "Smaller Config Serial Should Be Skipped",
 			initLocalMap:    make(registrationMap),
-			initUpstreamMap: make(registrationMap),
+			initUpstreamMap: make(registeredHandlerMap),
 			initSerial:      2,
 			inputSerial:     1,
 			shouldFail:      true,
@@ -115,7 +117,7 @@ func TestUpdateDNSServer(t *testing.T) {
 		{
 			name:            "Empty NS Group Domain Or Not Primary Element Should Fail",
 			initLocalMap:    make(registrationMap),
-			initUpstreamMap: make(registrationMap),
+			initUpstreamMap: make(registeredHandlerMap),
 			initSerial:      0,
 			inputSerial:     1,
 			inputUpdate: nbdns.Config{
@@ -137,7 +139,7 @@ func TestUpdateDNSServer(t *testing.T) {
 		{
 			name:            "Invalid NS Group Nameservers list Should Fail",
 			initLocalMap:    make(registrationMap),
-			initUpstreamMap: make(registrationMap),
+			initUpstreamMap: make(registeredHandlerMap),
 			initSerial:      0,
 			inputSerial:     1,
 			inputUpdate: nbdns.Config{
@@ -159,7 +161,7 @@ func TestUpdateDNSServer(t *testing.T) {
 		{
 			name:            "Invalid Custom Zone Records list Should Fail",
 			initLocalMap:    make(registrationMap),
-			initUpstreamMap: make(registrationMap),
+			initUpstreamMap: make(registeredHandlerMap),
 			initSerial:      0,
 			inputSerial:     1,
 			inputUpdate: nbdns.Config{
@@ -181,21 +183,21 @@ func TestUpdateDNSServer(t *testing.T) {
 		{
 			name:                "Empty Config Should Succeed and Clean Maps",
 			initLocalMap:        registrationMap{"netbird.cloud": struct{}{}},
-			initUpstreamMap:     registrationMap{zoneRecords[0].Name: struct{}{}},
+			initUpstreamMap:     registeredHandlerMap{zoneRecords[0].Name: dummyHandler},
 			initSerial:          0,
 			inputSerial:         1,
 			inputUpdate:         nbdns.Config{ServiceEnable: true},
-			expectedUpstreamMap: make(registrationMap),
+			expectedUpstreamMap: make(registeredHandlerMap),
 			expectedLocalMap:    make(registrationMap),
 		},
 		{
 			name:                "Disabled Service Should clean map",
 			initLocalMap:        registrationMap{"netbird.cloud": struct{}{}},
-			initUpstreamMap:     registrationMap{zoneRecords[0].Name: struct{}{}},
+			initUpstreamMap:     registeredHandlerMap{zoneRecords[0].Name: dummyHandler},
 			initSerial:          0,
 			inputSerial:         1,
 			inputUpdate:         nbdns.Config{ServiceEnable: false},
-			expectedUpstreamMap: make(registrationMap),
+			expectedUpstreamMap: make(registeredHandlerMap),
 			expectedLocalMap:    make(registrationMap),
 		},
 	}
@@ -431,7 +433,7 @@ func getDefaultServerWithNoHostManager(t *testing.T, addrPort string) *DefaultSe
 		ctxCancel: cancel,
 		server:    dnsServer,
 		dnsMux:    mux,
-		dnsMuxMap: make(registrationMap),
+		dnsMuxMap: make(registeredHandlerMap),
 		localResolver: &localResolver{
 			registeredMap: make(registrationMap),
 		},

client/internal/dns/systemd_linux.go

@@ -9,10 +9,11 @@ import (
 
 	"github.com/godbus/dbus/v5"
 	"github.com/miekg/dns"
-	nbdns "github.com/netbirdio/netbird/dns"
-	"github.com/netbirdio/netbird/iface"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
+
+	nbdns "github.com/netbirdio/netbird/dns"
+	"github.com/netbirdio/netbird/iface"
 )
 
 const (
@@ -75,6 +76,10 @@ func newSystemdDbusConfigurator(wgInterface *iface.WGIface) (hostManager, error)
 	}, nil
 }
 
+func (s *systemdDbusConfigurator) supportCustomPort() bool {
+	return true
+}
+
 func (s *systemdDbusConfigurator) applyDNSConfig(config hostDNSConfig) error {
 	parsedIP, err := netip.ParseAddr(config.serverIP)
 	if err != nil {

client/internal/dns/upstream.go

@@ -3,24 +3,31 @@ package dns
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"github.com/cenkalti/backoff/v4"
 	"github.com/miekg/dns"
 	log "github.com/sirupsen/logrus"
 )
 
 const (
-	failsTillDeact   = int32(3)
-	reactivatePeriod = time.Minute
+	failsTillDeact   = int32(5)
+	reactivatePeriod = 30 * time.Second
 	upstreamTimeout  = 15 * time.Second
 )
 
+type upstreamClient interface {
+	ExchangeContext(ctx context.Context, m *dns.Msg, a string) (r *dns.Msg, rtt time.Duration, err error)
+}
+
 type upstreamResolver struct {
 	ctx              context.Context
-	upstreamClient   *dns.Client
+	cancel           context.CancelFunc
+	upstreamClient   upstreamClient
 	upstreamServers  []string
 	disabled         bool
 	failsCount       atomic.Int32
@@ -33,9 +40,11 @@ type upstreamResolver struct {
 	reactivate func()
 }
 
-func newUpstreamResolver(ctx context.Context) *upstreamResolver {
+func newUpstreamResolver(parentCTX context.Context) *upstreamResolver {
+	ctx, cancel := context.WithCancel(parentCTX)
 	return &upstreamResolver{
 		ctx:              ctx,
+		cancel:           cancel,
 		upstreamClient:   &dns.Client{},
 		upstreamTimeout:  upstreamTimeout,
 		reactivatePeriod: reactivatePeriod,
@@ -43,6 +52,11 @@ func newUpstreamResolver(ctx context.Context) *upstreamResolver {
 	}
 }
 
+func (u *upstreamResolver) stop() {
+	log.Debugf("stoping serving DNS for upstreams %s", u.upstreamServers)
+	u.cancel()
+}
+
 // ServeDNS handles a DNS request
 func (u *upstreamResolver) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 	defer u.checkUpstreamFails()
@@ -107,28 +121,57 @@ func (u *upstreamResolver) checkUpstreamFails() {
 		log.Warnf("upstream resolving is disabled for %v", reactivatePeriod)
 		u.deactivate()
 		u.disabled = true
-		go u.waitUntilReactivation()
+		go u.waitUntilResponse()
 	}
 }
 
-// waitUntilReactivation reset fails counter and activates upstream resolving
-func (u *upstreamResolver) waitUntilReactivation() {
-	timer := time.NewTimer(u.reactivatePeriod)
-	defer func() {
-		if !timer.Stop() {
-			<-timer.C
-		}
-	}()
-
-	select {
-	case <-u.ctx.Done():
-		return
-	case <-timer.C:
-		log.Info("upstream resolving is reactivated")
-		u.failsCount.Store(0)
-		u.reactivate()
-		u.disabled = false
+// waitUntilResponse retries, in an exponential interval, querying the upstream servers until it gets a positive response
+func (u *upstreamResolver) waitUntilResponse() {
+	exponentialBackOff := &backoff.ExponentialBackOff{
+		InitialInterval:     500 * time.Millisecond,
+		RandomizationFactor: 0.5,
+		Multiplier:          1.1,
+		MaxInterval:         u.reactivatePeriod,
+		MaxElapsedTime:      0,
+		Stop:                backoff.Stop,
+		Clock:               backoff.SystemClock,
 	}
+
+	r := new(dns.Msg).SetQuestion("netbird.io.", dns.TypeA)
+
+	operation := func() error {
+		select {
+		case <-u.ctx.Done():
+			return backoff.Permanent(fmt.Errorf("exiting upstream retry loop for upstreams %s: parent context has been canceled", u.upstreamServers))
+		default:
+		}
+
+		var err error
+		for _, upstream := range u.upstreamServers {
+			ctx, cancel := context.WithTimeout(u.ctx, u.upstreamTimeout)
+			_, _, err = u.upstreamClient.ExchangeContext(ctx, r, upstream)
+
+			cancel()
+
+			if err == nil {
+				return nil
+			}
+		}
+
+		log.Tracef("checking connectivity with upstreams %s failed with error: %s. Retrying in %s", err, u.upstreamServers, exponentialBackOff.NextBackOff())
+		return fmt.Errorf("got an error from upstream check call")
+	}
+
+	err := backoff.Retry(operation, exponentialBackOff)
+	if err != nil {
+		log.Warn(err)
+		return
+	}
+
+	log.Infof("upstreams %s are responsive again. Adding them back to system", u.upstreamServers)
+	u.failsCount.Store(0)
+	u.reactivate()
+	u.disabled = false
 }
 
 // isTimeout returns true if the given error is a network timeout error.

client/internal/dns/upstream_test.go

@@ -2,10 +2,11 @@ package dns
 
 import (
 	"context"
-	"github.com/miekg/dns"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/miekg/dns"
 )
 
 func TestUpstreamResolver_ServeDNS(t *testing.T) {
@@ -106,8 +107,29 @@ func TestUpstreamResolver_ServeDNS(t *testing.T) {
 	}
 }
 
+type mockUpstreamResolver struct {
+	r   *dns.Msg
+	rtt time.Duration
+	err error
+}
+
+// ExchangeContext mock implementation of ExchangeContext from upstreamResolver
+func (c mockUpstreamResolver) ExchangeContext(_ context.Context, _ *dns.Msg, _ string) (r *dns.Msg, rtt time.Duration, err error) {
+	return c.r, c.rtt, c.err
+}
+
 func TestUpstreamResolver_DeactivationReactivation(t *testing.T) {
-	resolver := newUpstreamResolver(context.TODO())
+	resolver := &upstreamResolver{
+		ctx: context.TODO(),
+		upstreamClient: &mockUpstreamResolver{
+			err: nil,
+			r:   new(dns.Msg),
+			rtt: time.Millisecond,
+		},
+		upstreamTimeout:  upstreamTimeout,
+		reactivatePeriod: reactivatePeriod,
+		failsTillDeact:   failsTillDeact,
+	}
 	resolver.upstreamServers = []string{"0.0.0.0:-1"}
 	resolver.failsTillDeact = 0
 	resolver.reactivatePeriod = time.Microsecond * 100

client/internal/engine_test.go

@@ -3,8 +3,6 @@ package internal
 import (
 	"context"
 	"fmt"
-	"github.com/netbirdio/netbird/iface/bind"
-	"github.com/pion/transport/v2/stdnet"
 	"net"
 	"net/netip"
 	"os"
@@ -15,6 +13,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/pion/transport/v2/stdnet"
+
+	"github.com/netbirdio/netbird/iface/bind"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -1039,7 +1041,7 @@ func startManagement(dataDir string) (*grpc.Server, string, error) {
 		return nil, "", err
 	}
 	s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp))
-	store, err := server.NewFileStore(config.Datadir)
+	store, err := server.NewFileStore(config.Datadir, nil)
 	if err != nil {
 		log.Fatalf("failed creating a store: %s: %v", config.Datadir, err)
 	}

client/internal/routemanager/systemops_linux.go

@@ -62,6 +62,16 @@ func removeFromRouteTable(prefix netip.Prefix) error {
 }
 
 func enableIPForwarding() error {
-	err := os.WriteFile(ipv4ForwardingPath, []byte("1"), 0644)
-	return err
+	bytes, err := os.ReadFile(ipv4ForwardingPath)
+	if err != nil {
+		return err
+	}
+
+	// check if it is already enabled
+	// see more: https://github.com/netbirdio/netbird/issues/872
+	if len(bytes) > 0 && bytes[0] == 49 {
+		return nil
+	}
+
+	return os.WriteFile(ipv4ForwardingPath, []byte("1"), 0644)
 }

client/main.go

@@ -1,8 +1,9 @@
 package main
 
 import (
-	"github.com/netbirdio/netbird/client/cmd"
 	"os"
+
+	"github.com/netbirdio/netbird/client/cmd"
 )
 
 func main() {

client/server/server.go

@@ -236,7 +236,9 @@ func (s *Server) Login(callerCtx context.Context, msg *proto.LoginRequest) (*pro
 				}, nil
 			} else {
 				log.Warnf("canceling previous waiting execution")
-				s.oauthAuthFlow.waitCancel()
+				if s.oauthAuthFlow.waitCancel != nil {
+					s.oauthAuthFlow.waitCancel()
+				}
 			}
 		}
 

go.mod

@@ -21,14 +21,13 @@ require (
 	golang.org/x/sys v0.6.0
 	golang.zx2c4.com/wireguard v0.0.0-20230223181233-21636207a675
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20211215182854-7a385b3431de
-	golang.zx2c4.com/wireguard/windows v0.5.1
+	golang.zx2c4.com/wireguard/windows v0.5.3
 	google.golang.org/grpc v1.52.3
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
 require (
-	codeberg.org/ac/base62 v0.0.0-20210305150220-e793b546833a
 	fyne.io/fyne/v2 v2.1.4
 	github.com/c-robinson/iplib v1.0.3
 	github.com/coreos/go-iptables v0.6.0
@@ -57,6 +56,7 @@ require (
 	github.com/rs/xid v1.3.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/stretchr/testify v1.8.1
+	go.opentelemetry.io/otel v1.11.1
 	go.opentelemetry.io/otel/exporters/prometheus v0.33.0
 	go.opentelemetry.io/otel/metric v0.33.0
 	go.opentelemetry.io/otel/sdk/metric v0.33.0
@@ -124,7 +124,6 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/yashtewari/glob-intersection v0.1.0 // indirect
 	github.com/yuin/goldmark v1.4.13 // indirect
-	go.opentelemetry.io/otel v1.11.1 // indirect
 	go.opentelemetry.io/otel/sdk v1.11.1 // indirect
 	go.opentelemetry.io/otel/trace v1.11.1 // indirect
 	golang.org/x/image v0.0.0-20200430140353-33d19683fad8 // indirect
@@ -144,4 +143,4 @@ replace github.com/kardianos/service => github.com/netbirdio/service v0.0.0-2023
 
 replace github.com/getlantern/systray => github.com/netbirdio/systray v0.0.0-20221012095658-dc8eda872c0c
 
-replace golang.zx2c4.com/wireguard => github.com/netbirdio/wireguard-go v0.0.0-20230426151838-5c7986a94d53
+replace golang.zx2c4.com/wireguard => github.com/netbirdio/wireguard-go v0.0.0-20230524172305-5a498a82b33f

go.sum

@@ -31,8 +31,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-codeberg.org/ac/base62 v0.0.0-20210305150220-e793b546833a h1:U6cY/g6VSiy59vuvnBU6J/eSir0qVg4BeTnCDLaX+20=
-codeberg.org/ac/base62 v0.0.0-20210305150220-e793b546833a/go.mod h1:ykEpkLT4JtH3I4Rb4gwkDsNLfgUg803qRDeIX88t3e8=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 fyne.io/fyne/v2 v2.1.4 h1:bt1+28++kAzRzPB0GM2EuSV4cnl8rXNX4cjfd8G06Rc=
 fyne.io/fyne/v2 v2.1.4/go.mod h1:p+E/Dh+wPW8JwR2DVcsZ9iXgR9ZKde80+Y+40Is54AQ=
@@ -487,8 +485,8 @@ github.com/netbirdio/service v0.0.0-20230215170314-b923b89432b0 h1:hirFRfx3grVA/
 github.com/netbirdio/service v0.0.0-20230215170314-b923b89432b0/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
 github.com/netbirdio/systray v0.0.0-20221012095658-dc8eda872c0c h1:wK/s4nyZj/GF/kFJQjX6nqNfE0G3gcqd6hhnPCyp4sw=
 github.com/netbirdio/systray v0.0.0-20221012095658-dc8eda872c0c/go.mod h1:AecygODWIsBquJCJFop8MEQcJbWFfw/1yWbVabNgpCM=
-github.com/netbirdio/wireguard-go v0.0.0-20230426151838-5c7986a94d53 h1:OPbKpisDyMbOf/TDYS0Niw7yc/uoviED/pKyO+8A1C0=
-github.com/netbirdio/wireguard-go v0.0.0-20230426151838-5c7986a94d53/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
+github.com/netbirdio/wireguard-go v0.0.0-20230524172305-5a498a82b33f h1:WQXGYCKPkNs1KusFTLieV73UVTNfZVyez4CFRvlOruM=
+github.com/netbirdio/wireguard-go v0.0.0-20230524172305-5a498a82b33f/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -1039,8 +1037,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu
 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20211215182854-7a385b3431de h1:qDZ+lyO5jC9RNJ7ANJA0GWXk3pSn0Fu5SlcAIlgw+6w=
 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20211215182854-7a385b3431de/go.mod h1:Q2XNgour4QSkFj0BWCkVlW0HWJwQgNMsMahpSlI0Eno=
-golang.zx2c4.com/wireguard/windows v0.5.1 h1:OnYw96PF+CsIMrqWo5QP3Q59q5hY1rFErk/yN3cS+JQ=
-golang.zx2c4.com/wireguard/windows v0.5.1/go.mod h1:EApyTk/ZNrkbZjurHL1nleDYnsPpJYBO7LZEBCyDAHk=
+golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
+golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=

iface/bind/udp_mux_universal.go

@@ -209,7 +209,7 @@ func (m *UniversalUDPMuxDefault) GetXORMappedAddr(serverAddr net.Addr, deadline
 
 	// otherwise, make a STUN request to discover the address
 	// or wait for already sent request to complete
-	waitAddrReceived, err := m.sendStun(serverAddr)
+	waitAddrReceived, err := m.sendSTUN(serverAddr)
 	if err != nil {
 		return nil, fmt.Errorf("%s: %s", "failed to send STUN packet", err)
 	}
@@ -218,23 +218,31 @@ func (m *UniversalUDPMuxDefault) GetXORMappedAddr(serverAddr net.Addr, deadline
 	select {
 	case <-waitAddrReceived:
 		// when channel closed, addr was obtained
+		var addr *stun.XORMappedAddress
 		m.mu.Lock()
-		mappedAddr := *m.xorMappedMap[serverAddr.String()]
+		// A very odd case that mappedAddr is nil.
+		// Can happen when the deadline property is larger than params.XORMappedAddrCacheTTL.
+		// Or when we don't receive a response to our m.sendSTUN request (the response is handled asynchronously) and
+		// the XORMapped expires meanwhile triggering a closure of the waitAddrReceived channel.
+		// We protect the code from panic here.
+		if mappedAddr, ok := m.xorMappedMap[serverAddr.String()]; ok {
+			addr = mappedAddr.addr
+		}
 		m.mu.Unlock()
-		if mappedAddr.addr == nil {
+		if addr == nil {
 			return nil, fmt.Errorf("no XOR address mapping")
 		}
-		return mappedAddr.addr, nil
+		return addr, nil
 	case <-time.After(deadline):
 		return nil, fmt.Errorf("timeout while waiting for XORMappedAddr")
 	}
 }
 
-// sendStun sends a STUN request via UDP conn.
+// sendSTUN sends a STUN request via UDP conn.
 //
 // The returned channel is closed when the STUN response has been received.
 // Method is safe for concurrent use.
-func (m *UniversalUDPMuxDefault) sendStun(serverAddr net.Addr) (chan struct{}, error) {
+func (m *UniversalUDPMuxDefault) sendSTUN(serverAddr net.Addr) (chan struct{}, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 

iface/tun_windows.go

@@ -3,9 +3,11 @@ package iface
 import (
 	"fmt"
 	"net"
+	"net/netip"
 
 	"github.com/pion/transport/v2"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/sys/windows"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/ipc"
 	"golang.zx2c4.com/wireguard/tun"
@@ -50,7 +52,6 @@ func (c *tunDevice) createWithUserspace() (NetInterface, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	// We need to create a wireguard-go device and listen to configuration requests
 	tunDev := device.NewDevice(tunIface, c.iceBind, device.NewLogger(device.LogLevelSilent, "[netbird] "))
 	err = tunDev.Up()
@@ -59,6 +60,22 @@ func (c *tunDevice) createWithUserspace() (NetInterface, error) {
 		return nil, err
 	}
 
+	luid := winipcfg.LUID(tunIface.(*tun.NativeTun).LUID())
+
+	nbiface, err := luid.IPInterface(windows.AF_INET)
+	if err != nil {
+		_ = tunIface.Close()
+		return nil, fmt.Errorf("got error when getting ip interface %s", err)
+	}
+
+	nbiface.NLMTU = uint32(c.mtu)
+
+	err = nbiface.Set()
+	if err != nil {
+		_ = tunIface.Close()
+		return nil, fmt.Errorf("got error when getting setting the interface mtu: %s", err)
+	}
+
 	c.uapi, err = c.getUAPI(c.name)
 	if err != nil {
 		_ = tunIface.Close()
@@ -142,7 +159,7 @@ func (c *tunDevice) assignAddr() error {
 	tunDev := c.netInterface.(*tun.NativeTun)
 	luid := winipcfg.LUID(tunDev.LUID())
 	log.Debugf("adding address %s to interface: %s", c.address.IP, c.name)
-	return luid.SetIPAddresses([]net.IPNet{{c.address.IP, c.address.Network.Mask}})
+	return luid.SetIPAddresses([]netip.Prefix{netip.MustParsePrefix(c.address.String())})
 }
 
 // getUAPI returns a Listener

management/client/client_test.go

@@ -2,28 +2,31 @@ package client
 
 import (
 	"context"
-	"github.com/netbirdio/netbird/management/server/activity"
 	"net"
 	"path/filepath"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/netbirdio/netbird/management/server/activity"
+
 	"github.com/netbirdio/netbird/client/system"
 
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/netbirdio/netbird/encryption"
 	"github.com/netbirdio/netbird/management/proto"
 	mgmtProto "github.com/netbirdio/netbird/management/proto"
 	mgmt "github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/mock_server"
-	log "github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/assert"
 
-	"github.com/netbirdio/netbird/util"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+
+	"github.com/netbirdio/netbird/util"
 )
 
 const ValidKey = "A2C8E62B-38F5-4553-B31E-DD66C696CEBB"
@@ -50,7 +53,7 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) {
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	store, err := mgmt.NewFileStore(config.Datadir)
+	store, err := mgmt.NewFileStore(config.Datadir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

management/cmd/management.go

@@ -121,13 +121,6 @@ var (
 					return fmt.Errorf("failed creating datadir: %s: %v", config.Datadir, err)
 				}
 			}
-
-			store, err := server.NewFileStore(config.Datadir)
-			if err != nil {
-				return fmt.Errorf("failed creating Store: %s: %v", config.Datadir, err)
-			}
-			peersUpdateManager := server.NewPeersUpdateManager()
-
 			appMetrics, err := telemetry.NewDefaultAppMetrics(cmd.Context())
 			if err != nil {
 				return err
@@ -136,6 +129,11 @@ var (
 			if err != nil {
 				return err
 			}
+			store, err := server.NewFileStore(config.Datadir, appMetrics)
+			if err != nil {
+				return fmt.Errorf("failed creating Store: %s: %v", config.Datadir, err)
+			}
+			peersUpdateManager := server.NewPeersUpdateManager()
 
 			var idpManager idp.Manager
 			if config.IdpManagerConfig != nil {

management/server/account.go

@@ -15,13 +15,13 @@ import (
 	"sync"
 	"time"
 
-	"codeberg.org/ac/base62"
 	"github.com/eko/gocache/v3/cache"
 	cacheStore "github.com/eko/gocache/v3/store"
 	gocache "github.com/patrickmn/go-cache"
 	"github.com/rs/xid"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/netbirdio/netbird/base62"
 	nbdns "github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/management/server/activity"
 	"github.com/netbirdio/netbird/management/server/idp"

management/server/account_test.go

@@ -1869,7 +1869,7 @@ func createManager(t *testing.T) (*DefaultAccountManager, error) {
 
 func createStore(t *testing.T) (Store, error) {
 	dataDir := t.TempDir()
-	store, err := NewFileStore(dataDir)
+	store, err := NewFileStore(dataDir, nil)
 	if err != nil {
 		return nil, err
 	}

management/server/dns_test.go

@@ -1,10 +1,12 @@
 package server
 
 import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
 	"github.com/netbirdio/netbird/management/server/activity"
 	"github.com/netbirdio/netbird/management/server/status"
-	"github.com/stretchr/testify/require"
-	"testing"
 )
 
 const (
@@ -190,7 +192,7 @@ func createDNSManager(t *testing.T) (*DefaultAccountManager, error) {
 
 func createDNSStore(t *testing.T) (Store, error) {
 	dataDir := t.TempDir()
-	store, err := NewFileStore(dataDir)
+	store, err := NewFileStore(dataDir, nil)
 	if err != nil {
 		return nil, err
 	}

management/server/file_store.go

@@ -11,6 +11,7 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	"github.com/netbirdio/netbird/management/server/status"
+	"github.com/netbirdio/netbird/management/server/telemetry"
 
 	"github.com/netbirdio/netbird/util"
 )
@@ -37,13 +38,20 @@ type FileStore struct {
 	// sync.Mutex indexed by accountID
 	accountLocks      sync.Map   `json:"-"`
 	globalAccountLock sync.Mutex `json:"-"`
+
+	metrics telemetry.AppMetrics `json:"-"`
 }
 
 type StoredAccount struct{}
 
 // NewFileStore restores a store from the file located in the datadir
-func NewFileStore(dataDir string) (*FileStore, error) {
-	return restore(filepath.Join(dataDir, storeFileName))
+func NewFileStore(dataDir string, metrics telemetry.AppMetrics) (*FileStore, error) {
+	fs, err := restore(filepath.Join(dataDir, storeFileName))
+	if err != nil {
+		return nil, err
+	}
+	fs.metrics = metrics
+	return fs, nil
 }
 
 // restore the state of the store from the file.
@@ -221,7 +229,17 @@ func restore(file string) (*FileStore, error) {
 // persist account data to a file
 // It is recommended to call it with locking FileStore.mux
 func (s *FileStore) persist(file string) error {
-	return util.WriteJson(file, s)
+	start := time.Now()
+	err := util.WriteJson(file, s)
+	if err != nil {
+		return err
+	}
+	took := time.Since(start)
+	if s.metrics != nil {
+		s.metrics.StoreMetrics().CountPersistenceDuration(took)
+	}
+	log.Debugf("took %d ms to persist the FileStore", took.Milliseconds())
+	return nil
 }
 
 // AcquireGlobalLock acquires global lock across all the accounts and returns a function that releases the lock
@@ -235,6 +253,12 @@ func (s *FileStore) AcquireGlobalLock() (unlock func()) {
 		log.Debugf("released global lock in %v", time.Since(start))
 	}
 
+	took := time.Since(start)
+	log.Debugf("took %v to acquire global lock", took)
+	if s.metrics != nil {
+		s.metrics.StoreMetrics().CountGlobalLockAcquisitionDuration(took)
+	}
+
 	return unlock
 }
 

management/server/file_store_test.go

@@ -25,7 +25,7 @@ func TestStalePeerIndices(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		return
 	}
@@ -172,7 +172,7 @@ func TestStore(t *testing.T) {
 		return
 	}
 
-	restored, err := NewFileStore(store.storeFile)
+	restored, err := NewFileStore(store.storeFile, nil)
 	if err != nil {
 		return
 	}
@@ -232,7 +232,7 @@ func TestRestore(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		return
 	}
@@ -270,7 +270,7 @@ func TestRestorePolicies_Migration(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		return
 	}
@@ -307,7 +307,7 @@ func TestGetAccountByPrivateDomain(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		return
 	}
@@ -336,7 +336,7 @@ func TestFileStore_GetAccount(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -378,7 +378,7 @@ func TestFileStore_GetTokenIDByHashedToken(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -431,7 +431,7 @@ func TestFileStore_GetTokenIDByHashedToken_Failure(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -456,7 +456,7 @@ func TestFileStore_GetUserByTokenID(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -484,7 +484,7 @@ func TestFileStore_GetUserByTokenID_Failure(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -503,7 +503,7 @@ func TestFileStore_SavePeerStatus(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		return
 	}
@@ -548,7 +548,7 @@ func TestFileStore_SavePeerStatus(t *testing.T) {
 }
 
 func newStore(t *testing.T) *FileStore {
-	store, err := NewFileStore(t.TempDir())
+	store, err := NewFileStore(t.TempDir(), nil)
 	if err != nil {
 		t.Errorf("failed creating a new store")
 	}

management/server/grpcserver.go

@@ -114,6 +114,7 @@ func (s *GRPCServer) GetServerKey(ctx context.Context, req *proto.Empty) (*proto
 // Sync validates the existence of a connecting peer, sends an initial state (all available for the connecting peers) and
 // notifies the connected peer of any updates (e.g. new peers under the same account)
 func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementService_SyncServer) error {
+	reqStart := time.Now()
 	if s.appMetrics != nil {
 		s.appMetrics.GRPCMetrics().CountSyncRequest()
 	}
@@ -148,6 +149,11 @@ func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementServi
 	if s.config.TURNConfig.TimeBasedCredentials {
 		s.turnCredentialsManager.SetupRefresh(peer.ID)
 	}
+
+	if s.appMetrics != nil {
+		s.appMetrics.GRPCMetrics().CountSyncRequestDuration(time.Since(reqStart))
+	}
+
 	// keep a connection to the peer and send updates when available
 	for {
 		select {
@@ -262,6 +268,12 @@ func (s *GRPCServer) parseRequest(req *proto.EncryptedMessage, parsed pb.Message
 // In case it isn't, the endpoint checks whether setup key is provided within the request and tries to register a peer.
 // In case of the successful registration login is also successful
 func (s *GRPCServer) Login(ctx context.Context, req *proto.EncryptedMessage) (*proto.EncryptedMessage, error) {
+	reqStart := time.Now()
+	defer func() {
+		if s.appMetrics != nil {
+			s.appMetrics.GRPCMetrics().CountLoginRequestDuration(time.Since(reqStart))
+		}
+	}()
 	if s.appMetrics != nil {
 		s.appMetrics.GRPCMetrics().CountLoginRequest()
 	}

management/server/http/api/types.gen.go

@@ -30,6 +30,8 @@ const (
 	EventActivityCodePeerRename                               EventActivityCode = "peer.rename"
 	EventActivityCodePeerSshDisable                           EventActivityCode = "peer.ssh.disable"
 	EventActivityCodePeerSshEnable                            EventActivityCode = "peer.ssh.enable"
+	EventActivityCodePersonalAccessTokenCreate                EventActivityCode = "personal.access.token.create"
+	EventActivityCodePersonalAccessTokenDelete                EventActivityCode = "personal.access.token.delete"
 	EventActivityCodePolicyAdd                                EventActivityCode = "policy.add"
 	EventActivityCodePolicyDelete                             EventActivityCode = "policy.delete"
 	EventActivityCodePolicyUpdate                             EventActivityCode = "policy.update"
@@ -39,6 +41,8 @@ const (
 	EventActivityCodeRuleAdd                                  EventActivityCode = "rule.add"
 	EventActivityCodeRuleDelete                               EventActivityCode = "rule.delete"
 	EventActivityCodeRuleUpdate                               EventActivityCode = "rule.update"
+	EventActivityCodeServiceUserCreate                        EventActivityCode = "service.user.create"
+	EventActivityCodeServiceUserDelete                        EventActivityCode = "service.user.delete"
 	EventActivityCodeSetupkeyAdd                              EventActivityCode = "setupkey.add"
 	EventActivityCodeSetupkeyGroupAdd                         EventActivityCode = "setupkey.group.add"
 	EventActivityCodeSetupkeyGroupDelete                      EventActivityCode = "setupkey.group.delete"
@@ -151,6 +155,15 @@ type GroupMinimum struct {
 	PeersCount int `json:"peers_count"`
 }
 
+// GroupRequest defines model for GroupRequest.
+type GroupRequest struct {
+	// Name Group name identifier
+	Name string `json:"name"`
+
+	// Peers List of peers ids
+	Peers *[]string `json:"peers,omitempty"`
+}
+
 // Nameserver defines model for Nameserver.
 type Nameserver struct {
 	// Ip Nameserver IP
@@ -277,6 +290,13 @@ type PeerMinimum struct {
 	Name string `json:"name"`
 }
 
+// PeerRequest defines model for PeerRequest.
+type PeerRequest struct {
+	LoginExpirationEnabled bool   `json:"login_expiration_enabled"`
+	Name                   string `json:"name"`
+	SshEnabled             bool   `json:"ssh_enabled"`
+}
+
 // PersonalAccessToken defines model for PersonalAccessToken.
 type PersonalAccessToken struct {
 	// CreatedAt Date the token was created
@@ -480,6 +500,27 @@ type RuleMinimum struct {
 	Name string `json:"name"`
 }
 
+// RuleRequest defines model for RuleRequest.
+type RuleRequest struct {
+	// Description Rule friendly description
+	Description string `json:"description"`
+
+	// Destinations List of destination groups
+	Destinations *[]string `json:"destinations,omitempty"`
+
+	// Disabled Rules status
+	Disabled bool `json:"disabled"`
+
+	// Flow Rule flow, currently, only "bidirect" for bi-directional traffic is accepted
+	Flow string `json:"flow"`
+
+	// Name Rule name identifier
+	Name string `json:"name"`
+
+	// Sources List of source groups
+	Sources *[]string `json:"sources,omitempty"`
+}
+
 // SetupKey defines model for SetupKey.
 type SetupKey struct {
 	// AutoGroups Setup key groups to auto-assign to peers registered with this key
@@ -611,65 +652,6 @@ type PutApiAccountsAccountIdJSONBody struct {
 	Settings AccountSettings `json:"settings"`
 }
 
-// PostApiGroupsJSONBody defines parameters for PostApiGroups.
-type PostApiGroupsJSONBody struct {
-	Name  string    `json:"name"`
-	Peers *[]string `json:"peers,omitempty"`
-}
-
-// PutApiGroupsGroupIdJSONBody defines parameters for PutApiGroupsGroupId.
-type PutApiGroupsGroupIdJSONBody struct {
-	Name  *string   `json:"Name,omitempty"`
-	Peers *[]string `json:"Peers,omitempty"`
-}
-
-// PutApiPeersPeerIdJSONBody defines parameters for PutApiPeersPeerId.
-type PutApiPeersPeerIdJSONBody struct {
-	LoginExpirationEnabled bool   `json:"login_expiration_enabled"`
-	Name                   string `json:"name"`
-	SshEnabled             bool   `json:"ssh_enabled"`
-}
-
-// PostApiPoliciesJSONBody defines parameters for PostApiPolicies.
-type PostApiPoliciesJSONBody = PolicyMinimum
-
-// PutApiPoliciesPolicyIdJSONBody defines parameters for PutApiPoliciesPolicyId.
-type PutApiPoliciesPolicyIdJSONBody = PolicyMinimum
-
-// PostApiRulesJSONBody defines parameters for PostApiRules.
-type PostApiRulesJSONBody struct {
-	// Description Rule friendly description
-	Description  string    `json:"description"`
-	Destinations *[]string `json:"destinations,omitempty"`
-
-	// Disabled Rules status
-	Disabled bool `json:"disabled"`
-
-	// Flow Rule flow, currently, only "bidirect" for bi-directional traffic is accepted
-	Flow string `json:"flow"`
-
-	// Name Rule name identifier
-	Name    string    `json:"name"`
-	Sources *[]string `json:"sources,omitempty"`
-}
-
-// PutApiRulesRuleIdJSONBody defines parameters for PutApiRulesRuleId.
-type PutApiRulesRuleIdJSONBody struct {
-	// Description Rule friendly description
-	Description  string    `json:"description"`
-	Destinations *[]string `json:"destinations,omitempty"`
-
-	// Disabled Rules status
-	Disabled bool `json:"disabled"`
-
-	// Flow Rule flow, currently, only "bidirect" for bi-directional traffic is accepted
-	Flow string `json:"flow"`
-
-	// Name Rule name identifier
-	Name    string    `json:"name"`
-	Sources *[]string `json:"sources,omitempty"`
-}
-
 // GetApiUsersParams defines parameters for GetApiUsers.
 type GetApiUsersParams struct {
 	// ServiceUser Filters users and returns either regular users or service users
@@ -689,19 +671,19 @@ type PutApiDnsNameserversNsgroupIdJSONRequestBody = NameserverGroupRequest
 type PutApiDnsSettingsJSONRequestBody = DNSSettings
 
 // PostApiGroupsJSONRequestBody defines body for PostApiGroups for application/json ContentType.
-type PostApiGroupsJSONRequestBody PostApiGroupsJSONBody
+type PostApiGroupsJSONRequestBody = GroupRequest
 
 // PutApiGroupsGroupIdJSONRequestBody defines body for PutApiGroupsGroupId for application/json ContentType.
-type PutApiGroupsGroupIdJSONRequestBody PutApiGroupsGroupIdJSONBody
+type PutApiGroupsGroupIdJSONRequestBody = GroupRequest
 
 // PutApiPeersPeerIdJSONRequestBody defines body for PutApiPeersPeerId for application/json ContentType.
-type PutApiPeersPeerIdJSONRequestBody PutApiPeersPeerIdJSONBody
+type PutApiPeersPeerIdJSONRequestBody = PeerRequest
 
 // PostApiPoliciesJSONRequestBody defines body for PostApiPolicies for application/json ContentType.
-type PostApiPoliciesJSONRequestBody = PostApiPoliciesJSONBody
+type PostApiPoliciesJSONRequestBody = PolicyMinimum
 
 // PutApiPoliciesPolicyIdJSONRequestBody defines body for PutApiPoliciesPolicyId for application/json ContentType.
-type PutApiPoliciesPolicyIdJSONRequestBody = PutApiPoliciesPolicyIdJSONBody
+type PutApiPoliciesPolicyIdJSONRequestBody = PolicyMinimum
 
 // PostApiRoutesJSONRequestBody defines body for PostApiRoutes for application/json ContentType.
 type PostApiRoutesJSONRequestBody = RouteRequest
@@ -710,10 +692,10 @@ type PostApiRoutesJSONRequestBody = RouteRequest
 type PutApiRoutesRouteIdJSONRequestBody = RouteRequest
 
 // PostApiRulesJSONRequestBody defines body for PostApiRules for application/json ContentType.
-type PostApiRulesJSONRequestBody PostApiRulesJSONBody
+type PostApiRulesJSONRequestBody = RuleRequest
 
 // PutApiRulesRuleIdJSONRequestBody defines body for PutApiRulesRuleId for application/json ContentType.
-type PutApiRulesRuleIdJSONRequestBody PutApiRulesRuleIdJSONBody
+type PutApiRulesRuleIdJSONRequestBody = RuleRequest
 
 // PostApiSetupKeysJSONRequestBody defines body for PostApiSetupKeys for application/json ContentType.
 type PostApiSetupKeysJSONRequestBody = SetupKeyRequest

management/server/http/groups_handler.go

@@ -95,7 +95,7 @@ func (h *GroupsHandler) UpdateGroup(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if *req.Name == "" {
+	if req.Name == "" {
 		util.WriteError(status.Errorf(status.InvalidArgument, "group name shouldn't be empty"), w)
 		return
 	}
@@ -108,7 +108,7 @@ func (h *GroupsHandler) UpdateGroup(w http.ResponseWriter, r *http.Request) {
 	}
 	group := server.Group{
 		ID:    groupID,
-		Name:  *req.Name,
+		Name:  req.Name,
 		Peers: peers,
 	}
 

management/server/http/peers_handler.go

@@ -42,7 +42,7 @@ func (h *PeersHandler) getPeer(account *server.Account, peerID, userID string, w
 }
 
 func (h *PeersHandler) updatePeer(account *server.Account, user *server.User, peerID string, w http.ResponseWriter, r *http.Request) {
-	req := &api.PutApiPeersPeerIdJSONBody{}
+	req := &api.PeerRequest{}
 	err := json.NewDecoder(r.Body).Decode(&req)
 	if err != nil {
 		util.WriteErrorResponse("couldn't parse JSON request", http.StatusBadRequest, w)

management/server/idp/auth0.go

@@ -122,7 +122,7 @@ func NewAuth0Manager(oidcConfig OIDCConfig, config Auth0ClientConfig,
 	}
 
 	helper := JsonParser{}
-	config.AuthIssuer = oidcConfig.TokenEndpoint
+	config.AuthIssuer = oidcConfig.Issuer
 	config.GrantType = "client_credentials"
 
 	if config.ClientID == "" {

management/server/management_proto_test.go

@@ -2,7 +2,6 @@ package server
 
 import (
 	"context"
-	"github.com/netbirdio/netbird/management/server/activity"
 	"net"
 	"os"
 	"path/filepath"
@@ -10,14 +9,17 @@ import (
 	"testing"
 	"time"
 
-	"github.com/netbirdio/netbird/encryption"
-	mgmtProto "github.com/netbirdio/netbird/management/proto"
-	"github.com/netbirdio/netbird/util"
+	"github.com/netbirdio/netbird/management/server/activity"
+
 	"github.com/stretchr/testify/require"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/keepalive"
+
+	"github.com/netbirdio/netbird/encryption"
+	mgmtProto "github.com/netbirdio/netbird/management/proto"
+	"github.com/netbirdio/netbird/util"
 )
 
 var (
@@ -408,7 +410,7 @@ func startManagement(t *testing.T, config *Config) (*grpc.Server, string, error)
 		return nil, "", err
 	}
 	s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp))
-	store, err := NewFileStore(config.Datadir)
+	store, err := NewFileStore(config.Datadir, nil)
 	if err != nil {
 		return nil, "", err
 	}

management/server/management_test.go

@@ -496,7 +496,7 @@ func startServer(config *server.Config) (*grpc.Server, net.Listener) {
 	Expect(err).NotTo(HaveOccurred())
 	s := grpc.NewServer()
 
-	store, err := server.NewFileStore(config.Datadir)
+	store, err := server.NewFileStore(config.Datadir, nil)
 	if err != nil {
 		log.Fatalf("failed creating a store: %s: %v", config.Datadir, err)
 	}

management/server/nameserver_test.go

@@ -1,11 +1,13 @@
 package server
 
 import (
-	nbdns "github.com/netbirdio/netbird/dns"
-	"github.com/netbirdio/netbird/management/server/activity"
-	"github.com/stretchr/testify/require"
 	"net/netip"
 	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	nbdns "github.com/netbirdio/netbird/dns"
+	"github.com/netbirdio/netbird/management/server/activity"
 )
 
 const (
@@ -1064,7 +1066,7 @@ func createNSManager(t *testing.T) (*DefaultAccountManager, error) {
 
 func createNSStore(t *testing.T) (Store, error) {
 	dataDir := t.TempDir()
-	store, err := NewFileStore(dataDir)
+	store, err := NewFileStore(dataDir, nil)
 	if err != nil {
 		return nil, err
 	}

management/server/peer.go

@@ -28,6 +28,17 @@ type PeerSystemMeta struct {
 	UIVersion string
 }
 
+func (p PeerSystemMeta) isEqual(other PeerSystemMeta) bool {
+	return p.Hostname == other.Hostname &&
+		p.GoOS == other.GoOS &&
+		p.Kernel == other.Kernel &&
+		p.Core == other.Core &&
+		p.Platform == other.Platform &&
+		p.OS == other.OS &&
+		p.WtVersion == other.WtVersion &&
+		p.UIVersion == other.UIVersion
+}
+
 type PeerStatus struct {
 	// LastSeen is the last time peer was connected to the management service
 	LastSeen time.Time
@@ -114,13 +125,19 @@ func (p *Peer) Copy() *Peer {
 	}
 }
 
-// UpdateMeta updates peer's system meta data
-func (p *Peer) UpdateMeta(meta PeerSystemMeta) {
+// UpdateMetaIfNew updates peer's system metadata if new information is provided
+// returns true if meta was updated, false otherwise
+func (p *Peer) UpdateMetaIfNew(meta PeerSystemMeta) bool {
 	// Avoid overwriting UIVersion if the update was triggered sole by the CLI client
 	if meta.UIVersion == "" {
 		meta.UIVersion = p.Meta.UIVersion
 	}
+
+	if p.Meta.isEqual(meta) {
+		return false
+	}
 	p.Meta = meta
+	return true
 }
 
 // MarkLoginExpired marks peer's status expired or not
@@ -654,6 +671,8 @@ func (am *DefaultAccountManager) LoginPeer(login PeerLogin) (*Peer, *NetworkMap,
 		return nil, nil, err
 	}
 
+	// this flag prevents unnecessary calls to the persistent store.
+	shouldStoreAccount := false
 	updateRemotePeers := false
 	if peerLoginExpired(peer, account) {
 		err = checkAuth(login.UserID, peer)
@@ -664,19 +683,26 @@ func (am *DefaultAccountManager) LoginPeer(login PeerLogin) (*Peer, *NetworkMap,
 		// UserID is present, meaning that JWT validation passed successfully in the API layer.
 		updatePeerLastLogin(peer, account)
 		updateRemotePeers = true
+		shouldStoreAccount = true
 	}
 
-	peer = updatePeerMeta(peer, login.Meta, account)
+	peer, updated := updatePeerMeta(peer, login.Meta, account)
+	if updated {
+		shouldStoreAccount = true
+	}
 
 	peer, err = am.checkAndUpdatePeerSSHKey(peer, account, login.SSHKey)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	err = am.Store.SaveAccount(account)
-	if err != nil {
-		return nil, nil, err
+	if shouldStoreAccount {
+		err = am.Store.SaveAccount(account)
+		if err != nil {
+			return nil, nil, err
+		}
 	}
+
 	if updateRemotePeers {
 		err = am.updateAccountPeers(account)
 		if err != nil {
@@ -850,10 +876,12 @@ func (am *DefaultAccountManager) GetPeer(accountID, peerID, userID string) (*Pee
 	return nil, status.Errorf(status.Internal, "user %s has no access to peer %s under account %s", userID, peerID, accountID)
 }
 
-func updatePeerMeta(peer *Peer, meta PeerSystemMeta, account *Account) *Peer {
-	peer.UpdateMeta(meta)
-	account.UpdatePeer(peer)
-	return peer
+func updatePeerMeta(peer *Peer, meta PeerSystemMeta, account *Account) (*Peer, bool) {
+	if peer.UpdateMetaIfNew(meta) {
+		account.UpdatePeer(peer)
+		return peer, true
+	}
+	return peer, false
 }
 
 // GetPeerRules returns a list of source or destination rules of a given peer.

management/server/personal_access_token.go

@@ -7,9 +7,10 @@ import (
 	"hash/crc32"
 	"time"
 
-	"codeberg.org/ac/base62"
 	b "github.com/hashicorp/go-secure-stdlib/base62"
 	"github.com/rs/xid"
+
+	"github.com/netbirdio/netbird/base62"
 )
 
 const (

management/server/personal_access_token_test.go

@@ -4,11 +4,13 @@ import (
 	"crypto/sha256"
 	b64 "encoding/base64"
 	"hash/crc32"
+	"math/big"
 	"strings"
 	"testing"
 
-	"codeberg.org/ac/base62"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/netbirdio/netbird/base62"
 )
 
 func TestPAT_GenerateToken_Hashing(t *testing.T) {
@@ -33,6 +35,8 @@ func TestPAT_GenerateToken_Checksum(t *testing.T) {
 	secret := tokenWithoutPrefix[:len(tokenWithoutPrefix)-6]
 	tokenCheckSum := tokenWithoutPrefix[len(tokenWithoutPrefix)-6:]
 
+	var i big.Int
+	i.SetString(secret, 62)
 	expectedChecksum := crc32.ChecksumIEEE([]byte(secret))
 	actualChecksum, err := base62.Decode(tokenCheckSum)
 	if err != nil {

management/server/route_test.go

@@ -4,10 +4,11 @@ import (
 	"net/netip"
 	"testing"
 
-	"github.com/netbirdio/netbird/management/server/activity"
-	"github.com/netbirdio/netbird/route"
 	"github.com/rs/xid"
 	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/management/server/activity"
+	"github.com/netbirdio/netbird/route"
 )
 
 const (
@@ -946,7 +947,7 @@ func createRouterManager(t *testing.T) (*DefaultAccountManager, error) {
 
 func createRouterStore(t *testing.T) (Store, error) {
 	dataDir := t.TempDir()
-	store, err := NewFileStore(dataDir)
+	store, err := NewFileStore(dataDir, nil)
 	if err != nil {
 		return nil, err
 	}

management/server/telemetry/app_metrics.go

@@ -3,6 +3,10 @@ package telemetry
 import (
 	"context"
 	"fmt"
+	"net"
+	"net/http"
+	"reflect"
+
 	"github.com/gorilla/mux"
 	prometheus2 "github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -10,9 +14,6 @@ import (
 	"go.opentelemetry.io/otel/exporters/prometheus"
 	metric2 "go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/sdk/metric"
-	"net"
-	"net/http"
-	"reflect"
 )
 
 const defaultEndpoint = "/metrics"
@@ -25,6 +26,7 @@ type MockAppMetrics struct {
 	IDPMetricsFunc     func() *IDPMetrics
 	HTTPMiddlewareFunc func() *HTTPMiddleware
 	GRPCMetricsFunc    func() *GRPCMetrics
+	StoreMetricsFunc   func() *StoreMetrics
 }
 
 // GetMeter mocks the GetMeter function of the AppMetrics interface
@@ -75,6 +77,14 @@ func (mock *MockAppMetrics) GRPCMetrics() *GRPCMetrics {
 	return nil
 }
 
+// StoreMetrics mocks the MockAppMetrics function of the StoreMetrics interface
+func (mock *MockAppMetrics) StoreMetrics() *StoreMetrics {
+	if mock.StoreMetricsFunc != nil {
+		return mock.StoreMetricsFunc()
+	}
+	return nil
+}
+
 // AppMetrics is metrics interface
 type AppMetrics interface {
 	GetMeter() metric2.Meter
@@ -83,6 +93,7 @@ type AppMetrics interface {
 	IDPMetrics() *IDPMetrics
 	HTTPMiddleware() *HTTPMiddleware
 	GRPCMetrics() *GRPCMetrics
+	StoreMetrics() *StoreMetrics
 }
 
 // defaultAppMetrics are core application metrics based on OpenTelemetry https://opentelemetry.io/
@@ -94,6 +105,7 @@ type defaultAppMetrics struct {
 	idpMetrics     *IDPMetrics
 	httpMiddleware *HTTPMiddleware
 	grpcMetrics    *GRPCMetrics
+	storeMetrics   *StoreMetrics
 }
 
 // IDPMetrics returns metrics for the idp package
@@ -111,6 +123,11 @@ func (appMetrics *defaultAppMetrics) GRPCMetrics() *GRPCMetrics {
 	return appMetrics.grpcMetrics
 }
 
+// StoreMetrics returns metrics for the store
+func (appMetrics *defaultAppMetrics) StoreMetrics() *StoreMetrics {
+	return appMetrics.storeMetrics
+}
+
 // Close stop application metrics HTTP handler and closes listener.
 func (appMetrics *defaultAppMetrics) Close() error {
 	if appMetrics.listener == nil {
@@ -171,11 +188,17 @@ func NewDefaultAppMetrics(ctx context.Context) (AppMetrics, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	grpcMetrics, err := NewGRPCMetrics(ctx, meter)
 	if err != nil {
 		return nil, err
 	}
 
+	storeMetrics, err := NewStoreMetrics(ctx, meter)
+	if err != nil {
+		return nil, err
+	}
+
 	return &defaultAppMetrics{Meter: meter, ctx: ctx, idpMetrics: idpMetrics, httpMiddleware: middleware,
-		grpcMetrics: grpcMetrics}, nil
+		grpcMetrics: grpcMetrics, storeMetrics: storeMetrics}, nil
 }

management/server/telemetry/grpc_metrics.go

@@ -2,6 +2,8 @@ package telemetry
 
 import (
 	"context"
+	"time"
+
 	"go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/metric/instrument"
 	"go.opentelemetry.io/otel/metric/instrument/asyncint64"
@@ -15,6 +17,8 @@ type GRPCMetrics struct {
 	loginRequestsCounter  syncint64.Counter
 	getKeyRequestsCounter syncint64.Counter
 	activeStreamsGauge    asyncint64.Gauge
+	syncRequestDuration   syncint64.Histogram
+	loginRequestDuration  syncint64.Histogram
 	ctx                   context.Context
 }
 
@@ -38,12 +42,24 @@ func NewGRPCMetrics(ctx context.Context, meter metric.Meter) (*GRPCMetrics, erro
 		return nil, err
 	}
 
+	syncRequestDuration, err := meter.SyncInt64().Histogram("management.grpc.sync.request.duration.ms", instrument.WithUnit("milliseconds"))
+	if err != nil {
+		return nil, err
+	}
+
+	loginRequestDuration, err := meter.SyncInt64().Histogram("management.grpc.login.request.duration.ms", instrument.WithUnit("milliseconds"))
+	if err != nil {
+		return nil, err
+	}
+
 	return &GRPCMetrics{
 		meter:                 meter,
 		syncRequestsCounter:   syncRequestsCounter,
 		loginRequestsCounter:  loginRequestsCounter,
 		getKeyRequestsCounter: getKeyRequestsCounter,
 		activeStreamsGauge:    activeStreamsGauge,
+		syncRequestDuration:   syncRequestDuration,
+		loginRequestDuration:  loginRequestDuration,
 		ctx:                   ctx,
 	}, err
 }
@@ -63,6 +79,16 @@ func (grpcMetrics *GRPCMetrics) CountLoginRequest() {
 	grpcMetrics.loginRequestsCounter.Add(grpcMetrics.ctx, 1)
 }
 
+// CountLoginRequestDuration counts the duration of the login gRPC requests
+func (grpcMetrics *GRPCMetrics) CountLoginRequestDuration(duration time.Duration) {
+	grpcMetrics.loginRequestDuration.Record(grpcMetrics.ctx, duration.Milliseconds())
+}
+
+// CountSyncRequestDuration counts the duration of the sync gRPC requests
+func (grpcMetrics *GRPCMetrics) CountSyncRequestDuration(duration time.Duration) {
+	grpcMetrics.syncRequestDuration.Record(grpcMetrics.ctx, duration.Milliseconds())
+}
+
 // RegisterConnectedStreams registers a function that collects number of active streams and feeds it to the metrics gauge.
 func (grpcMetrics *GRPCMetrics) RegisterConnectedStreams(producer func() int64) error {
 	return grpcMetrics.meter.RegisterCallback(

management/server/telemetry/http_api_metrics.go

@@ -3,18 +3,22 @@ package telemetry
 import (
 	"context"
 	"fmt"
-	log "github.com/sirupsen/logrus"
-	"go.opentelemetry.io/otel/metric"
-	"go.opentelemetry.io/otel/metric/instrument"
-	"go.opentelemetry.io/otel/metric/instrument/syncint64"
 	"hash/fnv"
 	"net/http"
 	"strings"
+	time "time"
+
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/instrument"
+	"go.opentelemetry.io/otel/metric/instrument/syncint64"
 )
 
 const (
 	httpRequestCounterPrefix  = "management.http.request.counter"
 	httpResponseCounterPrefix = "management.http.response.counter"
+	httpRequestDurationPrefix = "management.http.request.duration.ms"
 )
 
 // WrappedResponseWriter is a wrapper for http.ResponseWriter that allows the
@@ -51,9 +55,9 @@ func (rw *WrappedResponseWriter) WriteHeader(code int) {
 type HTTPMiddleware struct {
 	meter metric.Meter
 	ctx   context.Context
-	// defaultEndpoint & method
+	// all HTTP requests by endpoint & method
 	httpRequestCounters map[string]syncint64.Counter
-	// defaultEndpoint & method & status code
+	// all HTTP responses by endpoint & method & status code
 	httpResponseCounters map[string]syncint64.Counter
 	// all HTTP requests
 	totalHTTPRequestsCounter syncint64.Counter
@@ -61,6 +65,48 @@ type HTTPMiddleware struct {
 	totalHTTPResponseCounter syncint64.Counter
 	// all HTTP responses by status code
 	totalHTTPResponseCodeCounters map[int]syncint64.Counter
+	// all HTTP requests durations by endpoint and method
+	httpRequestDurations map[string]syncint64.Histogram
+	// all HTTP requests durations
+	totalHTTPRequestDuration syncint64.Histogram
+}
+
+// NewMetricsMiddleware creates a new HTTPMiddleware
+func NewMetricsMiddleware(ctx context.Context, meter metric.Meter) (*HTTPMiddleware, error) {
+
+	totalHTTPRequestsCounter, err := meter.SyncInt64().Counter(
+		fmt.Sprintf("%s_total", httpRequestCounterPrefix),
+		instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	totalHTTPResponseCounter, err := meter.SyncInt64().Counter(
+		fmt.Sprintf("%s_total", httpResponseCounterPrefix),
+		instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+
+	totalHTTPRequestDuration, err := meter.SyncInt64().Histogram(
+		fmt.Sprintf("%s_total", httpRequestDurationPrefix),
+		instrument.WithUnit("milliseconds"))
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &HTTPMiddleware{
+			ctx:                           ctx,
+			httpRequestCounters:           map[string]syncint64.Counter{},
+			httpResponseCounters:          map[string]syncint64.Counter{},
+			httpRequestDurations:          map[string]syncint64.Histogram{},
+			totalHTTPResponseCodeCounters: map[int]syncint64.Counter{},
+			meter:                         meter,
+			totalHTTPRequestsCounter:      totalHTTPRequestsCounter,
+			totalHTTPResponseCounter:      totalHTTPResponseCounter,
+			totalHTTPRequestDuration:      totalHTTPRequestDuration,
+		},
+		nil
 }
 
 // AddHTTPRequestResponseCounter adds a new meter for an HTTP defaultEndpoint and Method (GET, POST, etc)
@@ -72,6 +118,12 @@ func (m *HTTPMiddleware) AddHTTPRequestResponseCounter(endpoint string, method s
 		return err
 	}
 	m.httpRequestCounters[meterKey] = httpReqCounter
+	durationKey := getRequestDurationKey(endpoint, method)
+	requestDuration, err := m.meter.SyncInt64().Histogram(durationKey, instrument.WithUnit("milliseconds"))
+	if err != nil {
+		return err
+	}
+	m.httpRequestDurations[durationKey] = requestDuration
 	respCodes := []int{200, 204, 400, 401, 403, 404, 500, 502, 503}
 	for _, code := range respCodes {
 		meterKey = getResponseCounterKey(endpoint, method, code)
@@ -92,38 +144,16 @@ func (m *HTTPMiddleware) AddHTTPRequestResponseCounter(endpoint string, method s
 	return nil
 }
 
-// NewMetricsMiddleware creates a new HTTPMiddleware
-func NewMetricsMiddleware(ctx context.Context, meter metric.Meter) (*HTTPMiddleware, error) {
-
-	totalHTTPRequestsCounter, err := meter.SyncInt64().Counter(
-		fmt.Sprintf("%s_total", httpRequestCounterPrefix),
-		instrument.WithUnit("1"))
-	if err != nil {
-		return nil, err
-	}
-	totalHTTPResponseCounter, err := meter.SyncInt64().Counter(
-		fmt.Sprintf("%s_total", httpResponseCounterPrefix),
-		instrument.WithUnit("1"))
-	if err != nil {
-		return nil, err
-	}
-	return &HTTPMiddleware{
-			ctx:                           ctx,
-			httpRequestCounters:           map[string]syncint64.Counter{},
-			httpResponseCounters:          map[string]syncint64.Counter{},
-			totalHTTPResponseCodeCounters: map[int]syncint64.Counter{},
-			meter:                         meter,
-			totalHTTPRequestsCounter:      totalHTTPRequestsCounter,
-			totalHTTPResponseCounter:      totalHTTPResponseCounter,
-		},
-		nil
-}
-
 func getRequestCounterKey(endpoint, method string) string {
 	return fmt.Sprintf("%s%s_%s", httpRequestCounterPrefix,
 		strings.ReplaceAll(endpoint, "/", "_"), method)
 }
 
+func getRequestDurationKey(endpoint, method string) string {
+	return fmt.Sprintf("%s%s_%s", httpRequestDurationPrefix,
+		strings.ReplaceAll(endpoint, "/", "_"), method)
+}
+
 func getResponseCounterKey(endpoint, method string, status int) string {
 	return fmt.Sprintf("%s%s_%s_%d", httpResponseCounterPrefix,
 		strings.ReplaceAll(endpoint, "/", "_"), method, status)
@@ -132,6 +162,7 @@ func getResponseCounterKey(endpoint, method string, status int) string {
 // Handler logs every request and response and adds the, to metrics.
 func (m *HTTPMiddleware) Handler(h http.Handler) http.Handler {
 	fn := func(rw http.ResponseWriter, r *http.Request) {
+		reqStart := time.Now()
 		traceID := hash(fmt.Sprintf("%v", r))
 		log.Tracef("HTTP request %v: %v %v", traceID, r.Method, r.URL)
 
@@ -161,6 +192,20 @@ func (m *HTTPMiddleware) Handler(h http.Handler) http.Handler {
 		if c, ok := m.totalHTTPResponseCodeCounters[w.Status()]; ok {
 			c.Add(m.ctx, 1)
 		}
+
+		durationKey := getRequestDurationKey(r.URL.Path, r.Method)
+		reqTook := time.Since(reqStart)
+		if c, ok := m.httpRequestDurations[durationKey]; ok {
+			c.Record(m.ctx, reqTook.Milliseconds())
+		}
+		log.Debugf("request %s %s took %d ms and finished with status %d", r.Method, r.URL.Path, reqTook.Milliseconds(), w.Status())
+
+		if w.Status() == 200 && (r.Method == http.MethodPut || r.Method == http.MethodPost || r.Method == http.MethodDelete) {
+			m.totalHTTPRequestDuration.Record(m.ctx, reqTook.Milliseconds(), attribute.String("type", "write"))
+		} else {
+			m.totalHTTPRequestDuration.Record(m.ctx, reqTook.Milliseconds(), attribute.String("type", "read"))
+		}
+
 	}
 
 	return http.HandlerFunc(fn)

management/server/telemetry/store_metrics.go

@@ -0,0 +1,47 @@
+package telemetry
+
+import (
+	"context"
+	"time"
+
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/instrument"
+	"go.opentelemetry.io/otel/metric/instrument/syncint64"
+)
+
+// StoreMetrics represents all metrics related to the FileStore
+type StoreMetrics struct {
+	globalLockAcquisitionDuration syncint64.Histogram
+	persistenceDuration           syncint64.Histogram
+	ctx                           context.Context
+}
+
+// NewStoreMetrics creates an instance of StoreMetrics
+func NewStoreMetrics(ctx context.Context, meter metric.Meter) (*StoreMetrics, error) {
+	globalLockAcquisitionDuration, err := meter.SyncInt64().Histogram("management.store.global.lock.acquisition.duration.micro",
+		instrument.WithUnit("microseconds"))
+	if err != nil {
+		return nil, err
+	}
+	persistenceDuration, err := meter.SyncInt64().Histogram("management.store.persistence.duration.micro",
+		instrument.WithUnit("microseconds"))
+	if err != nil {
+		return nil, err
+	}
+
+	return &StoreMetrics{
+		globalLockAcquisitionDuration: globalLockAcquisitionDuration,
+		persistenceDuration:           persistenceDuration,
+		ctx:                           ctx,
+	}, nil
+}
+
+// CountGlobalLockAcquisitionDuration counts the duration of the global lock acquisition
+func (metrics *StoreMetrics) CountGlobalLockAcquisitionDuration(duration time.Duration) {
+	metrics.globalLockAcquisitionDuration.Record(metrics.ctx, duration.Microseconds())
+}
+
+// CountPersistenceDuration counts the duration of a store persistence operation
+func (metrics *StoreMetrics) CountPersistenceDuration(duration time.Duration) {
+	metrics.persistenceDuration.Record(metrics.ctx, duration.Microseconds())
+}

management/server/user.go

@@ -565,6 +565,14 @@ func (am *DefaultAccountManager) SaveUser(accountID, initiatorUserID string, upd
 	}
 
 	defer func() {
+		if oldUser.IsBlocked() != update.IsBlocked() {
+			if update.IsBlocked() {
+				am.storeEvent(initiatorUserID, oldUser.Id, accountID, activity.UserBlocked, nil)
+			} else {
+				am.storeEvent(initiatorUserID, oldUser.Id, accountID, activity.UserUnblocked, nil)
+			}
+		}
+
 		// store activity logs
 		if oldUser.Role != newUser.Role {
 			am.storeEvent(initiatorUserID, oldUser.Id, accountID, activity.UserRoleUpdated, map[string]any{"role": newUser.Role})

release_files/darwin_pkg/postinstall

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+APP=/Applications/NetBird.app
+AGENT=/usr/local/bin/netbird
+LOG_FILE=/var/log/netbird/client_install.log
+
+{
+    echo "Installing NetBird..."
+
+    if test -d $APP; then
+        echo "NetBird app copied successfully."
+    else
+        echo "NetBird app could not be copied to the Applications folder."
+        exit 1
+    fi
+
+    ln -s $APP/Contents/MacOS/netbird $AGENT
+    if test -f $AGENT; then
+        echo "NetBird binary linked successfully."
+    else
+        echo "NetBird could not create symlink to /usr/local/bin"
+        exit 1
+    fi
+
+    $AGENT service install
+    $AGENT service start
+
+    open $APP
+
+    echo "Finished Netbird installation successfully"
+    exit 0 # all good
+} &> $LOG_FILE

release_files/darwin_pkg/preinstall

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+LOG_FILE=/var/log/netbird/client_install.log
+
+{
+    echo "Preinstall complete"
+    exit 0 # all good
+} &> $LOG_FILE

signal/client/grpc.go

@@ -24,6 +24,8 @@ import (
 	"github.com/netbirdio/netbird/signal/proto"
 )
 
+const defaultSendTimeout = 5 * time.Second
+
 // ConnStateNotifier is a wrapper interface of the status recorder
 type ConnStateNotifier interface {
 	MarkSignalDisconnected()
@@ -322,14 +324,28 @@ func (c *GrpcClient) Send(msg *proto.Message) error {
 		return err
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*2)
-	defer cancel()
-	_, err = c.realClient.Send(ctx, encryptedMessage)
-	if err != nil {
-		return err
+	attemptTimeout := defaultSendTimeout
+
+	for attempt := 0; attempt < 4; attempt++ {
+		if attempt > 1 {
+			attemptTimeout = time.Duration(attempt) * 5 * time.Second
+		}
+		ctx, cancel := context.WithTimeout(c.ctx, attemptTimeout)
+
+		_, err = c.realClient.Send(ctx, encryptedMessage)
+
+		cancel()
+
+		if s, ok := status.FromError(err); ok && s.Code() == codes.Canceled {
+			return err
+		}
+
+		if err == nil {
+			return nil
+		}
 	}
 
-	return nil
+	return err
 }
 
 // receive receives messages from other peers coming through the Signal Exchange