Add telemetry to measure app durations (#878)

Read and check ip_forward from proc before write

.goreleaser.yaml

@@ -12,11 +12,7 @@ builds:
       - arm
       - amd64
       - arm64
-      - mips
       - 386
-    gomips:
-      - hardfloat
-      - softfloat
     ignore:
       - goos: windows
         goarch: arm64
@@ -30,6 +26,26 @@ builds:
     tags:
       - load_wgnt_from_rsrc
 
+  - id: netbird-static
+    dir: client
+    binary: netbird
+    env: [CGO_ENABLED=0]
+    goos:
+      - linux
+    goarch:
+      - mips
+      - mipsle
+      - mips64
+      - mips64le
+    gomips:
+      - hardfloat
+      - softfloat
+    ldflags:
+      - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
+    mod_timestamp: '{{ .CommitTimestamp }}'
+    tags:
+      - load_wgnt_from_rsrc
+
   - id: netbird-mgmt
     dir: management
     env:
@@ -67,6 +83,7 @@ builds:
 archives:
   - builds:
       - netbird
+      - netbird-static
 
 nfpms:
 
@@ -359,4 +376,4 @@ uploads:
     mode: archive
     target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }}
     username: dev@wiretrustee.com
-    method: PUT
+    method: PUT

base62/base62.go

@@ -0,0 +1,59 @@
+package base62
+
+import (
+	"fmt"
+	"math"
+	"strings"
+)
+
+const (
+	alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+	base     = uint32(len(alphabet))
+)
+
+// Encode encodes a uint32 value to a base62 string.
+func Encode(num uint32) string {
+	if num == 0 {
+		return string(alphabet[0])
+	}
+
+	var encoded strings.Builder
+	remainder := uint32(0)
+
+	for num > 0 {
+		remainder = num % base
+		encoded.WriteByte(alphabet[remainder])
+		num /= base
+	}
+
+	// Reverse the encoded string
+	encodedString := encoded.String()
+	reversed := reverse(encodedString)
+	return reversed
+}
+
+// Decode decodes a base62 string to a uint32 value.
+func Decode(encoded string) (uint32, error) {
+	var decoded uint32
+	strLen := len(encoded)
+
+	for i, char := range encoded {
+		index := strings.IndexRune(alphabet, char)
+		if index < 0 {
+			return 0, fmt.Errorf("invalid character: %c", char)
+		}
+
+		decoded += uint32(index) * uint32(math.Pow(float64(base), float64(strLen-i-1)))
+	}
+
+	return decoded, nil
+}
+
+// Reverse a string.
+func reverse(s string) string {
+	runes := []rune(s)
+	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
+		runes[i], runes[j] = runes[j], runes[i]
+	}
+	return string(runes)
+}

base62/base62_test.go

@@ -0,0 +1,31 @@
+package base62
+
+import (
+	"testing"
+)
+
+func TestEncodeDecode(t *testing.T) {
+	tests := []struct {
+		num uint32
+	}{
+		{0},
+		{1},
+		{42},
+		{12345},
+		{99999},
+		{123456789},
+	}
+
+	for _, tt := range tests {
+		encoded := Encode(tt.num)
+		decoded, err := Decode(encoded)
+
+		if err != nil {
+			t.Errorf("Decode error: %v", err)
+		}
+
+		if decoded != tt.num {
+			t.Errorf("Decode(%v) = %v, want %v", encoded, decoded, tt.num)
+		}
+	}
+}

client/cmd/testutil.go

@@ -2,21 +2,23 @@ package cmd
 
 import (
 	"context"
-	"github.com/netbirdio/netbird/management/server/activity"
 	"net"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/netbirdio/netbird/management/server/activity"
+
 	"github.com/netbirdio/netbird/util"
 
+	"google.golang.org/grpc"
+
 	clientProto "github.com/netbirdio/netbird/client/proto"
 	client "github.com/netbirdio/netbird/client/server"
 	mgmtProto "github.com/netbirdio/netbird/management/proto"
 	mgmt "github.com/netbirdio/netbird/management/server"
 	sigProto "github.com/netbirdio/netbird/signal/proto"
 	sig "github.com/netbirdio/netbird/signal/server"
-	"google.golang.org/grpc"
 )
 
 func startTestingServices(t *testing.T) string {
@@ -63,7 +65,7 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	store, err := mgmt.NewFileStore(config.Datadir)
+	store, err := mgmt.NewFileStore(config.Datadir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

client/internal/dns/file_linux.go

@@ -32,6 +32,10 @@ func newFileConfigurator() (hostManager, error) {
 	return &fileConfigurator{}, nil
 }
 
+func (f *fileConfigurator) supportCustomPort() bool {
+	return false
+}
+
 func (f *fileConfigurator) applyDNSConfig(config hostDNSConfig) error {
 	backupFileExist := false
 	_, err := os.Stat(fileDefaultResolvConfBackupLocation)

client/internal/dns/host.go

@@ -10,6 +10,7 @@ import (
 type hostManager interface {
 	applyDNSConfig(config hostDNSConfig) error
 	restoreHostDNS() error
+	supportCustomPort() bool
 }
 
 type hostDNSConfig struct {
@@ -26,8 +27,9 @@ type domainConfig struct {
 }
 
 type mockHostConfigurator struct {
-	applyDNSConfigFunc func(config hostDNSConfig) error
+	applyDNSConfigFunc    func(config hostDNSConfig) error
-	restoreHostDNSFunc func() error
+	restoreHostDNSFunc    func() error
+	supportCustomPortFunc func() bool
 }
 
 func (m *mockHostConfigurator) applyDNSConfig(config hostDNSConfig) error {
@@ -44,10 +46,18 @@ func (m *mockHostConfigurator) restoreHostDNS() error {
 	return fmt.Errorf("method restoreHostDNS is not implemented")
 }
 
+func (m *mockHostConfigurator) supportCustomPort() bool {
+	if m.supportCustomPortFunc != nil {
+		return m.supportCustomPortFunc()
+	}
+	return false
+}
+
 func newNoopHostMocker() hostManager {
 	return &mockHostConfigurator{
-		applyDNSConfigFunc: func(config hostDNSConfig) error { return nil },
+		applyDNSConfigFunc:    func(config hostDNSConfig) error { return nil },
-		restoreHostDNSFunc: func() error { return nil },
+		restoreHostDNSFunc:    func() error { return nil },
+		supportCustomPortFunc: func() bool { return true },
 	}
 }
 

client/internal/dns/host_darwin.go

@@ -8,8 +8,9 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/netbirdio/netbird/iface"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/iface"
 )
 
 const (
@@ -39,6 +40,10 @@ func newHostManager(_ *iface.WGIface) (hostManager, error) {
 	}, nil
 }
 
+func (s *systemConfigurator) supportCustomPort() bool {
+	return true
+}
+
 func (s *systemConfigurator) applyDNSConfig(config hostDNSConfig) error {
 	var err error
 

client/internal/dns/host_windows.go

@@ -4,9 +4,10 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/netbirdio/netbird/iface"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows/registry"
+
+	"github.com/netbirdio/netbird/iface"
 )
 
 const (
@@ -42,6 +43,10 @@ func newHostManager(wgInterface *iface.WGIface) (hostManager, error) {
 	}, nil
 }
 
+func (s *registryConfigurator) supportCustomPort() bool {
+	return false
+}
+
 func (r *registryConfigurator) applyDNSConfig(config hostDNSConfig) error {
 	var err error
 	if config.routeAll {

client/internal/dns/network_manager_linux.go

@@ -11,8 +11,9 @@ import (
 	"github.com/godbus/dbus/v5"
 	"github.com/hashicorp/go-version"
 	"github.com/miekg/dns"
-	"github.com/netbirdio/netbird/iface"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/iface"
 )
 
 const (
@@ -88,6 +89,10 @@ func newNetworkManagerDbusConfigurator(wgInterface *iface.WGIface) (hostManager,
 	}, nil
 }
 
+func (n *networkManagerDbusConfigurator) supportCustomPort() bool {
+	return false
+}
+
 func (n *networkManagerDbusConfigurator) applyDNSConfig(config hostDNSConfig) error {
 	connSettings, configVersion, err := n.getAppliedConnectionSettings()
 	if err != nil {

client/internal/dns/resolvconf_linux.go

@@ -5,8 +5,9 @@ import (
 	"os/exec"
 	"strings"
 
-	"github.com/netbirdio/netbird/iface"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/iface"
 )
 
 const resolvconfCommand = "resolvconf"
@@ -21,6 +22,10 @@ func newResolvConfConfigurator(wgInterface *iface.WGIface) (hostManager, error)
 	}, nil
 }
 
+func (r *resolvconf) supportCustomPort() bool {
+	return false
+}
+
 func (r *resolvconf) applyDNSConfig(config hostDNSConfig) error {
 	var err error
 	if !config.routeAll {

client/internal/dns/server_nonandroid.go

@@ -13,9 +13,10 @@ import (
 
 	"github.com/miekg/dns"
 	"github.com/mitchellh/hashstructure/v2"
+	log "github.com/sirupsen/logrus"
+
 	nbdns "github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/iface"
-	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -254,7 +255,14 @@ func (s *DefaultServer) applyConfiguration(update nbdns.Config) error {
 	s.updateLocalResolver(localRecords)
 	s.currentConfig = dnsConfigToHostDNSConfig(update, s.runtimeIP, s.runtimePort)
 
-	if err = s.hostManager.applyDNSConfig(s.currentConfig); err != nil {
+	hostUpdate := s.currentConfig
+	if s.runtimePort != defaultPort && !s.hostManager.supportCustomPort() {
+		log.Warnf("the DNS manager of this peer doesn't support custom port. Disabling primary DNS setup. " +
+			"Learn more at: https://netbird.io/docs/how-to-guides/nameservers#local-resolver")
+		hostUpdate.routeAll = false
+	}
+
+	if err = s.hostManager.applyDNSConfig(hostUpdate); err != nil {
 		log.Error(err)
 	}
 

client/internal/dns/systemd_linux.go

@@ -9,10 +9,11 @@ import (
 
 	"github.com/godbus/dbus/v5"
 	"github.com/miekg/dns"
-	nbdns "github.com/netbirdio/netbird/dns"
-	"github.com/netbirdio/netbird/iface"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
+
+	nbdns "github.com/netbirdio/netbird/dns"
+	"github.com/netbirdio/netbird/iface"
 )
 
 const (
@@ -75,6 +76,10 @@ func newSystemdDbusConfigurator(wgInterface *iface.WGIface) (hostManager, error)
 	}, nil
 }
 
+func (s *systemdDbusConfigurator) supportCustomPort() bool {
+	return true
+}
+
 func (s *systemdDbusConfigurator) applyDNSConfig(config hostDNSConfig) error {
 	parsedIP, err := netip.ParseAddr(config.serverIP)
 	if err != nil {

client/internal/engine_test.go

@@ -3,8 +3,6 @@ package internal
 import (
 	"context"
 	"fmt"
-	"github.com/netbirdio/netbird/iface/bind"
-	"github.com/pion/transport/v2/stdnet"
 	"net"
 	"net/netip"
 	"os"
@@ -15,6 +13,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/pion/transport/v2/stdnet"
+
+	"github.com/netbirdio/netbird/iface/bind"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -1039,7 +1041,7 @@ func startManagement(dataDir string) (*grpc.Server, string, error) {
 		return nil, "", err
 	}
 	s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp))
-	store, err := server.NewFileStore(config.Datadir)
+	store, err := server.NewFileStore(config.Datadir, nil)
 	if err != nil {
 		log.Fatalf("failed creating a store: %s: %v", config.Datadir, err)
 	}

client/internal/routemanager/systemops_linux.go

@@ -62,6 +62,16 @@ func removeFromRouteTable(prefix netip.Prefix) error {
 }
 
 func enableIPForwarding() error {
-	err := os.WriteFile(ipv4ForwardingPath, []byte("1"), 0644)
+	bytes, err := os.ReadFile(ipv4ForwardingPath)
-	return err
+	if err != nil {
+		return err
+	}
+
+	// check if it is already enabled
+	// see more: https://github.com/netbirdio/netbird/issues/872
+	if len(bytes) > 0 && bytes[0] == 49 {
+		return nil
+	}
+
+	return os.WriteFile(ipv4ForwardingPath, []byte("1"), 0644)
 }

client/main.go

@@ -1,8 +1,9 @@
 package main
 
 import (
-	"github.com/netbirdio/netbird/client/cmd"
 	"os"
+
+	"github.com/netbirdio/netbird/client/cmd"
 )
 
 func main() {

go.mod

@@ -28,7 +28,6 @@ require (
 )
 
 require (
-	codeberg.org/ac/base62 v0.0.0-20210305150220-e793b546833a
 	fyne.io/fyne/v2 v2.1.4
 	github.com/c-robinson/iplib v1.0.3
 	github.com/coreos/go-iptables v0.6.0

go.sum

@@ -31,8 +31,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-codeberg.org/ac/base62 v0.0.0-20210305150220-e793b546833a h1:U6cY/g6VSiy59vuvnBU6J/eSir0qVg4BeTnCDLaX+20=
-codeberg.org/ac/base62 v0.0.0-20210305150220-e793b546833a/go.mod h1:ykEpkLT4JtH3I4Rb4gwkDsNLfgUg803qRDeIX88t3e8=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 fyne.io/fyne/v2 v2.1.4 h1:bt1+28++kAzRzPB0GM2EuSV4cnl8rXNX4cjfd8G06Rc=
 fyne.io/fyne/v2 v2.1.4/go.mod h1:p+E/Dh+wPW8JwR2DVcsZ9iXgR9ZKde80+Y+40Is54AQ=

iface/bind/udp_mux_universal.go

@@ -209,7 +209,7 @@ func (m *UniversalUDPMuxDefault) GetXORMappedAddr(serverAddr net.Addr, deadline
 
 	// otherwise, make a STUN request to discover the address
 	// or wait for already sent request to complete
-	waitAddrReceived, err := m.sendStun(serverAddr)
+	waitAddrReceived, err := m.sendSTUN(serverAddr)
 	if err != nil {
 		return nil, fmt.Errorf("%s: %s", "failed to send STUN packet", err)
 	}
@@ -218,23 +218,31 @@ func (m *UniversalUDPMuxDefault) GetXORMappedAddr(serverAddr net.Addr, deadline
 	select {
 	case <-waitAddrReceived:
 		// when channel closed, addr was obtained
+		var addr *stun.XORMappedAddress
 		m.mu.Lock()
-		mappedAddr := *m.xorMappedMap[serverAddr.String()]
+		// A very odd case that mappedAddr is nil.
+		// Can happen when the deadline property is larger than params.XORMappedAddrCacheTTL.
+		// Or when we don't receive a response to our m.sendSTUN request (the response is handled asynchronously) and
+		// the XORMapped expires meanwhile triggering a closure of the waitAddrReceived channel.
+		// We protect the code from panic here.
+		if mappedAddr, ok := m.xorMappedMap[serverAddr.String()]; ok {
+			addr = mappedAddr.addr
+		}
 		m.mu.Unlock()
-		if mappedAddr.addr == nil {
+		if addr == nil {
 			return nil, fmt.Errorf("no XOR address mapping")
 		}
-		return mappedAddr.addr, nil
+		return addr, nil
 	case <-time.After(deadline):
 		return nil, fmt.Errorf("timeout while waiting for XORMappedAddr")
 	}
 }
 
-// sendStun sends a STUN request via UDP conn.
+// sendSTUN sends a STUN request via UDP conn.
 //
 // The returned channel is closed when the STUN response has been received.
 // Method is safe for concurrent use.
-func (m *UniversalUDPMuxDefault) sendStun(serverAddr net.Addr) (chan struct{}, error) {
+func (m *UniversalUDPMuxDefault) sendSTUN(serverAddr net.Addr) (chan struct{}, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 

management/client/client_test.go

@@ -2,28 +2,31 @@ package client
 
 import (
 	"context"
-	"github.com/netbirdio/netbird/management/server/activity"
 	"net"
 	"path/filepath"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/netbirdio/netbird/management/server/activity"
+
 	"github.com/netbirdio/netbird/client/system"
 
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/netbirdio/netbird/encryption"
 	"github.com/netbirdio/netbird/management/proto"
 	mgmtProto "github.com/netbirdio/netbird/management/proto"
 	mgmt "github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/mock_server"
-	log "github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/assert"
 
-	"github.com/netbirdio/netbird/util"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+
+	"github.com/netbirdio/netbird/util"
 )
 
 const ValidKey = "A2C8E62B-38F5-4553-B31E-DD66C696CEBB"
@@ -50,7 +53,7 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) {
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	store, err := mgmt.NewFileStore(config.Datadir)
+	store, err := mgmt.NewFileStore(config.Datadir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

management/cmd/management.go

@@ -121,13 +121,6 @@ var (
 					return fmt.Errorf("failed creating datadir: %s: %v", config.Datadir, err)
 				}
 			}
-
-			store, err := server.NewFileStore(config.Datadir)
-			if err != nil {
-				return fmt.Errorf("failed creating Store: %s: %v", config.Datadir, err)
-			}
-			peersUpdateManager := server.NewPeersUpdateManager()
-
 			appMetrics, err := telemetry.NewDefaultAppMetrics(cmd.Context())
 			if err != nil {
 				return err
@@ -136,6 +129,11 @@ var (
 			if err != nil {
 				return err
 			}
+			store, err := server.NewFileStore(config.Datadir, appMetrics)
+			if err != nil {
+				return fmt.Errorf("failed creating Store: %s: %v", config.Datadir, err)
+			}
+			peersUpdateManager := server.NewPeersUpdateManager()
 
 			var idpManager idp.Manager
 			if config.IdpManagerConfig != nil {

management/server/account.go

@@ -15,13 +15,13 @@ import (
 	"sync"
 	"time"
 
-	"codeberg.org/ac/base62"
 	"github.com/eko/gocache/v3/cache"
 	cacheStore "github.com/eko/gocache/v3/store"
 	gocache "github.com/patrickmn/go-cache"
 	"github.com/rs/xid"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/netbirdio/netbird/base62"
 	nbdns "github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/management/server/activity"
 	"github.com/netbirdio/netbird/management/server/idp"

management/server/account_test.go

@@ -1869,7 +1869,7 @@ func createManager(t *testing.T) (*DefaultAccountManager, error) {
 
 func createStore(t *testing.T) (Store, error) {
 	dataDir := t.TempDir()
-	store, err := NewFileStore(dataDir)
+	store, err := NewFileStore(dataDir, nil)
 	if err != nil {
 		return nil, err
 	}

management/server/dns_test.go

@@ -1,10 +1,12 @@
 package server
 
 import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
 	"github.com/netbirdio/netbird/management/server/activity"
 	"github.com/netbirdio/netbird/management/server/status"
-	"github.com/stretchr/testify/require"
-	"testing"
 )
 
 const (
@@ -190,7 +192,7 @@ func createDNSManager(t *testing.T) (*DefaultAccountManager, error) {
 
 func createDNSStore(t *testing.T) (Store, error) {
 	dataDir := t.TempDir()
-	store, err := NewFileStore(dataDir)
+	store, err := NewFileStore(dataDir, nil)
 	if err != nil {
 		return nil, err
 	}

management/server/file_store.go

@@ -11,6 +11,7 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	"github.com/netbirdio/netbird/management/server/status"
+	"github.com/netbirdio/netbird/management/server/telemetry"
 
 	"github.com/netbirdio/netbird/util"
 )
@@ -37,13 +38,20 @@ type FileStore struct {
 	// sync.Mutex indexed by accountID
 	accountLocks      sync.Map   `json:"-"`
 	globalAccountLock sync.Mutex `json:"-"`
+
+	metrics telemetry.AppMetrics `json:"-"`
 }
 
 type StoredAccount struct{}
 
 // NewFileStore restores a store from the file located in the datadir
-func NewFileStore(dataDir string) (*FileStore, error) {
+func NewFileStore(dataDir string, metrics telemetry.AppMetrics) (*FileStore, error) {
-	return restore(filepath.Join(dataDir, storeFileName))
+	fs, err := restore(filepath.Join(dataDir, storeFileName))
+	if err != nil {
+		return nil, err
+	}
+	fs.metrics = metrics
+	return fs, nil
 }
 
 // restore the state of the store from the file.
@@ -221,7 +229,17 @@ func restore(file string) (*FileStore, error) {
 // persist account data to a file
 // It is recommended to call it with locking FileStore.mux
 func (s *FileStore) persist(file string) error {
-	return util.WriteJson(file, s)
+	start := time.Now()
+	err := util.WriteJson(file, s)
+	if err != nil {
+		return err
+	}
+	took := time.Since(start)
+	if s.metrics != nil {
+		s.metrics.StoreMetrics().CountPersistenceDuration(took)
+	}
+	log.Debugf("took %d ms to persist the FileStore", took.Milliseconds())
+	return nil
 }
 
 // AcquireGlobalLock acquires global lock across all the accounts and returns a function that releases the lock
@@ -235,6 +253,12 @@ func (s *FileStore) AcquireGlobalLock() (unlock func()) {
 		log.Debugf("released global lock in %v", time.Since(start))
 	}
 
+	took := time.Since(start)
+	log.Debugf("took %v to acquire global lock", took)
+	if s.metrics != nil {
+		s.metrics.StoreMetrics().CountGlobalLockAcquisitionDuration(took)
+	}
+
 	return unlock
 }
 

management/server/file_store_test.go

@@ -25,7 +25,7 @@ func TestStalePeerIndices(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		return
 	}
@@ -172,7 +172,7 @@ func TestStore(t *testing.T) {
 		return
 	}
 
-	restored, err := NewFileStore(store.storeFile)
+	restored, err := NewFileStore(store.storeFile, nil)
 	if err != nil {
 		return
 	}
@@ -232,7 +232,7 @@ func TestRestore(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		return
 	}
@@ -270,7 +270,7 @@ func TestRestorePolicies_Migration(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		return
 	}
@@ -307,7 +307,7 @@ func TestGetAccountByPrivateDomain(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		return
 	}
@@ -336,7 +336,7 @@ func TestFileStore_GetAccount(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -378,7 +378,7 @@ func TestFileStore_GetTokenIDByHashedToken(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -431,7 +431,7 @@ func TestFileStore_GetTokenIDByHashedToken_Failure(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -456,7 +456,7 @@ func TestFileStore_GetUserByTokenID(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -484,7 +484,7 @@ func TestFileStore_GetUserByTokenID_Failure(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -503,7 +503,7 @@ func TestFileStore_SavePeerStatus(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	store, err := NewFileStore(storeDir)
+	store, err := NewFileStore(storeDir, nil)
 	if err != nil {
 		return
 	}
@@ -548,7 +548,7 @@ func TestFileStore_SavePeerStatus(t *testing.T) {
 }
 
 func newStore(t *testing.T) *FileStore {
-	store, err := NewFileStore(t.TempDir())
+	store, err := NewFileStore(t.TempDir(), nil)
 	if err != nil {
 		t.Errorf("failed creating a new store")
 	}

management/server/grpcserver.go

@@ -114,6 +114,7 @@ func (s *GRPCServer) GetServerKey(ctx context.Context, req *proto.Empty) (*proto
 // Sync validates the existence of a connecting peer, sends an initial state (all available for the connecting peers) and
 // notifies the connected peer of any updates (e.g. new peers under the same account)
 func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementService_SyncServer) error {
+	reqStart := time.Now()
 	if s.appMetrics != nil {
 		s.appMetrics.GRPCMetrics().CountSyncRequest()
 	}
@@ -148,6 +149,11 @@ func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementServi
 	if s.config.TURNConfig.TimeBasedCredentials {
 		s.turnCredentialsManager.SetupRefresh(peer.ID)
 	}
+
+	if s.appMetrics != nil {
+		s.appMetrics.GRPCMetrics().CountSyncRequestDuration(time.Since(reqStart))
+	}
+
 	// keep a connection to the peer and send updates when available
 	for {
 		select {
@@ -262,6 +268,12 @@ func (s *GRPCServer) parseRequest(req *proto.EncryptedMessage, parsed pb.Message
 // In case it isn't, the endpoint checks whether setup key is provided within the request and tries to register a peer.
 // In case of the successful registration login is also successful
 func (s *GRPCServer) Login(ctx context.Context, req *proto.EncryptedMessage) (*proto.EncryptedMessage, error) {
+	reqStart := time.Now()
+	defer func() {
+		if s.appMetrics != nil {
+			s.appMetrics.GRPCMetrics().CountLoginRequestDuration(time.Since(reqStart))
+		}
+	}()
 	if s.appMetrics != nil {
 		s.appMetrics.GRPCMetrics().CountLoginRequest()
 	}

management/server/idp/auth0.go

@@ -122,7 +122,7 @@ func NewAuth0Manager(oidcConfig OIDCConfig, config Auth0ClientConfig,
 	}
 
 	helper := JsonParser{}
-	config.AuthIssuer = oidcConfig.TokenEndpoint
+	config.AuthIssuer = oidcConfig.Issuer
 	config.GrantType = "client_credentials"
 
 	if config.ClientID == "" {

management/server/management_proto_test.go

@@ -2,7 +2,6 @@ package server
 
 import (
 	"context"
-	"github.com/netbirdio/netbird/management/server/activity"
 	"net"
 	"os"
 	"path/filepath"
@@ -10,14 +9,17 @@ import (
 	"testing"
 	"time"
 
-	"github.com/netbirdio/netbird/encryption"
+	"github.com/netbirdio/netbird/management/server/activity"
-	mgmtProto "github.com/netbirdio/netbird/management/proto"
+
-	"github.com/netbirdio/netbird/util"
 	"github.com/stretchr/testify/require"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/keepalive"
+
+	"github.com/netbirdio/netbird/encryption"
+	mgmtProto "github.com/netbirdio/netbird/management/proto"
+	"github.com/netbirdio/netbird/util"
 )
 
 var (
@@ -408,7 +410,7 @@ func startManagement(t *testing.T, config *Config) (*grpc.Server, string, error)
 		return nil, "", err
 	}
 	s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp))
-	store, err := NewFileStore(config.Datadir)
+	store, err := NewFileStore(config.Datadir, nil)
 	if err != nil {
 		return nil, "", err
 	}

management/server/management_test.go

@@ -496,7 +496,7 @@ func startServer(config *server.Config) (*grpc.Server, net.Listener) {
 	Expect(err).NotTo(HaveOccurred())
 	s := grpc.NewServer()
 
-	store, err := server.NewFileStore(config.Datadir)
+	store, err := server.NewFileStore(config.Datadir, nil)
 	if err != nil {
 		log.Fatalf("failed creating a store: %s: %v", config.Datadir, err)
 	}

management/server/nameserver_test.go

@@ -1,11 +1,13 @@
 package server
 
 import (
-	nbdns "github.com/netbirdio/netbird/dns"
-	"github.com/netbirdio/netbird/management/server/activity"
-	"github.com/stretchr/testify/require"
 	"net/netip"
 	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	nbdns "github.com/netbirdio/netbird/dns"
+	"github.com/netbirdio/netbird/management/server/activity"
 )
 
 const (
@@ -1064,7 +1066,7 @@ func createNSManager(t *testing.T) (*DefaultAccountManager, error) {
 
 func createNSStore(t *testing.T) (Store, error) {
 	dataDir := t.TempDir()
-	store, err := NewFileStore(dataDir)
+	store, err := NewFileStore(dataDir, nil)
 	if err != nil {
 		return nil, err
 	}

management/server/peer.go

@@ -28,6 +28,17 @@ type PeerSystemMeta struct {
 	UIVersion string
 }
 
+func (p PeerSystemMeta) isEqual(other PeerSystemMeta) bool {
+	return p.Hostname == other.Hostname &&
+		p.GoOS == other.GoOS &&
+		p.Kernel == other.Kernel &&
+		p.Core == other.Core &&
+		p.Platform == other.Platform &&
+		p.OS == other.OS &&
+		p.WtVersion == other.WtVersion &&
+		p.UIVersion == other.UIVersion
+}
+
 type PeerStatus struct {
 	// LastSeen is the last time peer was connected to the management service
 	LastSeen time.Time
@@ -114,13 +125,19 @@ func (p *Peer) Copy() *Peer {
 	}
 }
 
-// UpdateMeta updates peer's system meta data
+// UpdateMetaIfNew updates peer's system metadata if new information is provided
-func (p *Peer) UpdateMeta(meta PeerSystemMeta) {
+// returns true if meta was updated, false otherwise
+func (p *Peer) UpdateMetaIfNew(meta PeerSystemMeta) bool {
 	// Avoid overwriting UIVersion if the update was triggered sole by the CLI client
 	if meta.UIVersion == "" {
 		meta.UIVersion = p.Meta.UIVersion
 	}
+
+	if p.Meta.isEqual(meta) {
+		return false
+	}
 	p.Meta = meta
+	return true
 }
 
 // MarkLoginExpired marks peer's status expired or not
@@ -654,6 +671,8 @@ func (am *DefaultAccountManager) LoginPeer(login PeerLogin) (*Peer, *NetworkMap,
 		return nil, nil, err
 	}
 
+	// this flag prevents unnecessary calls to the persistent store.
+	shouldStoreAccount := false
 	updateRemotePeers := false
 	if peerLoginExpired(peer, account) {
 		err = checkAuth(login.UserID, peer)
@@ -664,19 +683,26 @@ func (am *DefaultAccountManager) LoginPeer(login PeerLogin) (*Peer, *NetworkMap,
 		// UserID is present, meaning that JWT validation passed successfully in the API layer.
 		updatePeerLastLogin(peer, account)
 		updateRemotePeers = true
+		shouldStoreAccount = true
 	}
 
-	peer = updatePeerMeta(peer, login.Meta, account)
+	peer, updated := updatePeerMeta(peer, login.Meta, account)
+	if updated {
+		shouldStoreAccount = true
+	}
 
 	peer, err = am.checkAndUpdatePeerSSHKey(peer, account, login.SSHKey)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	err = am.Store.SaveAccount(account)
+	if shouldStoreAccount {
-	if err != nil {
+		err = am.Store.SaveAccount(account)
-		return nil, nil, err
+		if err != nil {
+			return nil, nil, err
+		}
 	}
+
 	if updateRemotePeers {
 		err = am.updateAccountPeers(account)
 		if err != nil {
@@ -850,10 +876,12 @@ func (am *DefaultAccountManager) GetPeer(accountID, peerID, userID string) (*Pee
 	return nil, status.Errorf(status.Internal, "user %s has no access to peer %s under account %s", userID, peerID, accountID)
 }
 
-func updatePeerMeta(peer *Peer, meta PeerSystemMeta, account *Account) *Peer {
+func updatePeerMeta(peer *Peer, meta PeerSystemMeta, account *Account) (*Peer, bool) {
-	peer.UpdateMeta(meta)
+	if peer.UpdateMetaIfNew(meta) {
-	account.UpdatePeer(peer)
+		account.UpdatePeer(peer)
-	return peer
+		return peer, true
+	}
+	return peer, false
 }
 
 // GetPeerRules returns a list of source or destination rules of a given peer.

management/server/personal_access_token.go

@@ -7,9 +7,10 @@ import (
 	"hash/crc32"
 	"time"
 
-	"codeberg.org/ac/base62"
 	b "github.com/hashicorp/go-secure-stdlib/base62"
 	"github.com/rs/xid"
+
+	"github.com/netbirdio/netbird/base62"
 )
 
 const (

management/server/personal_access_token_test.go

@@ -4,11 +4,13 @@ import (
 	"crypto/sha256"
 	b64 "encoding/base64"
 	"hash/crc32"
+	"math/big"
 	"strings"
 	"testing"
 
-	"codeberg.org/ac/base62"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/netbirdio/netbird/base62"
 )
 
 func TestPAT_GenerateToken_Hashing(t *testing.T) {
@@ -33,6 +35,8 @@ func TestPAT_GenerateToken_Checksum(t *testing.T) {
 	secret := tokenWithoutPrefix[:len(tokenWithoutPrefix)-6]
 	tokenCheckSum := tokenWithoutPrefix[len(tokenWithoutPrefix)-6:]
 
+	var i big.Int
+	i.SetString(secret, 62)
 	expectedChecksum := crc32.ChecksumIEEE([]byte(secret))
 	actualChecksum, err := base62.Decode(tokenCheckSum)
 	if err != nil {

management/server/route_test.go

@@ -4,10 +4,11 @@ import (
 	"net/netip"
 	"testing"
 
-	"github.com/netbirdio/netbird/management/server/activity"
-	"github.com/netbirdio/netbird/route"
 	"github.com/rs/xid"
 	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/management/server/activity"
+	"github.com/netbirdio/netbird/route"
 )
 
 const (
@@ -946,7 +947,7 @@ func createRouterManager(t *testing.T) (*DefaultAccountManager, error) {
 
 func createRouterStore(t *testing.T) (Store, error) {
 	dataDir := t.TempDir()
-	store, err := NewFileStore(dataDir)
+	store, err := NewFileStore(dataDir, nil)
 	if err != nil {
 		return nil, err
 	}

management/server/telemetry/app_metrics.go

@@ -3,6 +3,10 @@ package telemetry
 import (
 	"context"
 	"fmt"
+	"net"
+	"net/http"
+	"reflect"
+
 	"github.com/gorilla/mux"
 	prometheus2 "github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -10,9 +14,6 @@ import (
 	"go.opentelemetry.io/otel/exporters/prometheus"
 	metric2 "go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/sdk/metric"
-	"net"
-	"net/http"
-	"reflect"
 )
 
 const defaultEndpoint = "/metrics"
@@ -25,6 +26,7 @@ type MockAppMetrics struct {
 	IDPMetricsFunc     func() *IDPMetrics
 	HTTPMiddlewareFunc func() *HTTPMiddleware
 	GRPCMetricsFunc    func() *GRPCMetrics
+	StoreMetricsFunc   func() *StoreMetrics
 }
 
 // GetMeter mocks the GetMeter function of the AppMetrics interface
@@ -75,6 +77,14 @@ func (mock *MockAppMetrics) GRPCMetrics() *GRPCMetrics {
 	return nil
 }
 
+// StoreMetrics mocks the MockAppMetrics function of the StoreMetrics interface
+func (mock *MockAppMetrics) StoreMetrics() *StoreMetrics {
+	if mock.StoreMetricsFunc != nil {
+		return mock.StoreMetricsFunc()
+	}
+	return nil
+}
+
 // AppMetrics is metrics interface
 type AppMetrics interface {
 	GetMeter() metric2.Meter
@@ -83,6 +93,7 @@ type AppMetrics interface {
 	IDPMetrics() *IDPMetrics
 	HTTPMiddleware() *HTTPMiddleware
 	GRPCMetrics() *GRPCMetrics
+	StoreMetrics() *StoreMetrics
 }
 
 // defaultAppMetrics are core application metrics based on OpenTelemetry https://opentelemetry.io/
@@ -94,6 +105,7 @@ type defaultAppMetrics struct {
 	idpMetrics     *IDPMetrics
 	httpMiddleware *HTTPMiddleware
 	grpcMetrics    *GRPCMetrics
+	storeMetrics   *StoreMetrics
 }
 
 // IDPMetrics returns metrics for the idp package
@@ -111,6 +123,11 @@ func (appMetrics *defaultAppMetrics) GRPCMetrics() *GRPCMetrics {
 	return appMetrics.grpcMetrics
 }
 
+// StoreMetrics returns metrics for the store
+func (appMetrics *defaultAppMetrics) StoreMetrics() *StoreMetrics {
+	return appMetrics.storeMetrics
+}
+
 // Close stop application metrics HTTP handler and closes listener.
 func (appMetrics *defaultAppMetrics) Close() error {
 	if appMetrics.listener == nil {
@@ -171,11 +188,17 @@ func NewDefaultAppMetrics(ctx context.Context) (AppMetrics, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	grpcMetrics, err := NewGRPCMetrics(ctx, meter)
 	if err != nil {
 		return nil, err
 	}
 
+	storeMetrics, err := NewStoreMetrics(ctx, meter)
+	if err != nil {
+		return nil, err
+	}
+
 	return &defaultAppMetrics{Meter: meter, ctx: ctx, idpMetrics: idpMetrics, httpMiddleware: middleware,
-		grpcMetrics: grpcMetrics}, nil
+		grpcMetrics: grpcMetrics, storeMetrics: storeMetrics}, nil
 }

management/server/telemetry/grpc_metrics.go

@@ -2,6 +2,8 @@ package telemetry
 
 import (
 	"context"
+	"time"
+
 	"go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/metric/instrument"
 	"go.opentelemetry.io/otel/metric/instrument/asyncint64"
@@ -15,6 +17,8 @@ type GRPCMetrics struct {
 	loginRequestsCounter  syncint64.Counter
 	getKeyRequestsCounter syncint64.Counter
 	activeStreamsGauge    asyncint64.Gauge
+	syncRequestDuration   syncint64.Histogram
+	loginRequestDuration  syncint64.Histogram
 	ctx                   context.Context
 }
 
@@ -38,12 +42,24 @@ func NewGRPCMetrics(ctx context.Context, meter metric.Meter) (*GRPCMetrics, erro
 		return nil, err
 	}
 
+	syncRequestDuration, err := meter.SyncInt64().Histogram("management.grpc.sync.request.duration.ms", instrument.WithUnit("milliseconds"))
+	if err != nil {
+		return nil, err
+	}
+
+	loginRequestDuration, err := meter.SyncInt64().Histogram("management.grpc.login.request.duration.ms", instrument.WithUnit("milliseconds"))
+	if err != nil {
+		return nil, err
+	}
+
 	return &GRPCMetrics{
 		meter:                 meter,
 		syncRequestsCounter:   syncRequestsCounter,
 		loginRequestsCounter:  loginRequestsCounter,
 		getKeyRequestsCounter: getKeyRequestsCounter,
 		activeStreamsGauge:    activeStreamsGauge,
+		syncRequestDuration:   syncRequestDuration,
+		loginRequestDuration:  loginRequestDuration,
 		ctx:                   ctx,
 	}, err
 }
@@ -63,6 +79,16 @@ func (grpcMetrics *GRPCMetrics) CountLoginRequest() {
 	grpcMetrics.loginRequestsCounter.Add(grpcMetrics.ctx, 1)
 }
 
+// CountLoginRequestDuration counts the duration of the login gRPC requests
+func (grpcMetrics *GRPCMetrics) CountLoginRequestDuration(duration time.Duration) {
+	grpcMetrics.loginRequestDuration.Record(grpcMetrics.ctx, duration.Milliseconds())
+}
+
+// CountSyncRequestDuration counts the duration of the sync gRPC requests
+func (grpcMetrics *GRPCMetrics) CountSyncRequestDuration(duration time.Duration) {
+	grpcMetrics.syncRequestDuration.Record(grpcMetrics.ctx, duration.Milliseconds())
+}
+
 // RegisterConnectedStreams registers a function that collects number of active streams and feeds it to the metrics gauge.
 func (grpcMetrics *GRPCMetrics) RegisterConnectedStreams(producer func() int64) error {
 	return grpcMetrics.meter.RegisterCallback(

management/server/telemetry/http_api_metrics.go

@@ -3,18 +3,21 @@ package telemetry
 import (
 	"context"
 	"fmt"
+	"hash/fnv"
+	"net/http"
+	"strings"
+	time "time"
+
 	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/metric/instrument"
 	"go.opentelemetry.io/otel/metric/instrument/syncint64"
-	"hash/fnv"
-	"net/http"
-	"strings"
 )
 
 const (
 	httpRequestCounterPrefix  = "management.http.request.counter"
 	httpResponseCounterPrefix = "management.http.response.counter"
+	httpRequestDurationPrefix = "management.http.request.duration.ms"
 )
 
 // WrappedResponseWriter is a wrapper for http.ResponseWriter that allows the
@@ -51,9 +54,9 @@ func (rw *WrappedResponseWriter) WriteHeader(code int) {
 type HTTPMiddleware struct {
 	meter metric.Meter
 	ctx   context.Context
-	// defaultEndpoint & method
+	// all HTTP requests by endpoint & method
 	httpRequestCounters map[string]syncint64.Counter
-	// defaultEndpoint & method & status code
+	// all HTTP responses by endpoint & method & status code
 	httpResponseCounters map[string]syncint64.Counter
 	// all HTTP requests
 	totalHTTPRequestsCounter syncint64.Counter
@@ -61,6 +64,48 @@ type HTTPMiddleware struct {
 	totalHTTPResponseCounter syncint64.Counter
 	// all HTTP responses by status code
 	totalHTTPResponseCodeCounters map[int]syncint64.Counter
+	// all HTTP requests durations by endpoint and method
+	httpRequestDurations map[string]syncint64.Histogram
+	// all HTTP requests durations
+	totalHTTPRequestDuration syncint64.Histogram
+}
+
+// NewMetricsMiddleware creates a new HTTPMiddleware
+func NewMetricsMiddleware(ctx context.Context, meter metric.Meter) (*HTTPMiddleware, error) {
+
+	totalHTTPRequestsCounter, err := meter.SyncInt64().Counter(
+		fmt.Sprintf("%s_total", httpRequestCounterPrefix),
+		instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	totalHTTPResponseCounter, err := meter.SyncInt64().Counter(
+		fmt.Sprintf("%s_total", httpResponseCounterPrefix),
+		instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+
+	totalHTTPRequestDuration, err := meter.SyncInt64().Histogram(
+		fmt.Sprintf("%s_total", httpRequestDurationPrefix),
+		instrument.WithUnit("milliseconds"))
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &HTTPMiddleware{
+			ctx:                           ctx,
+			httpRequestCounters:           map[string]syncint64.Counter{},
+			httpResponseCounters:          map[string]syncint64.Counter{},
+			httpRequestDurations:          map[string]syncint64.Histogram{},
+			totalHTTPResponseCodeCounters: map[int]syncint64.Counter{},
+			meter:                         meter,
+			totalHTTPRequestsCounter:      totalHTTPRequestsCounter,
+			totalHTTPResponseCounter:      totalHTTPResponseCounter,
+			totalHTTPRequestDuration:      totalHTTPRequestDuration,
+		},
+		nil
 }
 
 // AddHTTPRequestResponseCounter adds a new meter for an HTTP defaultEndpoint and Method (GET, POST, etc)
@@ -72,6 +117,12 @@ func (m *HTTPMiddleware) AddHTTPRequestResponseCounter(endpoint string, method s
 		return err
 	}
 	m.httpRequestCounters[meterKey] = httpReqCounter
+	durationKey := getRequestDurationKey(endpoint, method)
+	requestDuration, err := m.meter.SyncInt64().Histogram(durationKey, instrument.WithUnit("milliseconds"))
+	if err != nil {
+		return err
+	}
+	m.httpRequestDurations[durationKey] = requestDuration
 	respCodes := []int{200, 204, 400, 401, 403, 404, 500, 502, 503}
 	for _, code := range respCodes {
 		meterKey = getResponseCounterKey(endpoint, method, code)
@@ -92,38 +143,16 @@ func (m *HTTPMiddleware) AddHTTPRequestResponseCounter(endpoint string, method s
 	return nil
 }
 
-// NewMetricsMiddleware creates a new HTTPMiddleware
-func NewMetricsMiddleware(ctx context.Context, meter metric.Meter) (*HTTPMiddleware, error) {
-
-	totalHTTPRequestsCounter, err := meter.SyncInt64().Counter(
-		fmt.Sprintf("%s_total", httpRequestCounterPrefix),
-		instrument.WithUnit("1"))
-	if err != nil {
-		return nil, err
-	}
-	totalHTTPResponseCounter, err := meter.SyncInt64().Counter(
-		fmt.Sprintf("%s_total", httpResponseCounterPrefix),
-		instrument.WithUnit("1"))
-	if err != nil {
-		return nil, err
-	}
-	return &HTTPMiddleware{
-			ctx:                           ctx,
-			httpRequestCounters:           map[string]syncint64.Counter{},
-			httpResponseCounters:          map[string]syncint64.Counter{},
-			totalHTTPResponseCodeCounters: map[int]syncint64.Counter{},
-			meter:                         meter,
-			totalHTTPRequestsCounter:      totalHTTPRequestsCounter,
-			totalHTTPResponseCounter:      totalHTTPResponseCounter,
-		},
-		nil
-}
-
 func getRequestCounterKey(endpoint, method string) string {
 	return fmt.Sprintf("%s%s_%s", httpRequestCounterPrefix,
 		strings.ReplaceAll(endpoint, "/", "_"), method)
 }
 
+func getRequestDurationKey(endpoint, method string) string {
+	return fmt.Sprintf("%s%s_%s", httpRequestDurationPrefix,
+		strings.ReplaceAll(endpoint, "/", "_"), method)
+}
+
 func getResponseCounterKey(endpoint, method string, status int) string {
 	return fmt.Sprintf("%s%s_%s_%d", httpResponseCounterPrefix,
 		strings.ReplaceAll(endpoint, "/", "_"), method, status)
@@ -132,6 +161,10 @@ func getResponseCounterKey(endpoint, method string, status int) string {
 // Handler logs every request and response and adds the, to metrics.
 func (m *HTTPMiddleware) Handler(h http.Handler) http.Handler {
 	fn := func(rw http.ResponseWriter, r *http.Request) {
+		reqStart := time.Now()
+		defer func() {
+			m.totalHTTPRequestDuration.Record(m.ctx, time.Since(reqStart).Milliseconds())
+		}()
 		traceID := hash(fmt.Sprintf("%v", r))
 		log.Tracef("HTTP request %v: %v %v", traceID, r.Method, r.URL)
 
@@ -161,6 +194,14 @@ func (m *HTTPMiddleware) Handler(h http.Handler) http.Handler {
 		if c, ok := m.totalHTTPResponseCodeCounters[w.Status()]; ok {
 			c.Add(m.ctx, 1)
 		}
+
+		durationKey := getRequestDurationKey(r.URL.Path, r.Method)
+		reqTook := time.Since(reqStart)
+		if c, ok := m.httpRequestDurations[durationKey]; ok {
+			c.Record(m.ctx, reqTook.Milliseconds())
+		}
+		log.Debugf("request %s %s took %d ms", r.Method, r.URL.Path, reqTook.Milliseconds())
+
 	}
 
 	return http.HandlerFunc(fn)

management/server/telemetry/store_metrics.go

@@ -0,0 +1,47 @@
+package telemetry
+
+import (
+	"context"
+	"time"
+
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/instrument"
+	"go.opentelemetry.io/otel/metric/instrument/syncint64"
+)
+
+// StoreMetrics represents all metrics related to the FileStore
+type StoreMetrics struct {
+	globalLockAcquisitionDuration syncint64.Histogram
+	persistenceDuration           syncint64.Histogram
+	ctx                           context.Context
+}
+
+// NewStoreMetrics creates an instance of StoreMetrics
+func NewStoreMetrics(ctx context.Context, meter metric.Meter) (*StoreMetrics, error) {
+	globalLockAcquisitionDuration, err := meter.SyncInt64().Histogram("management.store.global.lock.acquisition.duration.micro",
+		instrument.WithUnit("microseconds"))
+	if err != nil {
+		return nil, err
+	}
+	persistenceDuration, err := meter.SyncInt64().Histogram("management.store.persistence.duration.micro",
+		instrument.WithUnit("microseconds"))
+	if err != nil {
+		return nil, err
+	}
+
+	return &StoreMetrics{
+		globalLockAcquisitionDuration: globalLockAcquisitionDuration,
+		persistenceDuration:           persistenceDuration,
+		ctx:                           ctx,
+	}, nil
+}
+
+// CountGlobalLockAcquisitionDuration counts the duration of the global lock acquisition
+func (metrics *StoreMetrics) CountGlobalLockAcquisitionDuration(duration time.Duration) {
+	metrics.globalLockAcquisitionDuration.Record(metrics.ctx, duration.Microseconds())
+}
+
+// CountPersistenceDuration counts the duration of a store persistence operation
+func (metrics *StoreMetrics) CountPersistenceDuration(duration time.Duration) {
+	metrics.persistenceDuration.Record(metrics.ctx, duration.Microseconds())
+}

management/server/user.go

@@ -565,6 +565,14 @@ func (am *DefaultAccountManager) SaveUser(accountID, initiatorUserID string, upd
 	}
 
 	defer func() {
+		if oldUser.IsBlocked() != update.IsBlocked() {
+			if update.IsBlocked() {
+				am.storeEvent(initiatorUserID, oldUser.Id, accountID, activity.UserBlocked, nil)
+			} else {
+				am.storeEvent(initiatorUserID, oldUser.Id, accountID, activity.UserUnblocked, nil)
+			}
+		}
+
 		// store activity logs
 		if oldUser.Role != newUser.Role {
 			am.storeEvent(initiatorUserID, oldUser.Id, accountID, activity.UserRoleUpdated, map[string]any{"role": newUser.Role})