Update management/server/sqlite_store.go

* Add duplicate constants check

.github/workflows/golangci-lint.yml

@@ -33,6 +33,10 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
+      - name: Check for duplicate constants
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          ! awk '/const \(/,/)/{print $0}' management/server/activity/codes.go | grep -o '= [0-9]*' | sort | uniq -d | grep .
       - name: Install Go
         uses: actions/setup-go@v4
         with:

client/internal/peer/conn.go

@@ -466,7 +466,7 @@ func (conn *Conn) configureConnection(remoteConn net.Conn, remoteWgPort int, rem
 		LocalIceCandidateType:      pair.Local.Type().String(),
 		RemoteIceCandidateType:     pair.Remote.Type().String(),
 		LocalIceCandidateEndpoint:  fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()),
-		RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Local.Port()),
+		RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()),
 		Direct:                     !isRelayCandidate(pair.Local),
 		RosenpassEnabled:           rosenpassEnabled,
 		Mux:                        new(sync.RWMutex),

go.mod

@@ -60,7 +60,7 @@ require (
 	github.com/miekg/dns v1.1.43
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/nadoo/ipset v0.5.0
-	github.com/netbirdio/management-integrations/integrations v0.0.0-20240326083846-3682438fca98
+	github.com/netbirdio/management-integrations/integrations v0.0.0-20240415094251-369eb33c9b01
 	github.com/okta/okta-sdk-golang/v2 v2.18.0
 	github.com/oschwald/maxminddb-golang v1.12.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible

go.sum

@@ -383,8 +383,8 @@ github.com/nadoo/ipset v0.5.0 h1:5GJUAuZ7ITQQQGne5J96AmFjRtI8Avlbk6CabzYWVUc=
 github.com/nadoo/ipset v0.5.0/go.mod h1:rYF5DQLRGGoQ8ZSWeK+6eX5amAuPqwFkWjhQlEITGJQ=
 github.com/netbirdio/ice/v3 v3.0.0-20240315174635-e72a50fcb64e h1:PURA50S8u4mF6RrkYYCAvvPCixhqqEiEy3Ej6avh04c=
 github.com/netbirdio/ice/v3 v3.0.0-20240315174635-e72a50fcb64e/go.mod h1:YMLU7qbKfVjmEv7EoZPIVEI+kNYxWCdPK3VS0BU+U4Q=
-github.com/netbirdio/management-integrations/integrations v0.0.0-20240326083846-3682438fca98 h1:i6AtenTLu/CqhTmj0g1K/GWkkpMJMhQM6Vjs46x25nA=
+github.com/netbirdio/management-integrations/integrations v0.0.0-20240415094251-369eb33c9b01 h1:Fu9fq0ndfKVuFTEwbc8Etqui10BOkcMTv0UqcMy0RuY=
-github.com/netbirdio/management-integrations/integrations v0.0.0-20240326083846-3682438fca98/go.mod h1:kxks50DrZnhW+oRTdHOkVOJbcTcyo766am8RBugo+Yc=
+github.com/netbirdio/management-integrations/integrations v0.0.0-20240415094251-369eb33c9b01/go.mod h1:kxks50DrZnhW+oRTdHOkVOJbcTcyo766am8RBugo+Yc=
 github.com/netbirdio/service v0.0.0-20230215170314-b923b89432b0 h1:hirFRfx3grVA/9eEyjME5/z3nxdJlN9kfQpvWWPk32g=
 github.com/netbirdio/service v0.0.0-20230215170314-b923b89432b0/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
 github.com/netbirdio/systray v0.0.0-20231030152038-ef1ed2a27949 h1:xbWM9BU6mwZZLHxEjxIX/V8Hv3HurQt4mReIE4mY4DM=

management/cmd/management.go

@@ -251,7 +251,7 @@ var (
 
 			ctx, cancel := context.WithCancel(cmd.Context())
 			defer cancel()
-			httpAPIHandler, err := httpapi.APIHandler(ctx, accountManager, geo, *jwtValidator, appMetrics, httpAPIAuthCfg)
+			httpAPIHandler, err := httpapi.APIHandler(ctx, accountManager, geo, *jwtValidator, appMetrics, httpAPIAuthCfg, integratedPeerValidator)
 			if err != nil {
 				return fmt.Errorf("failed creating HTTP API handler: %v", err)
 			}

management/server/account.go

@@ -1473,7 +1473,7 @@ func (am *DefaultAccountManager) handleNewUserAccount(domainAcc *Account, claims
 	// if domain already has a primary account, add regular user
 	if domainAcc != nil {
 		account = domainAcc
-		account.Users[claims.UserId] = NewRegularUser(claims.UserId)
+		account.Users[claims.UserId] = NewRegularUser(claims.UserId, account.Id)
 		err = am.Store.SaveAccount(account)
 		if err != nil {
 			return nil, err
@@ -1849,6 +1849,7 @@ func (am *DefaultAccountManager) CheckUserAccessByJWTGroups(claims jwtclaims.Aut
 }
 
 func (am *DefaultAccountManager) onPeersInvalidated(accountID string) {
+	log.Debugf("validated peers has been invalidated for account %s", accountID)
 	updatedAccount, err := am.Store.GetAccount(accountID)
 	if err != nil {
 		log.Errorf("failed to get account %s: %v", accountID, err)
@@ -1861,9 +1862,10 @@ func (am *DefaultAccountManager) onPeersInvalidated(accountID string) {
 func addAllGroup(account *Account) error {
 	if len(account.Groups) == 0 {
 		allGroup := &nbgroup.Group{
-			ID:     xid.New().String(),
+			ID:        xid.New().String(),
-			Name:   "All",
+			Name:      "All",
-			Issued: nbgroup.GroupIssuedAPI,
+			Issued:    nbgroup.GroupIssuedAPI,
+			AccountID: account.Id,
 		}
 		for _, peer := range account.Peers {
 			allGroup.Peers = append(allGroup.Peers, peer.ID)
@@ -1907,7 +1909,7 @@ func newAccountWithId(accountID, userID, domain string) *Account {
 	routes := make(map[string]*route.Route)
 	setupKeys := map[string]*SetupKey{}
 	nameServersGroups := make(map[string]*nbdns.NameServerGroup)
-	users[userID] = NewOwnerUser(userID)
+	users[userID] = NewOwnerUser(userID, accountID)
 	dnsSettings := DNSSettings{
 		DisabledManagementGroups: make([]string, 0),
 	}

management/server/activity/codes.go

@@ -11,133 +11,134 @@ type Code struct {
 	Code    string
 }
 
+// Existing consts must not be changed, as this will break the compatibility with the existing data
 const (
 	// PeerAddedByUser indicates that a user added a new peer to the system
-	PeerAddedByUser Activity = iota
+	PeerAddedByUser Activity = 0
 	// PeerAddedWithSetupKey indicates that a new peer joined the system using a setup key
-	PeerAddedWithSetupKey
+	PeerAddedWithSetupKey Activity = 1
 	// UserJoined indicates that a new user joined the account
-	UserJoined
+	UserJoined Activity = 2
 	// UserInvited indicates that a new user was invited to join the account
-	UserInvited
+	UserInvited Activity = 3
 	// AccountCreated indicates that a new account has been created
-	AccountCreated
+	AccountCreated Activity = 4
 	// PeerRemovedByUser indicates that a user removed a peer from the system
-	PeerRemovedByUser
+	PeerRemovedByUser Activity = 5
 	// RuleAdded indicates that a user added a new rule
-	RuleAdded
+	RuleAdded Activity = 6
 	// RuleUpdated indicates that a user updated a rule
-	RuleUpdated
+	RuleUpdated Activity = 7
 	// RuleRemoved indicates that a user removed a rule
-	RuleRemoved
+	RuleRemoved Activity = 8
 	// PolicyAdded indicates that a user added a new policy
-	PolicyAdded
+	PolicyAdded Activity = 9
 	// PolicyUpdated indicates that a user updated a policy
-	PolicyUpdated
+	PolicyUpdated Activity = 10
 	// PolicyRemoved indicates that a user removed a policy
-	PolicyRemoved
+	PolicyRemoved Activity = 11
 	// SetupKeyCreated indicates that a user created a new setup key
-	SetupKeyCreated
+	SetupKeyCreated Activity = 12
 	// SetupKeyUpdated indicates that a user updated a setup key
-	SetupKeyUpdated
+	SetupKeyUpdated Activity = 13
 	// SetupKeyRevoked indicates that a user revoked a setup key
-	SetupKeyRevoked
+	SetupKeyRevoked Activity = 14
 	// SetupKeyOverused indicates that setup key usage exhausted
-	SetupKeyOverused
+	SetupKeyOverused Activity = 15
 	// GroupCreated indicates that a user created a group
-	GroupCreated
+	GroupCreated Activity = 16
 	// GroupUpdated indicates that a user updated a group
-	GroupUpdated
+	GroupUpdated Activity = 17
 	// GroupAddedToPeer indicates that a user added group to a peer
-	GroupAddedToPeer
+	GroupAddedToPeer Activity = 18
 	// GroupRemovedFromPeer indicates that a user removed peer group
-	GroupRemovedFromPeer
+	GroupRemovedFromPeer Activity = 19
 	// GroupAddedToUser indicates that a user added group to a user
-	GroupAddedToUser
+	GroupAddedToUser Activity = 20
 	// GroupRemovedFromUser indicates that a user removed a group from a user
-	GroupRemovedFromUser
+	GroupRemovedFromUser Activity = 21
 	// UserRoleUpdated indicates that a user changed the role of a user
-	UserRoleUpdated
+	UserRoleUpdated Activity = 22
 	// GroupAddedToSetupKey indicates that a user added group to a setup key
-	GroupAddedToSetupKey
+	GroupAddedToSetupKey Activity = 23
 	// GroupRemovedFromSetupKey indicates that a user removed a group from a setup key
-	GroupRemovedFromSetupKey
+	GroupRemovedFromSetupKey Activity = 24
 	// GroupAddedToDisabledManagementGroups indicates that a user added a group to the DNS setting Disabled management groups
-	GroupAddedToDisabledManagementGroups
+	GroupAddedToDisabledManagementGroups Activity = 25
 	// GroupRemovedFromDisabledManagementGroups indicates that a user removed a group from the DNS setting Disabled management groups
-	GroupRemovedFromDisabledManagementGroups
+	GroupRemovedFromDisabledManagementGroups Activity = 26
 	// RouteCreated indicates that a user created a route
-	RouteCreated
+	RouteCreated Activity = 27
 	// RouteRemoved indicates that a user deleted a route
-	RouteRemoved
+	RouteRemoved Activity = 28
 	// RouteUpdated indicates that a user updated a route
-	RouteUpdated
+	RouteUpdated Activity = 29
 	// PeerSSHEnabled indicates that a user enabled SSH server on a peer
-	PeerSSHEnabled
+	PeerSSHEnabled Activity = 30
 	// PeerSSHDisabled indicates that a user disabled SSH server on a peer
-	PeerSSHDisabled
+	PeerSSHDisabled Activity = 31
 	// PeerRenamed indicates that a user renamed a peer
-	PeerRenamed
+	PeerRenamed Activity = 32
 	// PeerLoginExpirationEnabled indicates that a user enabled login expiration of a peer
-	PeerLoginExpirationEnabled
+	PeerLoginExpirationEnabled Activity = 33
 	// PeerLoginExpirationDisabled indicates that a user disabled login expiration of a peer
-	PeerLoginExpirationDisabled
+	PeerLoginExpirationDisabled Activity = 34
 	// NameserverGroupCreated indicates that a user created a nameservers group
-	NameserverGroupCreated
+	NameserverGroupCreated Activity = 35
 	// NameserverGroupDeleted indicates that a user deleted a nameservers group
-	NameserverGroupDeleted
+	NameserverGroupDeleted Activity = 36
 	// NameserverGroupUpdated indicates that a user updated a nameservers group
-	NameserverGroupUpdated
+	NameserverGroupUpdated Activity = 37
 	// AccountPeerLoginExpirationEnabled indicates that a user enabled peer login expiration for the account
-	AccountPeerLoginExpirationEnabled
+	AccountPeerLoginExpirationEnabled Activity = 38
 	// AccountPeerLoginExpirationDisabled indicates that a user disabled peer login expiration for the account
-	AccountPeerLoginExpirationDisabled
+	AccountPeerLoginExpirationDisabled Activity = 39
 	// AccountPeerLoginExpirationDurationUpdated indicates that a user updated peer login expiration duration for the account
-	AccountPeerLoginExpirationDurationUpdated
+	AccountPeerLoginExpirationDurationUpdated Activity = 40
 	// PersonalAccessTokenCreated indicates that a user created a personal access token
-	PersonalAccessTokenCreated
+	PersonalAccessTokenCreated Activity = 41
 	// PersonalAccessTokenDeleted indicates that a user deleted a personal access token
-	PersonalAccessTokenDeleted
+	PersonalAccessTokenDeleted Activity = 42
 	// ServiceUserCreated indicates that a user created a service user
-	ServiceUserCreated
+	ServiceUserCreated Activity = 43
 	// ServiceUserDeleted indicates that a user deleted a service user
-	ServiceUserDeleted
+	ServiceUserDeleted Activity = 44
 	// UserBlocked indicates that a user blocked another user
-	UserBlocked
+	UserBlocked Activity = 45
 	// UserUnblocked indicates that a user unblocked another user
-	UserUnblocked
+	UserUnblocked Activity = 46
 	// UserDeleted indicates that a user deleted another user
-	UserDeleted
+	UserDeleted Activity = 47
 	// GroupDeleted indicates that a user deleted group
-	GroupDeleted
+	GroupDeleted Activity = 48
 	// UserLoggedInPeer indicates that user logged in their peer with an interactive SSO login
-	UserLoggedInPeer
+	UserLoggedInPeer Activity = 49
 	// PeerLoginExpired indicates that the user peer login has been expired and peer disconnected
-	PeerLoginExpired
+	PeerLoginExpired Activity = 50
 	// DashboardLogin indicates that the user logged in to the dashboard
-	DashboardLogin
+	DashboardLogin Activity = 51
 	// IntegrationCreated indicates that the user created an integration
-	IntegrationCreated
+	IntegrationCreated Activity = 52
 	// IntegrationUpdated indicates that the user updated an integration
-	IntegrationUpdated
+	IntegrationUpdated Activity = 53
 	// IntegrationDeleted indicates that the user deleted an integration
-	IntegrationDeleted
+	IntegrationDeleted Activity = 54
 	// AccountPeerApprovalEnabled indicates that the user enabled peer approval for the account
-	AccountPeerApprovalEnabled
+	AccountPeerApprovalEnabled Activity = 55
 	// AccountPeerApprovalDisabled indicates that the user disabled peer approval for the account
-	AccountPeerApprovalDisabled
+	AccountPeerApprovalDisabled Activity = 56
 	// PeerApproved indicates that the peer has been approved
-	PeerApproved
+	PeerApproved Activity = 57
 	// PeerApprovalRevoked indicates that the peer approval has been revoked
-	PeerApprovalRevoked
+	PeerApprovalRevoked Activity = 58
 	// TransferredOwnerRole indicates that the user transferred the owner role of the account
-	TransferredOwnerRole
+	TransferredOwnerRole Activity = 59
 	// PostureCheckCreated indicates that the user created a posture check
-	PostureCheckCreated
+	PostureCheckCreated Activity = 60
 	// PostureCheckUpdated indicates that the user updated a posture check
-	PostureCheckUpdated
+	PostureCheckUpdated Activity = 61
 	// PostureCheckDeleted indicates that the user deleted a posture check
-	PostureCheckDeleted
+	PostureCheckDeleted Activity = 62
 )
 
 var activityMap = map[Activity]Code{

management/server/http/accounts_handler_test.go

@@ -54,7 +54,7 @@ func initAccountsTestData(account *server.Account, admin *server.User) *Accounts
 
 func TestAccounts_AccountsHandler(t *testing.T) {
 	accountID := "test_account"
-	adminUser := server.NewAdminUser("test_user")
+	adminUser := server.NewAdminUser("test_user", "account_id")
 
 	sr := func(v string) *string { return &v }
 	br := func(v bool) *bool { return &v }

management/server/http/dns_settings_handler_test.go

@@ -34,7 +34,7 @@ var testingDNSSettingsAccount = &server.Account{
 	Id:     testDNSSettingsAccountID,
 	Domain: "hotmail.com",
 	Users: map[string]*server.User{
-		testDNSSettingsUserID: server.NewAdminUser("test_user"),
+		testDNSSettingsUserID: server.NewAdminUser("test_user", "account_id"),
 	},
 	DNSSettings: baseExistingDNSSettings,
 }

management/server/http/events_handler_test.go

@@ -196,7 +196,7 @@ func TestEvents_GetEvents(t *testing.T) {
 		},
 	}
 	accountID := "test_account"
-	adminUser := server.NewAdminUser("test_user")
+	adminUser := server.NewAdminUser("test_user", "account_id")
 	events := generateEvents(accountID, adminUser.Id)
 	handler := initEventsTestData(accountID, adminUser, events...)
 

management/server/http/geolocation_handler_test.go

@@ -42,7 +42,7 @@ func initGeolocationTestData(t *testing.T) *GeolocationsHandler {
 	return &GeolocationsHandler{
 		accountManager: &mock_server.MockAccountManager{
 			GetAccountFromTokenFunc: func(claims jwtclaims.AuthorizationClaims) (*server.Account, *server.User, error) {
-				user := server.NewAdminUser("test_user")
+				user := server.NewAdminUser("test_user", "account_id")
 				return &server.Account{
 					Id: claims.AccountId,
 					Users: map[string]*server.User{

management/server/http/groups_handler_test.go

@@ -124,7 +124,7 @@ func TestGetGroup(t *testing.T) {
 		Name: "Group",
 	}
 
-	adminUser := server.NewAdminUser("test_user")
+	adminUser := server.NewAdminUser("test_user", "account_id")
 	p := initGroupTestData(adminUser, group)
 
 	for _, tc := range tt {
@@ -246,7 +246,7 @@ func TestWriteGroup(t *testing.T) {
 		},
 	}
 
-	adminUser := server.NewAdminUser("test_user")
+	adminUser := server.NewAdminUser("test_user", "account_id")
 	p := initGroupTestData(adminUser)
 
 	for _, tc := range tt {
@@ -324,7 +324,7 @@ func TestDeleteGroup(t *testing.T) {
 		},
 	}
 
-	adminUser := server.NewAdminUser("test_user")
+	adminUser := server.NewAdminUser("test_user", "account_id")
 	p := initGroupTestData(adminUser)
 
 	for _, tc := range tt {

management/server/http/handler.go

@@ -12,6 +12,7 @@ import (
 	s "github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/geolocation"
 	"github.com/netbirdio/netbird/management/server/http/middleware"
+	"github.com/netbirdio/netbird/management/server/integrated_validator"
 	"github.com/netbirdio/netbird/management/server/jwtclaims"
 	"github.com/netbirdio/netbird/management/server/telemetry"
 )
@@ -38,7 +39,7 @@ type emptyObject struct {
 }
 
 // APIHandler creates the Management service HTTP API handler registering all the available endpoints.
-func APIHandler(ctx context.Context, accountManager s.AccountManager, LocationManager *geolocation.Geolocation, jwtValidator jwtclaims.JWTValidator, appMetrics telemetry.AppMetrics, authCfg AuthCfg) (http.Handler, error) {
+func APIHandler(ctx context.Context, accountManager s.AccountManager, LocationManager *geolocation.Geolocation, jwtValidator jwtclaims.JWTValidator, appMetrics telemetry.AppMetrics, authCfg AuthCfg, integratedValidator integrated_validator.IntegratedValidator) (http.Handler, error) {
 	claimsExtractor := jwtclaims.NewClaimsExtractor(
 		jwtclaims.WithAudience(authCfg.Audience),
 		jwtclaims.WithUserIDClaim(authCfg.UserIDClaim),
@@ -75,7 +76,7 @@ func APIHandler(ctx context.Context, accountManager s.AccountManager, LocationMa
 		AuthCfg:            authCfg,
 	}
 
-	if _, err := integrations.RegisterHandlers(ctx, prefix, api.Router, accountManager, claimsExtractor); err != nil {
+	if _, err := integrations.RegisterHandlers(ctx, prefix, api.Router, accountManager, claimsExtractor, integratedValidator); err != nil {
 		return nil, fmt.Errorf("register integrations endpoints: %w", err)
 	}
 

management/server/http/nameservers_handler_test.go

@@ -32,7 +32,7 @@ var testingNSAccount = &server.Account{
 	Id:     testNSGroupAccountID,
 	Domain: "hotmail.com",
 	Users: map[string]*server.User{
-		"test_user": server.NewAdminUser("test_user"),
+		"test_user": server.NewAdminUser("test_user", "account_id"),
 	},
 }
 

management/server/http/peers_handler_test.go

@@ -59,7 +59,7 @@ func initTestMetaData(peers ...*nbpeer.Peer) *PeersHandler {
 				return "netbird.selfhosted"
 			},
 			GetAccountFromTokenFunc: func(claims jwtclaims.AuthorizationClaims) (*server.Account, *server.User, error) {
-				user := server.NewAdminUser("test_user")
+				user := server.NewAdminUser("test_user", "account_id")
 				return &server.Account{
 					Id:     claims.AccountId,
 					Domain: "hotmail.com",

management/server/http/policies_handler_test.go

@@ -45,7 +45,7 @@ func initPoliciesTestData(policies ...*server.Policy) *Policies {
 				return nil
 			},
 			GetAccountFromTokenFunc: func(claims jwtclaims.AuthorizationClaims) (*server.Account, *server.User, error) {
-				user := server.NewAdminUser("test_user")
+				user := server.NewAdminUser("test_user", "account_id")
 				return &server.Account{
 					Id:     claims.AccountId,
 					Domain: "hotmail.com",

management/server/http/posture_checks_handler_test.go

@@ -62,7 +62,7 @@ func initPostureChecksTestData(postureChecks ...*posture.Checks) *PostureChecksH
 				return accountPostureChecks, nil
 			},
 			GetAccountFromTokenFunc: func(claims jwtclaims.AuthorizationClaims) (*server.Account, *server.User, error) {
-				user := server.NewAdminUser("test_user")
+				user := server.NewAdminUser("test_user", "account_id")
 				return &server.Account{
 					Id: claims.AccountId,
 					Users: map[string]*server.User{

management/server/http/routes_handler_test.go

@@ -75,7 +75,7 @@ var testingAccount = &server.Account{
 		},
 	},
 	Users: map[string]*server.User{
-		"test_user": server.NewAdminUser("test_user"),
+		"test_user": server.NewAdminUser("test_user", "account_id"),
 	},
 }
 

management/server/http/setupkeys_handler_test.go

@@ -97,7 +97,7 @@ func TestSetupKeysHandlers(t *testing.T) {
 	defaultSetupKey := server.GenerateDefaultSetupKey()
 	defaultSetupKey.Id = existingSetupKeyID
 
-	adminUser := server.NewAdminUser("test_user")
+	adminUser := server.NewAdminUser("test_user", "account_id")
 
 	newSetupKey := server.GenerateSetupKey(newSetupKeyName, server.SetupKeyReusable, 0, []string{"group-1"},
 		server.SetupKeyUnlimitedUsage, true)

management/server/scheduler.go

@@ -95,18 +95,18 @@ func (wm *DefaultScheduler) Schedule(in time.Duration, ID string, job func() (ne
 			case <-ticker.C:
 				select {
 				case <-cancel:
-					log.Debugf("scheduled job %s was canceled, stop timer", ID)
+					log.Tracef("scheduled job %s was canceled, stop timer", ID)
 					ticker.Stop()
 					return
 				default:
-					log.Debugf("time to do a scheduled job %s", ID)
+					log.Tracef("time to do a scheduled job %s", ID)
 				}
 				runIn, reschedule := job()
 				if !reschedule {
 					wm.mu.Lock()
 					defer wm.mu.Unlock()
 					delete(wm.jobs, ID)
-					log.Debugf("job %s is not scheduled to run again", ID)
+					log.Tracef("job %s is not scheduled to run again", ID)
 					ticker.Stop()
 					return
 				}
@@ -115,7 +115,7 @@ func (wm *DefaultScheduler) Schedule(in time.Duration, ID string, job func() (ne
 					ticker.Reset(runIn)
 				}
 			case <-cancel:
-				log.Debugf("job %s was canceled, stopping timer", ID)
+				log.Tracef("job %s was canceled, stopping timer", ID)
 				ticker.Stop()
 				return
 			}

management/server/sqlite_store.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"path/filepath"
+	"reflect"
 	"runtime"
 	"strings"
 	"sync"
@@ -134,72 +135,139 @@ func (s *SqliteStore) AcquireAccountLock(accountID string) (unlock func()) {
 	return unlock
 }
 
+func batchInsert(records interface{}, batchSize int, tx *gorm.DB) error {
+	// Get the reflect.Value of the records slice
+	v := reflect.ValueOf(records)
+	if v.Kind() != reflect.Slice {
+		return fmt.Errorf("provided input is not a slice")
+	}
+
+	// Insert records in batches
+	for i := 0; i < v.Len(); i += batchSize {
+		end := i + batchSize
+		if end > v.Len() {
+			end = v.Len()
+		}
+		// Use reflect.Slice to get a slice of the records for the current batch
+		batch := v.Slice(i, end).Interface()
+		if err := tx.CreateInBatches(batch, end-i).Debug().Error; err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (s *SqliteStore) SaveAccount(account *Account) error {
 	start := time.Now()
 
-	for _, key := range account.SetupKeys {
+	// operate over a fresh copy as we will modify its fields
-		account.SetupKeysG = append(account.SetupKeysG, *key)
+	accCopy := account.Copy()
+	accCopy.SetupKeysG = make([]SetupKey, 0, len(accCopy.SetupKeys))
+	for _, key := range accCopy.SetupKeys {
+		//we need an explicit reference to the account for gorm
+		key.AccountID = accCopy.Id
+		accCopy.SetupKeysG = append(accCopy.SetupKeysG, *key)
 	}
 
-	for id, peer := range account.Peers {
+	accCopy.PeersG = make([]nbpeer.Peer, 0, len(accCopy.Peers))
+	for id, peer := range accCopy.Peers {
 		peer.ID = id
-		account.PeersG = append(account.PeersG, *peer)
+		//we need an explicit reference to the account for gorm
+		peer.AccountID = accCopy.Id
+		accCopy.PeersG = append(accCopy.PeersG, *peer)
 	}
 
-	for id, user := range account.Users {
+	accCopy.UsersG = make([]User, 0, len(accCopy.Users))
+	for id, user := range accCopy.Users {
 		user.Id = id
+		//we need an explicit reference to the account for gorm
+		user.AccountID = accCopy.Id
+		user.PATsG = make([]PersonalAccessToken, 0, len(user.PATs))
 		for id, pat := range user.PATs {
 			pat.ID = id
 			user.PATsG = append(user.PATsG, *pat)
 		}
-		account.UsersG = append(account.UsersG, *user)
+		accCopy.UsersG = append(accCopy.UsersG, *user)
 	}
 
-	for id, group := range account.Groups {
+	accCopy.GroupsG = make([]nbgroup.Group, 0, len(accCopy.Groups))
+	for id, group := range accCopy.Groups {
 		group.ID = id
-		account.GroupsG = append(account.GroupsG, *group)
+		//we need an explicit reference to the account for gorm
+		group.AccountID = accCopy.Id
+		accCopy.GroupsG = append(accCopy.GroupsG, *group)
 	}
 
-	for id, route := range account.Routes {
+	accCopy.RoutesG = make([]route.Route, 0, len(accCopy.Routes))
+	for id, route := range accCopy.Routes {
 		route.ID = id
-		account.RoutesG = append(account.RoutesG, *route)
+		//we need an explicit reference to the account for gorm
+		route.AccountID = accCopy.Id
+		accCopy.RoutesG = append(accCopy.RoutesG, *route)
 	}
 
-	for id, ns := range account.NameServerGroups {
+	accCopy.NameServerGroupsG = make([]nbdns.NameServerGroup, 0, len(accCopy.NameServerGroups))
+	for id, ns := range accCopy.NameServerGroups {
 		ns.ID = id
-		account.NameServerGroupsG = append(account.NameServerGroupsG, *ns)
+		//we need an explicit reference to the account for gorm
+		ns.AccountID = accCopy.Id
+		accCopy.NameServerGroupsG = append(accCopy.NameServerGroupsG, *ns)
 	}
 
 	err := s.db.Transaction(func(tx *gorm.DB) error {
-		result := tx.Select(clause.Associations).Delete(account.Policies, "account_id = ?", account.Id)
+		result := tx.Select(clause.Associations).Delete(accCopy.Policies, "account_id = ?", accCopy.Id)
 		if result.Error != nil {
 			return result.Error
 		}
 
-		result = tx.Select(clause.Associations).Delete(account.UsersG, "account_id = ?", account.Id)
+		result = tx.Select(clause.Associations).Delete(accCopy.UsersG, "account_id = ?", accCopy.Id)
 		if result.Error != nil {
 			return result.Error
 		}
 
-		result = tx.Select(clause.Associations).Delete(account)
+		result = tx.Select(clause.Associations).Delete(accCopy)
 		if result.Error != nil {
 			return result.Error
 		}
 
 		result = tx.
 			Session(&gorm.Session{FullSaveAssociations: true}).
-			Clauses(clause.OnConflict{UpdateAll: true}).Create(account)
+			Clauses(clause.OnConflict{UpdateAll: true}).
+			Omit("PeersG", "GroupsG", "UsersG", "SetupKeysG", "RoutesG", "NameServerGroupsG").
+			Create(accCopy)
 		if result.Error != nil {
 			return result.Error
 		}
-		return nil
+
+		const batchSize = 500
+		err := batchInsert(accCopy.PeersG, batchSize, tx)
+		if err != nil {
+			return err
+		}
+		err = batchInsert(accCopy.UsersG, batchSize, tx)
+		if err != nil {
+			return err
+		}
+		err = batchInsert(accCopy.GroupsG, batchSize, tx)
+		if err != nil {
+			return err
+		}
+		err = batchInsert(accCopy.RoutesG, batchSize, tx)
+		if err != nil {
+			return err
+		}
+		err = batchInsert(accCopy.SetupKeysG, batchSize, tx)
+		if err != nil {
+			return err
+		}
+		return batchInsert(accCopy.NameServerGroupsG, batchSize, tx)
 	})
 
 	took := time.Since(start)
 	if s.metrics != nil {
 		s.metrics.StoreMetrics().CountPersistenceDuration(took)
 	}
-	log.Debugf("took %d ms to persist an account to the SQLite", took.Milliseconds())
+	log.Debugf("took %d ms to persist an account %s to the SQLite store", took.Milliseconds(), accCopy.Id)
 
 	return err
 }
@@ -207,6 +275,19 @@ func (s *SqliteStore) SaveAccount(account *Account) error {
 func (s *SqliteStore) DeleteAccount(account *Account) error {
 	start := time.Now()
 
+	account.UsersG = make([]User, 0, len(account.Users))
+	for id, user := range account.Users {
+		user.Id = id
+		//we need an explicit reference to an account as it is missing for some reason
+		user.AccountID = account.Id
+		user.PATsG = make([]PersonalAccessToken, 0, len(user.PATs))
+		for id, pat := range user.PATs {
+			pat.ID = id
+			user.PATsG = append(user.PATsG, *pat)
+		}
+		account.UsersG = append(account.UsersG, *user)
+	}
+
 	err := s.db.Transaction(func(tx *gorm.DB) error {
 		result := tx.Select(clause.Associations).Delete(account.Policies, "account_id = ?", account.Id)
 		if result.Error != nil {

management/server/sqlite_store_test.go

@@ -2,7 +2,12 @@ package server
 
 import (
 	"fmt"
+	nbdns "github.com/netbirdio/netbird/dns"
+	nbgroup "github.com/netbirdio/netbird/management/server/group"
+	route2 "github.com/netbirdio/netbird/route"
+	"math/rand"
 	"net"
+	"net/netip"
 	"path/filepath"
 	"runtime"
 	"testing"
@@ -29,6 +34,141 @@ func TestSqlite_NewStore(t *testing.T) {
 		t.Errorf("expected to create a new empty Accounts map when creating a new FileStore")
 	}
 }
+func TestSqlite_SaveAccount_Large(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("The SQLite store is not properly supported by Windows yet")
+	}
+
+	store := newSqliteStore(t)
+
+	account := newAccountWithId("account_id", "testuser", "")
+	groupALL, err := account.GetGroupAll()
+	if err != nil {
+		t.Fatal(err)
+	}
+	setupKey := GenerateDefaultSetupKey()
+	account.SetupKeys[setupKey.Key] = setupKey
+	const numPerAccount = 2000
+	for n := 0; n < numPerAccount; n++ {
+		netIP := randomIPv4()
+		peerID := fmt.Sprintf("%s-peer-%d", account.Id, n)
+
+		peer := &nbpeer.Peer{
+			ID:         peerID,
+			Key:        peerID,
+			SetupKey:   "",
+			IP:         netIP,
+			Name:       peerID,
+			DNSLabel:   peerID,
+			UserID:     userID,
+			Status:     &nbpeer.PeerStatus{Connected: false, LastSeen: time.Now()},
+			SSHEnabled: false,
+		}
+		account.Peers[peerID] = peer
+		group, _ := account.GetGroupAll()
+		group.Peers = append(group.Peers, peerID)
+		user := &User{
+			Id:        fmt.Sprintf("%s-user-%d", account.Id, n),
+			AccountID: account.Id,
+		}
+		account.Users[user.Id] = user
+		route := &route2.Route{
+			ID:          fmt.Sprintf("network-id-%d", n),
+			Description: "base route",
+			NetID:       fmt.Sprintf("network-id-%d", n),
+			Network:     netip.MustParsePrefix(netIP.String() + "/24"),
+			NetworkType: route2.IPv4Network,
+			Metric:      9999,
+			Masquerade:  false,
+			Enabled:     true,
+			Groups:      []string{groupALL.ID},
+		}
+		account.Routes[route.ID] = route
+
+		group = &nbgroup.Group{
+			ID:        fmt.Sprintf("group-id-%d", n),
+			AccountID: account.Id,
+			Name:      fmt.Sprintf("group-id-%d", n),
+			Issued:    "api",
+			Peers:     nil,
+		}
+		account.Groups[group.ID] = group
+
+		nameserver := &nbdns.NameServerGroup{
+			ID:                   fmt.Sprintf("nameserver-id-%d", n),
+			AccountID:            account.Id,
+			Name:                 fmt.Sprintf("nameserver-id-%d", n),
+			Description:          "",
+			NameServers:          []nbdns.NameServer{{IP: netip.MustParseAddr(netIP.String()), NSType: nbdns.UDPNameServerType}},
+			Groups:               []string{group.ID},
+			Primary:              false,
+			Domains:              nil,
+			Enabled:              false,
+			SearchDomainsEnabled: false,
+		}
+		account.NameServerGroups[nameserver.ID] = nameserver
+
+		setupKey := GenerateDefaultSetupKey()
+		account.SetupKeys[setupKey.Key] = setupKey
+	}
+
+	err = store.SaveAccount(account)
+	require.NoError(t, err)
+
+	if len(store.GetAllAccounts()) != 1 {
+		t.Errorf("expecting 1 Accounts to be stored after SaveAccount()")
+	}
+
+	a, err := store.GetAccount(account.Id)
+	if a == nil {
+		t.Errorf("expecting Account to be stored after SaveAccount(): %v", err)
+	}
+
+	if a != nil && len(a.Policies) != 1 {
+		t.Errorf("expecting Account to have one policy stored after SaveAccount(), got %d", len(a.Policies))
+	}
+
+	if a != nil && len(a.Policies[0].Rules) != 1 {
+		t.Errorf("expecting Account to have one policy rule stored after SaveAccount(), got %d", len(a.Policies[0].Rules))
+		return
+	}
+
+	if a != nil && len(a.Peers) != numPerAccount {
+		t.Errorf("expecting Account to have %d peers stored after SaveAccount(), got %d",
+			numPerAccount, len(a.Peers))
+		return
+	}
+
+	if a != nil && len(a.Users) != numPerAccount+1 {
+		t.Errorf("expecting Account to have %d users stored after SaveAccount(), got %d",
+			numPerAccount+1, len(a.Users))
+		return
+	}
+
+	if a != nil && len(a.Routes) != numPerAccount {
+		t.Errorf("expecting Account to have %d routes stored after SaveAccount(), got %d",
+			numPerAccount, len(a.Routes))
+		return
+	}
+
+	if a != nil && len(a.NameServerGroups) != numPerAccount {
+		t.Errorf("expecting Account to have %d NameServerGroups stored after SaveAccount(), got %d",
+			numPerAccount, len(a.NameServerGroups))
+		return
+	}
+
+	if a != nil && len(a.NameServerGroups) != numPerAccount {
+		t.Errorf("expecting Account to have %d NameServerGroups stored after SaveAccount(), got %d",
+			numPerAccount, len(a.NameServerGroups))
+		return
+	}
+
+	if a != nil && len(a.SetupKeys) != numPerAccount+1 {
+		t.Errorf("expecting Account to have %d SetupKeys stored after SaveAccount(), got %d",
+			numPerAccount+1, len(a.SetupKeys))
+		return
+	}
+}
 
 func TestSqlite_SaveAccount(t *testing.T) {
 	if runtime.GOOS == "windows" {
@@ -48,6 +188,12 @@ func TestSqlite_SaveAccount(t *testing.T) {
 		Name:     "peer name",
 		Status:   &nbpeer.PeerStatus{Connected: true, LastSeen: time.Now().UTC()},
 	}
+	admin := account.Users["testuser"]
+	admin.PATs = map[string]*PersonalAccessToken{"testtoken": {
+		ID:          "testtoken",
+		Name:        "test token",
+		HashedToken: "hashed token",
+	}}
 
 	err := store.SaveAccount(account)
 	require.NoError(t, err)
@@ -110,7 +256,7 @@ func TestSqlite_DeleteAccount(t *testing.T) {
 	store := newSqliteStore(t)
 
 	testUserID := "testuser"
-	user := NewAdminUser(testUserID)
+	user := NewAdminUser(testUserID, "account_id")
 	user.PATs = map[string]*PersonalAccessToken{"testtoken": {
 		ID:   "testtoken",
 		Name: "test token",
@@ -393,3 +539,12 @@ func newAccount(store Store, id int) error {
 
 	return store.SaveAccount(account)
 }
+
+func randomIPv4() net.IP {
+	rand.New(rand.NewSource(time.Now().UnixNano()))
+	b := make([]byte, 4)
+	for i := range b {
+		b[i] = byte(rand.Intn(256))
+	}
+	return net.IP(b)
+}

management/server/user.go

@@ -180,9 +180,11 @@ func (u *User) Copy() *User {
 }
 
 // NewUser creates a new user
-func NewUser(id string, role UserRole, isServiceUser bool, nonDeletable bool, serviceUserName string, autoGroups []string, issued string) *User {
+func NewUser(ID string, role UserRole, isServiceUser bool, nonDeletable bool, serviceUserName string, autoGroups []string, issued string,
+	accountID string) *User {
 	return &User{
-		Id:              id,
+		Id:              ID,
+		AccountID:       accountID,
 		Role:            role,
 		IsServiceUser:   isServiceUser,
 		NonDeletable:    nonDeletable,
@@ -194,22 +196,26 @@ func NewUser(id string, role UserRole, isServiceUser bool, nonDeletable bool, se
 }
 
 // NewRegularUser creates a new user with role UserRoleUser
-func NewRegularUser(id string) *User {
+func NewRegularUser(ID, accountID string) *User {
-	return NewUser(id, UserRoleUser, false, false, "", []string{}, UserIssuedAPI)
+	return NewUser(ID, UserRoleUser, false, false, "", []string{}, UserIssuedAPI,
+		accountID)
 }
 
 // NewAdminUser creates a new user with role UserRoleAdmin
-func NewAdminUser(id string) *User {
+func NewAdminUser(ID, accountID string) *User {
-	return NewUser(id, UserRoleAdmin, false, false, "", []string{}, UserIssuedAPI)
+	return NewUser(ID, UserRoleAdmin, false, false, "", []string{}, UserIssuedAPI,
+		accountID)
 }
 
 // NewOwnerUser creates a new user with role UserRoleOwner
-func NewOwnerUser(id string) *User {
+func NewOwnerUser(ID, accountID string) *User {
-	return NewUser(id, UserRoleOwner, false, false, "", []string{}, UserIssuedAPI)
+	return NewUser(ID, UserRoleOwner, false, false, "", []string{}, UserIssuedAPI,
+		accountID)
 }
 
 // createServiceUser creates a new service user under the given account.
-func (am *DefaultAccountManager) createServiceUser(accountID string, initiatorUserID string, role UserRole, serviceUserName string, nonDeletable bool, autoGroups []string) (*UserInfo, error) {
+func (am *DefaultAccountManager) createServiceUser(accountID string, initiatorUserID string, role UserRole,
+	serviceUserName string, nonDeletable bool, autoGroups []string) (*UserInfo, error) {
 	unlock := am.Store.AcquireAccountLock(accountID)
 	defer unlock()
 
@@ -231,7 +237,7 @@ func (am *DefaultAccountManager) createServiceUser(accountID string, initiatorUs
 	}
 
 	newUserID := uuid.New().String()
-	newUser := NewUser(newUserID, role, true, nonDeletable, serviceUserName, autoGroups, UserIssuedAPI)
+	newUser := NewUser(newUserID, role, true, nonDeletable, serviceUserName, autoGroups, UserIssuedAPI, accountID)
 	log.Debugf("New User: %v", newUser)
 	account.Users[newUserID] = newUser
 

management/server/user_test.go

@@ -679,8 +679,8 @@ func TestDefaultAccountManager_GetUser(t *testing.T) {
 func TestDefaultAccountManager_ListUsers(t *testing.T) {
 	store := newStore(t)
 	account := newAccountWithId(mockAccountID, mockUserID, "")
-	account.Users["normal_user1"] = NewRegularUser("normal_user1")
+	account.Users["normal_user1"] = NewRegularUser("normal_user1", mockAccountID)
-	account.Users["normal_user2"] = NewRegularUser("normal_user2")
+	account.Users["normal_user2"] = NewRegularUser("normal_user2", mockAccountID)
 
 	err := store.SaveAccount(account)
 	if err != nil {
@@ -760,7 +760,7 @@ func TestDefaultAccountManager_ListUsers_DashboardPermissions(t *testing.T) {
 		t.Run(testCase.name, func(t *testing.T) {
 			store := newStore(t)
 			account := newAccountWithId(mockAccountID, mockUserID, "")
-			account.Users["normal_user1"] = NewUser("normal_user1", testCase.role, false, false, "", []string{}, UserIssuedAPI)
+			account.Users["normal_user1"] = NewUser("normal_user1", testCase.role, false, false, "", []string{}, UserIssuedAPI, mockAccountID)
 			account.Settings.RegularUsersViewBlocked = testCase.limitedViewSettings
 			delete(account.Users, mockUserID)
 
@@ -844,10 +844,10 @@ func TestDefaultAccountManager_ExternalCache(t *testing.T) {
 
 func TestUser_IsAdmin(t *testing.T) {
 
-	user := NewAdminUser(mockUserID)
+	user := NewAdminUser(mockUserID, mockAccountID)
 	assert.True(t, user.HasAdminPower())
 
-	user = NewRegularUser(mockUserID)
+	user = NewRegularUser(mockUserID, mockAccountID)
 	assert.False(t, user.HasAdminPower())
 }
 
@@ -1055,8 +1055,8 @@ func TestDefaultAccountManager_SaveUser(t *testing.T) {
 			}
 
 			// create other users
-			account.Users[regularUserID] = NewRegularUser(regularUserID)
+			account.Users[regularUserID] = NewRegularUser(regularUserID, account.Id)
-			account.Users[adminUserID] = NewAdminUser(adminUserID)
+			account.Users[adminUserID] = NewAdminUser(adminUserID, account.Id)
 			account.Users[serviceUserID] = &User{IsServiceUser: true, Id: serviceUserID, Role: UserRoleAdmin, ServiceUserName: "service"}
 			err = manager.Store.SaveAccount(account)
 			if err != nil {