Log store engine type (#1234)

trigger test on management/cmd and signal/cmd changes.

---------

Co-authored-by: Maycon Santos <mlsmaycon@gmail.com>

.github/workflows/golang-test-darwin.yml

@@ -12,6 +12,9 @@ concurrency:
 
 jobs:
   test:
+    strategy:
+      matrix:
+        store: ['jsonfile', 'sqlite']
     runs-on: macos-latest
     steps:
       - name: Install Go
@@ -33,4 +36,4 @@ jobs:
         run: go mod tidy
 
       - name: Test
-        run: go test -exec 'sudo --preserve-env=CI' -timeout 5m -p 1 ./...
+        run: NETBIRD_STORE_ENGINE=${{ matrix.store }} go test -exec 'sudo --preserve-env=CI' -timeout 5m -p 1 ./...

.github/workflows/golang-test-linux.yml

@@ -15,6 +15,7 @@ jobs:
     strategy:
       matrix:
         arch: ['386','amd64']
+        store: ['jsonfile', 'sqlite']
     runs-on: ubuntu-latest
     steps:
       - name: Install Go
@@ -41,17 +42,16 @@ jobs:
         run: go mod tidy
 
       - name: Test
-        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} go test -exec 'sudo --preserve-env=CI' -timeout 5m -p 1 ./...
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} go test -exec 'sudo --preserve-env=CI' -timeout 5m -p 1 ./...
 
   test_client_on_docker:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
           go-version: "1.20.x"
 
-
       - name: Cache Go modules
         uses: actions/cache@v3
         with:
@@ -64,7 +64,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib
 
       - name: Install modules
         run: go mod tidy
@@ -82,7 +82,7 @@ jobs:
         run: CGO_ENABLED=0 go test -c -o nftablesmanager-testing.bin ./client/firewall/nftables/...
 
       - name: Generate Engine Test bin
-        run: CGO_ENABLED=0 go test -c -o engine-testing.bin ./client/internal
+        run: CGO_ENABLED=1 go test -c -o engine-testing.bin ./client/internal
 
       - name: Generate Peer Test bin
         run: CGO_ENABLED=0 go test -c -o peer-testing.bin ./client/internal/peer/...
@@ -95,15 +95,17 @@ jobs:
       - name: Run Iface tests in docker
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/iface --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/iface-testing.bin -test.timeout 5m -test.parallel 1
 
-
       - name: Run RouteManager tests in docker
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/routemanager --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/routemanager-testing.bin  -test.timeout 5m -test.parallel 1
 
       - name: Run nftables Manager tests in docker
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/firewall --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/nftablesmanager-testing.bin  -test.timeout 5m -test.parallel 1
 
-      - name: Run Engine tests in docker
-        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/engine-testing.bin  -test.timeout 5m -test.parallel 1
+      - name: Run Engine tests in docker with file store
+        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_ENGINE="jsonfile" --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/engine-testing.bin  -test.timeout 5m -test.parallel 1
+      
+      - name: Run Engine tests in docker with sqlite store
+        run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal -e NETBIRD_STORE_ENGINE="sqlite" --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/engine-testing.bin  -test.timeout 5m -test.parallel 1
 
       - name: Run Peer tests in docker
         run: docker run -t --cap-add=NET_ADMIN --privileged --rm -v $PWD:/ci -w /ci/client/internal/peer  --entrypoint /busybox/sh gcr.io/distroless/base:debug -c /ci/peer-testing.bin  -test.timeout 5m -test.parallel 1

.github/workflows/golang-test-windows.yml

@@ -39,7 +39,9 @@ jobs:
 
       - run: mv ${{ env.downloadPath }}/wintun/bin/amd64/wintun.dll 'C:\Windows\System32\'
 
-      - run: choco install -y sysinternals
+      - run: choco install -y sysinternals --ignore-checksums
+      - run: choco install -y mingw
+
       - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOMODCACHE=C:\Users\runneradmin\go\pkg\mod
       - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOCACHE=C:\Users\runneradmin\AppData\Local\go-build
 

.github/workflows/test-infrastructure-files.yml

@@ -8,6 +8,8 @@ on:
     paths:
       - 'infrastructure_files/**'
       - '.github/workflows/test-infrastructure-files.yml'
+      - 'management/cmd/**'
+      - 'signal/cmd/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.actor_id }}
@@ -56,6 +58,8 @@ jobs:
           CI_NETBIRD_IDP_MGMT_CLIENT_ID: testing.client.id
           CI_NETBIRD_IDP_MGMT_CLIENT_SECRET: testing.client.secret
           CI_NETBIRD_AUTH_SUPPORTED_SCOPES: "openid profile email offline_access api email_verified"
+          CI_NETBIRD_STORE_CONFIG_ENGINE: "sqlite"
+          CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false
 
       - name: check values
         working-directory: infrastructure_files
@@ -81,6 +85,8 @@ jobs:
           CI_NETBIRD_IDP_MGMT_CLIENT_ID: testing.client.id
           CI_NETBIRD_IDP_MGMT_CLIENT_SECRET: testing.client.secret
           CI_NETBIRD_SIGNAL_PORT: 12345
+          CI_NETBIRD_STORE_CONFIG_ENGINE: "sqlite"
+          CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false
 
         run: |
           grep AUTH_CLIENT_ID docker-compose.yml | grep $CI_NETBIRD_AUTH_CLIENT_ID
@@ -97,7 +103,9 @@ jobs:
           grep NETBIRD_TOKEN_SOURCE docker-compose.yml | grep $CI_NETBIRD_TOKEN_SOURCE
           grep AuthUserIDClaim management.json | grep $CI_NETBIRD_AUTH_USER_ID_CLAIM
           grep -A 3 DeviceAuthorizationFlow management.json | grep -A 1 ProviderConfig | grep Audience | grep $CI_NETBIRD_AUTH_DEVICE_AUTH_AUDIENCE
-          grep -A 8 DeviceAuthorizationFlow management.json | grep -A 6 ProviderConfig | grep Scope | grep "$CI_NETBIRD_AUTH_DEVICE_AUTH_SCOPE"
+          grep -A 3 DeviceAuthorizationFlow management.json | grep -A 1 ProviderConfig | grep Audience | grep $CI_NETBIRD_AUTH_DEVICE_AUTH_AUDIENCE
+          grep Engine management.json  | grep "$CI_NETBIRD_STORE_CONFIG_ENGINE"
+          grep IdpSignKeyRefreshEnabled management.json | grep "$CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH"
           grep UseIDToken management.json | grep false
           grep -A 1 IdpManagerConfig management.json | grep ManagerType | grep $CI_NETBIRD_MGMT_IDP 
           grep -A 3 IdpManagerConfig management.json | grep -A 1 ClientConfig | grep Issuer | grep $CI_NETBIRD_AUTH_AUTHORITY
@@ -105,12 +113,12 @@ jobs:
           grep -A 5 IdpManagerConfig management.json | grep -A 3 ClientConfig | grep ClientID | grep $CI_NETBIRD_IDP_MGMT_CLIENT_ID
           grep -A 6 IdpManagerConfig management.json | grep -A 4 ClientConfig | grep ClientSecret | grep $CI_NETBIRD_IDP_MGMT_CLIENT_SECRET
           grep -A 7 IdpManagerConfig management.json | grep -A 5 ClientConfig | grep GrantType | grep client_credentials
-          grep -A 2 PKCEAuthorizationFlow management.json | grep -A 1 ProviderConfig | grep Audience | grep $CI_NETBIRD_AUTH_AUDIENCE
-          grep -A 3 PKCEAuthorizationFlow management.json | grep -A 2 ProviderConfig | grep ClientID | grep $CI_NETBIRD_AUTH_CLIENT_ID
-          grep -A 4 PKCEAuthorizationFlow management.json | grep -A 3 ProviderConfig | grep ClientSecret | grep $CI_NETBIRD_AUTH_CLIENT_SECRET
-          grep -A 5 PKCEAuthorizationFlow management.json | grep -A 4 ProviderConfig | grep AuthorizationEndpoint | grep $CI_NETBIRD_AUTH_PKCE_AUTHORIZATION_ENDPOINT
-          grep -A 6 PKCEAuthorizationFlow management.json | grep -A 5 ProviderConfig | grep TokenEndpoint | grep $CI_NETBIRD_AUTH_TOKEN_ENDPOINT
-          grep -A 7 PKCEAuthorizationFlow management.json | grep -A 6 ProviderConfig | grep Scope | grep "$CI_NETBIRD_AUTH_SUPPORTED_SCOPES"
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep Audience | grep $CI_NETBIRD_AUTH_AUDIENCE
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep ClientID | grep $CI_NETBIRD_AUTH_CLIENT_ID
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep ClientSecret | grep $CI_NETBIRD_AUTH_CLIENT_SECRET
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep AuthorizationEndpoint | grep $CI_NETBIRD_AUTH_PKCE_AUTHORIZATION_ENDPOINT
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep TokenEndpoint | grep $CI_NETBIRD_AUTH_TOKEN_ENDPOINT
+          grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep Scope | grep "$CI_NETBIRD_AUTH_SUPPORTED_SCOPES"
 
       - name: Install modules
         run: go mod tidy

.gitignore

@@ -19,4 +19,5 @@ client/.distfiles/
 infrastructure_files/setup.env
 infrastructure_files/setup-*.env
 .vscode
-.DS_Store
+.DS_Store
+*.db

CONTRIBUTING.md

@@ -23,7 +23,6 @@ If you haven't already, join our slack workspace [here](https://join.slack.com/t
         - [Test suite](#test-suite)
     - [Checklist before submitting a PR](#checklist-before-submitting-a-pr)
     - [Other project repositories](#other-project-repositories)
-    - [Checklist before submitting a new node](#checklist-before-submitting-a-new-node)
     - [Contributor License Agreement](#contributor-license-agreement)
 
 ## Code of conduct
@@ -70,7 +69,7 @@ dependencies are installed. Here is a short guide on how that can be done.
 
 ### Requirements
 
-#### Go 1.19
+#### Go 1.21
 
 Follow the installation guide from https://go.dev/
 
@@ -139,15 +138,14 @@ checked out and set up:
 ### Build and start
 #### Client
 
-> Windows clients have a Wireguard driver requirement. We provide a bash script that can be executed in WLS 2 with docker support [wireguard_nt.sh](/client/wireguard_nt.sh).
-
 To start NetBird, execute:
 ```
 cd client
-# bash wireguard_nt.sh # if windows
-go build .
+CGO_ENABLED=0 go build .
 ```
 
+> Windows clients have a Wireguard driver requirement. You can downlowd the wintun driver from https://www.wintun.net/builds/wintun-0.14.1.zip, after decompressing, you can copy the file `windtun\bin\ARCH\wintun.dll` to the same path as your binary file or to `C:\Windows\System32\wintun.dll`.
+
 To start NetBird the client in the foreground:
 
 ```
@@ -215,4 +213,4 @@ NetBird project is composed of 3 main repositories:
 
 That we do not have any potential problems later it is sadly necessary to sign a [Contributor License Agreement](CONTRIBUTOR_LICENSE_AGREEMENT.md). That can be done literally with the push of a button.
 
-A bot will automatically comment on the pull request once it got opened asking for the agreement to be signed. Before it did not get signed it is sadly not possible to merge it in.
+A bot will automatically comment on the pull request once it got opened asking for the agreement to be signed. Before it did not get signed it is sadly not possible to merge it in.

client/cmd/testutil.go

@@ -65,7 +65,7 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	store, err := mgmt.NewFileStore(config.Datadir, nil)
+	store, err := mgmt.NewStoreFromJson(config.Datadir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

client/internal/dns/host.go

@@ -78,7 +78,7 @@ func dnsConfigToHostDNSConfig(dnsConfig nbdns.Config, ip string, port int) hostD
 		for _, domain := range nsConfig.Domains {
 			config.domains = append(config.domains, domainConfig{
 				domain:    strings.TrimSuffix(domain, "."),
-				matchOnly: true,
+				matchOnly: !nsConfig.SearchDomainsEnabled,
 			})
 		}
 	}

client/internal/dns/host_windows.go

@@ -22,13 +22,11 @@ const (
 	interfaceConfigPath          = "SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters\\Interfaces"
 	interfaceConfigNameServerKey = "NameServer"
 	interfaceConfigSearchListKey = "SearchList"
-	tcpipParametersPath          = "SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters"
 )
 
 type registryConfigurator struct {
-	guid                  string
-	routingAll            bool
-	existingSearchDomains []string
+	guid       string
+	routingAll bool
 }
 
 func newHostManager(wgInterface WGIface) (hostManager, error) {
@@ -148,30 +146,11 @@ func (r *registryConfigurator) restoreHostDNS() error {
 		log.Error(err)
 	}
 
-	return r.updateSearchDomains([]string{})
+	return r.deleteInterfaceRegistryKeyProperty(interfaceConfigSearchListKey)
 }
 
 func (r *registryConfigurator) updateSearchDomains(domains []string) error {
-	value, err := getLocalMachineRegistryKeyStringValue(tcpipParametersPath, interfaceConfigSearchListKey)
-	if err != nil {
-		return fmt.Errorf("unable to get current search domains failed with error: %s", err)
-	}
-
-	valueList := strings.Split(value, ",")
-	setExisting := false
-	if len(r.existingSearchDomains) == 0 {
-		r.existingSearchDomains = valueList
-		setExisting = true
-	}
-
-	if len(domains) == 0 && setExisting {
-		log.Infof("added %d search domains to the registry. Domain list: %s", len(domains), domains)
-		return nil
-	}
-
-	newList := append(r.existingSearchDomains, domains...)
-
-	err = setLocalMachineRegistryKeyStringValue(tcpipParametersPath, interfaceConfigSearchListKey, strings.Join(newList, ","))
+	err := r.setInterfaceRegistryKeyStringValue(interfaceConfigSearchListKey, strings.Join(domains, ","))
 	if err != nil {
 		return fmt.Errorf("adding search domain failed with error: %s", err)
 	}
@@ -235,33 +214,3 @@ func removeRegistryKeyFromDNSPolicyConfig(regKeyPath string) error {
 	}
 	return nil
 }
-
-func getLocalMachineRegistryKeyStringValue(keyPath, key string) (string, error) {
-	regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, keyPath, registry.QUERY_VALUE)
-	if err != nil {
-		return "", fmt.Errorf("unable to open existing key from registry, key path: HKEY_LOCAL_MACHINE\\%s, error: %s", keyPath, err)
-	}
-	defer regKey.Close()
-
-	val, _, err := regKey.GetStringValue(key)
-	if err != nil {
-		return "", fmt.Errorf("getting %s value for key path HKEY_LOCAL_MACHINE\\%s failed with error: %s", key, keyPath, err)
-	}
-
-	return val, nil
-}
-
-func setLocalMachineRegistryKeyStringValue(keyPath, key, value string) error {
-	regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, keyPath, registry.SET_VALUE)
-	if err != nil {
-		return fmt.Errorf("unable to open existing key from registry, key path: HKEY_LOCAL_MACHINE\\%s, error: %s", keyPath, err)
-	}
-	defer regKey.Close()
-
-	err = regKey.SetStringValue(key, value)
-	if err != nil {
-		return fmt.Errorf("setting %s value %s for key path HKEY_LOCAL_MACHINE\\%s failed with error: %s", key, value, keyPath, err)
-	}
-
-	return nil
-}

client/internal/engine.go

@@ -714,8 +714,9 @@ func toDNSConfig(protoDNSConfig *mgmProto.DNSConfig) nbdns.Config {
 
 	for _, nsGroup := range protoDNSConfig.GetNameServerGroups() {
 		dnsNSGroup := &nbdns.NameServerGroup{
-			Primary: nsGroup.GetPrimary(),
-			Domains: nsGroup.GetDomains(),
+			Primary:              nsGroup.GetPrimary(),
+			Domains:              nsGroup.GetDomains(),
+			SearchDomainsEnabled: nsGroup.GetSearchDomainsEnabled(),
 		}
 		for _, ns := range nsGroup.GetNameServers() {
 			dnsNS := nbdns.NameServer{

client/internal/engine_test.go

@@ -1039,10 +1039,11 @@ func startManagement(dataDir string) (*grpc.Server, string, error) {
 		return nil, "", err
 	}
 	s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp))
-	store, err := server.NewFileStore(config.Datadir, nil)
+	store, err := server.NewStoreFromJson(config.Datadir, nil)
 	if err != nil {
-		log.Fatalf("failed creating a store: %s: %v", config.Datadir, err)
+		return nil, "", err
 	}
+
 	peersUpdateManager := server.NewPeersUpdateManager()
 	eventStore := &activity.InMemoryEventStore{}
 	if err != nil {

dns/nameserver.go

@@ -50,21 +50,25 @@ func ToNameServerType(typeString string) NameServerType {
 // NameServerGroup group of nameservers and with group ids
 type NameServerGroup struct {
 	// ID identifier of group
-	ID string
+	ID string `gorm:"primaryKey"`
+	// AccountID is a reference to Account that this object belongs
+	AccountID string `gorm:"index"`
 	// Name group name
 	Name string
 	// Description group description
 	Description string
 	// NameServers list of nameservers
-	NameServers []NameServer
+	NameServers []NameServer `gorm:"serializer:json"`
 	// Groups list of peer group IDs to distribute the nameservers information
-	Groups []string
+	Groups []string `gorm:"serializer:json"`
 	// Primary indicates that the nameserver group is the primary resolver for any dns query
 	Primary bool
 	// Domains indicate the dns query domains to use with this nameserver group
-	Domains []string
+	Domains []string `gorm:"serializer:json"`
 	// Enabled group status
 	Enabled bool
+	// SearchDomainsEnabled indicates whether to add match domains to search domains list or not
+	SearchDomainsEnabled bool
 }
 
 // NameServer represents a DNS nameserver
@@ -131,14 +135,15 @@ func ParseNameServerURL(nsURL string) (NameServer, error) {
 // Copy copies a nameserver group object
 func (g *NameServerGroup) Copy() *NameServerGroup {
 	nsGroup := &NameServerGroup{
-		ID:          g.ID,
-		Name:        g.Name,
-		Description: g.Description,
-		NameServers: make([]NameServer, len(g.NameServers)),
-		Groups:      make([]string, len(g.Groups)),
-		Enabled:     g.Enabled,
-		Primary:     g.Primary,
-		Domains:     make([]string, len(g.Domains)),
+		ID:                   g.ID,
+		Name:                 g.Name,
+		Description:          g.Description,
+		NameServers:          make([]NameServer, len(g.NameServers)),
+		Groups:               make([]string, len(g.Groups)),
+		Enabled:              g.Enabled,
+		Primary:              g.Primary,
+		Domains:              make([]string, len(g.Domains)),
+		SearchDomainsEnabled: g.SearchDomainsEnabled,
 	}
 
 	copy(nsGroup.NameServers, g.NameServers)
@@ -154,6 +159,7 @@ func (g *NameServerGroup) IsEqual(other *NameServerGroup) bool {
 		other.Name == g.Name &&
 		other.Description == g.Description &&
 		other.Primary == g.Primary &&
+		other.SearchDomainsEnabled == g.SearchDomainsEnabled &&
 		compareNameServerList(g.NameServers, other.NameServers) &&
 		compareGroupsList(g.Groups, other.Groups) &&
 		compareGroupsList(g.Domains, other.Domains)

go.mod

@@ -46,11 +46,12 @@ require (
 	github.com/hashicorp/go-version v1.6.0
 	github.com/libp2p/go-netroute v0.2.0
 	github.com/magiconair/properties v1.8.5
-	github.com/mattn/go-sqlite3 v1.14.16
+	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/mdlayher/socket v0.4.0
 	github.com/miekg/dns v1.1.43
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/nadoo/ipset v0.5.0
+	github.com/netbirdio/management-integrations/integrations v0.0.0-20231017101406-322cbabed3da
 	github.com/okta/okta-sdk-golang/v2 v2.18.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pion/logging v0.2.2
@@ -74,6 +75,8 @@ require (
 	golang.org/x/term v0.8.0
 	google.golang.org/api v0.126.0
 	gopkg.in/yaml.v3 v3.0.1
+	gorm.io/driver/sqlite v1.5.3
+	gorm.io/gorm v1.25.4
 )
 
 require (
@@ -110,6 +113,8 @@ require (
 	github.com/googleapis/gax-go/v2 v2.10.0 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/native v1.0.0 // indirect
 	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect

go.sum

@@ -383,6 +383,10 @@ github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jackmordaunt/icns v0.0.0-20181231085925-4f16af745526/go.mod h1:UQkeMHVoNcyXYq9otUupF7/h/2tmHlhrS2zw7ZVvUqc=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/josephspurrier/goversioninfo v0.0.0-20200309025242-14b0ab84c6ca/go.mod h1:eJTEwMjXb7kZ633hO3Ln9mBUCOjX2+FlTljvpl9SYdE=
 github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
@@ -441,8 +445,8 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
-github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
-github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
+github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
@@ -491,6 +495,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/nadoo/ipset v0.5.0 h1:5GJUAuZ7ITQQQGne5J96AmFjRtI8Avlbk6CabzYWVUc=
 github.com/nadoo/ipset v0.5.0/go.mod h1:rYF5DQLRGGoQ8ZSWeK+6eX5amAuPqwFkWjhQlEITGJQ=
+github.com/netbirdio/management-integrations/integrations v0.0.0-20231017101406-322cbabed3da h1:S1RoPhLTw3+IhHGnyfcQlj4aqIIaQdVd3SqaiK+MYFY=
+github.com/netbirdio/management-integrations/integrations v0.0.0-20231017101406-322cbabed3da/go.mod h1:KSqjzHcqlodTWiuap5lRXxt5KT3vtYRoksL0KIrTK40=
 github.com/netbirdio/service v0.0.0-20230215170314-b923b89432b0 h1:hirFRfx3grVA/9eEyjME5/z3nxdJlN9kfQpvWWPk32g=
 github.com/netbirdio/service v0.0.0-20230215170314-b923b89432b0/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
 github.com/netbirdio/systray v0.0.0-20221012095658-dc8eda872c0c h1:wK/s4nyZj/GF/kFJQjX6nqNfE0G3gcqd6hhnPCyp4sw=
@@ -1189,6 +1195,10 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/sqlite v1.5.3 h1:7/0dUgX28KAcopdfbRWWl68Rflh6osa4rDh+m51KL2g=
+gorm.io/driver/sqlite v1.5.3/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4=
+gorm.io/gorm v1.25.4 h1:iyNd8fNAe8W9dvtlgeRI5zSVZPsq3OpcTu37cYcpCmw=
+gorm.io/gorm v1.25.4/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
 gvisor.dev/gvisor v0.0.0-20221203005347-703fd9b7fbc0 h1:Wobr37noukisGxpKo5jAsLREcpj61RxrWYzD8uwveOY=

infrastructure_files/base.setup.env

@@ -14,6 +14,7 @@ NETBIRD_MGMT_API_CERT_KEY_FILE="/etc/letsencrypt/live/$NETBIRD_LETSENCRYPT_DOMAI
 # By default Management single account mode is enabled and domain set to $NETBIRD_DOMAIN, you may want to set this to your user's email domain
 NETBIRD_MGMT_SINGLE_ACCOUNT_MODE_DOMAIN=$NETBIRD_DOMAIN
 NETBIRD_MGMT_DNS_DOMAIN=${NETBIRD_MGMT_DNS_DOMAIN:-netbird.selfhosted}
+NETBIRD_MGMT_IDP_SIGNKEY_REFRESH=${NETBIRD_MGMT_IDP_SIGNKEY_REFRESH:-false}
 
 # Signal
 NETBIRD_SIGNAL_PROTOCOL="http"
@@ -55,6 +56,9 @@ NETBIRD_AUTH_PKCE_AUDIENCE=$NETBIRD_AUTH_AUDIENCE
 NETBIRD_DASH_AUTH_USE_AUDIENCE=${NETBIRD_DASH_AUTH_USE_AUDIENCE:-true}
 NETBIRD_DASH_AUTH_AUDIENCE=$NETBIRD_AUTH_AUDIENCE
 
+# Store config
+NETBIRD_STORE_CONFIG_ENGINE=${NETBIRD_STORE_CONFIG_ENGINE:-"jsonfile"}
+
 # exports
 export NETBIRD_DOMAIN
 export NETBIRD_AUTH_CLIENT_ID
@@ -86,6 +90,7 @@ export LETSENCRYPT_VOLUMESUFFIX
 export NETBIRD_DISABLE_ANONYMOUS_METRICS
 export NETBIRD_MGMT_SINGLE_ACCOUNT_MODE_DOMAIN
 export NETBIRD_MGMT_DNS_DOMAIN
+export NETBIRD_MGMT_IDP_SIGNKEY_REFRESH
 export NETBIRD_SIGNAL_PROTOCOL
 export NETBIRD_SIGNAL_PORT
 export NETBIRD_AUTH_USER_ID_CLAIM
@@ -97,4 +102,5 @@ export NETBIRD_AUTH_PKCE_AUTHORIZATION_ENDPOINT
 export NETBIRD_AUTH_PKCE_USE_ID_TOKEN
 export NETBIRD_AUTH_PKCE_AUDIENCE
 export NETBIRD_DASH_AUTH_USE_AUDIENCE
-export NETBIRD_DASH_AUTH_AUDIENCE
+export NETBIRD_DASH_AUTH_AUDIENCE
+export NETBIRD_STORE_CONFIG_ENGINE

infrastructure_files/configure.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 if ! which curl >/dev/null 2>&1; then
   echo "This script uses curl fetch OpenID configuration from IDP."
@@ -154,6 +155,8 @@ if [ -n "$NETBIRD_MGMT_IDP" ]; then
   export NETBIRD_IDP_MGMT_CLIENT_ID
   export NETBIRD_IDP_MGMT_CLIENT_SECRET
   export NETBIRD_IDP_MGMT_EXTRA_CONFIG=$EXTRA_CONFIG
+else
+  export NETBIRD_IDP_MGMT_EXTRA_CONFIG={}
 fi
 
 IFS=',' read -r -a REDIRECT_URL_PORTS <<< "$NETBIRD_AUTH_PKCE_REDIRECT_URL_PORTS"
@@ -170,8 +173,29 @@ if [ "$NETBIRD_DASH_AUTH_USE_AUDIENCE" = "false" ]; then
     export NETBIRD_AUTH_PKCE_AUDIENCE=
 fi
 
+# Read the encryption key
+if test -f 'management.json'; then
+    encKey=$(jq -r  ".DataStoreEncryptionKey" management.json)
+    if [[ "$encKey" != "null" ]]; then
+        export NETBIRD_DATASTORE_ENC_KEY=$encKey
+
+    fi
+fi
+
 env | grep NETBIRD
 
+bkp_postfix="$(date +%s)"
+if test -f 'docker-compose.yml'; then
+    cp docker-compose.yml "docker-compose.yml.bkp.${bkp_postfix}"
+fi
+
+if test -f 'management.json'; then
+    cp management.json "management.json.bkp.${bkp_postfix}"
+fi
+
+if test -f 'turnserver.conf'; then
+    cp turnserver.conf "turnserver.conf.bpk.${bkp_postfix}"
+fi
 envsubst <docker-compose.yml.tmpl >docker-compose.yml
-envsubst <management.json.tmpl >management.json
-envsubst <turnserver.conf.tmpl >turnserver.conf
+envsubst <management.json.tmpl | jq . >management.json
+envsubst <turnserver.conf.tmpl >turnserver.conf

infrastructure_files/management.json.tmpl

@@ -27,6 +27,10 @@
         "Password": null
     },
     "Datadir": "",
+    "DataStoreEncryptionKey": "$NETBIRD_DATASTORE_ENC_KEY",
+    "StoreConfig": {
+        "Engine": "$NETBIRD_STORE_CONFIG_ENGINE"
+    },
     "HttpConfig": {
         "Address": "0.0.0.0:$NETBIRD_MGMT_API_PORT",
         "AuthIssuer": "$NETBIRD_AUTH_AUTHORITY",
@@ -35,6 +39,7 @@
         "AuthUserIDClaim": "$NETBIRD_AUTH_USER_ID_CLAIM",
         "CertFile":"$NETBIRD_MGMT_API_CERT_FILE",
         "CertKey":"$NETBIRD_MGMT_API_CERT_KEY_FILE",
+        "IdpSignKeyRefreshEnabled": $NETBIRD_MGMT_IDP_SIGNKEY_REFRESH,
         "OIDCConfigEndpoint":"$NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT"
     },
     "IdpManagerConfig": {
@@ -46,18 +51,25 @@
             "ClientSecret": "$NETBIRD_IDP_MGMT_CLIENT_SECRET",
             "GrantType": "client_credentials"
         },
-        "ExtraConfig": $NETBIRD_IDP_MGMT_EXTRA_CONFIG
+        "ExtraConfig": $NETBIRD_IDP_MGMT_EXTRA_CONFIG,
+        "Auth0ClientCredentials": null,
+        "AzureClientCredentials": null,
+        "KeycloakClientCredentials": null,
+        "ZitadelClientCredentials": null
      },
     "DeviceAuthorizationFlow": {
         "Provider": "$NETBIRD_AUTH_DEVICE_AUTH_PROVIDER",
         "ProviderConfig": {
           "Audience": "$NETBIRD_AUTH_DEVICE_AUTH_AUDIENCE",
+          "AuthorizationEndpoint": "",
           "Domain": "$NETBIRD_AUTH0_DOMAIN",
           "ClientID": "$NETBIRD_AUTH_DEVICE_AUTH_CLIENT_ID",
+          "ClientSecret": "",
           "TokenEndpoint": "$NETBIRD_AUTH_TOKEN_ENDPOINT",
           "DeviceAuthEndpoint": "$NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT",
           "Scope": "$NETBIRD_AUTH_DEVICE_AUTH_SCOPE",
-          "UseIDToken": $NETBIRD_AUTH_DEVICE_AUTH_USE_ID_TOKEN
+          "UseIDToken": $NETBIRD_AUTH_DEVICE_AUTH_USE_ID_TOKEN,
+          "RedirectURLs": null
          }
     },
     "PKCEAuthorizationFlow": {
@@ -65,11 +77,13 @@
             "Audience": "$NETBIRD_AUTH_PKCE_AUDIENCE",
             "ClientID": "$NETBIRD_AUTH_CLIENT_ID",
             "ClientSecret": "$NETBIRD_AUTH_CLIENT_SECRET",
+            "Domain": "",
             "AuthorizationEndpoint": "$NETBIRD_AUTH_PKCE_AUTHORIZATION_ENDPOINT",
             "TokenEndpoint": "$NETBIRD_AUTH_TOKEN_ENDPOINT",
             "Scope": "$NETBIRD_AUTH_SUPPORTED_SCOPES",
             "RedirectURLs": [$NETBIRD_AUTH_PKCE_REDIRECT_URLS],
-            "UseIDToken": $NETBIRD_AUTH_PKCE_USE_ID_TOKEN
+            "UseIDToken": $NETBIRD_AUTH_PKCE_USE_ID_TOKEN,
+            "RedirectURLs": null
         }
     }
 }

infrastructure_files/setup.env.example

@@ -53,6 +53,8 @@ NETBIRD_MGMT_IDP="none"
 # Some IDPs requires different client id and client secret for management api
 NETBIRD_IDP_MGMT_CLIENT_ID=$NETBIRD_AUTH_CLIENT_ID
 NETBIRD_IDP_MGMT_CLIENT_SECRET=""
+# With some IDPs may be needed enabling automatic refresh of signing keys on expire
+# NETBIRD_MGMT_IDP_SIGNKEY_REFRESH=false
 # NETBIRD_IDP_MGMT_EXTRA_ variables. See https://docs.netbird.io/selfhosted/identity-providers for more information about your IDP of choice.
 # -------------------------------------------
 # Letsencrypt

infrastructure_files/tests/setup.env

@@ -22,4 +22,6 @@ NETBIRD_AUTH_DEVICE_AUTH_SCOPE="openid email"
 NETBIRD_MGMT_IDP=$CI_NETBIRD_MGMT_IDP
 NETBIRD_IDP_MGMT_CLIENT_ID=$CI_NETBIRD_IDP_MGMT_CLIENT_ID
 NETBIRD_IDP_MGMT_CLIENT_SECRET=$CI_NETBIRD_IDP_MGMT_CLIENT_SECRET
-NETBIRD_SIGNAL_PORT=12345
+NETBIRD_SIGNAL_PORT=12345
+NETBIRD_STORE_CONFIG_ENGINE=$CI_NETBIRD_STORE_CONFIG_ENGINE
+NETBIRD_MGMT_IDP_SIGNKEY_REFRESH=$CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH

management/client/client_test.go

@@ -16,7 +16,6 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/netbirdio/netbird/encryption"
-	"github.com/netbirdio/netbird/management/proto"
 	mgmtProto "github.com/netbirdio/netbird/management/proto"
 	mgmt "github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/mock_server"
@@ -53,7 +52,7 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) {
 		t.Fatal(err)
 	}
 	s := grpc.NewServer()
-	store, err := mgmt.NewFileStore(config.Datadir, nil)
+	store, err := mgmt.NewStoreFromJson(config.Datadir, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -95,8 +94,8 @@ func startMockManagement(t *testing.T) (*grpc.Server, net.Listener, *mock_server
 	}
 
 	mgmtMockServer := &mock_server.ManagementServiceServerMock{
-		GetServerKeyFunc: func(context.Context, *proto.Empty) (*proto.ServerKeyResponse, error) {
-			response := &proto.ServerKeyResponse{
+		GetServerKeyFunc: func(context.Context, *mgmtProto.Empty) (*mgmtProto.ServerKeyResponse, error) {
+			response := &mgmtProto.ServerKeyResponse{
 				Key: serverKey.PublicKey().String(),
 			}
 			return response, nil
@@ -300,19 +299,19 @@ func Test_SystemMetaDataFromClient(t *testing.T) {
 		log.Fatalf("error while getting server public key from testclient, %v", err)
 	}
 
-	var actualMeta *proto.PeerSystemMeta
+	var actualMeta *mgmtProto.PeerSystemMeta
 	var actualValidKey string
 	var wg sync.WaitGroup
 	wg.Add(1)
 
-	mgmtMockServer.LoginFunc = func(ctx context.Context, msg *proto.EncryptedMessage) (*proto.EncryptedMessage, error) {
+	mgmtMockServer.LoginFunc = func(ctx context.Context, msg *mgmtProto.EncryptedMessage) (*mgmtProto.EncryptedMessage, error) {
 		peerKey, err := wgtypes.ParseKey(msg.GetWgPubKey())
 		if err != nil {
 			log.Warnf("error while parsing peer's Wireguard public key %s on Sync request.", msg.WgPubKey)
 			return nil, status.Errorf(codes.InvalidArgument, "provided wgPubKey %s is invalid", msg.WgPubKey)
 		}
 
-		loginReq := &proto.LoginRequest{}
+		loginReq := &mgmtProto.LoginRequest{}
 		err = encryption.DecryptMessage(peerKey, serverKey, msg.Body, loginReq)
 		if err != nil {
 			log.Fatal(err)
@@ -322,7 +321,7 @@ func Test_SystemMetaDataFromClient(t *testing.T) {
 		actualValidKey = loginReq.GetSetupKey()
 		wg.Done()
 
-		loginResp := &proto.LoginResponse{}
+		loginResp := &mgmtProto.LoginResponse{}
 		encryptedResp, err := encryption.EncryptMessage(peerKey, serverKey, loginResp)
 		if err != nil {
 			return nil, err
@@ -343,7 +342,7 @@ func Test_SystemMetaDataFromClient(t *testing.T) {
 
 	wg.Wait()
 
-	expectedMeta := &proto.PeerSystemMeta{
+	expectedMeta := &mgmtProto.PeerSystemMeta{
 		Hostname:           info.Hostname,
 		GoOS:               info.GoOS,
 		Kernel:             info.Kernel,
@@ -374,12 +373,12 @@ func Test_GetDeviceAuthorizationFlow(t *testing.T) {
 		log.Fatalf("error while creating testClient: %v", err)
 	}
 
-	expectedFlowInfo := &proto.DeviceAuthorizationFlow{
+	expectedFlowInfo := &mgmtProto.DeviceAuthorizationFlow{
 		Provider:       0,
-		ProviderConfig: &proto.ProviderConfig{ClientID: "client"},
+		ProviderConfig: &mgmtProto.ProviderConfig{ClientID: "client"},
 	}
 
-	mgmtMockServer.GetDeviceAuthorizationFlowFunc = func(ctx context.Context, req *mgmtProto.EncryptedMessage) (*proto.EncryptedMessage, error) {
+	mgmtMockServer.GetDeviceAuthorizationFlowFunc = func(ctx context.Context, req *mgmtProto.EncryptedMessage) (*mgmtProto.EncryptedMessage, error) {
 		encryptedResp, err := encryption.EncryptMessage(serverKey, client.key, expectedFlowInfo)
 		if err != nil {
 			return nil, err
@@ -418,14 +417,14 @@ func Test_GetPKCEAuthorizationFlow(t *testing.T) {
 		log.Fatalf("error while creating testClient: %v", err)
 	}
 
-	expectedFlowInfo := &proto.PKCEAuthorizationFlow{
-		ProviderConfig: &proto.ProviderConfig{
+	expectedFlowInfo := &mgmtProto.PKCEAuthorizationFlow{
+		ProviderConfig: &mgmtProto.ProviderConfig{
 			ClientID:     "client",
 			ClientSecret: "secret",
 		},
 	}
 
-	mgmtMockServer.GetPKCEAuthorizationFlowFunc = func(ctx context.Context, req *mgmtProto.EncryptedMessage) (*proto.EncryptedMessage, error) {
+	mgmtMockServer.GetPKCEAuthorizationFlowFunc = func(ctx context.Context, req *mgmtProto.EncryptedMessage) (*mgmtProto.EncryptedMessage, error) {
 		encryptedResp, err := encryption.EncryptMessage(serverKey, client.key, expectedFlowInfo)
 		if err != nil {
 			return nil, err

management/client/grpc.go

@@ -57,7 +57,7 @@ func NewClient(ctx context.Context, addr string, ourPrivateKey wgtypes.Key, tlsE
 		transportOption,
 		grpc.WithBlock(),
 		grpc.WithKeepaliveParams(keepalive.ClientParameters{
-			Time:    15 * time.Second,
+			Time:    30 * time.Second,
 			Timeout: 10 * time.Second,
 		}))
 	if err != nil {

management/cmd/management.go

@@ -126,7 +126,7 @@ var (
 			if err != nil {
 				return err
 			}
-			store, err := server.NewFileStore(config.Datadir, appMetrics)
+			store, err := server.NewStore(config.StoreConfig.Engine, config.Datadir, appMetrics)
 			if err != nil {
 				return fmt.Errorf("failed creating Store: %s: %v", config.Datadir, err)
 			}

management/cmd/migration_down.go

@@ -0,0 +1,66 @@
+package cmd
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/netbirdio/netbird/management/server"
+	"github.com/netbirdio/netbird/util"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var shortDown = "Rollback SQLite store to JSON file store. Please make a backup of the SQLite file before running this command."
+
+var downCmd = &cobra.Command{
+	Use:     "downgrade [--datadir directory] [--log-file console]",
+	Aliases: []string{"down"},
+	Short:   shortDown,
+	Long: shortDown +
+		"\n\n" +
+		"This command reads the content of {datadir}/store.db and migrates it to {datadir}/store.json that can be used by File store driver.",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		flag.Parse()
+		err := util.InitLog(logLevel, logFile)
+		if err != nil {
+			return fmt.Errorf("failed initializing log %v", err)
+		}
+
+		sqliteStorePath := path.Join(mgmtDataDir, "store.db")
+		if _, err := os.Stat(sqliteStorePath); errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf("%s doesn't exist, couldn't continue the operation", sqliteStorePath)
+		}
+
+		fileStorePath := path.Join(mgmtDataDir, "store.json")
+		if _, err := os.Stat(fileStorePath); err == nil {
+			return fmt.Errorf("%s already exists, couldn't continue the operation", fileStorePath)
+		}
+
+		sqlstore, err := server.NewSqliteStore(mgmtDataDir, nil)
+		if err != nil {
+			return fmt.Errorf("failed creating file store: %s: %v", mgmtDataDir, err)
+		}
+
+		sqliteStoreAccounts := len(sqlstore.GetAllAccounts())
+		log.Infof("%d account will be migrated from sqlite store %s to file store %s",
+			sqliteStoreAccounts, sqliteStorePath, fileStorePath)
+
+		store, err := server.NewFilestoreFromSqliteStore(sqlstore, mgmtDataDir, nil)
+		if err != nil {
+			return fmt.Errorf("failed creating file store: %s: %v", mgmtDataDir, err)
+		}
+
+		fsStoreAccounts := len(store.GetAllAccounts())
+		if fsStoreAccounts != sqliteStoreAccounts {
+			return fmt.Errorf("failed to migrate accounts from sqlite to file[]. Expected accounts: %d, got: %d",
+				sqliteStoreAccounts, fsStoreAccounts)
+		}
+
+		log.Info("Migration finished successfully")
+
+		return nil
+	},
+}

management/cmd/migration_up.go

@@ -0,0 +1,66 @@
+package cmd
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/netbirdio/netbird/management/server"
+	"github.com/netbirdio/netbird/util"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var shortUp = "Migrate JSON file store to SQLite store. Please make a backup of the JSON file before running this command."
+
+var upCmd = &cobra.Command{
+	Use:     "upgrade [--datadir directory] [--log-file console]",
+	Aliases: []string{"up"},
+	Short:   shortUp,
+	Long: shortUp +
+		"\n\n" +
+		"This command reads the content of {datadir}/store.json and migrates it to {datadir}/store.db that can be used by SQLite store driver.",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		flag.Parse()
+		err := util.InitLog(logLevel, logFile)
+		if err != nil {
+			return fmt.Errorf("failed initializing log %v", err)
+		}
+
+		fileStorePath := path.Join(mgmtDataDir, "store.json")
+		if _, err := os.Stat(fileStorePath); errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf("%s doesn't exist, couldn't continue the operation", fileStorePath)
+		}
+
+		sqlStorePath := path.Join(mgmtDataDir, "store.db")
+		if _, err := os.Stat(sqlStorePath); err == nil {
+			return fmt.Errorf("%s already exists, couldn't continue the operation", sqlStorePath)
+		}
+
+		fstore, err := server.NewFileStore(mgmtDataDir, nil)
+		if err != nil {
+			return fmt.Errorf("failed creating file store: %s: %v", mgmtDataDir, err)
+		}
+
+		fsStoreAccounts := len(fstore.GetAllAccounts())
+		log.Infof("%d account will be migrated from file store %s to sqlite store %s",
+			fsStoreAccounts, fileStorePath, sqlStorePath)
+
+		store, err := server.NewSqliteStoreFromFileStore(fstore, mgmtDataDir, nil)
+		if err != nil {
+			return fmt.Errorf("failed creating file store: %s: %v", mgmtDataDir, err)
+		}
+
+		sqliteStoreAccounts := len(store.GetAllAccounts())
+		if fsStoreAccounts != sqliteStoreAccounts {
+			return fmt.Errorf("failed to migrate accounts from file to sqlite. Expected accounts: %d, got: %d",
+				fsStoreAccounts, sqliteStoreAccounts)
+		}
+
+		log.Info("Migration finished successfully")
+
+		return nil
+	},
+}

management/cmd/root.go

@@ -34,6 +34,12 @@ var (
 		SilenceUsage: true,
 	}
 
+	migrationCmd = &cobra.Command{
+		Use:          "sqlite-migration",
+		Short:        "Contains sub-commands to perform JSON file store to SQLite store migration and rollback",
+		Long:         "",
+		SilenceUsage: true,
+	}
 	// Execution control channel for stopCh signal
 	stopCh chan int
 )
@@ -63,6 +69,14 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "")
 	rootCmd.PersistentFlags().StringVar(&logFile, "log-file", defaultLogFile, "sets Netbird log path. If console is specified the the log will be output to stdout")
 	rootCmd.AddCommand(mgmtCmd)
+
+	migrationCmd.PersistentFlags().StringVar(&mgmtDataDir, "datadir", defaultMgmtDataDir, "server data directory location")
+	migrationCmd.MarkFlagRequired("datadir") //nolint
+
+	migrationCmd.AddCommand(upCmd)
+	migrationCmd.AddCommand(downCmd)
+
+	rootCmd.AddCommand(migrationCmd)
 }
 
 // SetupCloseHandler handles SIGTERM signal and exits with success

management/proto/management.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v3.21.12
+// 	protoc        v3.21.9
 // source: management.proto
 
 package proto
@@ -1999,9 +1999,10 @@ type NameServerGroup struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	NameServers []*NameServer `protobuf:"bytes,1,rep,name=NameServers,proto3" json:"NameServers,omitempty"`
-	Primary     bool          `protobuf:"varint,2,opt,name=Primary,proto3" json:"Primary,omitempty"`
-	Domains     []string      `protobuf:"bytes,3,rep,name=Domains,proto3" json:"Domains,omitempty"`
+	NameServers          []*NameServer `protobuf:"bytes,1,rep,name=NameServers,proto3" json:"NameServers,omitempty"`
+	Primary              bool          `protobuf:"varint,2,opt,name=Primary,proto3" json:"Primary,omitempty"`
+	Domains              []string      `protobuf:"bytes,3,rep,name=Domains,proto3" json:"Domains,omitempty"`
+	SearchDomainsEnabled bool          `protobuf:"varint,4,opt,name=SearchDomainsEnabled,proto3" json:"SearchDomainsEnabled,omitempty"`
 }
 
 func (x *NameServerGroup) Reset() {
@@ -2057,6 +2058,13 @@ func (x *NameServerGroup) GetDomains() []string {
 	return nil
 }
 
+func (x *NameServerGroup) GetSearchDomainsEnabled() bool {
+	if x != nil {
+		return x.SearchDomainsEnabled
+	}
+	return false
+}
+
 // NameServer represents a dns.NameServer
 type NameServer struct {
 	state         protoimpl.MessageState
@@ -2444,73 +2452,76 @@ var file_management_proto_rawDesc = []byte{
 	0x52, 0x05, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x54, 0x54, 0x4c, 0x18, 0x04,
 	0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x54, 0x54, 0x4c, 0x12, 0x14, 0x0a, 0x05, 0x52, 0x44, 0x61,
 	0x74, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x52, 0x44, 0x61, 0x74, 0x61, 0x22,
-	0x7f, 0x0a, 0x0f, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f,
-	0x75, 0x70, 0x12, 0x38, 0x0a, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72,
-	0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65,
-	0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52,
-	0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07,
-	0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x50,
-	0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e,
-	0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73,
-	0x22, 0x48, 0x0a, 0x0a, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x0e,
-	0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16,
-	0x0a, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06,
-	0x4e, 0x53, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x03,
-	0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x22, 0xf0, 0x02, 0x0a, 0x0c, 0x46,
-	0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x50,
-	0x65, 0x65, 0x72, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x65, 0x65,
-	0x72, 0x49, 0x50, 0x12, 0x40, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d,
-	0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65,
-	0x2e, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x44, 0x69, 0x72, 0x65,
-	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18,
-	0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65,
+	0xb3, 0x01, 0x0a, 0x0f, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72,
+	0x6f, 0x75, 0x70, 0x12, 0x38, 0x0a, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65,
+	0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67,
+	0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x52, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a,
+	0x07, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07,
+	0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x6f, 0x6d, 0x61, 0x69,
+	0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e,
+	0x73, 0x12, 0x32, 0x0a, 0x14, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x44, 0x6f, 0x6d, 0x61, 0x69,
+	0x6e, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52,
+	0x14, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x45, 0x6e,
+	0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0x48, 0x0a, 0x0a, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72,
+	0x76, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x03, 0x52, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50,
+	0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x22,
+	0xf0, 0x02, 0x0a, 0x0c, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65,
+	0x12, 0x16, 0x0a, 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x50, 0x12, 0x40, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65,
+	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x6d, 0x61,
+	0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c,
+	0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52,
+	0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x06, 0x41, 0x63,
+	0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e,
+	0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c,
+	0x52, 0x75, 0x6c, 0x65, 0x2e, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74,
+	0x69, 0x6f, 0x6e, 0x12, 0x3d, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18,
+	0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65,
 	0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e,
-	0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3d,
-	0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e,
-	0x32, 0x21, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69,
-	0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x63, 0x6f, 0x6c, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x12, 0x0a,
-	0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x6f, 0x72,
-	0x74, 0x22, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x06,
-	0x0a, 0x02, 0x49, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4f, 0x55, 0x54, 0x10, 0x01, 0x22,
-	0x1e, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x43, 0x43,
-	0x45, 0x50, 0x54, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x52, 0x4f, 0x50, 0x10, 0x01, 0x22,
-	0x3c, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x0b, 0x0a, 0x07, 0x55,
-	0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4c, 0x4c, 0x10,
-	0x01, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x02, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x44,
-	0x50, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x43, 0x4d, 0x50, 0x10, 0x04, 0x32, 0xd1, 0x03,
-	0x0a, 0x11, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76,
-	0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1c, 0x2e, 0x6d,
-	0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70,
-	0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e,
-	0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65,
-	0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x04, 0x53, 0x79,
-	0x6e, 0x63, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e,
-	0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
-	0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e,
-	0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00,
-	0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b,
-	0x65, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e,
-	0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65,
-	0x6e, 0x74, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x09, 0x69, 0x73, 0x48, 0x65, 0x61, 0x6c,
-	0x74, 0x68, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74,
-	0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d,
-	0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x1a, 0x47,
-	0x65, 0x74, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a,
-	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61,
-	0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64,
-	0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63,
+	0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74,
+	0x69, 0x6f, 0x6e, 0x12, 0x06, 0x0a, 0x02, 0x49, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4f,
+	0x55, 0x54, 0x10, 0x01, 0x22, 0x1e, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a,
+	0x0a, 0x06, 0x41, 0x43, 0x43, 0x45, 0x50, 0x54, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x52,
+	0x4f, 0x50, 0x10, 0x01, 0x22, 0x3c, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c,
+	0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a,
+	0x03, 0x41, 0x4c, 0x4c, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x02, 0x12,
+	0x07, 0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x43, 0x4d, 0x50,
+	0x10, 0x04, 0x32, 0xd1, 0x03, 0x0a, 0x11, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e,
+	0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69,
+	0x6e, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45,
+	0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a,
+	0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63,
+	0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12,
+	0x46, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65,
 	0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65,
-	0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x50, 0x4b,
-	0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46,
-	0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74,
-	0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67,
-	0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45,
-	0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22,
-	0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f,
-	0x74, 0x6f, 0x33,
+	0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65,
+	0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73,
+	0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x53, 0x65,
+	0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65,
+	0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x6e,
+	0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65,
+	0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x09, 0x69,
+	0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67,
+	0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x11, 0x2e, 0x6d, 0x61,
+	0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00,
+	0x12, 0x5a, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74,
+	0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c,
+	0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72,
+	0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d,
+	0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70,
+	0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x18,
+	0x47, 0x65, 0x74, 0x50, 0x4b, 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67,
+	0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d,
+	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d,
+	0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73,
+	0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

management/proto/management.proto

@@ -317,6 +317,7 @@ message NameServerGroup {
   repeated NameServer NameServers = 1;
   bool Primary = 2;
   repeated string Domains = 3;
+  bool SearchDomainsEnabled = 4;
 }
 
 // NameServer represents a dns.NameServer

management/server/account.go

@@ -91,7 +91,7 @@ type AccountManager interface {
 	DeleteRoute(accountID, routeID, userID string) error
 	ListRoutes(accountID, userID string) ([]*route.Route, error)
 	GetNameServerGroup(accountID, nsGroupID string) (*nbdns.NameServerGroup, error)
-	CreateNameServerGroup(accountID string, name, description string, nameServerList []nbdns.NameServer, groups []string, primary bool, domains []string, enabled bool, userID string) (*nbdns.NameServerGroup, error)
+	CreateNameServerGroup(accountID string, name, description string, nameServerList []nbdns.NameServer, groups []string, primary bool, domains []string, enabled bool, userID string, searchDomainsEnabled bool) (*nbdns.NameServerGroup, error)
 	SaveNameServerGroup(accountID, userID string, nsGroupToSave *nbdns.NameServerGroup) error
 	DeleteNameServerGroup(accountID, nsGroupID, userID string) error
 	ListNameServerGroups(accountID string) ([]*nbdns.NameServerGroup, error)
@@ -103,6 +103,7 @@ type AccountManager interface {
 	UpdateAccountSettings(accountID, userID string, newSettings *Settings) (*Account, error)
 	LoginPeer(login PeerLogin) (*Peer, *NetworkMap, error) // used by peer gRPC API
 	SyncPeer(sync PeerSync) (*Peer, *NetworkMap, error)    // used by peer gRPC API
+	GetAllConnectedPeers() (map[string]struct{}, error)
 }
 
 type DefaultAccountManager struct {
@@ -164,24 +165,33 @@ func (s *Settings) Copy() *Settings {
 
 // Account represents a unique account of the system
 type Account struct {
-	Id string
+	// we have to name column to aid as it collides with Network.Id when work with associations
+	Id string `gorm:"primaryKey"`
+
 	// User.Id it was created by
 	CreatedBy              string
-	Domain                 string
+	Domain                 string `gorm:"index"`
 	DomainCategory         string
 	IsDomainPrimaryAccount bool
-	SetupKeys              map[string]*SetupKey
-	Network                *Network
-	Peers                  map[string]*Peer
-	Users                  map[string]*User
-	Groups                 map[string]*Group
-	Rules                  map[string]*Rule
-	Policies               []*Policy
-	Routes                 map[string]*route.Route
-	NameServerGroups       map[string]*nbdns.NameServerGroup
-	DNSSettings            *DNSSettings
+	SetupKeys              map[string]*SetupKey              `gorm:"-"`
+	SetupKeysG             []SetupKey                        `json:"-" gorm:"foreignKey:AccountID;references:id"`
+	Network                *Network                          `gorm:"embedded;embeddedPrefix:network_"`
+	Peers                  map[string]*Peer                  `gorm:"-"`
+	PeersG                 []Peer                            `json:"-" gorm:"foreignKey:AccountID;references:id"`
+	Users                  map[string]*User                  `gorm:"-"`
+	UsersG                 []User                            `json:"-" gorm:"foreignKey:AccountID;references:id"`
+	Groups                 map[string]*Group                 `gorm:"-"`
+	GroupsG                []Group                           `json:"-" gorm:"foreignKey:AccountID;references:id"`
+	Rules                  map[string]*Rule                  `gorm:"-"`
+	RulesG                 []Rule                            `json:"-" gorm:"foreignKey:AccountID;references:id"`
+	Policies               []*Policy                         `gorm:"foreignKey:AccountID;references:id"`
+	Routes                 map[string]*route.Route           `gorm:"-"`
+	RoutesG                []route.Route                     `json:"-" gorm:"foreignKey:AccountID;references:id"`
+	NameServerGroups       map[string]*nbdns.NameServerGroup `gorm:"-"`
+	NameServerGroupsG      []nbdns.NameServerGroup           `json:"-" gorm:"foreignKey:AccountID;references:id"`
+	DNSSettings            DNSSettings                       `gorm:"embedded;embeddedPrefix:dns_settings_"`
 	// Settings is a dictionary of Account settings
-	Settings *Settings
+	Settings *Settings `gorm:"embedded;embeddedPrefix:settings_"`
 }
 
 type UserInfo struct {
@@ -200,7 +210,7 @@ type UserInfo struct {
 // from the ACL peers that have distribution groups associated with the peer ID.
 // Please mind, that the returned route.Route objects will contain Peer.Key instead of Peer.ID.
 func (a *Account) getRoutesToSync(peerID string, aclPeers []*Peer) []*route.Route {
-	routes, peerDisabledRoutes := a.getEnabledAndDisabledRoutesByPeer(peerID)
+	routes, peerDisabledRoutes := a.getRoutingPeerRoutes(peerID)
 	peerRoutesMembership := make(lookupMap)
 	for _, r := range append(routes, peerDisabledRoutes...) {
 		peerRoutesMembership[route.GetHAUniqueID(r)] = struct{}{}
@@ -208,7 +218,7 @@ func (a *Account) getRoutesToSync(peerID string, aclPeers []*Peer) []*route.Rout
 
 	groupListMap := a.getPeerGroups(peerID)
 	for _, peer := range aclPeers {
-		activeRoutes, _ := a.getEnabledAndDisabledRoutesByPeer(peer.ID)
+		activeRoutes, _ := a.getRoutingPeerRoutes(peer.ID)
 		groupFilteredRoutes := a.filterRoutesByGroups(activeRoutes, groupListMap)
 		filteredRoutes := a.filterRoutesFromPeersOfSameHAGroup(groupFilteredRoutes, peerRoutesMembership)
 		routes = append(routes, filteredRoutes...)
@@ -244,20 +254,32 @@ func (a *Account) filterRoutesByGroups(routes []*route.Route, groupListMap looku
 	return filteredRoutes
 }
 
-// getEnabledAndDisabledRoutesByPeer returns the enabled and disabled lists of routes that belong to a peer.
+// getRoutingPeerRoutes returns the enabled and disabled lists of routes that the given routing peer serves
 // Please mind, that the returned route.Route objects will contain Peer.Key instead of Peer.ID.
-func (a *Account) getEnabledAndDisabledRoutesByPeer(peerID string) ([]*route.Route, []*route.Route) {
-	var enabledRoutes []*route.Route
-	var disabledRoutes []*route.Route
+// If the given is not a routing peer, then the lists are empty.
+func (a *Account) getRoutingPeerRoutes(peerID string) (enabledRoutes []*route.Route, disabledRoutes []*route.Route) {
+
+	peer := a.GetPeer(peerID)
+	if peer == nil {
+		log.Errorf("peer %s that doesn't exist under account %s", peerID, a.Id)
+		return enabledRoutes, disabledRoutes
+	}
+
+	// currently we support only linux routing peers
+	if peer.Meta.GoOS != "linux" {
+		return enabledRoutes, disabledRoutes
+	}
+
+	seenRoute := make(map[string]struct{})
 
 	takeRoute := func(r *route.Route, id string) {
-		peer := a.GetPeer(peerID)
-		if peer == nil {
-			log.Errorf("route %s has peer %s that doesn't exist under account %s", r.ID, peerID, a.Id)
+		if _, ok := seenRoute[r.ID]; ok {
 			return
 		}
+		seenRoute[r.ID] = struct{}{}
 
 		if r.Enabled {
+			r.Peer = peer.Key
 			enabledRoutes = append(enabledRoutes, r)
 			return
 		}
@@ -265,25 +287,30 @@ func (a *Account) getEnabledAndDisabledRoutesByPeer(peerID string) ([]*route.Rou
 	}
 
 	for _, r := range a.Routes {
-		if len(r.PeerGroups) != 0 {
-			for _, groupID := range r.PeerGroups {
-				group := a.GetGroup(groupID)
-				if group == nil {
-					log.Errorf("route %s has peers group %s that doesn't exist under account %s", r.ID, groupID, a.Id)
+		for _, groupID := range r.PeerGroups {
+			group := a.GetGroup(groupID)
+			if group == nil {
+				log.Errorf("route %s has peers group %s that doesn't exist under account %s", r.ID, groupID, a.Id)
+				continue
+			}
+			for _, id := range group.Peers {
+				if id != peerID {
 					continue
 				}
-				for _, id := range group.Peers {
-					if id == peerID {
-						takeRoute(r, id)
-						break
-					}
-				}
+
+				newPeerRoute := r.Copy()
+				newPeerRoute.Peer = id
+				newPeerRoute.PeerGroups = nil
+				newPeerRoute.ID = r.ID + ":" + id // we have to provide unique route id when distribute network map
+				takeRoute(newPeerRoute, id)
+				break
 			}
 		}
 		if r.Peer == peerID {
-			takeRoute(r, peerID)
+			takeRoute(r.Copy(), peerID)
 		}
 	}
+
 	return enabledRoutes, disabledRoutes
 }
 
@@ -319,50 +346,7 @@ func (a *Account) GetPeerNetworkMap(peerID, dnsDomain string) *NetworkMap {
 		peersToConnect = append(peersToConnect, p)
 	}
 
-	routes := a.getRoutesToSync(peerID, peersToConnect)
-
-	takePeer := func(id string) (*Peer, bool) {
-		peer := a.GetPeer(id)
-		if peer == nil || peer.Meta.GoOS != "linux" {
-			return nil, false
-		}
-		return peer, true
-	}
-
-	// We need to set Peer.Key instead of Peer.ID because this object will be sent to agents as part of a network map.
-	// Ideally we should have a separate field for that, but fine for now.
-	var routesUpdate []*route.Route
-	seenPeers := make(map[string]bool)
-	for _, r := range routes {
-		if r.Peer != "" {
-			peer, valid := takePeer(r.Peer)
-			if !valid {
-				continue
-			}
-			rCopy := r.Copy()
-			rCopy.Peer = peer.Key // client expects the key
-			routesUpdate = append(routesUpdate, rCopy)
-			continue
-		}
-		for _, groupID := range r.PeerGroups {
-			if group := a.GetGroup(groupID); group != nil {
-				for _, peerId := range group.Peers {
-					peer, valid := takePeer(peerId)
-					if !valid {
-						continue
-					}
-
-					if _, ok := seenPeers[peer.ID]; !ok {
-						rCopy := r.Copy()
-						rCopy.ID = r.ID + ":" + peer.ID // we have to provide unit route id when distribute network map
-						rCopy.Peer = peer.Key           // client expects the key
-						routesUpdate = append(routesUpdate, rCopy)
-					}
-					seenPeers[peer.ID] = true
-				}
-			}
-		}
-	}
+	routesUpdate := a.getRoutesToSync(peerID, peersToConnect)
 
 	dnsManagementStatus := a.getPeerDNSManagementStatus(peerID)
 	dnsUpdate := nbdns.Config{
@@ -538,13 +522,11 @@ func (a *Account) getUserGroups(userID string) ([]string, error) {
 func (a *Account) getPeerDNSManagementStatus(peerID string) bool {
 	peerGroups := a.getPeerGroups(peerID)
 	enabled := true
-	if a.DNSSettings != nil {
-		for _, groupID := range a.DNSSettings.DisabledManagementGroups {
-			_, found := peerGroups[groupID]
-			if found {
-				enabled = false
-				break
-			}
+	for _, groupID := range a.DNSSettings.DisabledManagementGroups {
+		_, found := peerGroups[groupID]
+		if found {
+			enabled = false
+			break
 		}
 	}
 	return enabled
@@ -631,10 +613,7 @@ func (a *Account) Copy() *Account {
 		nsGroups[id] = nsGroup.Copy()
 	}
 
-	var dnsSettings *DNSSettings
-	if a.DNSSettings != nil {
-		dnsSettings = a.DNSSettings.Copy()
-	}
+	dnsSettings := a.DNSSettings.Copy()
 
 	var settings *Settings
 	if a.Settings != nil {
@@ -972,6 +951,7 @@ func (am *DefaultAccountManager) warmupIDPCache() error {
 	if err != nil {
 		return err
 	}
+	log.Infof("%d entries received from IdP management", len(userData))
 
 	// If the Identity Provider does not support writing AppMetadata,
 	// in cases like this, we expect it to return all users in an "unset" field.
@@ -1071,6 +1051,7 @@ func (am *DefaultAccountManager) loadAccount(_ context.Context, accountID interf
 	if err != nil {
 		return nil, err
 	}
+	log.Debugf("%d entries received from IdP management", len(userData))
 
 	dataMap := make(map[string]*idp.UserData, len(userData))
 	for _, datum := range userData {
@@ -1582,6 +1563,11 @@ func (am *DefaultAccountManager) getAccountWithAuthorizationClaims(claims jwtcla
 	}
 }
 
+// GetAllConnectedPeers returns connected peers based on peersUpdateManager.GetAllConnectedPeers()
+func (am *DefaultAccountManager) GetAllConnectedPeers() (map[string]struct{}, error) {
+	return am.peersUpdateManager.GetAllConnectedPeers(), nil
+}
+
 func isDomainValid(domain string) bool {
 	re := regexp.MustCompile(`^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$`)
 	return re.Match([]byte(domain))
@@ -1636,7 +1622,7 @@ func newAccountWithId(accountID, userID, domain string) *Account {
 	setupKeys := map[string]*SetupKey{}
 	nameServersGroups := make(map[string]*nbdns.NameServerGroup)
 	users[userID] = NewAdminUser(userID)
-	dnsSettings := &DNSSettings{
+	dnsSettings := DNSSettings{
 		DisabledManagementGroups: make([]string, 0),
 	}
 	log.Debugf("created new account %s", accountID)

management/server/account_test.go

@@ -198,11 +198,11 @@ func TestAccount_GetPeerNetworkMap(t *testing.T) {
 	netIP := net.IP{100, 64, 0, 0}
 	netMask := net.IPMask{255, 255, 0, 0}
 	network := &Network{
-		Id:     "network",
-		Net:    net.IPNet{IP: netIP, Mask: netMask},
-		Dns:    "netbird.selfhosted",
-		Serial: 0,
-		mu:     sync.Mutex{},
+		Identifier: "network",
+		Net:        net.IPNet{IP: netIP, Mask: netMask},
+		Dns:        "netbird.selfhosted",
+		Serial:     0,
+		mu:         sync.Mutex{},
 	}
 
 	for _, testCase := range tt {
@@ -476,7 +476,7 @@ func TestDefaultAccountManager_GetGroupsFromTheToken(t *testing.T) {
 	// as initAccount was created without account id we have to take the id after account initialization
 	// that happens inside the GetAccountByUserOrAccountID where the id is getting generated
 	// it is important to set the id as it help to avoid creating additional account with empty Id and re-pointing indices to it
-	initAccount.Id = acc.Id
+	initAccount = acc
 
 	claims := jwtclaims.AuthorizationClaims{
 		AccountId:      accountID, // is empty as it is based on accountID right after initialization of initAccount
@@ -1025,7 +1025,6 @@ func TestAccountManager_NetworkUpdates(t *testing.T) {
 
 		wg.Wait()
 	})
-
 	t.Run("delete peer update", func(t *testing.T) {
 		wg.Add(1)
 		go func() {
@@ -1237,7 +1236,7 @@ func TestAccount_GetRoutesToSync(t *testing.T) {
 	}
 	account := &Account{
 		Peers: map[string]*Peer{
-			"peer-1": {Key: "peer-1"}, "peer-2": {Key: "peer-2"}, "peer-3": {Key: "peer-1"},
+			"peer-1": {Key: "peer-1", Meta: PeerSystemMeta{GoOS: "linux"}}, "peer-2": {Key: "peer-2", Meta: PeerSystemMeta{GoOS: "linux"}}, "peer-3": {Key: "peer-1", Meta: PeerSystemMeta{GoOS: "linux"}},
 		},
 		Groups: map[string]*Group{"group1": {ID: "group1", Peers: []string{"peer-1", "peer-2"}}},
 		Routes: map[string]*route.Route{
@@ -1309,7 +1308,7 @@ func TestAccount_Copy(t *testing.T) {
 			},
 		},
 		Network: &Network{
-			Id: "net1",
+			Identifier: "net1",
 		},
 		Peers: map[string]*Peer{
 			"peer1": {
@@ -1374,7 +1373,7 @@ func TestAccount_Copy(t *testing.T) {
 				NameServers: []nbdns.NameServer{},
 			},
 		},
-		DNSSettings: &DNSSettings{DisabledManagementGroups: []string{}},
+		DNSSettings: DNSSettings{DisabledManagementGroups: []string{}},
 		Settings:    &Settings{},
 	}
 	err := hasNilField(account)
@@ -1400,6 +1399,10 @@ func hasNilField(x interface{}) error {
 	rv := reflect.ValueOf(x)
 	rv = rv.Elem()
 	for i := 0; i < rv.NumField(); i++ {
+		// skip gorm internal fields
+		if json, ok := rv.Type().Field(i).Tag.Lookup("json"); ok && json == "-" {
+			continue
+		}
 		if f := rv.Field(i); f.IsValid() {
 			k := f.Kind()
 			switch k {
@@ -2045,7 +2048,7 @@ func createManager(t *testing.T) (*DefaultAccountManager, error) {
 
 func createStore(t *testing.T) (Store, error) {
 	dataDir := t.TempDir()
-	store, err := NewFileStore(dataDir, nil)
+	store, err := NewStoreFromJson(dataDir, nil)
 	if err != nil {
 		return nil, err
 	}

management/server/activity/sqlite/sqlite.go

@@ -45,6 +45,9 @@ const (
 		"VALUES(?, ?, ?, ?, ?, ?)"
 
 	insertDeleteUserQuery = `INSERT INTO deleted_users(id, email, name) VALUES(?, ?, ?)`
+
+	fallbackName  = "unknown"
+	fallbackEmail = "unknown@unknown.com"
 )
 
 // Store is the implementation of the activity.Store interface backed by SQLite
@@ -128,6 +131,7 @@ func NewSQLiteStore(dataDir string, encryptionKey string) (*Store, error) {
 
 func (store *Store) processResult(result *sql.Rows) ([]*activity.Event, error) {
 	events := make([]*activity.Event, 0)
+	var cryptErr error
 	for result.Next() {
 		var id int64
 		var operation activity.Activity
@@ -156,8 +160,8 @@ func (store *Store) processResult(result *sql.Rows) ([]*activity.Event, error) {
 		if targetUserName != nil {
 			name, err := store.fieldEncrypt.Decrypt(*targetUserName)
 			if err != nil {
-				log.Errorf("failed to decrypt username for target id: %s", target)
-				meta["username"] = ""
+				cryptErr = fmt.Errorf("failed to decrypt username for target id: %s", target)
+				meta["username"] = fallbackName
 			} else {
 				meta["username"] = name
 			}
@@ -166,8 +170,8 @@ func (store *Store) processResult(result *sql.Rows) ([]*activity.Event, error) {
 		if targetEmail != nil {
 			email, err := store.fieldEncrypt.Decrypt(*targetEmail)
 			if err != nil {
-				log.Errorf("failed to decrypt email address for target id: %s", target)
-				meta["email"] = ""
+				cryptErr = fmt.Errorf("failed to decrypt email address for target id: %s", target)
+				meta["email"] = fallbackEmail
 			} else {
 				meta["email"] = email
 			}
@@ -186,7 +190,8 @@ func (store *Store) processResult(result *sql.Rows) ([]*activity.Event, error) {
 		if initiatorName != nil {
 			name, err := store.fieldEncrypt.Decrypt(*initiatorName)
 			if err != nil {
-				log.Errorf("failed to decrypt username of initiator: %s", initiator)
+				cryptErr = fmt.Errorf("failed to decrypt username of initiator: %s", initiator)
+				event.InitiatorName = fallbackName
 			} else {
 				event.InitiatorName = name
 			}
@@ -195,7 +200,8 @@ func (store *Store) processResult(result *sql.Rows) ([]*activity.Event, error) {
 		if initiatorEmail != nil {
 			email, err := store.fieldEncrypt.Decrypt(*initiatorEmail)
 			if err != nil {
-				log.Errorf("failed to decrypt email address of initiator: %s", initiator)
+				cryptErr = fmt.Errorf("failed to decrypt email address of initiator: %s", initiator)
+				event.InitiatorEmail = fallbackEmail
 			} else {
 				event.InitiatorEmail = email
 			}
@@ -204,6 +210,10 @@ func (store *Store) processResult(result *sql.Rows) ([]*activity.Event, error) {
 		events = append(events, event)
 	}
 
+	if cryptErr != nil {
+		log.Warnf("%s", cryptErr)
+	}
+
 	return events, nil
 }
 

management/server/config.go

@@ -45,6 +45,8 @@ type Config struct {
 	DeviceAuthorizationFlow *DeviceAuthorizationFlow
 
 	PKCEAuthorizationFlow *PKCEAuthorizationFlow
+
+	StoreConfig StoreConfig
 }
 
 // GetAuthAudiences returns the audience from the http config and device authorization flow config
@@ -136,6 +138,11 @@ type ProviderConfig struct {
 	RedirectURLs []string
 }
 
+// StoreConfig contains Store configuration
+type StoreConfig struct {
+	Engine StoreEngine
+}
+
 // validateURL validates input http url
 func validateURL(httpURL string) bool {
 	_, err := url.ParseRequestURI(httpURL)

management/server/dns.go

@@ -20,23 +20,15 @@ type lookupMap map[string]struct{}
 // DNSSettings defines dns settings at the account level
 type DNSSettings struct {
 	// DisabledManagementGroups groups whose DNS management is disabled
-	DisabledManagementGroups []string
+	DisabledManagementGroups []string `gorm:"serializer:json"`
 }
 
 // Copy returns a copy of the DNS settings
-func (d *DNSSettings) Copy() *DNSSettings {
-	settings := &DNSSettings{
-		DisabledManagementGroups: make([]string, 0),
+func (d DNSSettings) Copy() DNSSettings {
+	settings := DNSSettings{
+		DisabledManagementGroups: make([]string, len(d.DisabledManagementGroups)),
 	}
-
-	if d == nil {
-		return settings
-	}
-
-	if d.DisabledManagementGroups != nil && len(d.DisabledManagementGroups) > 0 {
-		settings.DisabledManagementGroups = d.DisabledManagementGroups[:]
-	}
-
+	copy(settings.DisabledManagementGroups, d.DisabledManagementGroups)
 	return settings
 }
 
@@ -58,12 +50,8 @@ func (am *DefaultAccountManager) GetDNSSettings(accountID string, userID string)
 	if !user.IsAdmin() {
 		return nil, status.Errorf(status.PermissionDenied, "only admins are allowed to view DNS settings")
 	}
-
-	if account.DNSSettings == nil {
-		return &DNSSettings{}, nil
-	}
-
-	return account.DNSSettings.Copy(), nil
+	dnsSettings := account.DNSSettings.Copy()
+	return &dnsSettings, nil
 }
 
 // SaveDNSSettings validates a user role and updates the account's DNS settings
@@ -96,11 +84,7 @@ func (am *DefaultAccountManager) SaveDNSSettings(accountID string, userID string
 		}
 	}
 
-	oldSettings := &DNSSettings{}
-	if account.DNSSettings != nil {
-		oldSettings = account.DNSSettings.Copy()
-	}
-
+	oldSettings := account.DNSSettings.Copy()
 	account.DNSSettings = dnsSettingsToSave.Copy()
 
 	account.Network.IncSerial()
@@ -146,8 +130,9 @@ func toProtocolDNSConfig(update nbdns.Config) *proto.DNSConfig {
 
 	for _, nsGroup := range update.NameServerGroups {
 		protoGroup := &proto.NameServerGroup{
-			Primary: nsGroup.Primary,
-			Domains: nsGroup.Domains,
+			Primary:              nsGroup.Primary,
+			Domains:              nsGroup.Domains,
+			SearchDomainsEnabled: nsGroup.SearchDomainsEnabled,
 		}
 		for _, ns := range nsGroup.NameServers {
 			protoNS := &proto.NameServer{

management/server/dns_test.go

@@ -42,7 +42,7 @@ func TestGetDNSSettings(t *testing.T) {
 		t.Fatal("DNS settings for new accounts shouldn't return nil")
 	}
 
-	account.DNSSettings = &DNSSettings{
+	account.DNSSettings = DNSSettings{
 		DisabledManagementGroups: []string{group1ID},
 	}
 
@@ -196,7 +196,7 @@ func createDNSManager(t *testing.T) (*DefaultAccountManager, error) {
 
 func createDNSStore(t *testing.T) (Store, error) {
 	dataDir := t.TempDir()
-	store, err := NewFileStore(dataDir, nil)
+	store, err := NewStoreFromJson(dataDir, nil)
 	if err != nil {
 		return nil, err
 	}

management/server/file_store.go

@@ -54,6 +54,25 @@ func NewFileStore(dataDir string, metrics telemetry.AppMetrics) (*FileStore, err
 	return fs, nil
 }
 
+// NewFilestoreFromSqliteStore restores a store from Sqlite and stores to Filestore json in the file located in datadir
+func NewFilestoreFromSqliteStore(sqlitestore *SqliteStore, dataDir string, metrics telemetry.AppMetrics) (*FileStore, error) {
+	store, err := NewFileStore(dataDir, metrics)
+	if err != nil {
+		return nil, err
+	}
+
+	err = store.SaveInstallationID(sqlitestore.GetInstallationID())
+	if err != nil {
+		return nil, err
+	}
+
+	for _, account := range sqlitestore.GetAllAccounts() {
+		store.Accounts[account.Id] = account
+	}
+
+	return store, store.persist(store.storeFile)
+}
+
 // restore the state of the store from the file.
 // Creates a new empty store file if doesn't exist
 func restore(file string) (*FileStore, error) {
@@ -111,10 +130,6 @@ func restore(file string) (*FileStore, error) {
 		for _, peer := range account.Peers {
 			store.PeerKeyID2AccountID[peer.Key] = accountID
 			store.PeerID2AccountID[peer.ID] = accountID
-			// reset all peers to status = Disconnected
-			if peer.Status != nil && peer.Status.Connected {
-				peer.Status.Connected = false
-			}
 		}
 		for _, user := range account.Users {
 			store.UserID2AccountID[user.Id] = accountID
@@ -599,3 +614,8 @@ func (s *FileStore) Close() error {
 
 	return s.persist(s.storeFile)
 }
+
+// GetStoreEngine returns FileStoreEngine
+func (s *FileStore) GetStoreEngine() StoreEngine {
+	return FileStoreEngine
+}

management/server/file_store_test.go

@@ -387,7 +387,7 @@ func TestFileStore_GetAccount(t *testing.T) {
 	assert.Equal(t, expected.DomainCategory, account.DomainCategory)
 	assert.Equal(t, expected.Domain, account.Domain)
 	assert.Equal(t, expected.CreatedBy, account.CreatedBy)
-	assert.Equal(t, expected.Network.Id, account.Network.Id)
+	assert.Equal(t, expected.Network.Identifier, account.Network.Identifier)
 	assert.Len(t, account.Peers, len(expected.Peers))
 	assert.Len(t, account.Users, len(expected.Users))
 	assert.Len(t, account.SetupKeys, len(expected.SetupKeys))

management/server/group.go

@@ -23,6 +23,9 @@ type Group struct {
 	// ID of the group
 	ID string
 
+	// AccountID is a reference to Account that this object belongs
+	AccountID string `json:"-" gorm:"index"`
+
 	// Name visible in the UI
 	Name string
 
@@ -30,7 +33,7 @@ type Group struct {
 	Issued string
 
 	// Peers list of the group
-	Peers []string
+	Peers []string `gorm:"serializer:json"`
 }
 
 // EventMeta returns activity event meta related to the group

management/server/group_test.go

@@ -80,6 +80,7 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) {
 
 	groupForRoute := &Group{
 		"grp-for-route",
+		"account-id",
 		"Group for route",
 		GroupIssuedAPI,
 		make([]string, 0),
@@ -87,6 +88,7 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) {
 
 	groupForNameServerGroups := &Group{
 		"grp-for-name-server-grp",
+		"account-id",
 		"Group for name server groups",
 		GroupIssuedAPI,
 		make([]string, 0),
@@ -94,6 +96,7 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) {
 
 	groupForPolicies := &Group{
 		"grp-for-policies",
+		"account-id",
 		"Group for policies",
 		GroupIssuedAPI,
 		make([]string, 0),
@@ -101,6 +104,7 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) {
 
 	groupForSetupKeys := &Group{
 		"grp-for-keys",
+		"account-id",
 		"Group for setup keys",
 		GroupIssuedAPI,
 		make([]string, 0),
@@ -108,6 +112,7 @@ func initTestGroupAccount(am *DefaultAccountManager) (*Account, error) {
 
 	groupForUsers := &Group{
 		"grp-for-users",
+		"account-id",
 		"Group for users",
 		GroupIssuedAPI,
 		make([]string, 0),

management/server/http/api/openapi.yml

@@ -857,13 +857,17 @@ components:
           type: boolean
           example: true
         domains:
-          description: Nameserver group domain list
+          description: Nameserver group match domain list
           type: array
           items:
             type: string
             minLength: 1
             maxLength: 255
             example: "example.com"
+        search_domains_enabled:
+          description: Nameserver group search domain status for match domains. It should be true only if domains list is not empty.
+          type: boolean
+          example: true
       required:
         - name
         - description
@@ -872,6 +876,7 @@ components:
         - groups
         - primary
         - domains
+        - search_domains_enabled
     NameserverGroup:
       allOf:
         - type: object

management/server/http/api/types.gen.go

@@ -1,6 +1,6 @@
 // Package api provides primitives to interact with the openapi HTTP API.
 //
-// Code generated by github.com/deepmap/oapi-codegen version v1.15.0 DO NOT EDIT.
+// Code generated by github.com/deepmap/oapi-codegen version v1.11.1-0.20220912230023-4a1477f6a8ba DO NOT EDIT.
 package api
 
 import (
@@ -248,7 +248,7 @@ type NameserverGroup struct {
 	// Description Nameserver group  description
 	Description string `json:"description"`
 
-	// Domains Nameserver group domain list
+	// Domains Nameserver group match domain list
 	Domains []string `json:"domains"`
 
 	// Enabled Nameserver group status
@@ -268,6 +268,9 @@ type NameserverGroup struct {
 
 	// Primary Nameserver group primary status
 	Primary bool `json:"primary"`
+
+	// SearchDomainsEnabled Nameserver group search domain status for match domains. It should be true only if domains list is not empty.
+	SearchDomainsEnabled bool `json:"search_domains_enabled"`
 }
 
 // NameserverGroupRequest defines model for NameserverGroupRequest.
@@ -275,7 +278,7 @@ type NameserverGroupRequest struct {
 	// Description Nameserver group  description
 	Description string `json:"description"`
 
-	// Domains Nameserver group domain list
+	// Domains Nameserver group match domain list
 	Domains []string `json:"domains"`
 
 	// Enabled Nameserver group status
@@ -292,6 +295,9 @@ type NameserverGroupRequest struct {
 
 	// Primary Nameserver group primary status
 	Primary bool `json:"primary"`
+
+	// SearchDomainsEnabled Nameserver group search domain status for match domains. It should be true only if domains list is not empty.
+	SearchDomainsEnabled bool `json:"search_domains_enabled"`
 }
 
 // Peer defines model for Peer.

management/server/http/dns_settings_handler_test.go

@@ -26,7 +26,7 @@ const (
 	testDNSSettingsUserID        = "test_user"
 )
 
-var baseExistingDNSSettings = &server.DNSSettings{
+var baseExistingDNSSettings = server.DNSSettings{
 	DisabledManagementGroups: []string{testDNSSettingsExistingGroup},
 }
 
@@ -43,7 +43,7 @@ func initDNSSettingsTestData() *DNSSettingsHandler {
 	return &DNSSettingsHandler{
 		accountManager: &mock_server.MockAccountManager{
 			GetDNSSettingsFunc: func(accountID string, userID string) (*server.DNSSettings, error) {
-				return testingDNSSettingsAccount.DNSSettings, nil
+				return &testingDNSSettingsAccount.DNSSettings, nil
 			},
 			SaveDNSSettingsFunc: func(accountID string, userID string, dnsSettingsToSave *server.DNSSettings) error {
 				if dnsSettingsToSave != nil {

management/server/http/handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/rs/cors"
 
+	"github.com/netbirdio/management-integrations/integrations"
 	s "github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/http/middleware"
 	"github.com/netbirdio/netbird/management/server/jwtclaims"
@@ -58,6 +59,7 @@ func APIHandler(accountManager s.AccountManager, jwtValidator jwtclaims.JWTValid
 		AuthCfg:        authCfg,
 	}
 
+	integrations.RegisterHandlers(api.Router, accountManager)
 	api.addAccountsEndpoint()
 	api.addPeersEndpoint()
 	api.addUsersEndpoint()
@@ -73,8 +75,8 @@ func APIHandler(accountManager s.AccountManager, jwtValidator jwtclaims.JWTValid
 
 	err := api.Router.Walk(func(route *mux.Route, _ *mux.Router, _ []*mux.Route) error {
 		methods, err := route.GetMethods()
-		if err != nil {
-			return err
+		if err != nil { // we may have wildcard routes from integrations without methods, skip them for now
+			methods = []string{}
 		}
 		for _, method := range methods {
 			template, err := route.GetPathTemplate()

management/server/http/middleware/auth_middleware.go

@@ -57,10 +57,17 @@ func NewAuthMiddleware(getAccountFromPAT GetAccountFromPATFunc, validateAndParse
 func (m *AuthMiddleware) Handler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		auth := strings.Split(r.Header.Get("Authorization"), " ")
-		authType := auth[0]
-		switch strings.ToLower(authType) {
+		authType := strings.ToLower(auth[0])
+
+		// fallback to token when receive pat as bearer
+		if len(auth) >= 2 && authType == "bearer" && strings.HasPrefix(auth[1], "nbp_") {
+			authType = "token"
+			auth[0] = authType
+		}
+
+		switch authType {
 		case "bearer":
-			err := m.CheckJWTFromRequest(w, r)
+			err := m.checkJWTFromRequest(w, r, auth)
 			if err != nil {
 				log.Errorf("Error when validating JWT claims: %s", err.Error())
 				util.WriteError(status.Errorf(status.Unauthorized, "token invalid"), w)
@@ -68,7 +75,7 @@ func (m *AuthMiddleware) Handler(h http.Handler) http.Handler {
 			}
 			h.ServeHTTP(w, r)
 		case "token":
-			err := m.CheckPATFromRequest(w, r)
+			err := m.checkPATFromRequest(w, r, auth)
 			if err != nil {
 				log.Debugf("Error when validating PAT claims: %s", err.Error())
 				util.WriteError(status.Errorf(status.Unauthorized, "token invalid"), w)
@@ -83,9 +90,8 @@ func (m *AuthMiddleware) Handler(h http.Handler) http.Handler {
 }
 
 // CheckJWTFromRequest checks if the JWT is valid
-func (m *AuthMiddleware) CheckJWTFromRequest(w http.ResponseWriter, r *http.Request) error {
-
-	token, err := getTokenFromJWTRequest(r)
+func (m *AuthMiddleware) checkJWTFromRequest(w http.ResponseWriter, r *http.Request, auth []string) error {
+	token, err := getTokenFromJWTRequest(auth)
 
 	// If an error occurs, call the error handler and return an error
 	if err != nil {
@@ -110,8 +116,8 @@ func (m *AuthMiddleware) CheckJWTFromRequest(w http.ResponseWriter, r *http.Requ
 }
 
 // CheckPATFromRequest checks if the PAT is valid
-func (m *AuthMiddleware) CheckPATFromRequest(w http.ResponseWriter, r *http.Request) error {
-	token, err := getTokenFromPATRequest(r)
+func (m *AuthMiddleware) checkPATFromRequest(w http.ResponseWriter, r *http.Request, auth []string) error {
+	token, err := getTokenFromPATRequest(auth)
 
 	// If an error occurs, call the error handler and return an error
 	if err != nil {
@@ -143,16 +149,9 @@ func (m *AuthMiddleware) CheckPATFromRequest(w http.ResponseWriter, r *http.Requ
 	return nil
 }
 
-// getTokenFromJWTRequest is a "TokenExtractor" that takes a give request and extracts
+// getTokenFromJWTRequest is a "TokenExtractor" that takes auth header parts and extracts
 // the JWT token from the Authorization header.
-func getTokenFromJWTRequest(r *http.Request) (string, error) {
-	authHeader := r.Header.Get("Authorization")
-	if authHeader == "" {
-		return "", nil // No error, just no token
-	}
-
-	// TODO: Make this a bit more robust, parsing-wise
-	authHeaderParts := strings.Fields(authHeader)
+func getTokenFromJWTRequest(authHeaderParts []string) (string, error) {
 	if len(authHeaderParts) != 2 || strings.ToLower(authHeaderParts[0]) != "bearer" {
 		return "", errors.New("Authorization header format must be Bearer {token}")
 	}
@@ -160,16 +159,9 @@ func getTokenFromJWTRequest(r *http.Request) (string, error) {
 	return authHeaderParts[1], nil
 }
 
-// getTokenFromPATRequest is a "TokenExtractor" that takes a give request and extracts
+// getTokenFromPATRequest is a "TokenExtractor" that takes auth header parts and extracts
 // the PAT token from the Authorization header.
-func getTokenFromPATRequest(r *http.Request) (string, error) {
-	authHeader := r.Header.Get("Authorization")
-	if authHeader == "" {
-		return "", nil // No error, just no token
-	}
-
-	// TODO: Make this a bit more robust, parsing-wise
-	authHeaderParts := strings.Fields(authHeader)
+func getTokenFromPATRequest(authHeaderParts []string) (string, error) {
 	if len(authHeaderParts) != 2 || strings.ToLower(authHeaderParts[0]) != "token" {
 		return "", errors.New("Authorization header format must be Token {token}")
 	}

management/server/http/middleware/auth_middleware_test.go

@@ -19,7 +19,7 @@ const (
 	domain      = "domain"
 	userID      = "userID"
 	tokenID     = "tokenID"
-	PAT         = "PAT"
+	PAT         = "nbp_PAT"
 	JWT         = "JWT"
 	wrongToken  = "wrongToken"
 )
@@ -82,6 +82,11 @@ func TestAuthMiddleware_Handler(t *testing.T) {
 			authHeader:         "Token " + wrongToken,
 			expectedStatusCode: 401,
 		},
+		{
+			name:               "Fallback to PAT Token",
+			authHeader:         "Bearer " + PAT,
+			expectedStatusCode: 200,
+		},
 		{
 			name:               "Valid JWT Token",
 			authHeader:         "Bearer " + JWT,

management/server/http/nameservers_handler.go

@@ -79,7 +79,7 @@ func (h *NameserversHandler) CreateNameserverGroup(w http.ResponseWriter, r *htt
 		return
 	}
 
-	nsGroup, err := h.accountManager.CreateNameServerGroup(account.Id, req.Name, req.Description, nsList, req.Groups, req.Primary, req.Domains, req.Enabled, user.Id)
+	nsGroup, err := h.accountManager.CreateNameServerGroup(account.Id, req.Name, req.Description, nsList, req.Groups, req.Primary, req.Domains, req.Enabled, user.Id, req.SearchDomainsEnabled)
 	if err != nil {
 		util.WriteError(err, w)
 		return
@@ -119,14 +119,15 @@ func (h *NameserversHandler) UpdateNameserverGroup(w http.ResponseWriter, r *htt
 	}
 
 	updatedNSGroup := &nbdns.NameServerGroup{
-		ID:          nsGroupID,
-		Name:        req.Name,
-		Description: req.Description,
-		Primary:     req.Primary,
-		Domains:     req.Domains,
-		NameServers: nsList,
-		Groups:      req.Groups,
-		Enabled:     req.Enabled,
+		ID:                   nsGroupID,
+		Name:                 req.Name,
+		Description:          req.Description,
+		Primary:              req.Primary,
+		Domains:              req.Domains,
+		NameServers:          nsList,
+		Groups:               req.Groups,
+		Enabled:              req.Enabled,
+		SearchDomainsEnabled: req.SearchDomainsEnabled,
 	}
 
 	err = h.accountManager.SaveNameServerGroup(account.Id, user.Id, updatedNSGroup)
@@ -216,13 +217,14 @@ func toNameserverGroupResponse(serverNSGroup *nbdns.NameServerGroup) *api.Namese
 	}
 
 	return &api.NameserverGroup{
-		Id:          serverNSGroup.ID,
-		Name:        serverNSGroup.Name,
-		Description: serverNSGroup.Description,
-		Primary:     serverNSGroup.Primary,
-		Domains:     serverNSGroup.Domains,
-		Groups:      serverNSGroup.Groups,
-		Nameservers: nsList,
-		Enabled:     serverNSGroup.Enabled,
+		Id:                   serverNSGroup.ID,
+		Name:                 serverNSGroup.Name,
+		Description:          serverNSGroup.Description,
+		Primary:              serverNSGroup.Primary,
+		Domains:              serverNSGroup.Domains,
+		Groups:               serverNSGroup.Groups,
+		Nameservers:          nsList,
+		Enabled:              serverNSGroup.Enabled,
+		SearchDomainsEnabled: serverNSGroup.SearchDomainsEnabled,
 	}
 }

management/server/http/nameservers_handler_test.go

@@ -67,16 +67,17 @@ func initNameserversTestData() *NameserversHandler {
 				}
 				return nil, status.Errorf(status.NotFound, "nameserver group with ID %s not found", nsGroupID)
 			},
-			CreateNameServerGroupFunc: func(accountID string, name, description string, nameServerList []nbdns.NameServer, groups []string, primary bool, domains []string, enabled bool, _ string) (*nbdns.NameServerGroup, error) {
+			CreateNameServerGroupFunc: func(accountID string, name, description string, nameServerList []nbdns.NameServer, groups []string, primary bool, domains []string, enabled bool, _ string, searchDomains bool) (*nbdns.NameServerGroup, error) {
 				return &nbdns.NameServerGroup{
-					ID:          existingNSGroupID,
-					Name:        name,
-					Description: description,
-					NameServers: nameServerList,
-					Groups:      groups,
-					Enabled:     enabled,
-					Primary:     primary,
-					Domains:     domains,
+					ID:                   existingNSGroupID,
+					Name:                 name,
+					Description:          description,
+					NameServers:          nameServerList,
+					Groups:               groups,
+					Enabled:              enabled,
+					Primary:              primary,
+					Domains:              domains,
+					SearchDomainsEnabled: searchDomains,
 				}, nil
 			},
 			DeleteNameServerGroupFunc: func(accountID, nsGroupID, _ string) error {

management/server/http/peers_handler.go

@@ -31,6 +31,24 @@ func NewPeersHandler(accountManager server.AccountManager, authCfg AuthCfg) *Pee
 	}
 }
 
+func (h *PeersHandler) checkPeerStatus(peer *server.Peer) (*server.Peer, error) {
+	peerToReturn := peer.Copy()
+	if peer.Status.Connected {
+		statuses, err := h.accountManager.GetAllConnectedPeers()
+		if err != nil {
+			return peerToReturn, err
+		}
+
+		// Although we have online status in store we do not yet have an updated channel so have to show it as disconnected
+		// This may happen after server restart when not all peers are yet connected
+		if _, connected := statuses[peerToReturn.ID]; !connected {
+			peerToReturn.Status.Connected = false
+		}
+	}
+
+	return peerToReturn, nil
+}
+
 func (h *PeersHandler) getPeer(account *server.Account, peerID, userID string, w http.ResponseWriter) {
 	peer, err := h.accountManager.GetPeer(account.Id, peerID, userID)
 	if err != nil {
@@ -38,7 +56,13 @@ func (h *PeersHandler) getPeer(account *server.Account, peerID, userID string, w
 		return
 	}
 
-	util.WriteJSONObject(w, toPeerResponse(peer, account, h.accountManager.GetDNSDomain()))
+	peerToReturn, err := h.checkPeerStatus(peer)
+	if err != nil {
+		util.WriteError(err, w)
+		return
+	}
+
+	util.WriteJSONObject(w, toPeerResponse(peerToReturn, account, h.accountManager.GetDNSDomain()))
 }
 
 func (h *PeersHandler) updatePeer(account *server.Account, user *server.User, peerID string, w http.ResponseWriter, r *http.Request) {
@@ -120,7 +144,12 @@ func (h *PeersHandler) GetAllPeers(w http.ResponseWriter, r *http.Request) {
 
 		respBody := []*api.Peer{}
 		for _, peer := range peers {
-			respBody = append(respBody, toPeerResponse(peer, account, dnsDomain))
+			peerToReturn, err := h.checkPeerStatus(peer)
+			if err != nil {
+				util.WriteError(err, w)
+				return
+			}
+			respBody = append(respBody, toPeerResponse(peerToReturn, account, dnsDomain))
 		}
 		util.WriteJSONObject(w, respBody)
 		return

management/server/http/peers_handler_test.go

@@ -3,6 +3,7 @@ package http
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -23,19 +24,33 @@ import (
 )
 
 const testPeerID = "test_peer"
+const noUpdateChannelTestPeerID = "no-update-channel"
 
 func initTestMetaData(peers ...*server.Peer) *PeersHandler {
 	return &PeersHandler{
 		accountManager: &mock_server.MockAccountManager{
 			UpdatePeerFunc: func(accountID, userID string, update *server.Peer) (*server.Peer, error) {
-				p := peers[0].Copy()
+				var p *server.Peer
+				for _, peer := range peers {
+					if update.ID == peer.ID {
+						p = peer.Copy()
+						break
+					}
+				}
 				p.SSHEnabled = update.SSHEnabled
 				p.LoginExpirationEnabled = update.LoginExpirationEnabled
 				p.Name = update.Name
 				return p, nil
 			},
 			GetPeerFunc: func(accountID, peerID, userID string) (*server.Peer, error) {
-				return peers[0], nil
+				var p *server.Peer
+				for _, peer := range peers {
+					if peerID == peer.ID {
+						p = peer.Copy()
+						break
+					}
+				}
+				return p, nil
 			},
 			GetPeersFunc: func(accountID, userID string) ([]*server.Peer, error) {
 				return peers, nil
@@ -57,6 +72,16 @@ func initTestMetaData(peers ...*server.Peer) *PeersHandler {
 					},
 				}, user, nil
 			},
+			GetAllConnectedPeersFunc: func() (map[string]struct{}, error) {
+				statuses := make(map[string]struct{})
+				for _, peer := range peers {
+					if peer.ID == noUpdateChannelTestPeerID {
+						break
+					}
+					statuses[peer.ID] = struct{}{}
+				}
+				return statuses, nil
+			},
 		},
 		claimsExtractor: jwtclaims.NewClaimsExtractor(
 			jwtclaims.WithFromRequestContext(func(r *http.Request) jwtclaims.AuthorizationClaims {
@@ -79,7 +104,7 @@ func TestGetPeers(t *testing.T) {
 		Key:                    "key",
 		SetupKey:               "setupkey",
 		IP:                     net.ParseIP("100.64.0.1"),
-		Status:                 &server.PeerStatus{},
+		Status:                 &server.PeerStatus{Connected: true},
 		Name:                   "PeerName",
 		LoginExpirationEnabled: false,
 		Meta: server.PeerSystemMeta{
@@ -93,11 +118,17 @@ func TestGetPeers(t *testing.T) {
 		},
 	}
 
+	peer1 := peer.Copy()
+	peer1.ID = noUpdateChannelTestPeerID
+
 	expectedUpdatedPeer := peer.Copy()
 	expectedUpdatedPeer.LoginExpirationEnabled = true
 	expectedUpdatedPeer.SSHEnabled = true
 	expectedUpdatedPeer.Name = "New Name"
 
+	expectedPeer1 := peer1.Copy()
+	expectedPeer1.Status.Connected = false
+
 	tt := []struct {
 		name           string
 		expectedStatus int
@@ -116,13 +147,21 @@ func TestGetPeers(t *testing.T) {
 			expectedPeer:   peer,
 		},
 		{
-			name:           "GetPeer",
+			name:           "GetPeer with update channel",
 			requestType:    http.MethodGet,
 			requestPath:    "/api/peers/" + testPeerID,
 			expectedStatus: http.StatusOK,
 			expectedArray:  false,
 			expectedPeer:   peer,
 		},
+		{
+			name:           "GetPeer with no update channel",
+			requestType:    http.MethodGet,
+			requestPath:    "/api/peers/" + peer1.ID,
+			expectedStatus: http.StatusOK,
+			expectedArray:  false,
+			expectedPeer:   expectedPeer1,
+		},
 		{
 			name:           "PutPeer",
 			requestType:    http.MethodPut,
@@ -136,7 +175,7 @@ func TestGetPeers(t *testing.T) {
 
 	rr := httptest.NewRecorder()
 
-	p := initTestMetaData(peer)
+	p := initTestMetaData(peer, peer1)
 
 	for _, tc := range tt {
 		t.Run(tc.name, func(t *testing.T) {
@@ -171,6 +210,10 @@ func TestGetPeers(t *testing.T) {
 					t.Fatalf("Sent content is not in correct json format; %v", err)
 				}
 
+				// hardcode this check for now as we only have two peers in this suite
+				assert.Equal(t, len(respBody), 2)
+				assert.Equal(t, respBody[1].Connected, false)
+
 				got = respBody[0]
 			} else {
 				got = &api.Peer{}
@@ -180,12 +223,15 @@ func TestGetPeers(t *testing.T) {
 				}
 			}
 
+			fmt.Println(got)
+
 			assert.Equal(t, got.Name, tc.expectedPeer.Name)
 			assert.Equal(t, got.Version, tc.expectedPeer.Meta.WtVersion)
 			assert.Equal(t, got.Ip, tc.expectedPeer.IP.String())
 			assert.Equal(t, got.Os, "OS core")
 			assert.Equal(t, got.LoginExpirationEnabled, tc.expectedPeer.LoginExpirationEnabled)
 			assert.Equal(t, got.SshEnabled, tc.expectedPeer.SSHEnabled)
+			assert.Equal(t, got.Connected, tc.expectedPeer.Status.Connected)
 		})
 	}
 }

management/server/idp/authentik.go

@@ -251,34 +251,18 @@ func (am *AuthentikManager) GetUserDataByID(userID string, appMetadata AppMetada
 
 // GetAccount returns all the users for a given profile.
 func (am *AuthentikManager) GetAccount(accountID string) ([]*UserData, error) {
-	ctx, err := am.authenticationContext()
+	users, err := am.getAllUsers()
 	if err != nil {
 		return nil, err
 	}
 
-	userList, resp, err := am.apiClient.CoreApi.CoreUsersList(ctx).Execute()
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
 	if am.appMetrics != nil {
 		am.appMetrics.IDPMetrics().CountGetAccount()
 	}
 
-	if resp.StatusCode != http.StatusOK {
-		if am.appMetrics != nil {
-			am.appMetrics.IDPMetrics().CountRequestStatusError()
-		}
-		return nil, fmt.Errorf("unable to get account %s users, statusCode %d", accountID, resp.StatusCode)
-	}
-
-	users := make([]*UserData, 0)
-	for _, user := range userList.Results {
-		userData := parseAuthentikUser(user)
-		userData.AppMetadata.WTAccountID = accountID
-
-		users = append(users, userData)
+	for index, user := range users {
+		user.AppMetadata.WTAccountID = accountID
+		users[index] = user
 	}
 
 	return users, nil
@@ -287,37 +271,59 @@ func (am *AuthentikManager) GetAccount(accountID string) ([]*UserData, error) {
 // GetAllAccounts gets all registered accounts with corresponding user data.
 // It returns a list of users indexed by accountID.
 func (am *AuthentikManager) GetAllAccounts() (map[string][]*UserData, error) {
-	ctx, err := am.authenticationContext()
+	users, err := am.getAllUsers()
 	if err != nil {
 		return nil, err
 	}
 
-	userList, resp, err := am.apiClient.CoreApi.CoreUsersList(ctx).Execute()
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
+	indexedUsers := make(map[string][]*UserData)
+	indexedUsers[UnsetAccountID] = append(indexedUsers[UnsetAccountID], users...)
 
 	if am.appMetrics != nil {
 		am.appMetrics.IDPMetrics().CountGetAllAccounts()
 	}
 
-	if resp.StatusCode != http.StatusOK {
-		if am.appMetrics != nil {
-			am.appMetrics.IDPMetrics().CountRequestStatusError()
-		}
-		return nil, fmt.Errorf("unable to get all accounts, statusCode %d", resp.StatusCode)
-	}
-
-	indexedUsers := make(map[string][]*UserData)
-	for _, user := range userList.Results {
-		userData := parseAuthentikUser(user)
-		indexedUsers[UnsetAccountID] = append(indexedUsers[UnsetAccountID], userData)
-	}
-
 	return indexedUsers, nil
 }
 
+// getAllUsers returns all users in a Authentik account.
+func (am *AuthentikManager) getAllUsers() ([]*UserData, error) {
+	users := make([]*UserData, 0)
+
+	page := int32(1)
+	for {
+		ctx, err := am.authenticationContext()
+		if err != nil {
+			return nil, err
+		}
+
+		userList, resp, err := am.apiClient.CoreApi.CoreUsersList(ctx).Page(page).Execute()
+		if err != nil {
+			return nil, err
+		}
+		_ = resp.Body.Close()
+
+		if resp.StatusCode != http.StatusOK {
+			if am.appMetrics != nil {
+				am.appMetrics.IDPMetrics().CountRequestStatusError()
+			}
+			return nil, fmt.Errorf("unable to get all accounts, statusCode %d", resp.StatusCode)
+		}
+
+		for _, user := range userList.Results {
+			users = append(users, parseAuthentikUser(user))
+		}
+
+		page = int32(userList.GetPagination().Next)
+		if userList.GetPagination().Next == 0 {
+			break
+		}
+
+	}
+
+	return users, nil
+}
+
 // CreateUser creates a new user in authentik Idp and sends an invitation.
 func (am *AuthentikManager) CreateUser(_, _, _, _ string) (*UserData, error) {
 	return nil, fmt.Errorf("method CreateUser not implemented")

management/server/idp/azure.go

@@ -266,10 +266,7 @@ func (am *AzureManager) GetUserByEmail(email string) ([]*UserData, error) {
 
 // GetAccount returns all the users for a given profile.
 func (am *AzureManager) GetAccount(accountID string) ([]*UserData, error) {
-	q := url.Values{}
-	q.Add("$select", profileFields)
-
-	body, err := am.get("users", q)
+	users, err := am.getAllUsers()
 	if err != nil {
 		return nil, err
 	}
@@ -278,18 +275,9 @@ func (am *AzureManager) GetAccount(accountID string) ([]*UserData, error) {
 		am.appMetrics.IDPMetrics().CountGetAccount()
 	}
 
-	var profiles struct{ Value []azureProfile }
-	err = am.helper.Unmarshal(body, &profiles)
-	if err != nil {
-		return nil, err
-	}
-
-	users := make([]*UserData, 0)
-	for _, profile := range profiles.Value {
-		userData := profile.userData()
-		userData.AppMetadata.WTAccountID = accountID
-
-		users = append(users, userData)
+	for index, user := range users {
+		user.AppMetadata.WTAccountID = accountID
+		users[index] = user
 	}
 
 	return users, nil
@@ -298,28 +286,16 @@ func (am *AzureManager) GetAccount(accountID string) ([]*UserData, error) {
 // GetAllAccounts gets all registered accounts with corresponding user data.
 // It returns a list of users indexed by accountID.
 func (am *AzureManager) GetAllAccounts() (map[string][]*UserData, error) {
-	q := url.Values{}
-	q.Add("$select", profileFields)
-
-	body, err := am.get("users", q)
-	if err != nil {
-		return nil, err
-	}
-
-	if am.appMetrics != nil {
-		am.appMetrics.IDPMetrics().CountGetAllAccounts()
-	}
-
-	var profiles struct{ Value []azureProfile }
-	err = am.helper.Unmarshal(body, &profiles)
+	users, err := am.getAllUsers()
 	if err != nil {
 		return nil, err
 	}
 
 	indexedUsers := make(map[string][]*UserData)
-	for _, profile := range profiles.Value {
-		userData := profile.userData()
-		indexedUsers[UnsetAccountID] = append(indexedUsers[UnsetAccountID], userData)
+	indexedUsers[UnsetAccountID] = append(indexedUsers[UnsetAccountID], users...)
+
+	if am.appMetrics != nil {
+		am.appMetrics.IDPMetrics().CountGetAllAccounts()
 	}
 
 	return indexedUsers, nil
@@ -373,6 +349,39 @@ func (am *AzureManager) DeleteUser(userID string) error {
 	return nil
 }
 
+// getAllUsers returns all users in an Azure AD account.
+func (am *AzureManager) getAllUsers() ([]*UserData, error) {
+	users := make([]*UserData, 0)
+
+	q := url.Values{}
+	q.Add("$select", profileFields)
+	q.Add("$top", "500")
+
+	for nextLink := "users"; nextLink != ""; {
+		body, err := am.get(nextLink, q)
+		if err != nil {
+			return nil, err
+		}
+
+		var profiles struct {
+			Value    []azureProfile
+			NextLink string `json:"@odata.nextLink"`
+		}
+		err = am.helper.Unmarshal(body, &profiles)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, profile := range profiles.Value {
+			users = append(users, profile.userData())
+		}
+
+		nextLink = profiles.NextLink
+	}
+
+	return users, nil
+}
+
 // get perform Get requests.
 func (am *AzureManager) get(resource string, q url.Values) ([]byte, error) {
 	jwtToken, err := am.credentials.Authenticate()
@@ -380,7 +389,14 @@ func (am *AzureManager) get(resource string, q url.Values) ([]byte, error) {
 		return nil, err
 	}
 
-	reqURL := fmt.Sprintf("%s/%s?%s", am.GraphAPIEndpoint, resource, q.Encode())
+	var reqURL string
+	if strings.HasPrefix(resource, "https") {
+		// Already an absolute URL for paging
+		reqURL = resource
+	} else {
+		reqURL = fmt.Sprintf("%s/%s?%s", am.GraphAPIEndpoint, resource, q.Encode())
+	}
+
 	req, err := http.NewRequest(http.MethodGet, reqURL, nil)
 	if err != nil {
 		return nil, err

management/server/idp/google_workspace.go

@@ -96,7 +96,7 @@ func (gm *GoogleWorkspaceManager) UpdateUserAppMetadata(_ string, _ AppMetadata)
 
 // GetUserDataByID requests user data from Google Workspace via ID.
 func (gm *GoogleWorkspaceManager) GetUserDataByID(userID string, appMetadata AppMetadata) (*UserData, error) {
-	user, err := gm.usersService.Get(userID).Projection("full").Do()
+	user, err := gm.usersService.Get(userID).Do()
 	if err != nil {
 		return nil, err
 	}
@@ -113,43 +113,69 @@ func (gm *GoogleWorkspaceManager) GetUserDataByID(userID string, appMetadata App
 
 // GetAccount returns all the users for a given profile.
 func (gm *GoogleWorkspaceManager) GetAccount(accountID string) ([]*UserData, error) {
-	usersList, err := gm.usersService.List().Customer(gm.CustomerID).Projection("full").Do()
-	if err != nil {
-		return nil, err
-	}
-
-	usersData := make([]*UserData, 0)
-	for _, user := range usersList.Users {
-		userData := parseGoogleWorkspaceUser(user)
-		userData.AppMetadata.WTAccountID = accountID
-
-		usersData = append(usersData, userData)
-	}
-
-	return usersData, nil
-}
-
-// GetAllAccounts gets all registered accounts with corresponding user data.
-// It returns a list of users indexed by accountID.
-func (gm *GoogleWorkspaceManager) GetAllAccounts() (map[string][]*UserData, error) {
-	usersList, err := gm.usersService.List().Customer(gm.CustomerID).Projection("full").Do()
+	users, err := gm.getAllUsers()
 	if err != nil {
 		return nil, err
 	}
 
 	if gm.appMetrics != nil {
-		gm.appMetrics.IDPMetrics().CountGetAllAccounts()
+		gm.appMetrics.IDPMetrics().CountGetAccount()
+	}
+
+	for index, user := range users {
+		user.AppMetadata.WTAccountID = accountID
+		users[index] = user
+	}
+
+	return users, nil
+}
+
+// GetAllAccounts gets all registered accounts with corresponding user data.
+// It returns a list of users indexed by accountID.
+func (gm *GoogleWorkspaceManager) GetAllAccounts() (map[string][]*UserData, error) {
+	users, err := gm.getAllUsers()
+	if err != nil {
+		return nil, err
 	}
 
 	indexedUsers := make(map[string][]*UserData)
-	for _, user := range usersList.Users {
-		userData := parseGoogleWorkspaceUser(user)
-		indexedUsers[UnsetAccountID] = append(indexedUsers[UnsetAccountID], userData)
+	indexedUsers[UnsetAccountID] = append(indexedUsers[UnsetAccountID], users...)
+
+	if gm.appMetrics != nil {
+		gm.appMetrics.IDPMetrics().CountGetAllAccounts()
 	}
 
 	return indexedUsers, nil
 }
 
+// getAllUsers returns all users in a Google Workspace account filtered by customer ID.
+func (gm *GoogleWorkspaceManager) getAllUsers() ([]*UserData, error) {
+	users := make([]*UserData, 0)
+	pageToken := ""
+	for {
+		call := gm.usersService.List().Customer(gm.CustomerID).MaxResults(500)
+		if pageToken != "" {
+			call.PageToken(pageToken)
+		}
+
+		resp, err := call.Do()
+		if err != nil {
+			return nil, err
+		}
+
+		for _, user := range resp.Users {
+			users = append(users, parseGoogleWorkspaceUser(user))
+		}
+
+		pageToken = resp.NextPageToken
+		if pageToken == "" {
+			break
+		}
+	}
+
+	return users, nil
+}
+
 // CreateUser creates a new user in Google Workspace and sends an invitation.
 func (gm *GoogleWorkspaceManager) CreateUser(_, _, _, _ string) (*UserData, error) {
 	return nil, fmt.Errorf("method CreateUser not implemented")
@@ -158,7 +184,7 @@ func (gm *GoogleWorkspaceManager) CreateUser(_, _, _, _ string) (*UserData, erro
 // GetUserByEmail searches users with a given email.
 // If no users have been found, this function returns an empty list.
 func (gm *GoogleWorkspaceManager) GetUserByEmail(email string) ([]*UserData, error) {
-	user, err := gm.usersService.Get(email).Projection("full").Do()
+	user, err := gm.usersService.Get(email).Do()
 	if err != nil {
 		return nil, err
 	}

management/server/idp/okta.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/okta/okta-sdk-golang/v2/okta"
+	"github.com/okta/okta-sdk-golang/v2/okta/query"
 
 	"github.com/netbirdio/netbird/management/server/telemetry"
 )
@@ -160,7 +161,7 @@ func (om *OktaManager) GetUserByEmail(email string) ([]*UserData, error) {
 
 // GetAccount returns all the users for a given profile.
 func (om *OktaManager) GetAccount(accountID string) ([]*UserData, error) {
-	users, resp, err := om.client.User.ListUsers(context.Background(), nil)
+	users, err := om.getAllUsers()
 	if err != nil {
 		return nil, err
 	}
@@ -169,39 +170,40 @@ func (om *OktaManager) GetAccount(accountID string) ([]*UserData, error) {
 		om.appMetrics.IDPMetrics().CountGetAccount()
 	}
 
-	if resp.StatusCode != http.StatusOK {
-		if om.appMetrics != nil {
-			om.appMetrics.IDPMetrics().CountRequestStatusError()
-		}
-		return nil, fmt.Errorf("unable to get account, statusCode %d", resp.StatusCode)
+	for index, user := range users {
+		user.AppMetadata.WTAccountID = accountID
+		users[index] = user
 	}
 
-	list := make([]*UserData, 0)
-	for _, user := range users {
-		userData, err := parseOktaUser(user)
-		if err != nil {
-			return nil, err
-		}
-		userData.AppMetadata.WTAccountID = accountID
-
-		list = append(list, userData)
-	}
-
-	return list, nil
+	return users, nil
 }
 
 // GetAllAccounts gets all registered accounts with corresponding user data.
 // It returns a list of users indexed by accountID.
 func (om *OktaManager) GetAllAccounts() (map[string][]*UserData, error) {
-	users, resp, err := om.client.User.ListUsers(context.Background(), nil)
+	users, err := om.getAllUsers()
 	if err != nil {
 		return nil, err
 	}
 
+	indexedUsers := make(map[string][]*UserData)
+	indexedUsers[UnsetAccountID] = append(indexedUsers[UnsetAccountID], users...)
+
 	if om.appMetrics != nil {
 		om.appMetrics.IDPMetrics().CountGetAllAccounts()
 	}
 
+	return indexedUsers, nil
+}
+
+// getAllUsers returns all users in an Okta account.
+func (om *OktaManager) getAllUsers() ([]*UserData, error) {
+	qp := query.NewQueryParams(query.WithLimit(200))
+	userList, resp, err := om.client.User.ListUsers(context.Background(), qp)
+	if err != nil {
+		return nil, err
+	}
+
 	if resp.StatusCode != http.StatusOK {
 		if om.appMetrics != nil {
 			om.appMetrics.IDPMetrics().CountRequestStatusError()
@@ -209,17 +211,34 @@ func (om *OktaManager) GetAllAccounts() (map[string][]*UserData, error) {
 		return nil, fmt.Errorf("unable to get all accounts, statusCode %d", resp.StatusCode)
 	}
 
-	indexedUsers := make(map[string][]*UserData)
-	for _, user := range users {
+	for resp.HasNextPage() {
+		paginatedUsers := make([]*okta.User, 0)
+		resp, err = resp.Next(context.Background(), &paginatedUsers)
+		if err != nil {
+			return nil, err
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			if om.appMetrics != nil {
+				om.appMetrics.IDPMetrics().CountRequestStatusError()
+			}
+			return nil, fmt.Errorf("unable to get all accounts, statusCode %d", resp.StatusCode)
+		}
+
+		userList = append(userList, paginatedUsers...)
+	}
+
+	users := make([]*UserData, 0, len(userList))
+	for _, user := range userList {
 		userData, err := parseOktaUser(user)
 		if err != nil {
 			return nil, err
 		}
 
-		indexedUsers[UnsetAccountID] = append(indexedUsers[UnsetAccountID], userData)
+		users = append(users, userData)
 	}
 
-	return indexedUsers, nil
+	return users, nil
 }
 
 // UpdateUserAppMetadata updates user app metadata based on userID and metadata map.

management/server/management_proto_test.go

@@ -405,7 +405,7 @@ func startManagement(t *testing.T, config *Config) (*grpc.Server, string, error)
 		return nil, "", err
 	}
 	s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp))
-	store, err := NewFileStore(config.Datadir, nil)
+	store, err := NewStoreFromJson(config.Datadir, nil)
 	if err != nil {
 		return nil, "", err
 	}

management/server/management_test.go

@@ -393,6 +393,7 @@ var _ = Describe("Management service", func() {
 			ipChannel := make(chan string, 20)
 			for i := 0; i < initialPeers; i++ {
 				go func() {
+					defer GinkgoRecover()
 					key, _ := wgtypes.GenerateKey()
 					loginPeerWithValidSetupKey(serverPubKey, key, client)
 					encryptedBytes, err := encryption.EncryptMessage(serverPubKey, key, &mgmtProto.SyncRequest{})
@@ -496,7 +497,7 @@ func startServer(config *server.Config) (*grpc.Server, net.Listener) {
 	Expect(err).NotTo(HaveOccurred())
 	s := grpc.NewServer()
 
-	store, err := server.NewFileStore(config.Datadir, nil)
+	store, err := server.NewStoreFromJson(config.Datadir, nil)
 	if err != nil {
 		log.Fatalf("failed creating a store: %s: %v", config.Datadir, err)
 	}

management/server/metrics/selfhosted.go

@@ -48,6 +48,7 @@ type properties map[string]interface{}
 // DataSource metric data source
 type DataSource interface {
 	GetAllAccounts() []*server.Account
+	GetStoreEngine() server.StoreEngine
 }
 
 // ConnManager peer connection manager that holds state for current active connections
@@ -295,6 +296,7 @@ func (w *Worker) generateProperties() properties {
 	metricsProperties["max_active_peer_version"] = maxActivePeerVersion
 	metricsProperties["ui_clients"] = uiClient
 	metricsProperties["idp_manager"] = w.idpManager
+	metricsProperties["store_engine"] = w.dataSource.GetStoreEngine()
 
 	for protocol, count := range rulesProtocol {
 		metricsProperties["rules_protocol_"+protocol] = count

management/server/metrics/selfhosted_test.go

@@ -151,6 +151,11 @@ func (mockDatasource) GetAllAccounts() []*server.Account {
 	}
 }
 
+// GetStoreEngine returns FileStoreEngine
+func (mockDatasource) GetStoreEngine() server.StoreEngine {
+	return server.FileStoreEngine
+}
+
 // TestGenerateProperties tests and validate the properties generation by using the mockDatasource for the Worker.generateProperties
 func TestGenerateProperties(t *testing.T) {
 	ds := mockDatasource{}
@@ -236,4 +241,8 @@ func TestGenerateProperties(t *testing.T) {
 	if properties["user_peers"] != 2 {
 		t.Errorf("expected 2 user_peers, got %d", properties["user_peers"])
 	}
+
+	if properties["store_engine"] != server.FileStoreEngine {
+		t.Errorf("expected JsonFile, got %s", properties["store_engine"])
+	}
 }

management/server/mock_server/account_mock.go

@@ -60,7 +60,7 @@ type MockAccountManager struct {
 	GetPATFunc                      func(accountID string, initiatorUserID string, targetUserId string, tokenID string) (*server.PersonalAccessToken, error)
 	GetAllPATsFunc                  func(accountID string, initiatorUserID string, targetUserId string) ([]*server.PersonalAccessToken, error)
 	GetNameServerGroupFunc          func(accountID, nsGroupID string) (*nbdns.NameServerGroup, error)
-	CreateNameServerGroupFunc       func(accountID string, name, description string, nameServerList []nbdns.NameServer, groups []string, primary bool, domains []string, enabled bool, userID string) (*nbdns.NameServerGroup, error)
+	CreateNameServerGroupFunc       func(accountID string, name, description string, nameServerList []nbdns.NameServer, groups []string, primary bool, domains []string, enabled bool, userID string, searchDomainsEnabled bool) (*nbdns.NameServerGroup, error)
 	SaveNameServerGroupFunc         func(accountID, userID string, nsGroupToSave *nbdns.NameServerGroup) error
 	DeleteNameServerGroupFunc       func(accountID, nsGroupID, userID string) error
 	ListNameServerGroupsFunc        func(accountID string) ([]*nbdns.NameServerGroup, error)
@@ -75,6 +75,7 @@ type MockAccountManager struct {
 	LoginPeerFunc                   func(login server.PeerLogin) (*server.Peer, *server.NetworkMap, error)
 	SyncPeerFunc                    func(sync server.PeerSync) (*server.Peer, *server.NetworkMap, error)
 	InviteUserFunc                  func(accountID string, initiatorUserID string, targetUserEmail string) error
+	GetAllConnectedPeersFunc        func() (map[string]struct{}, error)
 }
 
 // GetUsersFromAccount mock implementation of GetUsersFromAccount from server.AccountManager interface
@@ -463,9 +464,9 @@ func (am *MockAccountManager) GetNameServerGroup(accountID, nsGroupID string) (*
 }
 
 // CreateNameServerGroup mocks CreateNameServerGroup of the AccountManager interface
-func (am *MockAccountManager) CreateNameServerGroup(accountID string, name, description string, nameServerList []nbdns.NameServer, groups []string, primary bool, domains []string, enabled bool, userID string) (*nbdns.NameServerGroup, error) {
+func (am *MockAccountManager) CreateNameServerGroup(accountID string, name, description string, nameServerList []nbdns.NameServer, groups []string, primary bool, domains []string, enabled bool, userID string, searchDomainsEnabled bool) (*nbdns.NameServerGroup, error) {
 	if am.CreateNameServerGroupFunc != nil {
-		return am.CreateNameServerGroupFunc(accountID, name, description, nameServerList, groups, primary, domains, enabled, userID)
+		return am.CreateNameServerGroupFunc(accountID, name, description, nameServerList, groups, primary, domains, enabled, userID, searchDomainsEnabled)
 	}
 	return nil, nil
 }
@@ -583,3 +584,11 @@ func (am *MockAccountManager) SyncPeer(sync server.PeerSync) (*server.Peer, *ser
 	}
 	return nil, nil, status.Errorf(codes.Unimplemented, "method SyncPeer is not implemented")
 }
+
+// GetAllConnectedPeers mocks GetAllConnectedPeers of the AccountManager interface
+func (am *MockAccountManager) GetAllConnectedPeers() (map[string]struct{}, error) {
+	if am.GetAllConnectedPeersFunc != nil {
+		return am.GetAllConnectedPeersFunc()
+	}
+	return nil, status.Errorf(codes.Unimplemented, "method GetAllConnectedPeers is not implemented")
+}

management/server/nameserver.go

@@ -35,7 +35,7 @@ func (am *DefaultAccountManager) GetNameServerGroup(accountID, nsGroupID string)
 }
 
 // CreateNameServerGroup creates and saves a new nameserver group
-func (am *DefaultAccountManager) CreateNameServerGroup(accountID string, name, description string, nameServerList []nbdns.NameServer, groups []string, primary bool, domains []string, enabled bool, userID string) (*nbdns.NameServerGroup, error) {
+func (am *DefaultAccountManager) CreateNameServerGroup(accountID string, name, description string, nameServerList []nbdns.NameServer, groups []string, primary bool, domains []string, enabled bool, userID string, searchDomainEnabled bool) (*nbdns.NameServerGroup, error) {
 
 	unlock := am.Store.AcquireAccountLock(accountID)
 	defer unlock()
@@ -46,14 +46,15 @@ func (am *DefaultAccountManager) CreateNameServerGroup(accountID string, name, d
 	}
 
 	newNSGroup := &nbdns.NameServerGroup{
-		ID:          xid.New().String(),
-		Name:        name,
-		Description: description,
-		NameServers: nameServerList,
-		Groups:      groups,
-		Enabled:     enabled,
-		Primary:     primary,
-		Domains:     domains,
+		ID:                   xid.New().String(),
+		Name:                 name,
+		Description:          description,
+		NameServers:          nameServerList,
+		Groups:               groups,
+		Enabled:              enabled,
+		Primary:              primary,
+		Domains:              domains,
+		SearchDomainsEnabled: searchDomainEnabled,
 	}
 
 	err = validateNameServerGroup(false, newNSGroup, account)
@@ -174,7 +175,7 @@ func validateNameServerGroup(existingGroup bool, nameserverGroup *nbdns.NameServ
 		}
 	}
 
-	err := validateDomainInput(nameserverGroup.Primary, nameserverGroup.Domains)
+	err := validateDomainInput(nameserverGroup.Primary, nameserverGroup.Domains, nameserverGroup.SearchDomainsEnabled)
 	if err != nil {
 		return err
 	}
@@ -197,7 +198,7 @@ func validateNameServerGroup(existingGroup bool, nameserverGroup *nbdns.NameServ
 	return nil
 }
 
-func validateDomainInput(primary bool, domains []string) error {
+func validateDomainInput(primary bool, domains []string, searchDomainsEnabled bool) error {
 	if !primary && len(domains) == 0 {
 		return status.Errorf(status.InvalidArgument, "nameserver group primary status is false and domains are empty,"+
 			" it should be primary or have at least one domain")
@@ -206,6 +207,12 @@ func validateDomainInput(primary bool, domains []string) error {
 		return status.Errorf(status.InvalidArgument, "nameserver group primary status is true and domains are not empty,"+
 			" you should set either primary or domain")
 	}
+
+	if primary && searchDomainsEnabled {
+		return status.Errorf(status.InvalidArgument, "nameserver group primary status is true and search domains is enabled,"+
+			" you should not set search domains for primary nameservers")
+	}
+
 	for _, domain := range domains {
 		if err := validateDomain(domain); err != nil {
 			return status.Errorf(status.InvalidArgument, "nameserver group got an invalid domain: %s %q", domain, err)

management/server/nameserver_test.go

@@ -23,13 +23,14 @@ const (
 
 func TestCreateNameServerGroup(t *testing.T) {
 	type input struct {
-		name        string
-		description string
-		enabled     bool
-		groups      []string
-		nameServers []nbdns.NameServer
-		primary     bool
-		domains     []string
+		name          string
+		description   string
+		enabled       bool
+		groups        []string
+		nameServers   []nbdns.NameServer
+		primary       bool
+		domains       []string
+		searchDomains bool
 	}
 
 	testCases := []struct {
@@ -383,6 +384,7 @@ func TestCreateNameServerGroup(t *testing.T) {
 				testCase.inputArgs.domains,
 				testCase.inputArgs.enabled,
 				userID,
+				testCase.inputArgs.searchDomains,
 			)
 
 			testCase.errFunc(t, err)
@@ -749,7 +751,7 @@ func createNSManager(t *testing.T) (*DefaultAccountManager, error) {
 
 func createNSStore(t *testing.T) (Store, error) {
 	dataDir := t.TempDir()
-	store, err := NewFileStore(dataDir, nil)
+	store, err := NewStoreFromJson(dataDir, nil)
 	if err != nil {
 		return nil, err
 	}

management/server/network.go

@@ -34,14 +34,14 @@ type NetworkMap struct {
 }
 
 type Network struct {
-	Id  string
-	Net net.IPNet
-	Dns string
+	Identifier string    `json:"id"`
+	Net        net.IPNet `gorm:"serializer:gob"`
+	Dns        string
 	// Serial is an ID that increments by 1 when any change to the network happened (e.g. new peer has been added).
 	// Used to synchronize state to the client apps.
 	Serial uint64
 
-	mu sync.Mutex `json:"-"`
+	mu sync.Mutex `json:"-" gorm:"-"`
 }
 
 // NewNetwork creates a new Network initializing it with a Serial=0
@@ -56,10 +56,10 @@ func NewNetwork() *Network {
 	intn := r.Intn(len(sub))
 
 	return &Network{
-		Id:     xid.New().String(),
-		Net:    sub[intn].IPNet,
-		Dns:    "",
-		Serial: 0}
+		Identifier: xid.New().String(),
+		Net:        sub[intn].IPNet,
+		Dns:        "",
+		Serial:     0}
 }
 
 // IncSerial increments Serial by 1 reflecting that the network state has been changed
@@ -78,10 +78,10 @@ func (n *Network) CurrentSerial() uint64 {
 
 func (n *Network) Copy() *Network {
 	return &Network{
-		Id:     n.Id,
-		Net:    n.Net,
-		Dns:    n.Dns,
-		Serial: n.Serial,
+		Identifier: n.Identifier,
+		Net:        n.Net,
+		Dns:        n.Dns,
+		Serial:     n.Serial,
 	}
 }
 

management/server/peer.go

@@ -72,22 +72,24 @@ type PeerLogin struct {
 // The Peer is a WireGuard peer identified by a public key
 type Peer struct {
 	// ID is an internal ID of the peer
-	ID string
+	ID string `gorm:"primaryKey"`
+	// AccountID is a reference to Account that this object belongs
+	AccountID string `json:"-" gorm:"index;uniqueIndex:idx_peers_account_id_ip"`
 	// WireGuard public key
-	Key string
+	Key string `gorm:"index"`
 	// A setup key this peer was registered with
 	SetupKey string
 	// IP address of the Peer
-	IP net.IP
+	IP net.IP `gorm:"uniqueIndex:idx_peers_account_id_ip"`
 	// Meta is a Peer system meta data
-	Meta PeerSystemMeta
+	Meta PeerSystemMeta `gorm:"embedded;embeddedPrefix:meta_"`
 	// Name is peer's name (machine name)
 	Name string
 	// DNSLabel is the parsed peer name for domain resolution. It is used to form an FQDN by appending the account's
 	// domain to the peer label. e.g. peer-dns-label.netbird.cloud
 	DNSLabel string
 	// Status peer's management connection status
-	Status *PeerStatus
+	Status *PeerStatus `gorm:"embedded;embeddedPrefix:peer_status_"`
 	// The user ID that registered the peer
 	UserID string
 	// SSHKey is a public SSH key of the peer
@@ -116,6 +118,7 @@ func (p *Peer) Copy() *Peer {
 	}
 	return &Peer{
 		ID:                     p.ID,
+		AccountID:              p.AccountID,
 		Key:                    p.Key,
 		SetupKey:               p.SetupKey,
 		IP:                     p.IP,

management/server/peer_test.go

@@ -369,8 +369,8 @@ func TestAccountManager_GetPeerNetwork(t *testing.T) {
 		return
 	}
 
-	if account.Network.Id != network.Id {
-		t.Errorf("expecting Account Networks ID to be equal, got %s expected %s", network.Id, account.Network.Id)
+	if account.Network.Identifier != network.Identifier {
+		t.Errorf("expecting Account Networks ID to be equal, got %s expected %s", network.Identifier, account.Network.Identifier)
 	}
 }
 

management/server/personal_access_token.go

@@ -26,7 +26,9 @@ const (
 
 // PersonalAccessToken holds all information about a PAT including a hashed version of it for verification
 type PersonalAccessToken struct {
-	ID             string
+	ID string `gorm:"primaryKey"`
+	// User is a reference to Account that this object belongs
+	UserID         string `gorm:"index"`
 	Name           string
 	HashedToken    string
 	ExpirationDate time.Time

management/server/policy.go

@@ -63,7 +63,10 @@ type PolicyUpdateOperation struct {
 // PolicyRule is the metadata of the policy
 type PolicyRule struct {
 	// ID of the policy rule
-	ID string
+	ID string `gorm:"primaryKey"`
+
+	// PolicyID is a reference to Policy that this object belongs
+	PolicyID string `json:"-" gorm:"index"`
 
 	// Name of the rule visible in the UI
 	Name string
@@ -78,10 +81,10 @@ type PolicyRule struct {
 	Action PolicyTrafficActionType
 
 	// Destinations policy destination groups
-	Destinations []string
+	Destinations []string `gorm:"serializer:json"`
 
 	// Sources policy source groups
-	Sources []string
+	Sources []string `gorm:"serializer:json"`
 
 	// Bidirectional define if the rule is applicable in both directions, sources, and destinations
 	Bidirectional bool
@@ -90,7 +93,7 @@ type PolicyRule struct {
 	Protocol PolicyRuleProtocolType
 
 	// Ports or it ranges list
-	Ports []string
+	Ports []string `gorm:"serializer:json"`
 }
 
 // Copy returns a copy of a policy rule
@@ -128,8 +131,11 @@ func (pm *PolicyRule) ToRule() *Rule {
 
 // Policy of the Rego query
 type Policy struct {
-	// ID of the policy
-	ID string
+	// ID of the policy'
+	ID string `gorm:"primaryKey"`
+
+	// AccountID is a reference to Account that this object belongs
+	AccountID string `json:"-" gorm:"index"`
 
 	// Name of the Policy
 	Name string
@@ -141,7 +147,7 @@ type Policy struct {
 	Enabled bool
 
 	// Rules of the policy
-	Rules []*PolicyRule
+	Rules []*PolicyRule `gorm:"foreignKey:PolicyID;references:id"`
 }
 
 // Copy returns a copy of the policy.
@@ -201,7 +207,6 @@ type FirewallRule struct {
 // This function returns the list of peers and firewall rules that are applicable to a given peer.
 func (a *Account) getPeerConnectionResources(peerID string) ([]*Peer, []*FirewallRule) {
 	generateResources, getAccumulatedResources := a.connResourcesGenerator()
-
 	for _, policy := range a.Policies {
 		if !policy.Enabled {
 			continue

management/server/route.go

@@ -98,7 +98,7 @@ func (am *DefaultAccountManager) checkRoutePrefixExistsForPeers(account *Account
 		// check that the peers from peerGroupIDs groups are not the same peers we saw in routesWithPrefix
 		for _, id := range group.Peers {
 			if _, ok := seenPeers[id]; ok {
-				peer := account.GetPeer(peerID)
+				peer := account.GetPeer(id)
 				if peer == nil {
 					return status.Errorf(status.InvalidArgument, "peer with ID %s not found", peerID)
 				}

management/server/route_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/rs/xid"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/netbirdio/netbird/management/server/activity"
@@ -48,11 +49,12 @@ func TestCreateRoute(t *testing.T) {
 	}
 
 	testCases := []struct {
-		name          string
-		inputArgs     input
-		shouldCreate  bool
-		errFunc       require.ErrorAssertionFunc
-		expectedRoute *route.Route
+		name            string
+		inputArgs       input
+		createInitRoute bool
+		shouldCreate    bool
+		errFunc         require.ErrorAssertionFunc
+		expectedRoute   *route.Route
 	}{
 		{
 			name: "Happy Path",
@@ -164,8 +166,9 @@ func TestCreateRoute(t *testing.T) {
 				enabled:     true,
 				groups:      []string{routeGroup1},
 			},
-			errFunc:      require.Error,
-			shouldCreate: false,
+			createInitRoute: true,
+			errFunc:         require.Error,
+			shouldCreate:    false,
 		},
 		{
 			name: "Bad Peers Group already has this route",
@@ -179,8 +182,9 @@ func TestCreateRoute(t *testing.T) {
 				enabled:      true,
 				groups:       []string{routeGroup1},
 			},
-			errFunc:      require.Error,
-			shouldCreate: false,
+			createInitRoute: true,
+			errFunc:         require.Error,
+			shouldCreate:    false,
 		},
 		{
 			name: "Empty Peer Should Create",
@@ -326,6 +330,18 @@ func TestCreateRoute(t *testing.T) {
 				t.Errorf("failed to init testing account: %s", err)
 			}
 
+			if testCase.createInitRoute {
+				groupAll, errInit := account.GetGroupAll()
+				if errInit != nil {
+					t.Errorf("failed to get group all: %s", errInit)
+				}
+				_, errInit = am.CreateRoute(account.Id, existingNetwork, "", []string{routeGroup3, routeGroup4},
+					"", existingRouteID, false, 1000, []string{groupAll.ID}, true, userID)
+				if errInit != nil {
+					t.Errorf("failed to create init route: %s", errInit)
+				}
+			}
+
 			outRoute, err := am.CreateRoute(
 				account.Id,
 				testCase.inputArgs.network,
@@ -370,17 +386,18 @@ func TestSaveRoute(t *testing.T) {
 	validGroupHA2 := routeGroupHA2
 
 	testCases := []struct {
-		name          string
-		existingRoute *route.Route
-		newPeer       *string
-		newPeerGroups []string
-		newMetric     *int
-		newPrefix     *netip.Prefix
-		newGroups     []string
-		skipCopying   bool
-		shouldCreate  bool
-		errFunc       require.ErrorAssertionFunc
-		expectedRoute *route.Route
+		name            string
+		existingRoute   *route.Route
+		createInitRoute bool
+		newPeer         *string
+		newPeerGroups   []string
+		newMetric       *int
+		newPrefix       *netip.Prefix
+		newGroups       []string
+		skipCopying     bool
+		shouldCreate    bool
+		errFunc         require.ErrorAssertionFunc
+		expectedRoute   *route.Route
 	}{
 		{
 			name: "Happy Path",
@@ -645,8 +662,9 @@ func TestSaveRoute(t *testing.T) {
 				Enabled:     true,
 				Groups:      []string{routeGroup1},
 			},
-			newPeer: &validUsedPeer,
-			errFunc: require.Error,
+			createInitRoute: true,
+			newPeer:         &validUsedPeer,
+			errFunc:         require.Error,
 		},
 		{
 			name: "Do not allow to modify existing route with a peers group from another route",
@@ -662,8 +680,9 @@ func TestSaveRoute(t *testing.T) {
 				Enabled:     true,
 				Groups:      []string{routeGroup1},
 			},
-			newPeerGroups: []string{routeGroup4},
-			errFunc:       require.Error,
+			createInitRoute: true,
+			newPeerGroups:   []string{routeGroup4},
+			errFunc:         require.Error,
 		},
 	}
 	for _, testCase := range testCases {
@@ -678,6 +697,21 @@ func TestSaveRoute(t *testing.T) {
 				t.Error("failed to init testing account")
 			}
 
+			if testCase.createInitRoute {
+				account.Routes["initRoute"] = &route.Route{
+					ID:          "initRoute",
+					Network:     netip.MustParsePrefix(existingNetwork),
+					NetID:       existingRouteID,
+					NetworkType: route.IPv4Network,
+					PeerGroups:  []string{routeGroup4},
+					Description: "super",
+					Masquerade:  false,
+					Metric:      9999,
+					Enabled:     true,
+					Groups:      []string{routeGroup1},
+				}
+			}
+
 			account.Routes[testCase.existingRoute.ID] = testCase.existingRoute
 
 			err = am.Store.SaveAccount(account)
@@ -811,15 +845,15 @@ func TestGetNetworkMap_RouteSyncPeerGroups(t *testing.T) {
 
 	peer1Routes, err := am.GetNetworkMap(peer1ID)
 	require.NoError(t, err)
-	require.Len(t, peer1Routes.Routes, 3, "HA route should have more than 1 routes")
+	assert.Len(t, peer1Routes.Routes, 1, "HA route should have 1 server route")
 
 	peer2Routes, err := am.GetNetworkMap(peer2ID)
 	require.NoError(t, err)
-	require.Len(t, peer2Routes.Routes, 3, "HA route should have more than 1 routes")
+	assert.Len(t, peer2Routes.Routes, 1, "HA route should have 1 server route")
 
 	peer4Routes, err := am.GetNetworkMap(peer4ID)
 	require.NoError(t, err)
-	require.Len(t, peer4Routes.Routes, 3, "HA route should have more than 1 routes")
+	assert.Len(t, peer4Routes.Routes, 1, "HA route should have 1 server route")
 
 	groups, err := am.ListGroups(account.Id)
 	require.NoError(t, err)
@@ -838,32 +872,32 @@ func TestGetNetworkMap_RouteSyncPeerGroups(t *testing.T) {
 
 	peer2RoutesAfterDelete, err := am.GetNetworkMap(peer2ID)
 	require.NoError(t, err)
-	require.Len(t, peer2RoutesAfterDelete.Routes, 2, "after peer deletion group should have only 2 route")
+	assert.Len(t, peer2RoutesAfterDelete.Routes, 2, "after peer deletion group should have 2 client routes")
 
 	err = am.GroupDeletePeer(account.Id, groupHA2.ID, peer4ID)
 	require.NoError(t, err)
 
 	peer2RoutesAfterDelete, err = am.GetNetworkMap(peer2ID)
 	require.NoError(t, err)
-	require.Len(t, peer2RoutesAfterDelete.Routes, 1, "after peer deletion group should have only 1 route")
+	assert.Len(t, peer2RoutesAfterDelete.Routes, 1, "after peer deletion group should have only 1 route")
 
 	err = am.GroupAddPeer(account.Id, groupHA2.ID, peer4ID)
 	require.NoError(t, err)
 
 	peer1RoutesAfterAdd, err := am.GetNetworkMap(peer1ID)
 	require.NoError(t, err)
-	require.Len(t, peer1RoutesAfterAdd.Routes, 2, "HA route should have more than 1 route")
+	assert.Len(t, peer1RoutesAfterAdd.Routes, 1, "HA route should have more than 1 route")
 
 	peer2RoutesAfterAdd, err := am.GetNetworkMap(peer2ID)
 	require.NoError(t, err)
-	require.Len(t, peer2RoutesAfterAdd.Routes, 2, "HA route should have more than 1 route")
+	assert.Len(t, peer2RoutesAfterAdd.Routes, 2, "HA route should have 2 client routes")
 
 	err = am.DeleteRoute(account.Id, newRoute.ID, userID)
 	require.NoError(t, err)
 
 	peer1DeletedRoute, err := am.GetNetworkMap(peer1ID)
 	require.NoError(t, err)
-	require.Len(t, peer1DeletedRoute.Routes, 0, "we should receive one route for peer1")
+	assert.Len(t, peer1DeletedRoute.Routes, 0, "we should receive one route for peer1")
 }
 
 func TestGetNetworkMap_RouteSync(t *testing.T) {
@@ -983,7 +1017,7 @@ func createRouterManager(t *testing.T) (*DefaultAccountManager, error) {
 
 func createRouterStore(t *testing.T) (Store, error) {
 	dataDir := t.TempDir()
-	store, err := NewFileStore(dataDir, nil)
+	store, err := NewStoreFromJson(dataDir, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -1193,11 +1227,5 @@ func initTestRouteAccount(t *testing.T, am *DefaultAccountManager) (*Account, er
 		}
 	}
 
-	_, err = am.CreateRoute(account.Id, existingNetwork, "", []string{routeGroup3, routeGroup4},
-		"", existingRouteID, false, 1000, []string{groupAll.ID}, true, userID)
-	if err != nil {
-		return nil, err
-	}
-
 	return am.Store.GetAccount(account.Id)
 }

management/server/rule.go

@@ -25,6 +25,9 @@ type Rule struct {
 	// ID of the rule
 	ID string
 
+	// AccountID is a reference to Account that this object belongs
+	AccountID string `json:"-" gorm:"index"`
+
 	// Name of the rule visible in the UI
 	Name string
 
@@ -35,10 +38,10 @@ type Rule struct {
 	Disabled bool
 
 	// Source list of groups IDs of peers
-	Source []string
+	Source []string `gorm:"serializer:json"`
 
 	// Destination list of groups IDs of peers
-	Destination []string
+	Destination []string `gorm:"serializer:json"`
 
 	// Flow of the traffic allowed by the rule
 	Flow TrafficFlowType

management/server/setupkey.go

@@ -68,13 +68,15 @@ type SetupKeyType string
 
 // SetupKey represents a pre-authorized key used to register machines (peers)
 type SetupKey struct {
-	Id        string
+	Id string
+	// AccountID is a reference to Account that this object belongs
+	AccountID string `json:"-" gorm:"index"`
 	Key       string
 	Name      string
 	Type      SetupKeyType
 	CreatedAt time.Time
 	ExpiresAt time.Time
-	UpdatedAt time.Time
+	UpdatedAt time.Time `gorm:"autoUpdateTime:false"`
 	// Revoked indicates whether the key was revoked or not (we don't remove them for tracking purposes)
 	Revoked bool
 	// UsedTimes indicates how many times the key was used
@@ -82,7 +84,7 @@ type SetupKey struct {
 	// LastUsed last time the key was used for peer registration
 	LastUsed time.Time
 	// AutoGroups is a list of Group IDs that are auto assigned to a Peer when it uses this key to register
-	AutoGroups []string
+	AutoGroups []string `gorm:"serializer:json"`
 	// UsageLimit indicates the number of times this key can be used to enroll a machine.
 	// The value of 0 indicates the unlimited usage.
 	UsageLimit int
@@ -99,6 +101,7 @@ func (key *SetupKey) Copy() *SetupKey {
 	}
 	return &SetupKey{
 		Id:         key.Id,
+		AccountID:  key.AccountID,
 		Key:        key.Key,
 		Name:       key.Name,
 		Type:       key.Type,

management/server/sqlite_store.go

@@ -0,0 +1,457 @@
+package server
+
+import (
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	nbdns "github.com/netbirdio/netbird/dns"
+	"github.com/netbirdio/netbird/management/server/status"
+	"github.com/netbirdio/netbird/management/server/telemetry"
+	"github.com/netbirdio/netbird/route"
+	log "github.com/sirupsen/logrus"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+	"gorm.io/gorm/logger"
+)
+
+// SqliteStore represents an account storage backed by a Sqlite DB persisted to disk
+type SqliteStore struct {
+	db                *gorm.DB
+	storeFile         string
+	accountLocks      sync.Map
+	globalAccountLock sync.Mutex
+	metrics           telemetry.AppMetrics
+	installationPK    int
+}
+
+type installation struct {
+	ID                  uint `gorm:"primaryKey"`
+	InstallationIDValue string
+}
+
+// NewSqliteStore restores a store from the file located in the datadir
+func NewSqliteStore(dataDir string, metrics telemetry.AppMetrics) (*SqliteStore, error) {
+	storeStr := "store.db?cache=shared"
+	if runtime.GOOS == "windows" {
+		// Vo avoid `The process cannot access the file because it is being used by another process` on Windows
+		storeStr = "store.db"
+	}
+
+	file := filepath.Join(dataDir, storeStr)
+	db, err := gorm.Open(sqlite.Open(file), &gorm.Config{
+		Logger:      logger.Default.LogMode(logger.Silent),
+		PrepareStmt: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	sql, err := db.DB()
+	if err != nil {
+		return nil, err
+	}
+	conns := runtime.NumCPU()
+	sql.SetMaxOpenConns(conns) // TODO: make it configurable
+
+	err = db.AutoMigrate(
+		&SetupKey{}, &Peer{}, &User{}, &PersonalAccessToken{}, &Group{}, &Rule{},
+		&Account{}, &Policy{}, &PolicyRule{}, &route.Route{}, &nbdns.NameServerGroup{},
+		&installation{},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &SqliteStore{db: db, storeFile: file, metrics: metrics, installationPK: 1}, nil
+}
+
+// NewSqliteStoreFromFileStore restores a store from FileStore and stores SQLite DB in the file located in datadir
+func NewSqliteStoreFromFileStore(filestore *FileStore, dataDir string, metrics telemetry.AppMetrics) (*SqliteStore, error) {
+	store, err := NewSqliteStore(dataDir, metrics)
+	if err != nil {
+		return nil, err
+	}
+
+	err = store.SaveInstallationID(filestore.InstallationID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, account := range filestore.GetAllAccounts() {
+		err := store.SaveAccount(account)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return store, nil
+}
+
+// AcquireGlobalLock acquires global lock across all the accounts and returns a function that releases the lock
+func (s *SqliteStore) AcquireGlobalLock() (unlock func()) {
+	log.Debugf("acquiring global lock")
+	start := time.Now()
+	s.globalAccountLock.Lock()
+
+	unlock = func() {
+		s.globalAccountLock.Unlock()
+		log.Debugf("released global lock in %v", time.Since(start))
+	}
+
+	took := time.Since(start)
+	log.Debugf("took %v to acquire global lock", took)
+	if s.metrics != nil {
+		s.metrics.StoreMetrics().CountGlobalLockAcquisitionDuration(took)
+	}
+
+	return unlock
+}
+
+func (s *SqliteStore) AcquireAccountLock(accountID string) (unlock func()) {
+	log.Debugf("acquiring lock for account %s", accountID)
+
+	start := time.Now()
+	value, _ := s.accountLocks.LoadOrStore(accountID, &sync.Mutex{})
+	mtx := value.(*sync.Mutex)
+	mtx.Lock()
+
+	unlock = func() {
+		mtx.Unlock()
+		log.Debugf("released lock for account %s in %v", accountID, time.Since(start))
+	}
+
+	return unlock
+}
+
+func (s *SqliteStore) SaveAccount(account *Account) error {
+	start := time.Now()
+
+	for _, key := range account.SetupKeys {
+		account.SetupKeysG = append(account.SetupKeysG, *key)
+	}
+
+	for id, peer := range account.Peers {
+		peer.ID = id
+		account.PeersG = append(account.PeersG, *peer)
+	}
+
+	for id, user := range account.Users {
+		user.Id = id
+		for id, pat := range user.PATs {
+			pat.ID = id
+			user.PATsG = append(user.PATsG, *pat)
+		}
+		account.UsersG = append(account.UsersG, *user)
+	}
+
+	for id, group := range account.Groups {
+		group.ID = id
+		account.GroupsG = append(account.GroupsG, *group)
+	}
+
+	for id, rule := range account.Rules {
+		rule.ID = id
+		account.RulesG = append(account.RulesG, *rule)
+	}
+
+	for id, route := range account.Routes {
+		route.ID = id
+		account.RoutesG = append(account.RoutesG, *route)
+	}
+
+	for id, ns := range account.NameServerGroups {
+		ns.ID = id
+		account.NameServerGroupsG = append(account.NameServerGroupsG, *ns)
+	}
+
+	err := s.db.Transaction(func(tx *gorm.DB) error {
+		result := tx.Select(clause.Associations).Delete(account.Policies, "account_id = ?", account.Id)
+		if result.Error != nil {
+			return result.Error
+		}
+
+		result = tx.Select(clause.Associations).Delete(account.UsersG, "account_id = ?", account.Id)
+		if result.Error != nil {
+			return result.Error
+		}
+
+		result = tx.Select(clause.Associations).Delete(account)
+		if result.Error != nil {
+			return result.Error
+		}
+
+		result = tx.
+			Session(&gorm.Session{FullSaveAssociations: true}).
+			Clauses(clause.OnConflict{UpdateAll: true}).Create(account)
+		if result.Error != nil {
+			return result.Error
+		}
+		return nil
+	})
+
+	took := time.Since(start)
+	if s.metrics != nil {
+		s.metrics.StoreMetrics().CountPersistenceDuration(took)
+	}
+	log.Debugf("took %d ms to persist an account to the SQLite", took.Milliseconds())
+
+	return err
+}
+
+func (s *SqliteStore) SaveInstallationID(ID string) error {
+	installation := installation{InstallationIDValue: ID}
+	installation.ID = uint(s.installationPK)
+
+	return s.db.Clauses(clause.OnConflict{UpdateAll: true}).Create(&installation).Error
+}
+
+func (s *SqliteStore) GetInstallationID() string {
+	var installation installation
+
+	if result := s.db.First(&installation, "id = ?", s.installationPK); result.Error != nil {
+		return ""
+	}
+
+	return installation.InstallationIDValue
+}
+
+func (s *SqliteStore) SavePeerStatus(accountID, peerID string, peerStatus PeerStatus) error {
+	var peer Peer
+
+	result := s.db.First(&peer, "account_id = ? and id = ?", accountID, peerID)
+	if result.Error != nil {
+		return status.Errorf(status.NotFound, "peer %s not found", peerID)
+	}
+
+	peer.Status = &peerStatus
+
+	return s.db.Save(peer).Error
+}
+
+// DeleteHashedPAT2TokenIDIndex is noop in Sqlite
+func (s *SqliteStore) DeleteHashedPAT2TokenIDIndex(hashedToken string) error {
+	return nil
+}
+
+// DeleteTokenID2UserIDIndex is noop in Sqlite
+func (s *SqliteStore) DeleteTokenID2UserIDIndex(tokenID string) error {
+	return nil
+}
+
+func (s *SqliteStore) GetAccountByPrivateDomain(domain string) (*Account, error) {
+	var account Account
+
+	result := s.db.First(&account, "domain = ?", strings.ToLower(domain))
+	if result.Error != nil {
+		return nil, status.Errorf(status.NotFound, "account not found: provided domain is not registered or is not private")
+	}
+
+	// TODO:  rework to not call GetAccount
+	return s.GetAccount(account.Id)
+}
+
+func (s *SqliteStore) GetAccountBySetupKey(setupKey string) (*Account, error) {
+	var key SetupKey
+	result := s.db.Select("account_id").First(&key, "key = ?", strings.ToUpper(setupKey))
+	if result.Error != nil {
+		return nil, status.Errorf(status.NotFound, "account not found: index lookup failed")
+	}
+
+	if key.AccountID == "" {
+		return nil, status.Errorf(status.NotFound, "account not found: index lookup failed")
+	}
+
+	return s.GetAccount(key.AccountID)
+}
+
+func (s *SqliteStore) GetTokenIDByHashedToken(hashedToken string) (string, error) {
+	var token PersonalAccessToken
+	result := s.db.First(&token, "hashed_token = ?", hashedToken)
+	if result.Error != nil {
+		return "", status.Errorf(status.NotFound, "account not found: index lookup failed")
+	}
+
+	return token.ID, nil
+}
+
+func (s *SqliteStore) GetUserByTokenID(tokenID string) (*User, error) {
+	var token PersonalAccessToken
+	result := s.db.First(&token, "id = ?", tokenID)
+	if result.Error != nil {
+		return nil, status.Errorf(status.NotFound, "account not found: index lookup failed")
+	}
+
+	if token.UserID == "" {
+		return nil, status.Errorf(status.NotFound, "account not found: index lookup failed")
+	}
+
+	var user User
+	result = s.db.Preload("PATsG").First(&user, "id = ?", token.UserID)
+	if result.Error != nil {
+		return nil, status.Errorf(status.NotFound, "account not found: index lookup failed")
+	}
+
+	user.PATs = make(map[string]*PersonalAccessToken, len(user.PATsG))
+	for _, pat := range user.PATsG {
+		user.PATs[pat.ID] = &pat
+	}
+
+	return &user, nil
+}
+
+func (s *SqliteStore) GetAllAccounts() (all []*Account) {
+	var accounts []Account
+	result := s.db.Find(&accounts)
+	if result.Error != nil {
+		return all
+	}
+
+	for _, account := range accounts {
+		if acc, err := s.GetAccount(account.Id); err == nil {
+			all = append(all, acc)
+		}
+	}
+
+	return all
+}
+
+func (s *SqliteStore) GetAccount(accountID string) (*Account, error) {
+	var account Account
+
+	result := s.db.Model(&account).
+		Preload("UsersG.PATsG"). // have to be specifies as this is nester reference
+		Preload(clause.Associations).
+		First(&account, "id = ?", accountID)
+	if result.Error != nil {
+		return nil, status.Errorf(status.NotFound, "account not found")
+	}
+
+	// we have to manually preload policy rules as it seems that gorm preloading doesn't do it for us
+	for i, policy := range account.Policies {
+		var rules []*PolicyRule
+		err := s.db.Model(&PolicyRule{}).Find(&rules, "policy_id = ?", policy.ID).Error
+		if err != nil {
+			return nil, status.Errorf(status.NotFound, "account not found")
+		}
+		account.Policies[i].Rules = rules
+	}
+
+	account.SetupKeys = make(map[string]*SetupKey, len(account.SetupKeysG))
+	for _, key := range account.SetupKeysG {
+		account.SetupKeys[key.Key] = key.Copy()
+	}
+	account.SetupKeysG = nil
+
+	account.Peers = make(map[string]*Peer, len(account.PeersG))
+	for _, peer := range account.PeersG {
+		account.Peers[peer.ID] = peer.Copy()
+	}
+	account.PeersG = nil
+
+	account.Users = make(map[string]*User, len(account.UsersG))
+	for _, user := range account.UsersG {
+		user.PATs = make(map[string]*PersonalAccessToken, len(user.PATs))
+		for _, pat := range user.PATsG {
+			user.PATs[pat.ID] = pat.Copy()
+		}
+		account.Users[user.Id] = user.Copy()
+	}
+	account.UsersG = nil
+
+	account.Groups = make(map[string]*Group, len(account.GroupsG))
+	for _, group := range account.GroupsG {
+		account.Groups[group.ID] = group.Copy()
+	}
+	account.GroupsG = nil
+
+	account.Rules = make(map[string]*Rule, len(account.RulesG))
+	for _, rule := range account.RulesG {
+		account.Rules[rule.ID] = rule.Copy()
+	}
+	account.RulesG = nil
+
+	account.Routes = make(map[string]*route.Route, len(account.RoutesG))
+	for _, route := range account.RoutesG {
+		account.Routes[route.ID] = route.Copy()
+	}
+	account.RoutesG = nil
+
+	account.NameServerGroups = make(map[string]*nbdns.NameServerGroup, len(account.NameServerGroupsG))
+	for _, ns := range account.NameServerGroupsG {
+		account.NameServerGroups[ns.ID] = ns.Copy()
+	}
+	account.NameServerGroupsG = nil
+
+	return &account, nil
+}
+
+func (s *SqliteStore) GetAccountByUser(userID string) (*Account, error) {
+	var user User
+	result := s.db.Select("account_id").First(&user, "id = ?", userID)
+	if result.Error != nil {
+		return nil, status.Errorf(status.NotFound, "account not found: index lookup failed")
+	}
+
+	if user.AccountID == "" {
+		return nil, status.Errorf(status.NotFound, "account not found: index lookup failed")
+	}
+
+	return s.GetAccount(user.AccountID)
+}
+
+func (s *SqliteStore) GetAccountByPeerID(peerID string) (*Account, error) {
+	var peer Peer
+	result := s.db.Select("account_id").First(&peer, "id = ?", peerID)
+	if result.Error != nil {
+		return nil, status.Errorf(status.NotFound, "account not found: index lookup failed")
+	}
+
+	if peer.AccountID == "" {
+		return nil, status.Errorf(status.NotFound, "account not found: index lookup failed")
+	}
+
+	return s.GetAccount(peer.AccountID)
+}
+
+func (s *SqliteStore) GetAccountByPeerPubKey(peerKey string) (*Account, error) {
+	var peer Peer
+
+	result := s.db.Select("account_id").First(&peer, "key = ?", peerKey)
+	if result.Error != nil {
+		return nil, status.Errorf(status.NotFound, "account not found: index lookup failed")
+	}
+
+	if peer.AccountID == "" {
+		return nil, status.Errorf(status.NotFound, "account not found: index lookup failed")
+	}
+
+	return s.GetAccount(peer.AccountID)
+}
+
+// SaveUserLastLogin stores the last login time for a user in DB.
+func (s *SqliteStore) SaveUserLastLogin(accountID, userID string, lastLogin time.Time) error {
+	var peer Peer
+
+	result := s.db.First(&peer, "account_id = ? and user_id = ?", accountID, userID)
+	if result.Error != nil {
+		return status.Errorf(status.NotFound, "user %s not found", userID)
+	}
+
+	peer.LastLogin = lastLogin
+
+	return s.db.Save(peer).Error
+}
+
+// Close is noop in Sqlite
+func (s *SqliteStore) Close() error {
+	return nil
+}
+
+// GetStoreEngine returns SqliteStoreEngine
+func (s *SqliteStore) GetStoreEngine() StoreEngine {
+	return SqliteStoreEngine
+}

management/server/sqlite_store_test.go

@@ -0,0 +1,229 @@
+package server
+
+import (
+	"fmt"
+	"net"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/netbirdio/netbird/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSqlite_NewStore(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("The SQLite store is not properly supported by Windows yet")
+	}
+
+	store := newSqliteStore(t)
+
+	if len(store.GetAllAccounts()) != 0 {
+		t.Errorf("expected to create a new empty Accounts map when creating a new FileStore")
+	}
+}
+
+func TestSqlite_SaveAccount(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("The SQLite store is not properly supported by Windows yet")
+	}
+
+	store := newSqliteStore(t)
+
+	account := newAccountWithId("account_id", "testuser", "")
+	setupKey := GenerateDefaultSetupKey()
+	account.SetupKeys[setupKey.Key] = setupKey
+	account.Peers["testpeer"] = &Peer{
+		Key:      "peerkey",
+		SetupKey: "peerkeysetupkey",
+		IP:       net.IP{127, 0, 0, 1},
+		Meta:     PeerSystemMeta{},
+		Name:     "peer name",
+		Status:   &PeerStatus{Connected: true, LastSeen: time.Now().UTC()},
+	}
+
+	err := store.SaveAccount(account)
+	require.NoError(t, err)
+
+	account2 := newAccountWithId("account_id2", "testuser2", "")
+	setupKey = GenerateDefaultSetupKey()
+	account2.SetupKeys[setupKey.Key] = setupKey
+	account2.Peers["testpeer2"] = &Peer{
+		Key:      "peerkey2",
+		SetupKey: "peerkeysetupkey2",
+		IP:       net.IP{127, 0, 0, 2},
+		Meta:     PeerSystemMeta{},
+		Name:     "peer name 2",
+		Status:   &PeerStatus{Connected: true, LastSeen: time.Now().UTC()},
+	}
+
+	err = store.SaveAccount(account2)
+	require.NoError(t, err)
+
+	if len(store.GetAllAccounts()) != 2 {
+		t.Errorf("expecting 2 Accounts to be stored after SaveAccount()")
+	}
+
+	a, err := store.GetAccount(account.Id)
+	if a == nil {
+		t.Errorf("expecting Account to be stored after SaveAccount(): %v", err)
+	}
+
+	if a != nil && len(a.Policies) != 1 {
+		t.Errorf("expecting Account to have one policy stored after SaveAccount(), got %d", len(a.Policies))
+	}
+
+	if a != nil && len(a.Policies[0].Rules) != 1 {
+		t.Errorf("expecting Account to have one policy rule stored after SaveAccount(), got %d", len(a.Policies[0].Rules))
+		return
+	}
+
+	if a, err := store.GetAccountByPeerPubKey("peerkey"); a == nil {
+		t.Errorf("expecting PeerKeyID2AccountID index updated after SaveAccount(): %v", err)
+	}
+
+	if a, err := store.GetAccountByUser("testuser"); a == nil {
+		t.Errorf("expecting UserID2AccountID index updated after SaveAccount(): %v", err)
+	}
+
+	if a, err := store.GetAccountByPeerID("testpeer"); a == nil {
+		t.Errorf("expecting PeerID2AccountID index updated after SaveAccount(): %v", err)
+	}
+
+	if a, err := store.GetAccountBySetupKey(setupKey.Key); a == nil {
+		t.Errorf("expecting SetupKeyID2AccountID index updated after SaveAccount(): %v", err)
+	}
+}
+
+func TestSqlite_SavePeerStatus(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("The SQLite store is not properly supported by Windows yet")
+	}
+
+	store := newSqliteStoreFromFile(t, "testdata/store.json")
+
+	account, err := store.GetAccount("bf1c8084-ba50-4ce7-9439-34653001fc3b")
+	require.NoError(t, err)
+
+	// save status of non-existing peer
+	newStatus := PeerStatus{Connected: true, LastSeen: time.Now().UTC()}
+	err = store.SavePeerStatus(account.Id, "non-existing-peer", newStatus)
+	assert.Error(t, err)
+
+	// save new status of existing peer
+	account.Peers["testpeer"] = &Peer{
+		Key:      "peerkey",
+		ID:       "testpeer",
+		SetupKey: "peerkeysetupkey",
+		IP:       net.IP{127, 0, 0, 1},
+		Meta:     PeerSystemMeta{},
+		Name:     "peer name",
+		Status:   &PeerStatus{Connected: false, LastSeen: time.Now().UTC()},
+	}
+
+	err = store.SaveAccount(account)
+	require.NoError(t, err)
+
+	err = store.SavePeerStatus(account.Id, "testpeer", newStatus)
+	require.NoError(t, err)
+
+	account, err = store.GetAccount(account.Id)
+	require.NoError(t, err)
+
+	actual := account.Peers["testpeer"].Status
+	assert.Equal(t, newStatus, *actual)
+}
+
+func TestSqlite_TestGetAccountByPrivateDomain(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("The SQLite store is not properly supported by Windows yet")
+	}
+
+	store := newSqliteStoreFromFile(t, "testdata/store.json")
+
+	existingDomain := "test.com"
+
+	account, err := store.GetAccountByPrivateDomain(existingDomain)
+	require.NoError(t, err, "should found account")
+	require.Equal(t, existingDomain, account.Domain, "domains should match")
+
+	_, err = store.GetAccountByPrivateDomain("missing-domain.com")
+	require.Error(t, err, "should return error on domain lookup")
+}
+
+func TestSqlite_GetTokenIDByHashedToken(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("The SQLite store is not properly supported by Windows yet")
+	}
+
+	store := newSqliteStoreFromFile(t, "testdata/store.json")
+
+	hashed := "SoMeHaShEdToKeN"
+	id := "9dj38s35-63fb-11ec-90d6-0242ac120003"
+
+	token, err := store.GetTokenIDByHashedToken(hashed)
+	require.NoError(t, err)
+	require.Equal(t, id, token)
+}
+
+func TestSqlite_GetUserByTokenID(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("The SQLite store is not properly supported by Windows yet")
+	}
+
+	store := newSqliteStoreFromFile(t, "testdata/store.json")
+
+	id := "9dj38s35-63fb-11ec-90d6-0242ac120003"
+
+	user, err := store.GetUserByTokenID(id)
+	require.NoError(t, err)
+	require.Equal(t, id, user.PATs[id].ID)
+}
+
+func newSqliteStore(t *testing.T) *SqliteStore {
+	t.Helper()
+
+	store, err := NewSqliteStore(t.TempDir(), nil)
+	require.NoError(t, err)
+	require.NotNil(t, store)
+
+	return store
+}
+
+func newSqliteStoreFromFile(t *testing.T, filename string) *SqliteStore {
+	t.Helper()
+
+	storeDir := t.TempDir()
+
+	err := util.CopyFileContents(filename, filepath.Join(storeDir, "store.json"))
+	require.NoError(t, err)
+
+	fStore, err := NewFileStore(storeDir, nil)
+	require.NoError(t, err)
+
+	store, err := NewSqliteStoreFromFileStore(fStore, storeDir, nil)
+	require.NoError(t, err)
+	require.NotNil(t, store)
+
+	return store
+}
+
+func newAccount(store Store, id int) error {
+	str := fmt.Sprintf("%s-%d", uuid.New().String(), id)
+	account := newAccountWithId(str, str+"-testuser", "example.com")
+	setupKey := GenerateDefaultSetupKey()
+	account.SetupKeys[setupKey.Key] = setupKey
+	account.Peers["p"+str] = &Peer{
+		Key:      "peerkey" + str,
+		SetupKey: "peerkeysetupkey",
+		IP:       net.IP{127, 0, 0, 1},
+		Meta:     PeerSystemMeta{},
+		Name:     "peer name",
+		Status:   &PeerStatus{Connected: true, LastSeen: time.Now().UTC()},
+	}
+
+	return store.SaveAccount(account)
+}

management/server/store.go

@@ -1,6 +1,15 @@
 package server
 
-import "time"
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/management/server/telemetry"
+)
 
 type Store interface {
 	GetAllAccounts() []*Account
@@ -25,4 +34,65 @@ type Store interface {
 	SaveUserLastLogin(accountID, userID string, lastLogin time.Time) error
 	// Close should close the store persisting all unsaved data.
 	Close() error
+	// GetStoreEngine should return StoreEngine of the current store implementation.
+	// This is also a method of metrics.DataSource interface.
+	GetStoreEngine() StoreEngine
+}
+
+type StoreEngine string
+
+const (
+	FileStoreEngine   StoreEngine = "jsonfile"
+	SqliteStoreEngine StoreEngine = "sqlite"
+)
+
+func getStoreEngineFromEnv() StoreEngine {
+	// NETBIRD_STORE_ENGINE supposed to be used in tests. Otherwise rely on the config file.
+	kind, ok := os.LookupEnv("NETBIRD_STORE_ENGINE")
+	if !ok {
+		return FileStoreEngine
+	}
+
+	value := StoreEngine(strings.ToLower(kind))
+
+	if value == FileStoreEngine || value == SqliteStoreEngine {
+		return value
+	}
+
+	return FileStoreEngine
+}
+
+func NewStore(kind StoreEngine, dataDir string, metrics telemetry.AppMetrics) (Store, error) {
+	if kind == "" {
+		// fallback to env. Normally this only should be used from tests
+		kind = getStoreEngineFromEnv()
+	}
+	switch kind {
+	case FileStoreEngine:
+		log.Info("using JSON file store engine")
+		return NewFileStore(dataDir, metrics)
+	case SqliteStoreEngine:
+		log.Info("using SQLite store engine")
+		return NewSqliteStore(dataDir, metrics)
+	default:
+		return nil, fmt.Errorf("unsupported kind of store %s", kind)
+	}
+}
+
+func NewStoreFromJson(dataDir string, metrics telemetry.AppMetrics) (Store, error) {
+	fstore, err := NewFileStore(dataDir, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	kind := getStoreEngineFromEnv()
+
+	switch kind {
+	case FileStoreEngine:
+		return fstore, nil
+	case SqliteStoreEngine:
+		return NewSqliteStoreFromFileStore(fstore, dataDir, metrics)
+	default:
+		return nil, fmt.Errorf("unsupported store engine %s", kind)
+	}
 }

management/server/store_test.go

@@ -0,0 +1,88 @@
+package server
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type benchCase struct {
+	name    string
+	storeFn func(b *testing.B) Store
+	size    int
+}
+
+var newFs = func(b *testing.B) Store {
+	store, _ := NewFileStore(b.TempDir(), nil)
+	return store
+}
+
+var newSqlite = func(b *testing.B) Store {
+	store, _ := NewSqliteStore(b.TempDir(), nil)
+	return store
+}
+
+func BenchmarkTest_StoreWrite(b *testing.B) {
+	cases := []benchCase{
+		{name: "FileStore_Write", storeFn: newFs, size: 100},
+		{name: "SqliteStore_Write", storeFn: newSqlite, size: 100},
+		{name: "FileStore_Write", storeFn: newFs, size: 500},
+		{name: "SqliteStore_Write", storeFn: newSqlite, size: 500},
+		{name: "FileStore_Write", storeFn: newFs, size: 1000},
+		{name: "SqliteStore_Write", storeFn: newSqlite, size: 1000},
+		{name: "FileStore_Write", storeFn: newFs, size: 2000},
+		{name: "SqliteStore_Write", storeFn: newSqlite, size: 2000},
+	}
+
+	for _, c := range cases {
+		name := fmt.Sprintf("%s_%d", c.name, c.size)
+		store := c.storeFn(b)
+
+		for i := 0; i < c.size; i++ {
+			_ = newAccount(store, i)
+		}
+
+		b.Run(name, func(b *testing.B) {
+			b.RunParallel(func(pb *testing.PB) {
+				i := c.size
+				for pb.Next() {
+					i++
+					err := newAccount(store, i)
+					require.NoError(b, err)
+				}
+			})
+		})
+	}
+}
+
+func BenchmarkTest_StoreRead(b *testing.B) {
+	cases := []benchCase{
+		{name: "FileStore_Read", storeFn: newFs, size: 100},
+		{name: "SqliteStore_Read", storeFn: newSqlite, size: 100},
+		{name: "FileStore_Read", storeFn: newFs, size: 500},
+		{name: "SqliteStore_Read", storeFn: newSqlite, size: 500},
+		{name: "FileStore_Read", storeFn: newFs, size: 1000},
+		{name: "SqliteStore_Read", storeFn: newSqlite, size: 1000},
+	}
+
+	for _, c := range cases {
+		name := fmt.Sprintf("%s_%d", c.name, c.size)
+		store := c.storeFn(b)
+
+		for i := 0; i < c.size; i++ {
+			_ = newAccount(store, i)
+		}
+
+		accounts := store.GetAllAccounts()
+		id := accounts[c.size-1].Id
+
+		b.Run(name, func(b *testing.B) {
+			b.RunParallel(func(pb *testing.PB) {
+				for pb.Next() {
+					_, _ = store.GetAccount(id)
+				}
+			})
+		})
+	}
+}

management/server/testdata/store.json

@@ -2,52 +2,87 @@
     "Accounts": {
         "bf1c8084-ba50-4ce7-9439-34653001fc3b": {
             "Id": "bf1c8084-ba50-4ce7-9439-34653001fc3b",
+            "CreatedBy": "",
             "Domain": "test.com",
             "DomainCategory": "private",
             "IsDomainPrimaryAccount": true,
             "SetupKeys": {
                 "A2C8E62B-38F5-4553-B31E-DD66C696CEBB": {
+                    "Id": "",
+                    "AccountID": "",
                     "Key": "A2C8E62B-38F5-4553-B31E-DD66C696CEBB",
                     "Name": "Default key",
                     "Type": "reusable",
                     "CreatedAt": "2021-08-19T20:46:20.005936822+02:00",
                     "ExpiresAt": "2321-09-18T20:46:20.005936822+02:00",
+                    "UpdatedAt": "0001-01-01T00:00:00Z",
                     "Revoked": false,
-                    "UsedTimes": 0
-
+                    "UsedTimes": 0,
+                    "LastUsed": "0001-01-01T00:00:00Z",
+                    "AutoGroups": null,
+                    "UsageLimit": 0,
+                    "Ephemeral": false
                 }
             },
             "Network": {
-                "Id": "af1c8024-ha40-4ce2-9418-34653101fc3c",
+                "id": "af1c8024-ha40-4ce2-9418-34653101fc3c",
                 "Net": {
                     "IP": "100.64.0.0",
                     "Mask": "//8AAA=="
                 },
-                "Dns": null
+                "Dns": "",
+                "Serial": 0
             },
             "Peers": {},
             "Users": {
                 "edafee4e-63fb-11ec-90d6-0242ac120003": {
                     "Id": "edafee4e-63fb-11ec-90d6-0242ac120003",
+                    "AccountID": "",
                     "Role": "admin",
-                    "PATs": {}
+                    "IsServiceUser": false,
+                    "ServiceUserName": "",
+                    "AutoGroups": null,
+                    "PATs": {},
+                    "Blocked": false,
+                    "LastLogin": "0001-01-01T00:00:00Z"
                 },
                 "f4f6d672-63fb-11ec-90d6-0242ac120003": {
                     "Id": "f4f6d672-63fb-11ec-90d6-0242ac120003",
+                    "AccountID": "",
                     "Role": "user",
+                    "IsServiceUser": false,
+                    "ServiceUserName": "",
+                    "AutoGroups": null,
                     "PATs": {
                         "9dj38s35-63fb-11ec-90d6-0242ac120003": {
-                            "ID":"9dj38s35-63fb-11ec-90d6-0242ac120003",
-                            "Description":"some Description",
-                            "HashedToken":"SoMeHaShEdToKeN",
-                            "ExpirationDate":"2023-02-27T00:00:00Z",
-                            "CreatedBy":"user",
-                            "CreatedAt":"2023-01-01T00:00:00Z",
-                            "LastUsed":"2023-02-01T00:00:00Z"
+                            "ID": "9dj38s35-63fb-11ec-90d6-0242ac120003",
+                            "UserID": "",
+                            "Name": "",
+                            "HashedToken": "SoMeHaShEdToKeN",
+                            "ExpirationDate": "2023-02-27T00:00:00Z",
+                            "CreatedBy": "user",
+                            "CreatedAt": "2023-01-01T00:00:00Z",
+                            "LastUsed": "2023-02-01T00:00:00Z"
                         }
-                    }
+                    },
+                    "Blocked": false,
+                    "LastLogin": "0001-01-01T00:00:00Z"
                 }
+            },
+            "Groups": null,
+            "Rules": null,
+            "Policies": [],
+            "Routes": null,
+            "NameServerGroups": null,
+            "DNSSettings": null,
+            "Settings": {
+                "PeerLoginExpirationEnabled": false,
+                "PeerLoginExpiration": 86400000000000,
+                "GroupsPropagationEnabled": false,
+                "JWTGroupsEnabled": false,
+                "JWTGroupsClaimName": ""
             }
         }
-    }
+    },
+    "InstallationID": ""
 }

management/server/user.go

@@ -44,14 +44,17 @@ type UserRole string
 
 // User represents a user of the system
 type User struct {
-	Id            string
+	Id string `gorm:"primaryKey"`
+	// AccountID is a reference to Account that this object belongs
+	AccountID     string `json:"-" gorm:"index"`
 	Role          UserRole
 	IsServiceUser bool
 	// ServiceUserName is only set if IsServiceUser is true
 	ServiceUserName string
 	// AutoGroups is a list of Group IDs to auto-assign to peers registered by this user
-	AutoGroups []string
-	PATs       map[string]*PersonalAccessToken
+	AutoGroups []string                        `gorm:"serializer:json"`
+	PATs       map[string]*PersonalAccessToken `gorm:"-"`
+	PATsG      []PersonalAccessToken           `json:"-" gorm:"foreignKey:UserID;references:id"`
 	// Blocked indicates whether the user is blocked. Blocked users can't use the system.
 	Blocked bool
 	// LastLogin is the last time the user logged in to IdP
@@ -124,6 +127,7 @@ func (u *User) Copy() *User {
 	}
 	return &User{
 		Id:              u.Id,
+		AccountID:       u.AccountID,
 		Role:            u.Role,
 		AutoGroups:      autoGroups,
 		IsServiceUser:   u.IsServiceUser,
@@ -224,10 +228,20 @@ func (am *DefaultAccountManager) inviteNewUser(accountID, userID string, invite
 		return nil, status.Errorf(status.NotFound, "account %s doesn't exist", accountID)
 	}
 
-	// initiator is the one who is inviting the new user
-	initiatorUser, err := am.lookupUserInCache(userID, account)
+	initiatorUser, err := account.FindUser(userID)
 	if err != nil {
-		return nil, status.Errorf(status.NotFound, "user %s doesn't exist in IdP", userID)
+		return nil, status.Errorf(status.NotFound, "initiator user with ID %s doesn't exist", userID)
+	}
+
+	inviterID := userID
+	if initiatorUser.IsServiceUser {
+		inviterID = account.CreatedBy
+	}
+
+	// inviterUser is the one who is inviting the new user
+	inviterUser, err := am.lookupUserInCache(inviterID, account)
+	if err != nil || inviterUser == nil {
+		return nil, status.Errorf(status.NotFound, "inviter user with ID %s doesn't exist in IdP", inviterID)
 	}
 
 	// check if the user is already registered with this email => reject
@@ -249,7 +263,7 @@ func (am *DefaultAccountManager) inviteNewUser(accountID, userID string, invite
 		return nil, status.Errorf(status.UserAlreadyExists, "can't invite a user with an existing NetBird account")
 	}
 
-	idpUser, err := am.idpManager.CreateUser(invite.Email, invite.Name, accountID, initiatorUser.Email)
+	idpUser, err := am.idpManager.CreateUser(invite.Email, invite.Name, accountID, inviterUser.Email)
 	if err != nil {
 		return nil, err
 	}

management/server/user_test.go

@@ -251,6 +251,7 @@ func TestUser_Copy(t *testing.T) {
 	// this is an imaginary case which will never be in DB this way
 	user := User{
 		Id:              "userId",
+		AccountID:       "accountId",
 		Role:            "role",
 		IsServiceUser:   true,
 		ServiceUserName: "servicename",
@@ -291,6 +292,11 @@ func validateStruct(s interface{}) (err error) {
 		field := structVal.Field(i)
 		fieldName := structType.Field(i).Name
 
+		// skip gorm internal fields
+		if json, ok := structType.Field(i).Tag.Lookup("json"); ok && json == "-" {
+			continue
+		}
+
 		isSet := field.IsValid() && (!field.IsZero() || field.Type().String() == "bool")
 
 		if !isSet {

release_files/install.sh

@@ -66,9 +66,14 @@ download_release_binary() {
     if [ "$OS_TYPE" = "darwin" ] && [ "$1" = "$UI_APP" ]; then
         INSTALL_DIR="/Applications/NetBird UI.app"
 
+        if test -d "$INSTALL_DIR" ; then
+          echo "removing $INSTALL_DIR"
+          rm -rfv "$INSTALL_DIR"
+        fi
+
         # Unzip the app and move to INSTALL_DIR
         unzip -q -o "$BINARY_NAME"
-        mv "netbird_ui_${OS_TYPE}_${ARCH}" "$INSTALL_DIR"
+        mv "netbird_ui_${OS_TYPE}_${ARCH}/" "$INSTALL_DIR/"
     else
         ${SUDO} mkdir -p "$INSTALL_DIR"
         tar -xzvf "$BINARY_NAME"
@@ -184,16 +189,6 @@ install_netbird() {
         fi
     fi
 
-    # Checks if SKIP_UI_APP env is set
-    if [ -z "$SKIP_UI_APP" ]; then
-        SKIP_UI_APP=false
-    else
-        if $SKIP_UI_APP; then
-            echo "SKIP_UI_APP has been set to true in the environment"
-            echo "NetBird UI installation will be omitted based on your preference"
-        fi
-    fi
-
     # Run the installation, if a desktop environment is not detected
     # only the CLI will be installed
     case "$PACKAGE_MANAGER" in
@@ -294,6 +289,14 @@ is_bin_package_manager() {
   fi
 }
 
+stop_running_netbird_ui() {
+  NB_UI_PROC=$(ps -ef | grep "[n]etbird-ui" | awk '{print $2}')
+  if [ -n "$NB_UI_PROC" ]; then
+    echo "NetBird UI is running with PID $NB_UI_PROC. Stopping it..."
+    kill -9 "$NB_UI_PROC"
+  fi
+}
+
 update_netbird() {
   if is_bin_package_manager "$CONFIG_FILE"; then
     latest_release=$(get_latest_release)
@@ -301,7 +304,7 @@ update_netbird() {
     installed_version=$(netbird version)
 
     if [ "$latest_version" = "$installed_version" ]; then
-      echo "Installed netbird version ($installed_version) is up-to-date"
+      echo "Installed NetBird version ($installed_version) is up-to-date"
       exit 0
     fi
 
@@ -310,8 +313,9 @@ update_netbird() {
       echo ""
       echo "Initiating NetBird update. This will stop the netbird service and restart it after the update"
 
-      ${SUDO} netbird service stop
-      ${SUDO} netbird service uninstall
+      ${SUDO} netbird service stop || true
+      ${SUDO} netbird service uninstall || true
+      stop_running_netbird_ui
       install_native_binaries
 
       ${SUDO} netbird service install
@@ -322,6 +326,16 @@ update_netbird() {
   fi
 }
 
+# Checks if SKIP_UI_APP env is set
+if [ -z "$SKIP_UI_APP" ]; then
+    SKIP_UI_APP=false
+else
+    if $SKIP_UI_APP; then
+      echo "SKIP_UI_APP has been set to true in the environment"
+      echo "NetBird UI installation will be omitted based on your preference"
+    fi
+fi
+
 # Identify OS name and default package manager
 if type uname >/dev/null 2>&1; then
 	case "$(uname)" in
@@ -334,7 +348,7 @@ if type uname >/dev/null 2>&1; then
             if [ "$ARCH" != "amd64" ] && [ "$ARCH" != "arm64" ] \
                 && [ "$ARCH" != "x86_64" ];then
                 SKIP_UI_APP=true
-                echo "NetBird UI installation will be omitted as $ARCH is not a compactible architecture"
+                echo "NetBird UI installation will be omitted as $ARCH is not a compatible architecture"
             fi
 
             # Allow netbird UI installation for linux running desktop enviroment
@@ -376,7 +390,13 @@ if type uname >/dev/null 2>&1; then
 	esac
 fi
 
-case "$1" in
+UPDATE_FLAG=$1
+
+if [ "${UPDATE_NETBIRD}-x" = "true-x" ]; then
+  UPDATE_FLAG="--update"
+fi
+
+case "$UPDATE_FLAG" in
     --update)
       update_netbird
     ;;

route/route.go

@@ -65,17 +65,19 @@ func ToPrefixType(prefix string) NetworkType {
 
 // Route represents a route
 type Route struct {
-	ID          string
-	Network     netip.Prefix
+	ID string `gorm:"primaryKey"`
+	// AccountID is a reference to Account that this object belongs
+	AccountID   string       `gorm:"index"`
+	Network     netip.Prefix `gorm:"serializer:gob"`
 	NetID       string
 	Description string
 	Peer        string
-	PeerGroups  []string
+	PeerGroups  []string `gorm:"serializer:gob"`
 	NetworkType NetworkType
 	Masquerade  bool
 	Metric      int
 	Enabled     bool
-	Groups      []string
+	Groups      []string `gorm:"serializer:json"`
 }
 
 // EventMeta returns activity event meta related to the route

signal/client/grpc.go

@@ -79,7 +79,7 @@ func NewClient(ctx context.Context, addr string, key wgtypes.Key, tlsEnabled boo
 		transportOption,
 		grpc.WithBlock(),
 		grpc.WithKeepaliveParams(keepalive.ClientParameters{
-			Time:    15 * time.Second,
+			Time:    30 * time.Second,
 			Timeout: 10 * time.Second,
 		}))