[management, infrastructure, idp] Simplified IdP Management - Embedded IdP (#5008)

Embed Dex as a built-in IdP to simplify self-hosting setup.
Adds an embedded OIDC Identity Provider (Dex) with local user management and optional external IdP connectors (Google/GitHub/OIDC/SAML), plus device-auth flow for CLI login. Introduces instance onboarding/setup endpoints (including owner creation), field-level encryption for sensitive user data, a streamlined self-hosting provisioning script, and expanded APIs + test coverage for IdP management.

more at https://github.com/netbirdio/netbird/pull/5008#issuecomment-3718987393

management/server/store/file_store.go

@@ -16,6 +16,7 @@ import (
 	"github.com/netbirdio/netbird/management/server/types"
 	nbutil "github.com/netbirdio/netbird/management/server/util"
 	"github.com/netbirdio/netbird/util"
+	"github.com/netbirdio/netbird/util/crypt"
 )
 
 // storeFileName Store file name. Stored in the datadir
@@ -263,3 +264,8 @@ func (s *FileStore) Close(ctx context.Context) error {
 func (s *FileStore) GetStoreEngine() types.Engine {
 	return types.FileStoreEngine
 }
+
+// SetFieldEncrypt is a no-op for FileStore as it doesn't support field encryption.
+func (s *FileStore) SetFieldEncrypt(_ *crypt.FieldEncrypt) {
+	// no-op: FileStore stores data in plaintext JSON; encryption is not supported
+}

management/server/store/sql_store.go

@@ -37,6 +37,7 @@ import (
 	"github.com/netbirdio/netbird/management/server/util"
 	"github.com/netbirdio/netbird/route"
 	"github.com/netbirdio/netbird/shared/management/status"
+	"github.com/netbirdio/netbird/util/crypt"
 )
 
 const (
@@ -57,13 +58,13 @@ const (
 
 // SqlStore represents an account storage backed by a Sql DB persisted to disk
 type SqlStore struct {
-	db                *gorm.DB
-	globalAccountLock sync.Mutex
-	metrics           telemetry.AppMetrics
-	installationPK    int
-	storeEngine       types.Engine
-	pool              *pgxpool.Pool
-
+	db                 *gorm.DB
+	globalAccountLock  sync.Mutex
+	metrics            telemetry.AppMetrics
+	installationPK     int
+	storeEngine        types.Engine
+	pool               *pgxpool.Pool
+	fieldEncrypt       *crypt.FieldEncrypt
 	transactionTimeout time.Duration
 }
 
@@ -175,6 +176,13 @@ func (s *SqlStore) SaveAccount(ctx context.Context, account *types.Account) erro
 
 	generateAccountSQLTypes(account)
 
+	// Encrypt sensitive user data before saving
+	for i := range account.UsersG {
+		if err := account.UsersG[i].EncryptSensitiveData(s.fieldEncrypt); err != nil {
+			return fmt.Errorf("encrypt user: %w", err)
+		}
+	}
+
 	for _, group := range account.GroupsG {
 		group.StoreGroupPeers()
 	}
@@ -440,7 +448,18 @@ func (s *SqlStore) SaveUsers(ctx context.Context, users []*types.User) error {
 		return nil
 	}
 
-	result := s.db.Clauses(clause.OnConflict{UpdateAll: true}).Create(&users)
+	usersCopy := make([]*types.User, len(users))
+	for i, user := range users {
+		userCopy := user.Copy()
+		userCopy.Email = user.Email
+		userCopy.Name = user.Name
+		if err := userCopy.EncryptSensitiveData(s.fieldEncrypt); err != nil {
+			return fmt.Errorf("encrypt user: %w", err)
+		}
+		usersCopy[i] = userCopy
+	}
+
+	result := s.db.Clauses(clause.OnConflict{UpdateAll: true}).Create(&usersCopy)
 	if result.Error != nil {
 		log.WithContext(ctx).Errorf("failed to save users to store: %s", result.Error)
 		return status.Errorf(status.Internal, "failed to save users to store")
@@ -450,7 +469,15 @@ func (s *SqlStore) SaveUsers(ctx context.Context, users []*types.User) error {
 
 // SaveUser saves the given user to the database.
 func (s *SqlStore) SaveUser(ctx context.Context, user *types.User) error {
-	result := s.db.Save(user)
+	userCopy := user.Copy()
+	userCopy.Email = user.Email
+	userCopy.Name = user.Name
+
+	if err := userCopy.EncryptSensitiveData(s.fieldEncrypt); err != nil {
+		return fmt.Errorf("encrypt user: %w", err)
+	}
+
+	result := s.db.Save(userCopy)
 	if result.Error != nil {
 		log.WithContext(ctx).Errorf("failed to save user to store: %s", result.Error)
 		return status.Errorf(status.Internal, "failed to save user to store")
@@ -600,6 +627,10 @@ func (s *SqlStore) GetUserByPATID(ctx context.Context, lockStrength LockingStren
 		return nil, status.NewGetUserFromStoreError()
 	}
 
+	if err := user.DecryptSensitiveData(s.fieldEncrypt); err != nil {
+		return nil, fmt.Errorf("decrypt user: %w", err)
+	}
+
 	return &user, nil
 }
 
@@ -618,6 +649,10 @@ func (s *SqlStore) GetUserByUserID(ctx context.Context, lockStrength LockingStre
 		return nil, status.NewGetUserFromStoreError()
 	}
 
+	if err := user.DecryptSensitiveData(s.fieldEncrypt); err != nil {
+		return nil, fmt.Errorf("decrypt user: %w", err)
+	}
+
 	return &user, nil
 }
 
@@ -654,6 +689,12 @@ func (s *SqlStore) GetAccountUsers(ctx context.Context, lockStrength LockingStre
 		return nil, status.Errorf(status.Internal, "issue getting users from store")
 	}
 
+	for _, user := range users {
+		if err := user.DecryptSensitiveData(s.fieldEncrypt); err != nil {
+			return nil, fmt.Errorf("decrypt user: %w", err)
+		}
+	}
+
 	return users, nil
 }
 
@@ -672,6 +713,10 @@ func (s *SqlStore) GetAccountOwner(ctx context.Context, lockStrength LockingStre
 		return nil, status.Errorf(status.Internal, "failed to get account owner from the store")
 	}
 
+	if err := user.DecryptSensitiveData(s.fieldEncrypt); err != nil {
+		return nil, fmt.Errorf("decrypt user: %w", err)
+	}
+
 	return &user, nil
 }
 
@@ -866,6 +911,9 @@ func (s *SqlStore) getAccountGorm(ctx context.Context, accountID string) (*types
 		if user.AutoGroups == nil {
 			user.AutoGroups = []string{}
 		}
+		if err := user.DecryptSensitiveData(s.fieldEncrypt); err != nil {
+			return nil, fmt.Errorf("decrypt user: %w", err)
+		}
 		account.Users[user.Id] = &user
 		user.PATsG = nil
 	}
@@ -1141,6 +1189,9 @@ func (s *SqlStore) getAccountPgx(ctx context.Context, accountID string) (*types.
 	account.Users = make(map[string]*types.User, len(account.UsersG))
 	for i := range account.UsersG {
 		user := &account.UsersG[i]
+		if err := user.DecryptSensitiveData(s.fieldEncrypt); err != nil {
+			return nil, fmt.Errorf("decrypt user: %w", err)
+		}
 		user.PATs = make(map[string]*types.PersonalAccessToken)
 		if userPats, ok := patsByUserID[user.Id]; ok {
 			for j := range userPats {
@@ -1545,7 +1596,7 @@ func (s *SqlStore) getPeers(ctx context.Context, accountID string) ([]nbpeer.Pee
 }
 
 func (s *SqlStore) getUsers(ctx context.Context, accountID string) ([]types.User, error) {
-	const query = `SELECT id, account_id, role, is_service_user, non_deletable, service_user_name, auto_groups, blocked, pending_approval, last_login, created_at, issued, integration_ref_id, integration_ref_integration_type FROM users WHERE account_id = $1`
+	const query = `SELECT id, account_id, role, is_service_user, non_deletable, service_user_name, auto_groups, blocked, pending_approval, last_login, created_at, issued, integration_ref_id, integration_ref_integration_type, email, name FROM users WHERE account_id = $1`
 	rows, err := s.pool.Query(ctx, query, accountID)
 	if err != nil {
 		return nil, err
@@ -1555,7 +1606,7 @@ func (s *SqlStore) getUsers(ctx context.Context, accountID string) ([]types.User
 		var autoGroups []byte
 		var lastLogin, createdAt sql.NullTime
 		var isServiceUser, nonDeletable, blocked, pendingApproval sql.NullBool
-		err := row.Scan(&u.Id, &u.AccountID, &u.Role, &isServiceUser, &nonDeletable, &u.ServiceUserName, &autoGroups, &blocked, &pendingApproval, &lastLogin, &createdAt, &u.Issued, &u.IntegrationReference.ID, &u.IntegrationReference.IntegrationType)
+		err := row.Scan(&u.Id, &u.AccountID, &u.Role, &isServiceUser, &nonDeletable, &u.ServiceUserName, &autoGroups, &blocked, &pendingApproval, &lastLogin, &createdAt, &u.Issued, &u.IntegrationReference.ID, &u.IntegrationReference.IntegrationType, &u.Email, &u.Name)
 		if err == nil {
 			if lastLogin.Valid {
 				u.LastLogin = &lastLogin.Time
@@ -3012,6 +3063,11 @@ func (s *SqlStore) GetDB() *gorm.DB {
 	return s.db
 }
 
+// SetFieldEncrypt sets the field encryptor for encrypting sensitive user data.
+func (s *SqlStore) SetFieldEncrypt(enc *crypt.FieldEncrypt) {
+	s.fieldEncrypt = enc
+}
+
 func (s *SqlStore) GetAccountDNSSettings(ctx context.Context, lockStrength LockingStrength, accountID string) (*types.DNSSettings, error) {
 	tx := s.db
 	if lockStrength != LockingStrengthNone {

management/server/store/sql_store_test.go

@@ -32,6 +32,7 @@ import (
 	nbroute "github.com/netbirdio/netbird/route"
 	route2 "github.com/netbirdio/netbird/route"
 	"github.com/netbirdio/netbird/shared/management/status"
+	"github.com/netbirdio/netbird/util/crypt"
 )
 
 func runTestForAllEngines(t *testing.T, testDataFile string, f func(t *testing.T, store Store)) {
@@ -2090,7 +2091,7 @@ func newAccountWithId(ctx context.Context, accountID, userID, domain string) *ty
 	setupKeys := map[string]*types.SetupKey{}
 	nameServersGroups := make(map[string]*nbdns.NameServerGroup)
 
-	owner := types.NewOwnerUser(userID)
+	owner := types.NewOwnerUser(userID, "", "")
 	owner.AccountID = accountID
 	users[userID] = owner
 
@@ -3114,6 +3115,138 @@ func TestSqlStore_SaveUsers(t *testing.T) {
 	require.Equal(t, users[1].AutoGroups, user.AutoGroups)
 }
 
+func TestSqlStore_SaveUserWithEncryption(t *testing.T) {
+	store, cleanup, err := NewTestStoreFromSQL(context.Background(), "../testdata/extended-store.sql", t.TempDir())
+	t.Cleanup(cleanup)
+	require.NoError(t, err)
+
+	// Enable encryption
+	key, err := crypt.GenerateKey()
+	require.NoError(t, err)
+	fieldEncrypt, err := crypt.NewFieldEncrypt(key)
+	require.NoError(t, err)
+	store.SetFieldEncrypt(fieldEncrypt)
+
+	accountID := "bf1c8084-ba50-4ce7-9439-34653001fc3b"
+
+	// rawUser is used to read raw (potentially encrypted) data from the database
+	// without any gorm hooks or automatic decryption
+	type rawUser struct {
+		Id    string
+		Email string
+		Name  string
+	}
+
+	t.Run("save user with empty email and name", func(t *testing.T) {
+		user := &types.User{
+			Id:         "user-empty-fields",
+			AccountID:  accountID,
+			Role:       types.UserRoleUser,
+			Email:      "",
+			Name:       "",
+			AutoGroups: []string{"groupA"},
+		}
+		err = store.SaveUser(context.Background(), user)
+		require.NoError(t, err)
+
+		// Verify using direct database query that empty strings remain empty (not encrypted)
+		var raw rawUser
+		err = store.(*SqlStore).db.Table("users").Select("id, email, name").Where("id = ?", user.Id).First(&raw).Error
+		require.NoError(t, err)
+		require.Equal(t, "", raw.Email, "empty email should remain empty in database")
+		require.Equal(t, "", raw.Name, "empty name should remain empty in database")
+
+		// Verify manual decryption returns empty strings
+		decryptedEmail, err := fieldEncrypt.Decrypt(raw.Email)
+		require.NoError(t, err)
+		require.Equal(t, "", decryptedEmail)
+
+		decryptedName, err := fieldEncrypt.Decrypt(raw.Name)
+		require.NoError(t, err)
+		require.Equal(t, "", decryptedName)
+	})
+
+	t.Run("save user with email and name", func(t *testing.T) {
+		user := &types.User{
+			Id:         "user-with-fields",
+			AccountID:  accountID,
+			Role:       types.UserRoleAdmin,
+			Email:      "test@example.com",
+			Name:       "Test User",
+			AutoGroups: []string{"groupB"},
+		}
+		err = store.SaveUser(context.Background(), user)
+		require.NoError(t, err)
+
+		// Verify using direct database query that the data is encrypted (not plaintext)
+		var raw rawUser
+		err = store.(*SqlStore).db.Table("users").Select("id, email, name").Where("id = ?", user.Id).First(&raw).Error
+		require.NoError(t, err)
+		require.NotEqual(t, "test@example.com", raw.Email, "email should be encrypted in database")
+		require.NotEqual(t, "Test User", raw.Name, "name should be encrypted in database")
+
+		// Verify manual decryption returns correct values
+		decryptedEmail, err := fieldEncrypt.Decrypt(raw.Email)
+		require.NoError(t, err)
+		require.Equal(t, "test@example.com", decryptedEmail)
+
+		decryptedName, err := fieldEncrypt.Decrypt(raw.Name)
+		require.NoError(t, err)
+		require.Equal(t, "Test User", decryptedName)
+	})
+
+	t.Run("save multiple users with mixed fields", func(t *testing.T) {
+		users := []*types.User{
+			{
+				Id:        "batch-user-1",
+				AccountID: accountID,
+				Email:     "",
+				Name:      "",
+			},
+			{
+				Id:        "batch-user-2",
+				AccountID: accountID,
+				Email:     "batch@example.com",
+				Name:      "Batch User",
+			},
+		}
+		err = store.SaveUsers(context.Background(), users)
+		require.NoError(t, err)
+
+		// Verify first user (empty fields) using direct database query
+		var raw1 rawUser
+		err = store.(*SqlStore).db.Table("users").Select("id, email, name").Where("id = ?", "batch-user-1").First(&raw1).Error
+		require.NoError(t, err)
+		require.Equal(t, "", raw1.Email, "empty email should remain empty in database")
+		require.Equal(t, "", raw1.Name, "empty name should remain empty in database")
+
+		// Verify second user (with fields) using direct database query
+		var raw2 rawUser
+		err = store.(*SqlStore).db.Table("users").Select("id, email, name").Where("id = ?", "batch-user-2").First(&raw2).Error
+		require.NoError(t, err)
+		require.NotEqual(t, "batch@example.com", raw2.Email, "email should be encrypted in database")
+		require.NotEqual(t, "Batch User", raw2.Name, "name should be encrypted in database")
+
+		// Verify manual decryption returns empty strings for first user
+		decryptedEmail1, err := fieldEncrypt.Decrypt(raw1.Email)
+		require.NoError(t, err)
+		require.Equal(t, "", decryptedEmail1)
+
+		decryptedName1, err := fieldEncrypt.Decrypt(raw1.Name)
+		require.NoError(t, err)
+		require.Equal(t, "", decryptedName1)
+
+		// Verify manual decryption returns correct values for second user
+		decryptedEmail2, err := fieldEncrypt.Decrypt(raw2.Email)
+		require.NoError(t, err)
+		require.Equal(t, "batch@example.com", decryptedEmail2)
+
+		decryptedName2, err := fieldEncrypt.Decrypt(raw2.Name)
+		require.NoError(t, err)
+		require.Equal(t, "Batch User", decryptedName2)
+	})
+}
+
 func TestSqlStore_DeleteUser(t *testing.T) {
 	store, cleanup, err := NewTestStoreFromSQL(context.Background(), "../testdata/extended-store.sql", t.TempDir())
 	t.Cleanup(cleanup)

management/server/store/store.go

@@ -27,6 +27,7 @@ import (
 	"github.com/netbirdio/netbird/management/server/testutil"
 	"github.com/netbirdio/netbird/management/server/types"
 	"github.com/netbirdio/netbird/util"
+	"github.com/netbirdio/netbird/util/crypt"
 
 	"github.com/netbirdio/netbird/management/server/migration"
 	resourceTypes "github.com/netbirdio/netbird/management/server/networks/resources/types"
@@ -204,6 +205,9 @@ type Store interface {
 	MarkAccountPrimary(ctx context.Context, accountID string) error
 	UpdateAccountNetwork(ctx context.Context, accountID string, ipNet net.IPNet) error
 	GetPolicyRulesByResourceID(ctx context.Context, lockStrength LockingStrength, accountID string, peerID string) ([]*types.PolicyRule, error)
+
+	// SetFieldEncrypt sets the field encryptor for encrypting sensitive user data.
+	SetFieldEncrypt(enc *crypt.FieldEncrypt)
 	GetUserIDByPeerKey(ctx context.Context, lockStrength LockingStrength, peerKey string) (string, error)
 }
 
@@ -340,6 +344,12 @@ func getMigrationsPreAuto(ctx context.Context) []migrationFunc {
 		func(db *gorm.DB) error {
 			return migration.DropIndex[routerTypes.NetworkRouter](ctx, db, "idx_network_routers_id")
 		},
+		func(db *gorm.DB) error {
+			return migration.MigrateNewField[types.User](ctx, db, "name", "")
+		},
+		func(db *gorm.DB) error {
+			return migration.MigrateNewField[types.User](ctx, db, "email", "")
+		},
 	}
 } // migratePostAuto migrates the SQLite database to the latest schema
 func migratePostAuto(ctx context.Context, db *gorm.DB) error {