[management, infrastructure, idp] Simplified IdP Management - Embedded IdP (#5008)

Embed Dex as a built-in IdP to simplify self-hosting setup.
Adds an embedded OIDC Identity Provider (Dex) with local user management and optional external IdP connectors (Google/GitHub/OIDC/SAML), plus device-auth flow for CLI login. Introduces instance onboarding/setup endpoints (including owner creation), field-level encryption for sensitive user data, a streamlined self-hosting provisioning script, and expanded APIs + test coverage for IdP management.

more at https://github.com/netbirdio/netbird/pull/5008#issuecomment-3718987393

management/internals/modules/peers/ephemeral/manager/ephemeral_test.go

@@ -309,7 +309,7 @@ func newAccountWithId(ctx context.Context, accountID, userID, domain string, dis
 	setupKeys := map[string]*types.SetupKey{}
 	nameServersGroups := make(map[string]*nbdns.NameServerGroup)
 
-	owner := types.NewOwnerUser(userID)
+	owner := types.NewOwnerUser(userID, "", "")
 	owner.AccountID = accountID
 	users[userID] = owner
 

management/internals/server/boot.go

@@ -21,7 +21,6 @@ import (
 	"github.com/netbirdio/management-integrations/integrations"
 	"github.com/netbirdio/netbird/encryption"
 	"github.com/netbirdio/netbird/formatter/hook"
-	nbconfig "github.com/netbirdio/netbird/management/internals/server/config"
 	nbgrpc "github.com/netbirdio/netbird/management/internals/shared/grpc"
 	"github.com/netbirdio/netbird/management/server/activity"
 	nbContext "github.com/netbirdio/netbird/management/server/context"
@@ -29,6 +28,7 @@ import (
 	"github.com/netbirdio/netbird/management/server/store"
 	"github.com/netbirdio/netbird/management/server/telemetry"
 	mgmtProto "github.com/netbirdio/netbird/shared/management/proto"
+	"github.com/netbirdio/netbird/util/crypt"
 )
 
 var (
@@ -62,6 +62,14 @@ func (s *BaseServer) Store() store.Store {
 			log.Fatalf("failed to create store: %v", err)
 		}
 
+		if s.Config.DataStoreEncryptionKey != "" {
+			fieldEncrypt, err := crypt.NewFieldEncrypt(s.Config.DataStoreEncryptionKey)
+			if err != nil {
+				log.Fatalf("failed to create field encryptor: %v", err)
+			}
+			store.SetFieldEncrypt(fieldEncrypt)
+		}
+
 		return store
 	})
 }
@@ -73,27 +81,18 @@ func (s *BaseServer) EventStore() activity.Store {
 			log.Fatalf("failed to initialize integration metrics: %v", err)
 		}
 
-		eventStore, key, err := integrations.InitEventStore(context.Background(), s.Config.Datadir, s.Config.DataStoreEncryptionKey, integrationMetrics)
+		eventStore, _, err := integrations.InitEventStore(context.Background(), s.Config.Datadir, s.Config.DataStoreEncryptionKey, integrationMetrics)
 		if err != nil {
 			log.Fatalf("failed to initialize event store: %v", err)
 		}
 
-		if s.Config.DataStoreEncryptionKey != key {
-			log.WithContext(context.Background()).Infof("update Config with activity store key")
-			s.Config.DataStoreEncryptionKey = key
-			err := updateMgmtConfig(context.Background(), nbconfig.MgmtConfigPath, s.Config)
-			if err != nil {
-				log.Fatalf("failed to update Config with activity store: %v", err)
-			}
-		}
-
 		return eventStore
 	})
 }
 
 func (s *BaseServer) APIHandler() http.Handler {
 	return Create(s, func() http.Handler {
-		httpAPIHandler, err := nbhttp.NewAPIHandler(context.Background(), s.AccountManager(), s.NetworksManager(), s.ResourcesManager(), s.RoutesManager(), s.GroupsManager(), s.GeoLocationManager(), s.AuthManager(), s.Metrics(), s.IntegratedValidator(), s.ProxyController(), s.PermissionsManager(), s.PeersManager(), s.SettingsManager(), s.NetworkMapController())
+		httpAPIHandler, err := nbhttp.NewAPIHandler(context.Background(), s.AccountManager(), s.NetworksManager(), s.ResourcesManager(), s.RoutesManager(), s.GroupsManager(), s.GeoLocationManager(), s.AuthManager(), s.Metrics(), s.IntegratedValidator(), s.ProxyController(), s.PermissionsManager(), s.PeersManager(), s.SettingsManager(), s.NetworkMapController(), s.IdpManager())
 		if err != nil {
 			log.Fatalf("failed to create API handler: %v", err)
 		}
@@ -145,7 +144,7 @@ func (s *BaseServer) GRPCServer() *grpc.Server {
 		}
 
 		gRPCAPIHandler := grpc.NewServer(gRPCOpts...)
-		srv, err := nbgrpc.NewServer(s.Config, s.AccountManager(), s.SettingsManager(), s.SecretsManager(), s.Metrics(), s.AuthManager(), s.IntegratedValidator(), s.NetworkMapController())
+		srv, err := nbgrpc.NewServer(s.Config, s.AccountManager(), s.SettingsManager(), s.SecretsManager(), s.Metrics(), s.AuthManager(), s.IntegratedValidator(), s.NetworkMapController(), s.OAuthConfigProvider())
 		if err != nil {
 			log.Fatalf("failed to create management server: %v", err)
 		}

management/internals/server/config/config.go

@@ -57,6 +57,10 @@ type Config struct {
 
 	// disable default all-to-all policy
 	DisableDefaultPolicy bool
+
+	// EmbeddedIdP contains configuration for the embedded Dex OIDC provider.
+	// When set, Dex will be embedded in the management server and serve requests at /oauth2/
+	EmbeddedIdP *idp.EmbeddedIdPConfig
 }
 
 // GetAuthAudiences returns the audience from the http config and device authorization flow config

management/internals/server/container.go

@@ -44,6 +44,9 @@ func maybeCreateNamed[T any](s Server, name string, createFunc func() T) (result
 
 func maybeCreateKeyed[T any](s Server, key string, createFunc func() T) (result T, isNew bool) {
 	if t, ok := s.GetContainer(key); ok {
+		if t == nil {
+			return result, false
+		}
 		return t.(T), false
 	}
 

management/internals/server/controllers.go

@@ -55,14 +55,33 @@ func (s *BaseServer) SecretsManager() grpc.SecretsManager {
 }
 
 func (s *BaseServer) AuthManager() auth.Manager {
+	audiences := s.Config.GetAuthAudiences()
+	audience := s.Config.HttpConfig.AuthAudience
+	keysLocation := s.Config.HttpConfig.AuthKeysLocation
+	signingKeyRefreshEnabled := s.Config.HttpConfig.IdpSignKeyRefreshEnabled
+	issuer := s.Config.HttpConfig.AuthIssuer
+	userIDClaim := s.Config.HttpConfig.AuthUserIDClaim
+
+	// Use embedded IdP configuration if available
+	if oauthProvider := s.OAuthConfigProvider(); oauthProvider != nil {
+		audiences = oauthProvider.GetClientIDs()
+		if len(audiences) > 0 {
+			audience = audiences[0] // Use the first client ID as the primary audience
+		}
+		keysLocation = oauthProvider.GetKeysLocation()
+		signingKeyRefreshEnabled = true
+		issuer = oauthProvider.GetIssuer()
+		userIDClaim = oauthProvider.GetUserIDClaim()
+	}
+
 	return Create(s, func() auth.Manager {
 		return auth.NewManager(s.Store(),
-			s.Config.HttpConfig.AuthIssuer,
-			s.Config.HttpConfig.AuthAudience,
-			s.Config.HttpConfig.AuthKeysLocation,
-			s.Config.HttpConfig.AuthUserIDClaim,
-			s.Config.GetAuthAudiences(),
-			s.Config.HttpConfig.IdpSignKeyRefreshEnabled)
+			issuer,
+			audience,
+			keysLocation,
+			userIDClaim,
+			audiences,
+			signingKeyRefreshEnabled)
 	})
 }
 

management/internals/server/modules.go

@@ -95,6 +95,17 @@ func (s *BaseServer) IdpManager() idp.Manager {
 	return Create(s, func() idp.Manager {
 		var idpManager idp.Manager
 		var err error
+		// Use embedded IdP manager if embedded Dex is configured and enabled.
+		// Legacy IdpManager won't be used anymore even if configured.
+		if s.Config.EmbeddedIdP != nil && s.Config.EmbeddedIdP.Enabled {
+			idpManager, err = idp.NewEmbeddedIdPManager(context.Background(), s.Config.EmbeddedIdP, s.Metrics())
+			if err != nil {
+				log.Fatalf("failed to create embedded IDP manager: %v", err)
+			}
+			return idpManager
+		}
+
+		// Fall back to external IdP manager
 		if s.Config.IdpManagerConfig != nil {
 			idpManager, err = idp.NewManager(context.Background(), *s.Config.IdpManagerConfig, s.Metrics())
 			if err != nil {
@@ -105,6 +116,25 @@ func (s *BaseServer) IdpManager() idp.Manager {
 	})
 }
 
+// OAuthConfigProvider is only relevant when we have an embedded IdP manager. Otherwise must be nil
+func (s *BaseServer) OAuthConfigProvider() idp.OAuthConfigProvider {
+	if s.Config.EmbeddedIdP == nil || !s.Config.EmbeddedIdP.Enabled {
+		return nil
+	}
+
+	idpManager := s.IdpManager()
+	if idpManager == nil {
+		return nil
+	}
+
+	// Reuse the EmbeddedIdPManager instance from IdpManager
+	// EmbeddedIdPManager implements both idp.Manager and idp.OAuthConfigProvider
+	if provider, ok := idpManager.(idp.OAuthConfigProvider); ok {
+		return provider
+	}
+	return nil
+}
+
 func (s *BaseServer) GroupsManager() groups.Manager {
 	return Create(s, func() groups.Manager {
 		return groups.NewManager(s.Store(), s.PermissionsManager(), s.AccountManager())

management/internals/server/server.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/netbirdio/netbird/management/server/idp"
 	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/metric"
 	"golang.org/x/crypto/acme/autocert"
@@ -22,7 +23,6 @@ import (
 	nbconfig "github.com/netbirdio/netbird/management/internals/server/config"
 	"github.com/netbirdio/netbird/management/server/metrics"
 	"github.com/netbirdio/netbird/management/server/store"
-	"github.com/netbirdio/netbird/util"
 	"github.com/netbirdio/netbird/util/wsproxy"
 	wsproxyserver "github.com/netbirdio/netbird/util/wsproxy/server"
 	"github.com/netbirdio/netbird/version"
@@ -40,7 +40,7 @@ type Server interface {
 	SetContainer(key string, container any)
 }
 
-// Server holds the HTTP BaseServer instance.
+// BaseServer holds the HTTP server instance.
 // Add any additional fields you need, such as database connections, Config, etc.
 type BaseServer struct {
 	// Config holds the server configuration
@@ -144,7 +144,7 @@ func (s *BaseServer) Start(ctx context.Context) error {
 		log.WithContext(srvCtx).Infof("running gRPC backward compatibility server: %s", compatListener.Addr().String())
 	}
 
-	rootHandler := s.handlerFunc(s.GRPCServer(), s.APIHandler(), s.Metrics().GetMeter())
+	rootHandler := s.handlerFunc(srvCtx, s.GRPCServer(), s.APIHandler(), s.Metrics().GetMeter())
 	switch {
 	case s.certManager != nil:
 		// a call to certManager.Listener() always creates a new listener so we do it once
@@ -215,6 +215,10 @@ func (s *BaseServer) Stop() error {
 	if s.update != nil {
 		s.update.StopWatch()
 	}
+	// Stop embedded IdP if configured
+	if embeddedIdP, ok := s.IdpManager().(*idp.EmbeddedIdPManager); ok {
+		_ = embeddedIdP.Stop(ctx)
+	}
 
 	select {
 	case <-s.Errors():
@@ -246,11 +250,7 @@ func (s *BaseServer) SetContainer(key string, container any) {
 	log.Tracef("container with key %s set successfully", key)
 }
 
-func updateMgmtConfig(ctx context.Context, path string, config *nbconfig.Config) error {
-	return util.DirectWriteJson(ctx, path, config)
-}
-
-func (s *BaseServer) handlerFunc(gRPCHandler *grpc.Server, httpHandler http.Handler, meter metric.Meter) http.Handler {
+func (s *BaseServer) handlerFunc(_ context.Context, gRPCHandler *grpc.Server, httpHandler http.Handler, meter metric.Meter) http.Handler {
 	wsProxy := wsproxyserver.New(gRPCHandler, wsproxyserver.WithOTelMeter(meter))
 
 	return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {

management/internals/shared/grpc/server.go

@@ -16,6 +16,7 @@ import (
 	pb "github.com/golang/protobuf/proto" // nolint
 	"github.com/golang/protobuf/ptypes/timestamp"
 	"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/realip"
+	"github.com/netbirdio/netbird/shared/management/client/common"
 	log "github.com/sirupsen/logrus"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	"google.golang.org/grpc/codes"
@@ -24,6 +25,7 @@ import (
 
 	"github.com/netbirdio/netbird/management/internals/controllers/network_map"
 	nbconfig "github.com/netbirdio/netbird/management/internals/server/config"
+	"github.com/netbirdio/netbird/management/server/idp"
 
 	"github.com/netbirdio/netbird/management/server/integrations/integrated_validator"
 	"github.com/netbirdio/netbird/management/server/store"
@@ -69,6 +71,8 @@ type Server struct {
 
 	networkMapController network_map.Controller
 
+	oAuthConfigProvider idp.OAuthConfigProvider
+
 	syncSem atomic.Int32
 	syncLim int32
 }
@@ -83,6 +87,7 @@ func NewServer(
 	authManager auth.Manager,
 	integratedPeerValidator integrated_validator.IntegratedValidator,
 	networkMapController network_map.Controller,
+	oAuthConfigProvider idp.OAuthConfigProvider,
 ) (*Server, error) {
 	if appMetrics != nil {
 		// update gauge based on number of connected peers which is equal to open gRPC streams
@@ -119,6 +124,7 @@ func NewServer(
 		blockPeersWithSameConfig: blockPeersWithSameConfig,
 		integratedPeerValidator:  integratedPeerValidator,
 		networkMapController:     networkMapController,
+		oAuthConfigProvider:      oAuthConfigProvider,
 
 		loginFilter: newLoginFilter(),
 
@@ -761,32 +767,48 @@ func (s *Server) GetDeviceAuthorizationFlow(ctx context.Context, req *proto.Encr
 		return nil, status.Error(codes.InvalidArgument, errMSG)
 	}
 
-	if s.config.DeviceAuthorizationFlow == nil || s.config.DeviceAuthorizationFlow.Provider == string(nbconfig.NONE) {
-		return nil, status.Error(codes.NotFound, "no device authorization flow information available")
-	}
+	var flowInfoResp *proto.DeviceAuthorizationFlow
 
-	provider, ok := proto.DeviceAuthorizationFlowProvider_value[strings.ToUpper(s.config.DeviceAuthorizationFlow.Provider)]
-	if !ok {
-		return nil, status.Errorf(codes.InvalidArgument, "no provider found in the protocol for %s", s.config.DeviceAuthorizationFlow.Provider)
-	}
+	// Use embedded IdP configuration if available
+	if s.oAuthConfigProvider != nil {
+		flowInfoResp = &proto.DeviceAuthorizationFlow{
+			Provider: proto.DeviceAuthorizationFlow_HOSTED,
+			ProviderConfig: &proto.ProviderConfig{
+				ClientID:           s.oAuthConfigProvider.GetCLIClientID(),
+				Audience:           s.oAuthConfigProvider.GetCLIClientID(),
+				DeviceAuthEndpoint: s.oAuthConfigProvider.GetDeviceAuthEndpoint(),
+				TokenEndpoint:      s.oAuthConfigProvider.GetTokenEndpoint(),
+				Scope:              s.oAuthConfigProvider.GetDefaultScopes(),
+			},
+		}
+	} else {
+		if s.config.DeviceAuthorizationFlow == nil || s.config.DeviceAuthorizationFlow.Provider == string(nbconfig.NONE) {
+			return nil, status.Error(codes.NotFound, "no device authorization flow information available")
+		}
 
-	flowInfoResp := &proto.DeviceAuthorizationFlow{
-		Provider: proto.DeviceAuthorizationFlowProvider(provider),
-		ProviderConfig: &proto.ProviderConfig{
-			ClientID:           s.config.DeviceAuthorizationFlow.ProviderConfig.ClientID,
-			ClientSecret:       s.config.DeviceAuthorizationFlow.ProviderConfig.ClientSecret,
-			Domain:             s.config.DeviceAuthorizationFlow.ProviderConfig.Domain,
-			Audience:           s.config.DeviceAuthorizationFlow.ProviderConfig.Audience,
-			DeviceAuthEndpoint: s.config.DeviceAuthorizationFlow.ProviderConfig.DeviceAuthEndpoint,
-			TokenEndpoint:      s.config.DeviceAuthorizationFlow.ProviderConfig.TokenEndpoint,
-			Scope:              s.config.DeviceAuthorizationFlow.ProviderConfig.Scope,
-			UseIDToken:         s.config.DeviceAuthorizationFlow.ProviderConfig.UseIDToken,
-		},
+		provider, ok := proto.DeviceAuthorizationFlowProvider_value[strings.ToUpper(s.config.DeviceAuthorizationFlow.Provider)]
+		if !ok {
+			return nil, status.Errorf(codes.InvalidArgument, "no provider found in the protocol for %s", s.config.DeviceAuthorizationFlow.Provider)
+		}
+
+		flowInfoResp = &proto.DeviceAuthorizationFlow{
+			Provider: proto.DeviceAuthorizationFlowProvider(provider),
+			ProviderConfig: &proto.ProviderConfig{
+				ClientID:           s.config.DeviceAuthorizationFlow.ProviderConfig.ClientID,
+				ClientSecret:       s.config.DeviceAuthorizationFlow.ProviderConfig.ClientSecret,
+				Domain:             s.config.DeviceAuthorizationFlow.ProviderConfig.Domain,
+				Audience:           s.config.DeviceAuthorizationFlow.ProviderConfig.Audience,
+				DeviceAuthEndpoint: s.config.DeviceAuthorizationFlow.ProviderConfig.DeviceAuthEndpoint,
+				TokenEndpoint:      s.config.DeviceAuthorizationFlow.ProviderConfig.TokenEndpoint,
+				Scope:              s.config.DeviceAuthorizationFlow.ProviderConfig.Scope,
+				UseIDToken:         s.config.DeviceAuthorizationFlow.ProviderConfig.UseIDToken,
+			},
+		}
 	}
 
 	encryptedResp, err := encryption.EncryptMessage(peerKey, key, flowInfoResp)
 	if err != nil {
-		return nil, status.Error(codes.Internal, "failed to encrypt no device authorization flow information")
+		return nil, status.Error(codes.Internal, "failed to encrypt device authorization flow information")
 	}
 
 	return &proto.EncryptedMessage{
@@ -820,30 +842,47 @@ func (s *Server) GetPKCEAuthorizationFlow(ctx context.Context, req *proto.Encryp
 		return nil, status.Error(codes.InvalidArgument, errMSG)
 	}
 
-	if s.config.PKCEAuthorizationFlow == nil {
-		return nil, status.Error(codes.NotFound, "no pkce authorization flow information available")
-	}
+	var initInfoFlow *proto.PKCEAuthorizationFlow
 
-	initInfoFlow := &proto.PKCEAuthorizationFlow{
-		ProviderConfig: &proto.ProviderConfig{
-			Audience:              s.config.PKCEAuthorizationFlow.ProviderConfig.Audience,
-			ClientID:              s.config.PKCEAuthorizationFlow.ProviderConfig.ClientID,
-			ClientSecret:          s.config.PKCEAuthorizationFlow.ProviderConfig.ClientSecret,
-			TokenEndpoint:         s.config.PKCEAuthorizationFlow.ProviderConfig.TokenEndpoint,
-			AuthorizationEndpoint: s.config.PKCEAuthorizationFlow.ProviderConfig.AuthorizationEndpoint,
-			Scope:                 s.config.PKCEAuthorizationFlow.ProviderConfig.Scope,
-			RedirectURLs:          s.config.PKCEAuthorizationFlow.ProviderConfig.RedirectURLs,
-			UseIDToken:            s.config.PKCEAuthorizationFlow.ProviderConfig.UseIDToken,
-			DisablePromptLogin:    s.config.PKCEAuthorizationFlow.ProviderConfig.DisablePromptLogin,
-			LoginFlag:             uint32(s.config.PKCEAuthorizationFlow.ProviderConfig.LoginFlag),
-		},
+	// Use embedded IdP configuration if available
+	if s.oAuthConfigProvider != nil {
+		initInfoFlow = &proto.PKCEAuthorizationFlow{
+			ProviderConfig: &proto.ProviderConfig{
+				Audience:              s.oAuthConfigProvider.GetCLIClientID(),
+				ClientID:              s.oAuthConfigProvider.GetCLIClientID(),
+				TokenEndpoint:         s.oAuthConfigProvider.GetTokenEndpoint(),
+				AuthorizationEndpoint: s.oAuthConfigProvider.GetAuthorizationEndpoint(),
+				Scope:                 s.oAuthConfigProvider.GetDefaultScopes(),
+				RedirectURLs:          s.oAuthConfigProvider.GetCLIRedirectURLs(),
+				LoginFlag:             uint32(common.LoginFlagPromptLogin),
+			},
+		}
+	} else {
+		if s.config.PKCEAuthorizationFlow == nil {
+			return nil, status.Error(codes.NotFound, "no pkce authorization flow information available")
+		}
+
+		initInfoFlow = &proto.PKCEAuthorizationFlow{
+			ProviderConfig: &proto.ProviderConfig{
+				Audience:              s.config.PKCEAuthorizationFlow.ProviderConfig.Audience,
+				ClientID:              s.config.PKCEAuthorizationFlow.ProviderConfig.ClientID,
+				ClientSecret:          s.config.PKCEAuthorizationFlow.ProviderConfig.ClientSecret,
+				TokenEndpoint:         s.config.PKCEAuthorizationFlow.ProviderConfig.TokenEndpoint,
+				AuthorizationEndpoint: s.config.PKCEAuthorizationFlow.ProviderConfig.AuthorizationEndpoint,
+				Scope:                 s.config.PKCEAuthorizationFlow.ProviderConfig.Scope,
+				RedirectURLs:          s.config.PKCEAuthorizationFlow.ProviderConfig.RedirectURLs,
+				UseIDToken:            s.config.PKCEAuthorizationFlow.ProviderConfig.UseIDToken,
+				DisablePromptLogin:    s.config.PKCEAuthorizationFlow.ProviderConfig.DisablePromptLogin,
+				LoginFlag:             uint32(s.config.PKCEAuthorizationFlow.ProviderConfig.LoginFlag),
+			},
+		}
 	}
 
 	flowInfoResp := s.integratedPeerValidator.ValidateFlowResponse(ctx, peerKey.String(), initInfoFlow)
 
 	encryptedResp, err := encryption.EncryptMessage(peerKey, key, flowInfoResp)
 	if err != nil {
-		return nil, status.Error(codes.Internal, "failed to encrypt no pkce authorization flow information")
+		return nil, status.Error(codes.Internal, "failed to encrypt pkce authorization flow information")
 	}
 
 	return &proto.EncryptedMessage{