[management, infrastructure, idp] Simplified IdP Management - Embedded IdP (#5008)

Embed Dex as a built-in IdP to simplify self-hosting setup.
Adds an embedded OIDC Identity Provider (Dex) with local user management and optional external IdP connectors (Google/GitHub/OIDC/SAML), plus device-auth flow for CLI login. Introduces instance onboarding/setup endpoints (including owner creation), field-level encryption for sensitive user data, a streamlined self-hosting provisioning script, and expanded APIs + test coverage for IdP management.

more at https://github.com/netbirdio/netbird/pull/5008#issuecomment-3718987393

management/internals/server/boot.go

@@ -21,7 +21,6 @@ import (
 	"github.com/netbirdio/management-integrations/integrations"
 	"github.com/netbirdio/netbird/encryption"
 	"github.com/netbirdio/netbird/formatter/hook"
-	nbconfig "github.com/netbirdio/netbird/management/internals/server/config"
 	nbgrpc "github.com/netbirdio/netbird/management/internals/shared/grpc"
 	"github.com/netbirdio/netbird/management/server/activity"
 	nbContext "github.com/netbirdio/netbird/management/server/context"
@@ -29,6 +28,7 @@ import (
 	"github.com/netbirdio/netbird/management/server/store"
 	"github.com/netbirdio/netbird/management/server/telemetry"
 	mgmtProto "github.com/netbirdio/netbird/shared/management/proto"
+	"github.com/netbirdio/netbird/util/crypt"
 )
 
 var (
@@ -62,6 +62,14 @@ func (s *BaseServer) Store() store.Store {
 			log.Fatalf("failed to create store: %v", err)
 		}
 
+		if s.Config.DataStoreEncryptionKey != "" {
+			fieldEncrypt, err := crypt.NewFieldEncrypt(s.Config.DataStoreEncryptionKey)
+			if err != nil {
+				log.Fatalf("failed to create field encryptor: %v", err)
+			}
+			store.SetFieldEncrypt(fieldEncrypt)
+		}
+
 		return store
 	})
 }
@@ -73,27 +81,18 @@ func (s *BaseServer) EventStore() activity.Store {
 			log.Fatalf("failed to initialize integration metrics: %v", err)
 		}
 
-		eventStore, key, err := integrations.InitEventStore(context.Background(), s.Config.Datadir, s.Config.DataStoreEncryptionKey, integrationMetrics)
+		eventStore, _, err := integrations.InitEventStore(context.Background(), s.Config.Datadir, s.Config.DataStoreEncryptionKey, integrationMetrics)
 		if err != nil {
 			log.Fatalf("failed to initialize event store: %v", err)
 		}
 
-		if s.Config.DataStoreEncryptionKey != key {
-			log.WithContext(context.Background()).Infof("update Config with activity store key")
-			s.Config.DataStoreEncryptionKey = key
-			err := updateMgmtConfig(context.Background(), nbconfig.MgmtConfigPath, s.Config)
-			if err != nil {
-				log.Fatalf("failed to update Config with activity store: %v", err)
-			}
-		}
-
 		return eventStore
 	})
 }
 
 func (s *BaseServer) APIHandler() http.Handler {
 	return Create(s, func() http.Handler {
-		httpAPIHandler, err := nbhttp.NewAPIHandler(context.Background(), s.AccountManager(), s.NetworksManager(), s.ResourcesManager(), s.RoutesManager(), s.GroupsManager(), s.GeoLocationManager(), s.AuthManager(), s.Metrics(), s.IntegratedValidator(), s.ProxyController(), s.PermissionsManager(), s.PeersManager(), s.SettingsManager(), s.NetworkMapController())
+		httpAPIHandler, err := nbhttp.NewAPIHandler(context.Background(), s.AccountManager(), s.NetworksManager(), s.ResourcesManager(), s.RoutesManager(), s.GroupsManager(), s.GeoLocationManager(), s.AuthManager(), s.Metrics(), s.IntegratedValidator(), s.ProxyController(), s.PermissionsManager(), s.PeersManager(), s.SettingsManager(), s.NetworkMapController(), s.IdpManager())
 		if err != nil {
 			log.Fatalf("failed to create API handler: %v", err)
 		}
@@ -145,7 +144,7 @@ func (s *BaseServer) GRPCServer() *grpc.Server {
 		}
 
 		gRPCAPIHandler := grpc.NewServer(gRPCOpts...)
-		srv, err := nbgrpc.NewServer(s.Config, s.AccountManager(), s.SettingsManager(), s.SecretsManager(), s.Metrics(), s.AuthManager(), s.IntegratedValidator(), s.NetworkMapController())
+		srv, err := nbgrpc.NewServer(s.Config, s.AccountManager(), s.SettingsManager(), s.SecretsManager(), s.Metrics(), s.AuthManager(), s.IntegratedValidator(), s.NetworkMapController(), s.OAuthConfigProvider())
 		if err != nil {
 			log.Fatalf("failed to create management server: %v", err)
 		}

management/internals/server/config/config.go

@@ -57,6 +57,10 @@ type Config struct {
 
 	// disable default all-to-all policy
 	DisableDefaultPolicy bool
+
+	// EmbeddedIdP contains configuration for the embedded Dex OIDC provider.
+	// When set, Dex will be embedded in the management server and serve requests at /oauth2/
+	EmbeddedIdP *idp.EmbeddedIdPConfig
 }
 
 // GetAuthAudiences returns the audience from the http config and device authorization flow config

management/internals/server/container.go

@@ -44,6 +44,9 @@ func maybeCreateNamed[T any](s Server, name string, createFunc func() T) (result
 
 func maybeCreateKeyed[T any](s Server, key string, createFunc func() T) (result T, isNew bool) {
 	if t, ok := s.GetContainer(key); ok {
+		if t == nil {
+			return result, false
+		}
 		return t.(T), false
 	}
 

management/internals/server/controllers.go

@@ -55,14 +55,33 @@ func (s *BaseServer) SecretsManager() grpc.SecretsManager {
 }
 
 func (s *BaseServer) AuthManager() auth.Manager {
+	audiences := s.Config.GetAuthAudiences()
+	audience := s.Config.HttpConfig.AuthAudience
+	keysLocation := s.Config.HttpConfig.AuthKeysLocation
+	signingKeyRefreshEnabled := s.Config.HttpConfig.IdpSignKeyRefreshEnabled
+	issuer := s.Config.HttpConfig.AuthIssuer
+	userIDClaim := s.Config.HttpConfig.AuthUserIDClaim
+
+	// Use embedded IdP configuration if available
+	if oauthProvider := s.OAuthConfigProvider(); oauthProvider != nil {
+		audiences = oauthProvider.GetClientIDs()
+		if len(audiences) > 0 {
+			audience = audiences[0] // Use the first client ID as the primary audience
+		}
+		keysLocation = oauthProvider.GetKeysLocation()
+		signingKeyRefreshEnabled = true
+		issuer = oauthProvider.GetIssuer()
+		userIDClaim = oauthProvider.GetUserIDClaim()
+	}
+
 	return Create(s, func() auth.Manager {
 		return auth.NewManager(s.Store(),
-			s.Config.HttpConfig.AuthIssuer,
-			s.Config.HttpConfig.AuthAudience,
-			s.Config.HttpConfig.AuthKeysLocation,
-			s.Config.HttpConfig.AuthUserIDClaim,
-			s.Config.GetAuthAudiences(),
-			s.Config.HttpConfig.IdpSignKeyRefreshEnabled)
+			issuer,
+			audience,
+			keysLocation,
+			userIDClaim,
+			audiences,
+			signingKeyRefreshEnabled)
 	})
 }
 

management/internals/server/modules.go

@@ -95,6 +95,17 @@ func (s *BaseServer) IdpManager() idp.Manager {
 	return Create(s, func() idp.Manager {
 		var idpManager idp.Manager
 		var err error
+		// Use embedded IdP manager if embedded Dex is configured and enabled.
+		// Legacy IdpManager won't be used anymore even if configured.
+		if s.Config.EmbeddedIdP != nil && s.Config.EmbeddedIdP.Enabled {
+			idpManager, err = idp.NewEmbeddedIdPManager(context.Background(), s.Config.EmbeddedIdP, s.Metrics())
+			if err != nil {
+				log.Fatalf("failed to create embedded IDP manager: %v", err)
+			}
+			return idpManager
+		}
+
+		// Fall back to external IdP manager
 		if s.Config.IdpManagerConfig != nil {
 			idpManager, err = idp.NewManager(context.Background(), *s.Config.IdpManagerConfig, s.Metrics())
 			if err != nil {
@@ -105,6 +116,25 @@ func (s *BaseServer) IdpManager() idp.Manager {
 	})
 }
 
+// OAuthConfigProvider is only relevant when we have an embedded IdP manager. Otherwise must be nil
+func (s *BaseServer) OAuthConfigProvider() idp.OAuthConfigProvider {
+	if s.Config.EmbeddedIdP == nil || !s.Config.EmbeddedIdP.Enabled {
+		return nil
+	}
+
+	idpManager := s.IdpManager()
+	if idpManager == nil {
+		return nil
+	}
+
+	// Reuse the EmbeddedIdPManager instance from IdpManager
+	// EmbeddedIdPManager implements both idp.Manager and idp.OAuthConfigProvider
+	if provider, ok := idpManager.(idp.OAuthConfigProvider); ok {
+		return provider
+	}
+	return nil
+}
+
 func (s *BaseServer) GroupsManager() groups.Manager {
 	return Create(s, func() groups.Manager {
 		return groups.NewManager(s.Store(), s.PermissionsManager(), s.AccountManager())

management/internals/server/server.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/netbirdio/netbird/management/server/idp"
 	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel/metric"
 	"golang.org/x/crypto/acme/autocert"
@@ -22,7 +23,6 @@ import (
 	nbconfig "github.com/netbirdio/netbird/management/internals/server/config"
 	"github.com/netbirdio/netbird/management/server/metrics"
 	"github.com/netbirdio/netbird/management/server/store"
-	"github.com/netbirdio/netbird/util"
 	"github.com/netbirdio/netbird/util/wsproxy"
 	wsproxyserver "github.com/netbirdio/netbird/util/wsproxy/server"
 	"github.com/netbirdio/netbird/version"
@@ -40,7 +40,7 @@ type Server interface {
 	SetContainer(key string, container any)
 }
 
-// Server holds the HTTP BaseServer instance.
+// BaseServer holds the HTTP server instance.
 // Add any additional fields you need, such as database connections, Config, etc.
 type BaseServer struct {
 	// Config holds the server configuration
@@ -144,7 +144,7 @@ func (s *BaseServer) Start(ctx context.Context) error {
 		log.WithContext(srvCtx).Infof("running gRPC backward compatibility server: %s", compatListener.Addr().String())
 	}
 
-	rootHandler := s.handlerFunc(s.GRPCServer(), s.APIHandler(), s.Metrics().GetMeter())
+	rootHandler := s.handlerFunc(srvCtx, s.GRPCServer(), s.APIHandler(), s.Metrics().GetMeter())
 	switch {
 	case s.certManager != nil:
 		// a call to certManager.Listener() always creates a new listener so we do it once
@@ -215,6 +215,10 @@ func (s *BaseServer) Stop() error {
 	if s.update != nil {
 		s.update.StopWatch()
 	}
+	// Stop embedded IdP if configured
+	if embeddedIdP, ok := s.IdpManager().(*idp.EmbeddedIdPManager); ok {
+		_ = embeddedIdP.Stop(ctx)
+	}
 
 	select {
 	case <-s.Errors():
@@ -246,11 +250,7 @@ func (s *BaseServer) SetContainer(key string, container any) {
 	log.Tracef("container with key %s set successfully", key)
 }
 
-func updateMgmtConfig(ctx context.Context, path string, config *nbconfig.Config) error {
-	return util.DirectWriteJson(ctx, path, config)
-}
-
-func (s *BaseServer) handlerFunc(gRPCHandler *grpc.Server, httpHandler http.Handler, meter metric.Meter) http.Handler {
+func (s *BaseServer) handlerFunc(_ context.Context, gRPCHandler *grpc.Server, httpHandler http.Handler, meter metric.Meter) http.Handler {
 	wsProxy := wsproxyserver.New(gRPCHandler, wsproxyserver.WithOTelMeter(meter))
 
 	return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {