Merge branch 'main' of github.com:netbirdio/netbird into feat/admin-cli

* Add vLLM to Agent Network

* Add vllm e2e test

.github/workflows/release.yml

@@ -293,8 +293,11 @@ jobs:
           ${{ steps.goreleaser.outputs.artifacts }}
           JSON
 
+          # dockers_v2 artifacts have no top-level goarch field, so match the
+          # per-platform -amd64 tag suffix instead; it works for both the old
+          # dockers and the new dockers_v2 image naming.
           mapfile -t src_images < <(
-            jq -r '.[] | select(.type == "Docker Image") | select(.goarch == "amd64") | .name | select(startswith("ghcr.io/"))' /tmp/goreleaser-artifacts.json
+            jq -r '.[] | select(.type == "Docker Image") | .name | select(startswith("ghcr.io/") and endswith("-amd64"))' /tmp/goreleaser-artifacts.json
           )
 
           for src in "${src_images[@]}"; do

combined/cmd/admin.go

@@ -0,0 +1,91 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/netbirdio/netbird/formatter/hook"
+	admincmd "github.com/netbirdio/netbird/management/cmd/admin"
+	tokencmd "github.com/netbirdio/netbird/management/cmd/token"
+	"github.com/netbirdio/netbird/management/server/store"
+	"github.com/netbirdio/netbird/management/server/types"
+	"github.com/netbirdio/netbird/util"
+)
+
+// newAdminCommands creates the admin command tree with combined-specific resource openers.
+func newAdminCommands() *cobra.Command {
+	cmd := admincmd.NewCommands(withAdminResources)
+	cmd.AddCommand(tokencmd.NewCommands(withAdminTokenStore))
+	return cmd
+}
+
+// withAdminResources loads the combined YAML config, initializes stores, and calls fn.
+func withAdminResources(cmd *cobra.Command, fn func(ctx context.Context, resources admincmd.Resources) error) error {
+	return withAdminStore(cmd, func(ctx context.Context, managementStore store.Store, cfg *CombinedConfig) error {
+		mgmtConfig, err := cfg.ToManagementConfig()
+		if err != nil {
+			return fmt.Errorf("create management config: %w", err)
+		}
+
+		idpStorage, err := admincmd.OpenEmbeddedIDPStorage(mgmtConfig.EmbeddedIdP)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := idpStorage.Close(); err != nil {
+				log.Debugf("close embedded IdP storage: %v", err)
+			}
+		}()
+
+		return fn(ctx, admincmd.Resources{Store: managementStore, IDPStorage: idpStorage})
+	})
+}
+
+// withAdminTokenStore opens only the management store for admin token commands.
+func withAdminTokenStore(cmd *cobra.Command, fn func(ctx context.Context, s store.Store) error) error {
+	return withAdminStore(cmd, func(ctx context.Context, managementStore store.Store, _ *CombinedConfig) error {
+		return fn(ctx, managementStore)
+	})
+}
+
+func withAdminStore(cmd *cobra.Command, fn func(ctx context.Context, s store.Store, cfg *CombinedConfig) error) error {
+	if err := util.InitLog("error", "console"); err != nil {
+		return fmt.Errorf("init log: %w", err)
+	}
+
+	ctx := context.WithValue(cmd.Context(), hook.ExecutionContextKey, hook.SystemSource) //nolint:staticcheck
+
+	cfg, err := LoadConfig(configPath)
+	if err != nil {
+		return fmt.Errorf("load config: %w", err)
+	}
+
+	if dsn := cfg.Server.Store.DSN; dsn != "" {
+		switch strings.ToLower(cfg.Server.Store.Engine) {
+		case "postgres":
+			os.Setenv("NB_STORE_ENGINE_POSTGRES_DSN", dsn)
+		case "mysql":
+			os.Setenv("NB_STORE_ENGINE_MYSQL_DSN", dsn)
+		}
+	}
+	if file := cfg.Server.Store.File; file != "" {
+		os.Setenv("NB_STORE_ENGINE_SQLITE_FILE", file)
+	}
+
+	managementStore, err := store.NewStore(ctx, types.Engine(cfg.Management.Store.Engine), cfg.Management.DataDir, nil, true)
+	if err != nil {
+		return fmt.Errorf("create store: %w", err)
+	}
+	defer func() {
+		if err := managementStore.Close(ctx); err != nil {
+			log.Debugf("close store: %v", err)
+		}
+	}()
+
+	return fn(ctx, managementStore, cfg)
+}

combined/cmd/root.go

@@ -64,7 +64,7 @@ func init() {
 	rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "path to YAML configuration file (required)")
 	_ = rootCmd.MarkPersistentFlagRequired("config")
 
-	rootCmd.AddCommand(newTokenCommands())
+	rootCmd.AddCommand(newAdminCommands())
 }
 
 func RootCmd() *cobra.Command {

combined/cmd/token.go

@@ -1,63 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"strings"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/netbirdio/netbird/formatter/hook"
-	tokencmd "github.com/netbirdio/netbird/management/cmd/token"
-	"github.com/netbirdio/netbird/management/server/store"
-	"github.com/netbirdio/netbird/management/server/types"
-	"github.com/netbirdio/netbird/util"
-)
-
-// newTokenCommands creates the token command tree with combined-specific store opener.
-func newTokenCommands() *cobra.Command {
-	return tokencmd.NewCommands(withTokenStore)
-}
-
-// withTokenStore loads the combined YAML config, initializes the store, and calls fn.
-func withTokenStore(cmd *cobra.Command, fn func(ctx context.Context, s store.Store) error) error {
-	if err := util.InitLog("error", "console"); err != nil {
-		return fmt.Errorf("init log: %w", err)
-	}
-
-	ctx := context.WithValue(cmd.Context(), hook.ExecutionContextKey, hook.SystemSource) //nolint:staticcheck
-
-	cfg, err := LoadConfig(configPath)
-	if err != nil {
-		return fmt.Errorf("load config: %w", err)
-	}
-
-	if dsn := cfg.Server.Store.DSN; dsn != "" {
-		switch strings.ToLower(cfg.Server.Store.Engine) {
-		case "postgres":
-			os.Setenv("NB_STORE_ENGINE_POSTGRES_DSN", dsn)
-		case "mysql":
-			os.Setenv("NB_STORE_ENGINE_MYSQL_DSN", dsn)
-		}
-	}
-	if file := cfg.Server.Store.File; file != "" {
-		os.Setenv("NB_STORE_ENGINE_SQLITE_FILE", file)
-	}
-
-	datadir := cfg.Management.DataDir
-	engine := types.Engine(cfg.Management.Store.Engine)
-
-	s, err := store.NewStore(ctx, engine, datadir, nil, true)
-	if err != nil {
-		return fmt.Errorf("create store: %w", err)
-	}
-	defer func() {
-		if err := s.Close(ctx); err != nil {
-			log.Debugf("close store: %v", err)
-		}
-	}()
-
-	return fn(ctx, s)
-}

e2e/agentnetwork/vllm_test.go

@@ -0,0 +1,171 @@
+//go:build e2e
+
+package agentnetwork
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/e2e/harness"
+	"github.com/netbirdio/netbird/shared/management/http/api"
+)
+
+// TestVLLMProvider proves the proxy supports a self-hosted vLLM backend. vLLM is
+// OpenAI-compatible, so it uses the "vllm" catalog entry (KindCustom) and is
+// reached over plain HTTP — no TLS anywhere on the path:
+//
+//	client --tunnel--> netbird proxy --http--> vllm (:8000, OpenAI-compatible)
+//
+// The mock vLLM server answers /v1/chat/completions with an OpenAI-shaped
+// completion carrying a non-zero usage block. The test asserts the chat returns
+// 200 with the completion, that the request is recorded in the access log by its
+// session id, and that vLLM's usage block is metered into a consumption row —
+// which together prove request routing, response parsing, and token accounting
+// all work for a self-hosted OpenAI-compatible provider.
+//
+// It needs no external credentials (the mock ignores auth), so it always runs.
+func TestVLLMProvider(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	defer cancel()
+
+	vllm, err := harness.StartVLLM(ctx, srv)
+	require.NoError(t, err, "start mock vLLM server")
+	t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
+
+	grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-vllm"})
+	require.NoError(t, err, "create group")
+	t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
+
+	ephemeral := false
+	sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
+		Name:       "e2e-vllm-client",
+		Type:       "reusable",
+		ExpiresIn:  86400,
+		UsageLimit: 0,
+		AutoGroups: []string{grp.Id},
+		Ephemeral:  &ephemeral,
+	})
+	require.NoError(t, err, "mint setup key")
+	require.NotEmpty(t, sk.Key, "setup key plaintext")
+
+	// vLLM provider pointed at the mock over plain HTTP. The mock ignores auth,
+	// so a dummy key satisfies the "Bearer ${API_KEY}" template. The served model
+	// is enumerated so the router dispatches this model string to this provider.
+	dummyKey := "sk-vllm-e2e"
+	prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
+		Name:             "vllm",
+		ProviderId:       "vllm",
+		UpstreamUrl:      vllm.URL,
+		ApiKey:           &dummyKey,
+		Enabled:          ptr(true),
+		BootstrapCluster: ptr(harness.AgentNetworkCluster),
+		Models: &[]api.AgentNetworkProviderModel{
+			{Id: harness.VLLMModel, InputPer1k: 0.001, OutputPer1k: 0.002},
+		},
+	})
+	require.NoError(t, err, "create vllm provider")
+	t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
+
+	// Token limit far above the handful of tokens this test drives, so it never
+	// blocks but switches on usage metering — the switch that makes consumption
+	// rows get recorded.
+	enabled := true
+	pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
+		Name:                   "e2e-vllm-allow",
+		Enabled:                &enabled,
+		SourceGroups:           []string{grp.Id},
+		DestinationProviderIds: []string{prov.Id},
+		Limits: &api.AgentNetworkPolicyLimits{
+			TokenLimit: api.AgentNetworkPolicyTokenLimit{
+				Enabled:       true,
+				GroupCap:      10_000_000,
+				UserCap:       10_000_000,
+				WindowSeconds: 60,
+			},
+		},
+	})
+	require.NoError(t, err, "create policy")
+	t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
+
+	settings, err := srv.GetSettings(ctx)
+	require.NoError(t, err, "read settings")
+	require.NotEmpty(t, settings.Endpoint, "endpoint must be assigned")
+
+	proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-vllm-proxy")
+	require.NoError(t, err, "mint proxy token")
+	px, err := harness.StartProxy(ctx, srv, proxyToken)
+	require.NoError(t, err, "start proxy")
+	t.Cleanup(func() { _ = px.Terminate(context.Background()) })
+
+	cl, err := harness.StartClient(ctx, srv, sk.Key)
+	require.NoError(t, err, "start client")
+	t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
+
+	require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
+	if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
+		t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background()))
+	}
+	proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
+	require.NoError(t, err, "resolve endpoint to proxy IP")
+
+	before, _ := srv.ListAccessLogs(ctx)
+	sessionID := "e2e-session-vllm"
+
+	// Retry to absorb tunnel/DNS jitter on the first call.
+	var code int
+	var body string
+	deadline := time.Now().Add(90 * time.Second)
+	for time.Now().Before(deadline) {
+		c, b, cerr := cl.Chat(ctx, settings.Endpoint, proxyIP, harness.WireChat, harness.VLLMModel, "Reply with exactly: pong", sessionID)
+		if cerr == nil {
+			code, body = c, b
+			if code == 200 {
+				break
+			}
+		}
+		time.Sleep(5 * time.Second)
+	}
+	require.Equal(t, 200, code,
+		"chat through the vLLM provider must return 200; body: %s\n=== vllm logs ===\n%s\n=== proxy logs ===\n%s",
+		body, vllm.Logs(context.Background()), px.Logs(context.Background()))
+	require.True(t, strings.Contains(body, "chat.completion"),
+		"body should be an OpenAI-compatible chat completion; got: %s", body)
+
+	// The request must surface as an access-log row carrying our session id.
+	require.Eventually(t, func() bool {
+		logs, lerr := srv.ListAccessLogs(ctx)
+		return lerr == nil && logs.TotalRecords > before.TotalRecords
+	}, 30*time.Second, 2*time.Second, "an access-log row should be ingested for the vLLM provider")
+
+	require.Eventually(t, func() bool {
+		logs, lerr := srv.ListAccessLogs(ctx)
+		if lerr != nil {
+			return false
+		}
+		for _, r := range logs.Data {
+			if r.SessionId != nil && *r.SessionId == sessionID {
+				return true
+			}
+		}
+		return false
+	}, 30*time.Second, 2*time.Second, "session id %q must be recorded in an access-log row", sessionID)
+
+	// vLLM's usage block (prompt_tokens=11, completion_tokens=2) must be parsed
+	// and metered into a consumption row with positive token counts.
+	require.Eventually(t, func() bool {
+		rows, lerr := srv.ListConsumption(ctx)
+		if lerr != nil {
+			return false
+		}
+		for _, r := range rows {
+			if r.TokensInput > 0 && r.TokensOutput > 0 {
+				return true
+			}
+		}
+		return false
+	}, 60*time.Second, 3*time.Second, "vLLM usage must be metered into a consumption row")
+}

e2e/harness/vllm.go

@@ -0,0 +1,113 @@
+//go:build e2e
+
+package harness
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/wait"
+)
+
+const (
+	vllmImage = "nginx:alpine"
+	vllmAlias = "vllm"
+	vllmPort  = "8000/tcp"
+	// VLLMModel is the served model id the mock advertises and echoes back. It
+	// matches a real small model commonly served by vLLM so the provider's
+	// enumerated model and the client's request line up.
+	VLLMModel = "Qwen/Qwen2.5-0.5B-Instruct"
+)
+
+// vllmNginxConf emulates a vLLM OpenAI-compatible server over plain HTTP (vLLM's
+// default: no TLS, port 8000). It answers /v1/models with a one-model list and
+// any chat/completions path with a canned OpenAI-shaped chat completion carrying
+// a non-zero usage block, so the proxy's OpenAI parser records real token
+// consumption. Running actual vLLM in CI is infeasible (GPU + multi-GB model
+// download), so this stands in for the wire contract the proxy depends on.
+const vllmNginxConf = `pid /tmp/nginx.pid;
+events {}
+http {
+  server {
+    listen 8000;
+    location = /v1/models {
+      default_type application/json;
+      return 200 '{"object":"list","data":[{"id":"Qwen/Qwen2.5-0.5B-Instruct","object":"model","owned_by":"vllm"}]}';
+    }
+    location / {
+      default_type application/json;
+      return 200 '{"id":"chatcmpl-e2e-vllm","object":"chat.completion","created":1700000000,"model":"Qwen/Qwen2.5-0.5B-Instruct","choices":[{"index":0,"message":{"role":"assistant","content":"pong"},"finish_reason":"stop"}],"usage":{"prompt_tokens":11,"completion_tokens":2,"total_tokens":13}}';
+    }
+  }
+}
+`
+
+// VLLM is a mock vLLM OpenAI-compatible server on the combined server's network,
+// reachable at http://vllm:8000. A "vllm" provider points at it to exercise the
+// proxy's support for self-hosted OpenAI-compatible backends.
+type VLLM struct {
+	container testcontainers.Container
+	workDir   string
+	// URL is the upstream URL the vllm provider points at (http://<alias>:8000).
+	URL string
+}
+
+// StartVLLM runs the mock vLLM server on the shared network over plain HTTP.
+func StartVLLM(ctx context.Context, c *Combined) (*VLLM, error) {
+	workDir, err := os.MkdirTemp("/tmp", "nb-e2e-vllm-*")
+	if err != nil {
+		return nil, fmt.Errorf("create vllm work dir: %w", err)
+	}
+	// Widen so the (non-root worker) nginx container can traverse the bind mount.
+	if err := os.Chmod(workDir, 0o755); err != nil { //nolint:gosec // throwaway e2e config dir
+		return nil, fmt.Errorf("chmod vllm dir: %w", err)
+	}
+	if err := os.WriteFile(filepath.Join(workDir, "nginx.conf"), []byte(vllmNginxConf), 0o644); err != nil { //nolint:gosec // non-secret e2e config
+		return nil, fmt.Errorf("write nginx conf: %w", err)
+	}
+
+	req := testcontainers.ContainerRequest{
+		Image:          vllmImage,
+		ExposedPorts:   []string{vllmPort},
+		Networks:       []string{c.network.Name},
+		NetworkAliases: map[string][]string{c.network.Name: {vllmAlias}},
+		Cmd:            []string{"nginx", "-c", "/conf/nginx.conf", "-g", "daemon off;"},
+		HostConfigModifier: func(hc *container.HostConfig) {
+			hc.Binds = append(hc.Binds, workDir+":/conf:ro")
+		},
+		WaitingFor: wait.ForListeningPort(vllmPort).WithStartupTimeout(60 * time.Second),
+	}
+
+	ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	if err != nil {
+		_ = os.RemoveAll(workDir)
+		return nil, fmt.Errorf("start vllm container: %w", err)
+	}
+
+	return &VLLM{container: ctr, workDir: workDir, URL: "http://" + vllmAlias + ":8000"}, nil
+}
+
+// Logs returns the vLLM container logs, for diagnostics on failure.
+func (v *VLLM) Logs(ctx context.Context) string {
+	return containerLogs(ctx, v.container)
+}
+
+// Terminate stops the vLLM container and cleans its work dir.
+func (v *VLLM) Terminate(ctx context.Context) error {
+	var err error
+	if v.container != nil {
+		err = v.container.Terminate(ctx)
+	}
+	if v.workDir != "" {
+		_ = os.RemoveAll(v.workDir)
+	}
+	return err
+}

management/cmd/admin.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/netbirdio/netbird/formatter/hook"
+	admincmd "github.com/netbirdio/netbird/management/cmd/admin"
+	tokencmd "github.com/netbirdio/netbird/management/cmd/token"
+	nbconfig "github.com/netbirdio/netbird/management/internals/server/config"
+	"github.com/netbirdio/netbird/management/server/store"
+	"github.com/netbirdio/netbird/util"
+)
+
+var adminDatadir string
+
+// newAdminCommands creates the admin command tree with management-specific resource openers.
+func newAdminCommands() *cobra.Command {
+	cmd := admincmd.NewCommands(withAdminResources)
+	cmd.PersistentFlags().StringVar(&adminDatadir, "datadir", "", "Override the data directory from config (used for store.db and the default idp.db)")
+	cmd.AddCommand(tokencmd.NewCommands(withAdminTokenStore))
+	return cmd
+}
+
+// withAdminResources initializes logging, loads config, opens the management store
+// and embedded IdP storage, and calls fn.
+func withAdminResources(cmd *cobra.Command, fn func(ctx context.Context, resources admincmd.Resources) error) error {
+	return withAdminStore(cmd, func(ctx context.Context, managementStore store.Store, config *nbconfig.Config) error {
+		idpStorage, err := admincmd.OpenEmbeddedIDPStorage(config.EmbeddedIdP)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := idpStorage.Close(); err != nil {
+				log.Debugf("close embedded IdP storage: %v", err)
+			}
+		}()
+
+		return fn(ctx, admincmd.Resources{Store: managementStore, IDPStorage: idpStorage})
+	})
+}
+
+// withAdminTokenStore opens only the management store for admin token commands.
+func withAdminTokenStore(cmd *cobra.Command, fn func(ctx context.Context, s store.Store) error) error {
+	return withAdminStore(cmd, func(ctx context.Context, managementStore store.Store, _ *nbconfig.Config) error {
+		return fn(ctx, managementStore)
+	})
+}
+
+func withAdminStore(cmd *cobra.Command, fn func(ctx context.Context, s store.Store, config *nbconfig.Config) error) error {
+	if err := util.InitLog("error", "console"); err != nil {
+		return fmt.Errorf("init log: %w", err)
+	}
+
+	ctx := context.WithValue(cmd.Context(), hook.ExecutionContextKey, hook.SystemSource) //nolint:staticcheck
+
+	config, err := LoadMgmtConfig(ctx, nbconfig.MgmtConfigPath)
+	if err != nil {
+		return fmt.Errorf("load config: %w", err)
+	}
+
+	datadir := config.Datadir
+	if adminDatadir != "" {
+		oldDatadir := datadir
+		datadir = adminDatadir
+		if config.EmbeddedIdP != nil && config.EmbeddedIdP.Storage.Type == "sqlite3" {
+			defaultIDPFile := filepath.Join(oldDatadir, "idp.db")
+			if config.EmbeddedIdP.Storage.Config.File == "" || config.EmbeddedIdP.Storage.Config.File == defaultIDPFile {
+				config.EmbeddedIdP.Storage.Config.File = filepath.Join(datadir, "idp.db")
+			}
+		}
+	}
+
+	managementStore, err := store.NewStore(ctx, config.StoreConfig.Engine, datadir, nil, true)
+	if err != nil {
+		return fmt.Errorf("create store: %w", err)
+	}
+	defer func() {
+		if err := managementStore.Close(ctx); err != nil {
+			log.Debugf("close store: %v", err)
+		}
+	}()
+
+	return fn(ctx, managementStore, config)
+}

management/cmd/admin/admin.go

@@ -0,0 +1,441 @@
+// Package admincmd provides reusable cobra commands for self-hosted administrator helpers.
+// Both the management and combined binaries use these commands, each providing
+// their own opener to handle config loading and storage initialization.
+package admincmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+	"strings"
+
+	"github.com/dexidp/dex/storage"
+	"github.com/spf13/cobra"
+	"golang.org/x/crypto/bcrypt"
+
+	nbdex "github.com/netbirdio/netbird/idp/dex"
+	"github.com/netbirdio/netbird/management/server"
+	"github.com/netbirdio/netbird/management/server/idp"
+	"github.com/netbirdio/netbird/management/server/store"
+	"github.com/netbirdio/netbird/management/server/types"
+)
+
+const (
+	localConnectorID           = "local"
+	dashboardClientID          = "netbird-dashboard"
+	cliClientID                = "netbird-cli"
+	defaultTOTPAuthenticatorID = "default-totp"
+)
+
+// Resources contains the storages required by the admin commands.
+type Resources struct {
+	Store      store.Store
+	IDPStorage storage.Storage
+}
+
+// Opener initializes command resources from the command context and calls fn.
+type Opener func(cmd *cobra.Command, fn func(ctx context.Context, resources Resources) error) error
+
+type userSelector struct {
+	email  string
+	userID string
+}
+
+func (s userSelector) normalized() userSelector {
+	return userSelector{
+		email:  strings.TrimSpace(s.email),
+		userID: strings.TrimSpace(s.userID),
+	}
+}
+
+func (s userSelector) validate() error {
+	s = s.normalized()
+	if (s.email == "") == (s.userID == "") {
+		return fmt.Errorf("provide exactly one of --email or --user-id")
+	}
+	return nil
+}
+
+// NewCommands creates the admin command tree with the given resource opener.
+func NewCommands(opener Opener) *cobra.Command {
+	adminCmd := &cobra.Command{
+		Use:   "admin",
+		Short: "Self-hosted administrator helpers",
+		Long:  "Administrative helpers for self-hosted deployments using the embedded identity provider.",
+	}
+
+	userCmd := &cobra.Command{
+		Use:   "user",
+		Short: "Manage local embedded IdP users",
+	}
+
+	var passwordSelector userSelector
+	var password string
+	var passwordFile string
+	passwordCmd := &cobra.Command{
+		Use:     "change-password (--email email | --user-id id) (--password password | --password-file path)",
+		Aliases: []string{"set-password"},
+		Short:   "Change a local user's password",
+		Args:    cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			newPassword, err := resolvePasswordInput(cmd, password, passwordFile)
+			if err != nil {
+				return err
+			}
+			return opener(cmd, func(ctx context.Context, resources Resources) error {
+				return runChangePassword(ctx, resources.IDPStorage, cmd.OutOrStdout(), passwordSelector, newPassword)
+			})
+		},
+	}
+	addUserSelectorFlags(passwordCmd, &passwordSelector)
+	passwordCmd.Flags().StringVar(&password, "password", "", "New password for the user")
+	passwordCmd.Flags().StringVar(&passwordFile, "password-file", "", "Read new password from file ('-' for stdin)")
+
+	var resetSelector userSelector
+	resetMFACmd := &cobra.Command{
+		Use:   "reset-mfa (--email email | --user-id id)",
+		Short: "Reset a local user's MFA enrollment",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			return opener(cmd, func(ctx context.Context, resources Resources) error {
+				return runResetMFA(ctx, resources.IDPStorage, cmd.OutOrStdout(), resetSelector)
+			})
+		},
+	}
+	addUserSelectorFlags(resetMFACmd, &resetSelector)
+
+	userCmd.AddCommand(passwordCmd, resetMFACmd)
+
+	mfaCmd := &cobra.Command{
+		Use:   "mfa",
+		Short: "Manage local MFA for embedded IdP users",
+	}
+
+	enableCmd := &cobra.Command{
+		Use:   "enable",
+		Short: "Enable MFA for local embedded IdP users",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			return opener(cmd, func(ctx context.Context, resources Resources) error {
+				return runSetMFAEnabled(ctx, resources, cmd.OutOrStdout(), true)
+			})
+		},
+	}
+
+	disableCmd := &cobra.Command{
+		Use:   "disable",
+		Short: "Disable MFA for local embedded IdP users",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			return opener(cmd, func(ctx context.Context, resources Resources) error {
+				return runSetMFAEnabled(ctx, resources, cmd.OutOrStdout(), false)
+			})
+		},
+	}
+
+	statusCmd := &cobra.Command{
+		Use:   "status",
+		Short: "Show local MFA status",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			return opener(cmd, func(ctx context.Context, resources Resources) error {
+				return runMFAStatus(ctx, resources, cmd.OutOrStdout())
+			})
+		},
+	}
+
+	mfaCmd.AddCommand(enableCmd, disableCmd, statusCmd)
+	adminCmd.AddCommand(userCmd, mfaCmd)
+	return adminCmd
+}
+
+// OpenEmbeddedIDPStorage opens the Dex storage configured for the embedded IdP.
+func OpenEmbeddedIDPStorage(cfg *idp.EmbeddedIdPConfig) (storage.Storage, error) {
+	if cfg == nil || !cfg.Enabled {
+		return nil, fmt.Errorf("admin commands require the embedded IdP to be enabled")
+	}
+
+	yamlConfig, err := cfg.ToYAMLConfig()
+	if err != nil {
+		return nil, fmt.Errorf("build embedded IdP config: %w", err)
+	}
+
+	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+	st, err := yamlConfig.Storage.OpenStorage(logger)
+	if err != nil {
+		return nil, fmt.Errorf("open embedded IdP storage: %w", err)
+	}
+	return st, nil
+}
+
+func addUserSelectorFlags(cmd *cobra.Command, selector *userSelector) {
+	cmd.Flags().StringVar(&selector.email, "email", "", "User email")
+	cmd.Flags().StringVar(&selector.userID, "user-id", "", "User ID")
+}
+
+func resolvePasswordInput(cmd *cobra.Command, password, passwordFile string) (string, error) {
+	if password != "" && passwordFile != "" {
+		return "", fmt.Errorf("provide only one of --password or --password-file")
+	}
+	if passwordFile == "" {
+		return password, nil
+	}
+
+	var data []byte
+	var err error
+	if passwordFile == "-" {
+		data, err = io.ReadAll(cmd.InOrStdin())
+	} else {
+		data, err = os.ReadFile(passwordFile)
+	}
+	if err != nil {
+		return "", fmt.Errorf("read password: %w", err)
+	}
+	return strings.TrimRight(string(data), "\r\n"), nil
+}
+
+func runChangePassword(ctx context.Context, idpStorage storage.Storage, w io.Writer, selector userSelector, password string) error {
+	if idpStorage == nil {
+		return fmt.Errorf("embedded IdP storage is required")
+	}
+	selector = selector.normalized()
+	if err := selector.validate(); err != nil {
+		return err
+	}
+	if password == "" {
+		return fmt.Errorf("password is required")
+	}
+	if err := server.ValidatePassword(password); err != nil {
+		return fmt.Errorf("invalid password: %w", err)
+	}
+
+	user, err := findLocalUser(ctx, idpStorage, selector)
+	if err != nil {
+		return err
+	}
+
+	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
+	if err != nil {
+		return fmt.Errorf("hash password: %w", err)
+	}
+
+	if err := idpStorage.UpdatePassword(ctx, user.Email, func(old storage.Password) (storage.Password, error) {
+		old.Hash = hash
+		return old, nil
+	}); err != nil {
+		return fmt.Errorf("update password for %s: %w", user.Email, err)
+	}
+
+	if err := deleteLocalAuthSession(ctx, idpStorage, user.UserID); err != nil {
+		return err
+	}
+
+	_, _ = fmt.Fprintf(w, "Password updated for %s.\n", user.Email)
+	return nil
+}
+
+func runResetMFA(ctx context.Context, idpStorage storage.Storage, w io.Writer, selector userSelector) error {
+	if idpStorage == nil {
+		return fmt.Errorf("embedded IdP storage is required")
+	}
+	selector = selector.normalized()
+	if err := selector.validate(); err != nil {
+		return err
+	}
+
+	user, err := findLocalUser(ctx, idpStorage, selector)
+	if err != nil {
+		return err
+	}
+
+	reset := false
+	err = idpStorage.UpdateUserIdentity(ctx, user.UserID, localConnectorID, func(old storage.UserIdentity) (storage.UserIdentity, error) {
+		reset = reset || len(old.MFASecrets) > 0 || len(old.WebAuthnCredentials) > 0
+		old.MFASecrets = map[string]*storage.MFASecret{}
+		old.WebAuthnCredentials = map[string][]storage.WebAuthnCredential{}
+		return old, nil
+	})
+	if errors.Is(err, storage.ErrNotFound) {
+		if err := deleteLocalAuthSession(ctx, idpStorage, user.UserID); err != nil {
+			return err
+		}
+		_, _ = fmt.Fprintf(w, "No MFA enrollment found for %s.\n", user.Email)
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("reset MFA for %s: %w", user.Email, err)
+	}
+
+	if err := deleteLocalAuthSession(ctx, idpStorage, user.UserID); err != nil {
+		return err
+	}
+
+	if reset {
+		_, _ = fmt.Fprintf(w, "MFA reset for %s. The user will re-enroll at next login.\n", user.Email)
+	} else {
+		_, _ = fmt.Fprintf(w, "No MFA enrollment found for %s.\n", user.Email)
+	}
+	return nil
+}
+
+func runSetMFAEnabled(ctx context.Context, resources Resources, w io.Writer, enabled bool) error {
+	if resources.Store == nil {
+		return fmt.Errorf("management store is required")
+	}
+	if resources.IDPStorage == nil {
+		return fmt.Errorf("embedded IdP storage is required")
+	}
+
+	accounts := resources.Store.GetAllAccounts(ctx)
+	if len(accounts) != 1 {
+		return fmt.Errorf("expected exactly one account, got %d; local MFA is supported only in single-account embedded IdP deployments", len(accounts))
+	}
+
+	settings := &types.Settings{}
+	if accounts[0].Settings != nil {
+		settings = accounts[0].Settings.Copy()
+	}
+	settings.LocalMfaEnabled = enabled
+	if err := resources.Store.SaveAccountSettings(ctx, accounts[0].Id, settings); err != nil {
+		return fmt.Errorf("save local MFA account setting: %w", err)
+	}
+
+	if err := setIDPClientsMFA(ctx, resources.IDPStorage, enabled); err != nil {
+		return err
+	}
+
+	state := "disabled"
+	if enabled {
+		state = "enabled"
+	}
+	_, _ = fmt.Fprintf(w, "Local MFA %s.\n", state)
+	return nil
+}
+
+func runMFAStatus(ctx context.Context, resources Resources, w io.Writer) error {
+	if resources.Store == nil {
+		return fmt.Errorf("management store is required")
+	}
+	if resources.IDPStorage == nil {
+		return fmt.Errorf("embedded IdP storage is required")
+	}
+
+	accounts := resources.Store.GetAllAccounts(ctx)
+	accountStatus := "unknown"
+	if len(accounts) == 1 && accounts[0].Settings != nil {
+		accountStatus = "disabled"
+		if accounts[0].Settings.LocalMfaEnabled {
+			accountStatus = "enabled"
+		}
+	}
+
+	clientStatus, err := idpClientsMFAStatus(ctx, resources.IDPStorage)
+	if err != nil {
+		return err
+	}
+
+	_, _ = fmt.Fprintf(w, "Account setting: %s\n", accountStatus)
+	_, _ = fmt.Fprintf(w, "Embedded IdP clients: %s\n", clientStatus)
+	return nil
+}
+
+func findLocalUser(ctx context.Context, idpStorage storage.Storage, selector userSelector) (storage.Password, error) {
+	selector = selector.normalized()
+	if err := selector.validate(); err != nil {
+		return storage.Password{}, err
+	}
+
+	if selector.email != "" {
+		user, err := idpStorage.GetPassword(ctx, selector.email)
+		if errors.Is(err, storage.ErrNotFound) {
+			return storage.Password{}, fmt.Errorf("local user with email %q not found", selector.email)
+		}
+		if err != nil {
+			return storage.Password{}, fmt.Errorf("get local user by email %q: %w", selector.email, err)
+		}
+		return user, nil
+	}
+
+	rawUserID := selector.userID
+	if decodedUserID, _, err := nbdex.DecodeDexUserID(selector.userID); err == nil && decodedUserID != "" {
+		rawUserID = decodedUserID
+	}
+
+	users, err := idpStorage.ListPasswords(ctx)
+	if err != nil {
+		return storage.Password{}, fmt.Errorf("list local users: %w", err)
+	}
+	for _, user := range users {
+		if user.UserID == rawUserID || user.UserID == selector.userID {
+			return user, nil
+		}
+	}
+
+	return storage.Password{}, fmt.Errorf("local user with ID %q not found", selector.userID)
+}
+
+func deleteLocalAuthSession(ctx context.Context, idpStorage storage.Storage, userID string) error {
+	err := idpStorage.DeleteAuthSession(ctx, userID, localConnectorID)
+	if err == nil || errors.Is(err, storage.ErrNotFound) {
+		return nil
+	}
+	return fmt.Errorf("delete local auth session for user %s: %w", userID, err)
+}
+
+func setIDPClientsMFA(ctx context.Context, idpStorage storage.Storage, enabled bool) error {
+	var mfaChain []string
+	if enabled {
+		mfaChain = []string{defaultTOTPAuthenticatorID}
+	}
+
+	for _, clientID := range []string{cliClientID, dashboardClientID} {
+		if err := idpStorage.UpdateClient(ctx, clientID, func(old storage.Client) (storage.Client, error) {
+			old.MFAChain = mfaChain
+			return old, nil
+		}); err != nil {
+			if errors.Is(err, storage.ErrNotFound) {
+				return fmt.Errorf("embedded IdP client %q not found; start the management server once before toggling MFA", clientID)
+			}
+			return fmt.Errorf("update MFA chain on embedded IdP client %q: %w", clientID, err)
+		}
+	}
+	return nil
+}
+
+func idpClientsMFAStatus(ctx context.Context, idpStorage storage.Storage) (string, error) {
+	clientIDs := []string{cliClientID, dashboardClientID}
+	enabledCount := 0
+	for _, clientID := range clientIDs {
+		client, err := idpStorage.GetClient(ctx, clientID)
+		if errors.Is(err, storage.ErrNotFound) {
+			return "unknown", fmt.Errorf("embedded IdP client %q not found", clientID)
+		}
+		if err != nil {
+			return "unknown", fmt.Errorf("get embedded IdP client %q: %w", clientID, err)
+		}
+		if hasAuthenticator(client.MFAChain, defaultTOTPAuthenticatorID) {
+			enabledCount++
+		}
+	}
+
+	switch enabledCount {
+	case 0:
+		return "disabled", nil
+	case len(clientIDs):
+		return "enabled", nil
+	default:
+		return "partially enabled", nil
+	}
+}
+
+func hasAuthenticator(chain []string, authenticatorID string) bool {
+	for _, id := range chain {
+		if id == authenticatorID {
+			return true
+		}
+	}
+	return false
+}

management/cmd/admin/admin_test.go

@@ -0,0 +1,160 @@
+package admincmd
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"log/slog"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/dexidp/dex/storage"
+	"github.com/dexidp/dex/storage/memory"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/crypto/bcrypt"
+
+	nbdex "github.com/netbirdio/netbird/idp/dex"
+)
+
+func newTestIDPStorage(t *testing.T) storage.Storage {
+	t.Helper()
+
+	st := memory.New(slog.New(slog.NewTextHandler(io.Discard, nil)))
+	hash, err := bcrypt.GenerateFromPassword([]byte("OldPass1!"), bcrypt.DefaultCost)
+	require.NoError(t, err)
+
+	require.NoError(t, st.CreatePassword(context.Background(), storage.Password{
+		Email:    "user@example.com",
+		Username: "User",
+		UserID:   "user-1",
+		Hash:     hash,
+	}))
+	require.NoError(t, st.CreateUserIdentity(context.Background(), storage.UserIdentity{
+		UserID:      "user-1",
+		ConnectorID: localConnectorID,
+		MFASecrets: map[string]*storage.MFASecret{
+			defaultTOTPAuthenticatorID: {
+				AuthenticatorID: defaultTOTPAuthenticatorID,
+				Type:            "TOTP",
+				Secret:          "otpauth://totp/NetBird:user@example.com?secret=ABC",
+				Confirmed:       true,
+				CreatedAt:       time.Now(),
+			},
+		},
+		WebAuthnCredentials: map[string][]storage.WebAuthnCredential{
+			"webauthn": {{CredentialID: []byte("credential")}},
+		},
+	}))
+	require.NoError(t, st.CreateAuthSession(context.Background(), storage.AuthSession{
+		UserID:      "user-1",
+		ConnectorID: localConnectorID,
+		Nonce:       "nonce",
+	}))
+	require.NoError(t, st.CreateClient(context.Background(), storage.Client{ID: cliClientID, Name: "CLI"}))
+	require.NoError(t, st.CreateClient(context.Background(), storage.Client{ID: dashboardClientID, Name: "Dashboard"}))
+
+	return st
+}
+
+func TestRunChangePassword(t *testing.T) {
+	ctx := context.Background()
+	st := newTestIDPStorage(t)
+	var out bytes.Buffer
+
+	err := runChangePassword(ctx, st, &out, userSelector{email: "user@example.com"}, "NewPass1!")
+	require.NoError(t, err)
+	require.Contains(t, out.String(), "Password updated")
+
+	user, err := st.GetPassword(ctx, "user@example.com")
+	require.NoError(t, err)
+	require.NoError(t, bcrypt.CompareHashAndPassword(user.Hash, []byte("NewPass1!")))
+
+	_, err = st.GetAuthSession(ctx, "user-1", localConnectorID)
+	require.ErrorIs(t, err, storage.ErrNotFound)
+}
+
+func TestRunChangePasswordValidatesPassword(t *testing.T) {
+	st := newTestIDPStorage(t)
+	err := runChangePassword(context.Background(), st, io.Discard, userSelector{email: "user@example.com"}, "short")
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "invalid password")
+}
+
+func TestRunResetMFA(t *testing.T) {
+	ctx := context.Background()
+	st := newTestIDPStorage(t)
+	var out bytes.Buffer
+
+	encodedUserID := nbdex.EncodeDexUserID("user-1", localConnectorID)
+	err := runResetMFA(ctx, st, &out, userSelector{userID: encodedUserID})
+	require.NoError(t, err)
+	require.Contains(t, out.String(), "MFA reset")
+
+	identity, err := st.GetUserIdentity(ctx, "user-1", localConnectorID)
+	require.NoError(t, err)
+	require.Empty(t, identity.MFASecrets)
+	require.Empty(t, identity.WebAuthnCredentials)
+
+	_, err = st.GetAuthSession(ctx, "user-1", localConnectorID)
+	require.ErrorIs(t, err, storage.ErrNotFound)
+}
+
+func TestRunResetMFAWithoutEnrollment(t *testing.T) {
+	ctx := context.Background()
+	st := newTestIDPStorage(t)
+	require.NoError(t, st.UpdateUserIdentity(ctx, "user-1", localConnectorID, func(old storage.UserIdentity) (storage.UserIdentity, error) {
+		old.MFASecrets = nil
+		old.WebAuthnCredentials = nil
+		return old, nil
+	}))
+
+	var out bytes.Buffer
+	err := runResetMFA(ctx, st, &out, userSelector{email: "user@example.com"})
+	require.NoError(t, err)
+	require.Contains(t, out.String(), "No MFA enrollment found")
+}
+
+func TestSetIDPClientsMFA(t *testing.T) {
+	ctx := context.Background()
+	st := newTestIDPStorage(t)
+
+	require.NoError(t, setIDPClientsMFA(ctx, st, true))
+	status, err := idpClientsMFAStatus(ctx, st)
+	require.NoError(t, err)
+	require.Equal(t, "enabled", status)
+
+	require.NoError(t, setIDPClientsMFA(ctx, st, false))
+	status, err = idpClientsMFAStatus(ctx, st)
+	require.NoError(t, err)
+	require.Equal(t, "disabled", status)
+}
+
+func TestUserSelectorValidate(t *testing.T) {
+	require.NoError(t, userSelector{email: " user@example.com "}.validate())
+	require.NoError(t, userSelector{userID: "user-1"}.validate())
+	require.Error(t, userSelector{}.validate())
+	require.Error(t, userSelector{email: "user@example.com", userID: "user-1"}.validate())
+}
+
+func TestFindLocalUserNotFound(t *testing.T) {
+	st := newTestIDPStorage(t)
+	_, err := findLocalUser(context.Background(), st, userSelector{email: "missing@example.com"})
+	require.Error(t, err)
+	require.True(t, strings.Contains(err.Error(), "not found"))
+}
+
+func TestResolvePasswordInputFromStdin(t *testing.T) {
+	cmd := &cobra.Command{}
+	cmd.SetIn(strings.NewReader("NewPass1!\n"))
+
+	password, err := resolvePasswordInput(cmd, "", "-")
+	require.NoError(t, err)
+	require.Equal(t, "NewPass1!", password)
+}
+
+func TestResolvePasswordInputRejectsMultipleSources(t *testing.T) {
+	_, err := resolvePasswordInput(&cobra.Command{}, "NewPass1!", "-")
+	require.Error(t, err)
+}

management/cmd/root.go

@@ -83,7 +83,7 @@ func init() {
 
 	rootCmd.AddCommand(migrationCmd)
 
-	tc := newTokenCommands()
-	tc.PersistentFlags().StringVar(&nbconfig.MgmtConfigPath, "config", defaultMgmtConfig, "Netbird config file location")
-	rootCmd.AddCommand(tc)
+	ac := newAdminCommands()
+	ac.PersistentFlags().StringVar(&nbconfig.MgmtConfigPath, "config", defaultMgmtConfig, "Netbird config file location")
+	rootCmd.AddCommand(ac)
 }

management/cmd/token.go

@@ -1,55 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/netbirdio/netbird/formatter/hook"
-	tokencmd "github.com/netbirdio/netbird/management/cmd/token"
-	nbconfig "github.com/netbirdio/netbird/management/internals/server/config"
-	"github.com/netbirdio/netbird/management/server/store"
-	"github.com/netbirdio/netbird/util"
-)
-
-var tokenDatadir string
-
-// newTokenCommands creates the token command tree with management-specific store opener.
-func newTokenCommands() *cobra.Command {
-	cmd := tokencmd.NewCommands(withTokenStore)
-	cmd.PersistentFlags().StringVar(&tokenDatadir, "datadir", "", "Override the data directory from config (where store.db is located)")
-	return cmd
-}
-
-// withTokenStore initializes logging, loads config, opens the store, and calls fn.
-func withTokenStore(cmd *cobra.Command, fn func(ctx context.Context, s store.Store) error) error {
-	if err := util.InitLog("error", "console"); err != nil {
-		return fmt.Errorf("init log: %w", err)
-	}
-
-	ctx := context.WithValue(cmd.Context(), hook.ExecutionContextKey, hook.SystemSource) //nolint:staticcheck
-
-	config, err := LoadMgmtConfig(ctx, nbconfig.MgmtConfigPath)
-	if err != nil {
-		return fmt.Errorf("load config: %w", err)
-	}
-
-	datadir := config.Datadir
-	if tokenDatadir != "" {
-		datadir = tokenDatadir
-	}
-
-	s, err := store.NewStore(ctx, config.StoreConfig.Engine, datadir, nil, true)
-	if err != nil {
-		return fmt.Errorf("create store: %w", err)
-	}
-	defer func() {
-		if err := s.Close(ctx); err != nil {
-			log.Debugf("close store: %v", err)
-		}
-	}()
-
-	return fn(ctx, s)
-}

management/server/user.go

@@ -1849,12 +1849,17 @@ func (am *DefaultAccountManager) DeleteUserInvite(ctx context.Context, accountID
 
 const minPasswordLength = 8
 
-// validatePassword checks password strength requirements:
+// validatePassword checks password strength requirements.
+func validatePassword(password string) error {
+	return ValidatePassword(password)
+}
+
+// ValidatePassword checks password strength requirements:
 // - Minimum 8 characters
 // - At least 1 digit
 // - At least 1 uppercase letter
 // - At least 1 special character
-func validatePassword(password string) error {
+func ValidatePassword(password string) error {
 	if len(password) < minPasswordLength {
 		return errors.New("password must be at least 8 characters long")
 	}

shared/management/client/grpc.go

@@ -33,10 +33,15 @@ const ConnectTimeout = 10 * time.Second
 const healthCheckTimeout = 5 * time.Second
 
 const (
-	// EnvMaxRecvMsgSize overrides the default gRPC max receive message size (4 MB)
+	// EnvMaxRecvMsgSize overrides the default gRPC max receive message size
 	// for the management client connection. Value is in bytes.
 	EnvMaxRecvMsgSize = "NB_MANAGEMENT_GRPC_MAX_MSG_SIZE"
 
+	// defaultMaxRecvMsgSize is the max gRPC receive message size used for the
+	// management client connection when EnvMaxRecvMsgSize is unset or invalid.
+	// It overrides the gRPC library default of 4 MB.
+	defaultMaxRecvMsgSize = 1024 * 1024 * 16
+
 	errMsgMgmtPublicKey    = "failed getting Management Service public key: %s"
 	errMsgNoMgmtConnection = "no connection to management"
 )
@@ -84,22 +89,22 @@ type ExposeResponse struct {
 }
 
 // MaxRecvMsgSize returns the configured max gRPC receive message size from
-// the environment, or 0 if unset (which uses the gRPC default of 4 MB).
+// the environment, or defaultMaxRecvMsgSize (16 MB) if unset or invalid.
 func MaxRecvMsgSize() int {
 	val := os.Getenv(EnvMaxRecvMsgSize)
 	if val == "" {
-		return 0
+		return defaultMaxRecvMsgSize
 	}
 
 	size, err := strconv.Atoi(val)
 	if err != nil {
 		log.Warnf("invalid %s value %q, using default: %v", EnvMaxRecvMsgSize, val, err)
-		return 0
+		return defaultMaxRecvMsgSize
 	}
 
 	if size <= 0 {
 		log.Warnf("invalid %s value %d, must be positive, using default", EnvMaxRecvMsgSize, size)
-		return 0
+		return defaultMaxRecvMsgSize
 	}
 
 	return size

shared/management/client/grpc_test.go

@@ -21,11 +21,11 @@ func TestMaxRecvMsgSize(t *testing.T) {
 		envValue string
 		expected int
 	}{
-		{name: "unset returns 0", envValue: "", expected: 0},
+		{name: "unset returns default", envValue: "", expected: defaultMaxRecvMsgSize},
 		{name: "valid value", envValue: "10485760", expected: 10485760},
-		{name: "non-numeric returns 0", envValue: "abc", expected: 0},
-		{name: "negative returns 0", envValue: "-1", expected: 0},
-		{name: "zero returns 0", envValue: "0", expected: 0},
+		{name: "non-numeric returns default", envValue: "abc", expected: defaultMaxRecvMsgSize},
+		{name: "negative returns default", envValue: "-1", expected: defaultMaxRecvMsgSize},
+		{name: "zero returns default", envValue: "0", expected: defaultMaxRecvMsgSize},
 	}
 
 	for _, tt := range tests {