[client] Add debug cpu start and stop commands (#7749)

* Add debug cpu start and stop commands to profile the daemon without a restart * Restore test globals on every exit and stop the daemon in the cpu profile test * Add a no-updown flag to debug for * Enable sync response persistence with --no-updown and reset flags between debug test runs * Reset flags of every command between debug test runs * Reset slice flags with Replace in the debug test helper * Explain a running CPU profile in debug for and document cpu start and no-updown limits
2026-09-30 18:49:08 +02:00 · 2026-09-30 17:37:49 +02:00
parent 0dc729c4ea
commit 6c453a0f97
3 changed files with 305 additions and 26 deletions
@@ -23,7 +23,10 @@ import (
 	"github.com/netbirdio/netbird/version"
 )

-const errCloseConnection = "Failed to close connection: %v"
+const (
+	errCloseConnection = "Failed to close connection: %v"
+	noUpDownFlag       = "no-updown"
+)

 var (
 	logFileCount             uint32
@@ -257,13 +260,14 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 	}

 	stateWasDown := stat.Status != string(internal.StatusConnected) && stat.Status != string(internal.StatusConnecting)
+	noUpDown, _ := cmd.Flags().GetBool(noUpDownFlag)

 	initialLogLevel, err := client.GetLogLevel(cmd.Context(), &proto.GetLogLevelRequest{})
 	if err != nil {
 		return fmt.Errorf("failed to get log level: %v", status.Convert(err).Message())
 	}

-	if stateWasDown {
+	if stateWasDown && !noUpDown {
 		if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
 			cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message())
 		} else {
@@ -284,34 +288,20 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 	}

 	needsRestoreUp := false
-	if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
-		cmd.PrintErrf("Failed to bring service down: %v\n", status.Convert(err).Message())
+	if noUpDown {
+		enableSyncResponsePersistence(cmd, client)
 	} else {
-		needsRestoreUp = !stateWasDown
-		cmd.Println("netbird down")
+		needsRestoreUp = restartDaemon(cmd, client, stateWasDown)
 	}

-	time.Sleep(1 * time.Second)
-
-	// Enable sync response persistence before bringing the service up
-	if _, err := client.SetSyncResponsePersistence(cmd.Context(), &proto.SetSyncResponsePersistenceRequest{
-		Enabled: true,
-	}); err != nil {
-		cmd.PrintErrf("Failed to enable sync response persistence: %v\n", status.Convert(err).Message())
-	}
-
-	if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
-		cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message())
-	} else {
-		needsRestoreUp = false
-		cmd.Println("netbird up")
-	}
-
-	time.Sleep(3 * time.Second)
-
 	cpuProfilingStarted := false
 	if _, err := client.StartCPUProfile(cmd.Context(), &proto.StartCPUProfileRequest{}); err != nil {
-		cmd.PrintErrf("Failed to start CPU profiling: %v\n", err)
+		if msg := status.Convert(err).Message(); strings.Contains(msg, "already in progress") {
+			cmd.PrintErrln("CPU profiling is already running (started with `netbird debug cpu start`). " +
+				"It is left running and is included in a bundle created after `netbird debug cpu stop`.")
+		} else {
+			cmd.PrintErrf("Failed to start CPU profiling: %v\n", msg)
+		}
 	} else {
 		cpuProfilingStarted = true
 		defer func() {
@@ -401,7 +391,7 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 		}
 	}

-	if stateWasDown {
+	if stateWasDown && !noUpDown {
 		if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
 			cmd.PrintErrf("Failed to restore service down state: %v\n", status.Convert(err).Message())
 		} else {
@@ -458,6 +448,47 @@ func setSyncResponsePersistence(cmd *cobra.Command, args []string) error {
 	return nil
 }

+// enableSyncResponsePersistence asks the daemon to keep the latest sync
+// response so the bundle carries the network map. With a running daemon only
+// syncs received after the call are kept.
+func enableSyncResponsePersistence(cmd *cobra.Command, client proto.DaemonServiceClient) {
+	if _, err := client.SetSyncResponsePersistence(cmd.Context(), &proto.SetSyncResponsePersistenceRequest{
+		Enabled: true,
+	}); err != nil {
+		cmd.PrintErrf("Failed to enable sync response persistence: %v\n", status.Convert(err).Message())
+	}
+}
+
+// restartDaemon cycles the daemon down and up with sync response persistence
+// enabled so the bundle carries the network map. It reports whether the
+// daemon was left down although it was running before, so the caller can
+// bring it back up.
+func restartDaemon(cmd *cobra.Command, client proto.DaemonServiceClient, stateWasDown bool) bool {
+	needsRestoreUp := false
+	if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
+		cmd.PrintErrf("Failed to bring service down: %v\n", status.Convert(err).Message())
+	} else {
+		needsRestoreUp = !stateWasDown
+		cmd.Println("netbird down")
+	}
+
+	time.Sleep(1 * time.Second)
+
+	// Enable sync response persistence before bringing the service up
+	enableSyncResponsePersistence(cmd, client)
+
+	if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
+		cmd.PrintErrf("Failed to bring service up: %v\n", status.Convert(err).Message())
+	} else {
+		needsRestoreUp = false
+		cmd.Println("netbird up")
+	}
+
+	time.Sleep(3 * time.Second)
+
+	return needsRestoreUp
+}
+
 func waitForDurationOrCancel(ctx context.Context, duration time.Duration, cmd *cobra.Command) error {
 	ticker := time.NewTicker(1 * time.Second)
 	defer ticker.Stop()
@@ -546,4 +577,5 @@ func init() {
 	forCmd.Flags().StringVar(&uploadBundleURLFlag, "upload-bundle-url", types.DefaultBundleURL, "Service URL to get an URL to upload the debug bundle")
 	forCmd.Flags().BoolVar(&uploadBundleInsecureFlag, "upload-bundle-insecure", false, "Allow uploading to an http or untrusted-TLS upload server (self-hosted); requires root")
 	forCmd.Flags().Bool("capture", false, "Capture packets during the debug duration and include in bundle")
+	forCmd.Flags().Bool(noUpDownFlag, false, "Keep the daemon running instead of bringing it down and up before collecting. The bundle only includes the network map if a sync arrives during the run")
 }
@@ -0,0 +1,83 @@
+package cmd
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/status"
+
+	"github.com/netbirdio/netbird/client/proto"
+)
+
+var debugCPUCmd = &cobra.Command{
+	Use:   "cpu",
+	Short: "Profile the daemon's CPU usage",
+	Long: `Starts and stops CPU profiling in the running daemon without restarting it.
+The profile is included in the next debug bundle as cpu.prof.
+
+Profiling is not time limited: it keeps running, and keeps costing CPU, until
+"netbird debug cpu stop" is run.`,
+}
+
+var debugCPUStartCmd = &cobra.Command{
+	Use:     "start",
+	Short:   "Start CPU profiling in the daemon",
+	Example: "  netbird debug cpu start",
+	Args:    cobra.NoArgs,
+	RunE:    debugCPUStart,
+}
+
+var debugCPUStopCmd = &cobra.Command{
+	Use:   "stop",
+	Short: "Stop CPU profiling in the daemon",
+	Long: `Stops CPU profiling. The captured profile stays in the daemon until the next
+debug bundle is created, which includes it as cpu.prof.`,
+	Example: "  netbird debug cpu stop && netbird debug bundle",
+	Args:    cobra.NoArgs,
+	RunE:    debugCPUStop,
+}
+
+func debugCPUStart(cmd *cobra.Command, _ []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	if _, err := proto.NewDaemonServiceClient(conn).StartCPUProfile(cmd.Context(), &proto.StartCPUProfileRequest{}); err != nil {
+		return fmt.Errorf("start CPU profiling: %v", status.Convert(err).Message())
+	}
+
+	cmd.Println("CPU profiling started and runs until stopped. Run `netbird debug cpu stop` and then `netbird debug bundle` to collect it.")
+	return nil
+}
+
+func debugCPUStop(cmd *cobra.Command, _ []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	if _, err := proto.NewDaemonServiceClient(conn).StopCPUProfile(cmd.Context(), &proto.StopCPUProfileRequest{}); err != nil {
+		return fmt.Errorf("stop CPU profiling: %v", status.Convert(err).Message())
+	}
+
+	cmd.Println("CPU profiling stopped. Run `netbird debug bundle` to include cpu.prof.")
+	return nil
+}
+
+func init() {
+	debugCPUCmd.AddCommand(debugCPUStartCmd)
+	debugCPUCmd.AddCommand(debugCPUStopCmd)
+	debugCmd.AddCommand(debugCPUCmd)
+}
@@ -0,0 +1,164 @@
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"os/user"
+	"strings"
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/client/internal"
+	"github.com/netbirdio/netbird/client/internal/profilemanager"
+)
+
+// startDebugTestDaemon starts an in-process daemon with an isolated profile
+// directory and returns the address the CLI should dial.
+func startDebugTestDaemon(t *testing.T) string {
+	t.Helper()
+
+	tempDir := t.TempDir()
+	origDefaultProfileDir := profilemanager.DefaultConfigPathDir
+	origActiveProfileStatePath := profilemanager.ActiveProfileStatePath
+	origConfigDirOverride := profilemanager.ConfigDirOverride
+	origDaemonAddr := daemonAddr
+	t.Cleanup(func() {
+		profilemanager.DefaultConfigPathDir = origDefaultProfileDir
+		profilemanager.ActiveProfileStatePath = origActiveProfileStatePath
+		profilemanager.ConfigDirOverride = origConfigDirOverride
+		daemonAddr = origDaemonAddr
+	})
+
+	profilemanager.DefaultConfigPathDir = tempDir
+	profilemanager.ActiveProfileStatePath = tempDir + "/active_profile.json"
+	profilemanager.ConfigDirOverride = tempDir
+
+	currUser, err := user.Current()
+	require.NoError(t, err)
+	sm := profilemanager.ServiceManager{}
+	created, err := sm.AddProfile("test1", currUser.Username)
+	require.NoError(t, err)
+	require.NoError(t, sm.SetActiveProfileState(&profilemanager.ActiveProfileState{
+		ID:       created.ID,
+		Username: currUser.Username,
+	}))
+
+	ctx, cancel := context.WithCancel(internal.CtxInitState(context.Background()))
+	srv, lis := startClientDaemon(t, ctx, "", tempDir+"/config.json")
+	t.Cleanup(func() {
+		cancel()
+		srv.Stop()
+	})
+
+	return "tcp://" + lis.Addr().String()
+}
+
+// runDebugCmd runs `netbird debug <args>` against the daemon at addr and
+// returns everything the command printed.
+func runDebugCmd(addr string, args ...string) (string, error) {
+	daemonAddr = addr
+	var out bytes.Buffer
+	rootCmd.SetOut(&out)
+	rootCmd.SetErr(&out)
+	rootCmd.SetArgs(append(append([]string{"debug"}, args...), "--daemon-addr", addr, "--log-file", ""))
+	err := rootCmd.Execute()
+	rootCmd.SetOut(nil)
+	rootCmd.SetErr(nil)
+	rootCmd.SetArgs(nil)
+	resetFlags(rootCmd)
+	return out.String(), err
+}
+
+// resetFlags puts every flag of the command and its subcommands back to its
+// default so a value parsed in one run does not leak into the next in-process
+// execution.
+func resetFlags(cmd *cobra.Command) {
+	reset := func(f *pflag.Flag) {
+		// Set appends to a slice flag and would parse the "[a,b]" default
+		// text as elements, so slices are replaced instead.
+		if sv, ok := f.Value.(pflag.SliceValue); ok {
+			var def []string
+			if trimmed := strings.Trim(f.DefValue, "[]"); trimmed != "" {
+				def = strings.Split(trimmed, ",")
+			}
+			_ = sv.Replace(def)
+		} else {
+			_ = f.Value.Set(f.DefValue)
+		}
+		f.Changed = false
+	}
+	cmd.Flags().VisitAll(reset)
+	cmd.PersistentFlags().VisitAll(reset)
+	// Commands pin their writers to the buffer of the run that first used
+	// them, so a later run would print into the old buffer.
+	cmd.SetOut(nil)
+	cmd.SetErr(nil)
+	for _, sub := range cmd.Commands() {
+		resetFlags(sub)
+	}
+}
+
+// TestResetFlagsSliceDefault guards against Set("[]") on slice flags, which
+// stores a literal "[]" element instead of the empty default.
+func TestResetFlagsSliceDefault(t *testing.T) {
+	cmd := &cobra.Command{Use: "x"}
+	var env, withDefault []string
+	cmd.Flags().StringSliceVar(&env, "env", nil, "")
+	cmd.Flags().StringSliceVar(&withDefault, "names", []string{"a", "b"}, "")
+	require.NoError(t, cmd.Flags().Parse([]string{"--env", "K=V", "--names", "c"}))
+
+	resetFlags(cmd)
+
+	assert.Empty(t, env, "slice flag with no default must reset to empty")
+	assert.Equal(t, []string{"a", "b"}, withDefault, "slice flag must reset to its default")
+}
+
+func TestDebugCPUStartStop(t *testing.T) {
+	addr := startDebugTestDaemon(t)
+
+	run := func(args ...string) error {
+		_, err := runDebugCmd(addr, append([]string{"cpu"}, args...)...)
+		return err
+	}
+
+	require.Error(t, run("stop"), "stop without a running profile must fail")
+	require.NoError(t, run("start"))
+	assert.Error(t, run("start"), "second start must be rejected while profiling")
+	require.NoError(t, run("stop"))
+	assert.Error(t, run("stop"), "second stop must be rejected")
+	assert.NoError(t, run("start"), "profiling can be started again after a stop")
+	assert.NoError(t, run("stop"))
+}
+
+// TestDebugForKeepsRunningCPUProfile covers `debug for` started while a
+// profile from `debug cpu start` is running: it must say so, leave the
+// profile alone, and still create the bundle.
+func TestDebugForKeepsRunningCPUProfile(t *testing.T) {
+	addr := startDebugTestDaemon(t)
+
+	_, err := runDebugCmd(addr, "cpu", "start")
+	require.NoError(t, err)
+
+	out, err := runDebugCmd(addr, "for", "1s", "-S=false", "--no-updown")
+	require.NoError(t, err, "output: %s", out)
+	assert.Contains(t, out, "CPU profiling is already running", "the conflict must be explained")
+	assert.NotContains(t, out, "rpc error", "the raw RPC error must not reach the user")
+	assert.Contains(t, out, "Local file:", "the bundle must still be created")
+
+	_, err = runDebugCmd(addr, "cpu", "stop")
+	assert.NoError(t, err, "the profile started by the user must still be running")
+}
+
+func TestDebugForNoUpDown(t *testing.T) {
+	addr := startDebugTestDaemon(t)
+
+	out, err := runDebugCmd(addr, "for", "1s", "-S=false", "--no-updown")
+	require.NoError(t, err, "output: %s", out)
+	assert.NotContains(t, out, "netbird down", "--no-updown must not bring the daemon down")
+	assert.NotContains(t, out, "netbird up", "--no-updown must not bring the daemon up")
+	assert.Contains(t, out, "Local file:", "the bundle must still be created")
+}