[client] Prevent corruption from competing log rotation and improve debug bundle (#6214)

* Adds heuristic to detect an edge case on Linux where a system has configured logrotate as a separate service to rotate log files which would mangle our client log files. If we detect logrotate being configured for netbird, we disable our rotation.

* Adds new env var to disable log rotation: NB_LOG_DISABLE_ROTATION

* Adds compressed and plain logrotate files to debug bundle.

* Replaces lumberjack with timberjack (maintained fork with bug fixes and extra features).

* Clarifies which daemon version is running in the bundle stats.

* Change logging for client service status to console

client/cmd/debug.go

@@ -19,6 +19,7 @@ import (
 	"github.com/netbirdio/netbird/client/server"
 	mgmProto "github.com/netbirdio/netbird/shared/management/proto"
 	"github.com/netbirdio/netbird/upload-server/types"
+	"github.com/netbirdio/netbird/version"
 )
 
 const errCloseConnection = "Failed to close connection: %v"
@@ -100,6 +101,7 @@ func debugBundle(cmd *cobra.Command, _ []string) error {
 		Anonymize:    anonymizeFlag,
 		SystemInfo:   systemInfoFlag,
 		LogFileCount: logFileCount,
+		CliVersion:   version.NetbirdVersion(),
 	}
 	if uploadBundleFlag {
 		request.UploadURL = uploadBundleURLFlag
@@ -298,6 +300,7 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 		Anonymize:    anonymizeFlag,
 		SystemInfo:   systemInfoFlag,
 		LogFileCount: logFileCount,
+		CliVersion:   version.NetbirdVersion(),
 	}
 	if uploadBundleFlag {
 		request.UploadURL = uploadBundleURLFlag
@@ -432,6 +435,7 @@ func generateDebugBundle(config *profilemanager.Config, recorder *peer.Status, c
 			SyncResponse:   syncResponse,
 			LogPath:        logFilePath,
 			CPUProfile:     nil,
+			DaemonVersion:  version.NetbirdVersion(), // acting as daemon
 		},
 		debug.BundleConfig{
 			IncludeSystemInfo: true,

client/cmd/service_controller.go

@@ -102,7 +102,7 @@ func (p *program) Stop(srv service.Service) error {
 }
 
 // Common setup for service control commands
-func setupServiceControlCommand(cmd *cobra.Command, ctx context.Context, cancel context.CancelFunc) (service.Service, error) {
+func setupServiceControlCommand(cmd *cobra.Command, ctx context.Context, cancel context.CancelFunc, consoleLog bool) (service.Service, error) {
 	// rootCmd env vars are already applied by PersistentPreRunE.
 	SetFlagsFromEnvVars(serviceCmd)
 
@@ -112,8 +112,14 @@ func setupServiceControlCommand(cmd *cobra.Command, ctx context.Context, cancel
 		return nil, err
 	}
 
-	if err := util.InitLog(logLevel, logFiles...); err != nil {
-		return nil, fmt.Errorf("init log: %w", err)
+	if consoleLog {
+		if err := util.InitLog(logLevel, util.LogConsole); err != nil {
+			return nil, fmt.Errorf("init log: %w", err)
+		}
+	} else {
+		if err := util.InitLog(logLevel, logFiles...); err != nil {
+			return nil, fmt.Errorf("init log: %w", err)
+		}
 	}
 
 	cfg, err := newSVCConfig()
@@ -138,7 +144,7 @@ var runCmd = &cobra.Command{
 		SetupCloseHandler(ctx, cancel)
 		SetupDebugHandler(ctx, nil, nil, nil, util.FindFirstLogPath(logFiles))
 
-		s, err := setupServiceControlCommand(cmd, ctx, cancel)
+		s, err := setupServiceControlCommand(cmd, ctx, cancel, false)
 		if err != nil {
 			return err
 		}
@@ -152,7 +158,7 @@ var startCmd = &cobra.Command{
 	Short: "starts NetBird service",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		ctx, cancel := context.WithCancel(cmd.Context())
-		s, err := setupServiceControlCommand(cmd, ctx, cancel)
+		s, err := setupServiceControlCommand(cmd, ctx, cancel, false)
 		if err != nil {
 			return err
 		}
@@ -170,7 +176,7 @@ var stopCmd = &cobra.Command{
 	Short: "stops NetBird service",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		ctx, cancel := context.WithCancel(cmd.Context())
-		s, err := setupServiceControlCommand(cmd, ctx, cancel)
+		s, err := setupServiceControlCommand(cmd, ctx, cancel, false)
 		if err != nil {
 			return err
 		}
@@ -188,7 +194,7 @@ var restartCmd = &cobra.Command{
 	Short: "restarts NetBird service",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		ctx, cancel := context.WithCancel(cmd.Context())
-		s, err := setupServiceControlCommand(cmd, ctx, cancel)
+		s, err := setupServiceControlCommand(cmd, ctx, cancel, false)
 		if err != nil {
 			return err
 		}
@@ -206,7 +212,7 @@ var svcStatusCmd = &cobra.Command{
 	Short: "shows NetBird service status",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		ctx, cancel := context.WithCancel(cmd.Context())
-		s, err := setupServiceControlCommand(cmd, ctx, cancel)
+		s, err := setupServiceControlCommand(cmd, ctx, cancel, true)
 		if err != nil {
 			return err
 		}

client/internal/debug/debug.go

@@ -254,6 +254,8 @@ type BundleGenerator struct {
 	capturePath    string
 	refreshStatus  func() // Optional callback to refresh status before bundle generation
 	clientMetrics  MetricsExporter
+	daemonVersion  string
+	cliVersion     string
 
 	anonymize         bool
 	includeSystemInfo bool
@@ -278,6 +280,8 @@ type GeneratorDependencies struct {
 	CapturePath    string
 	RefreshStatus  func()
 	ClientMetrics  MetricsExporter
+	DaemonVersion  string
+	CliVersion     string
 }
 
 func NewBundleGenerator(deps GeneratorDependencies, cfg BundleConfig) *BundleGenerator {
@@ -299,6 +303,8 @@ func NewBundleGenerator(deps GeneratorDependencies, cfg BundleConfig) *BundleGen
 		capturePath:    deps.CapturePath,
 		refreshStatus:  deps.RefreshStatus,
 		clientMetrics:  deps.ClientMetrics,
+		daemonVersion:  deps.DaemonVersion,
+		cliVersion:     deps.CliVersion,
 
 		anonymize:         cfg.Anonymize,
 		includeSystemInfo: cfg.IncludeSystemInfo,
@@ -459,9 +465,11 @@ func (g *BundleGenerator) addStatus() error {
 		protoFullStatus := nbstatus.ToProtoFullStatus(fullStatus)
 		protoFullStatus.Events = g.statusRecorder.GetEventHistory()
 		overview := nbstatus.ConvertToStatusOutputOverview(protoFullStatus, nbstatus.ConvertOptions{
-			Anonymize:   g.anonymize,
-			ProfileName: profName,
+			Anonymize:     g.anonymize,
+			ProfileName:   profName,
+			DaemonVersion: g.daemonVersion,
 		})
+		overview.CliVersion = g.cliVersion
 		statusOutput := overview.FullDetailSummary()
 
 		statusReader := strings.NewReader(statusOutput)
@@ -1039,7 +1047,8 @@ func (g *BundleGenerator) addRotatedLogFiles(logDir string) {
 		return
 	}
 
-	pattern := filepath.Join(logDir, "client-*.log.gz")
+	// This regex will match both logs rotated by us and logrotate on linux
+	pattern := filepath.Join(logDir, "client*.log.*")
 	files, err := filepath.Glob(pattern)
 	if err != nil {
 		log.Warnf("failed to glob rotated logs: %v", err)
@@ -1072,7 +1081,12 @@ func (g *BundleGenerator) addRotatedLogFiles(logDir string) {
 
 	for i := 0; i < maxFiles; i++ {
 		name := filepath.Base(files[i])
-		if err := g.addSingleLogFileGz(files[i], name); err != nil {
+		if strings.HasSuffix(name, ".gz") {
+			err = g.addSingleLogFileGz(files[i], name)
+		} else {
+			err = g.addSingleLogfile(files[i], name)
+		}
+		if err != nil {
 			log.Warnf("failed to add rotated log %s: %v", name, err)
 		}
 	}

client/internal/debug/debug_logfiles_test.go

@@ -0,0 +1,103 @@
+package debug
+
+import (
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestAddRotatedLogFiles_PicksUpAllVariants asserts that the rotated-log
+// glob picks up logs rotated by timberjack (gzipped) and by logrotate (plain
+// and gzipped), and skips unrelated files.
+func TestAddRotatedLogFiles_PicksUpAllVariants(t *testing.T) {
+	dir := t.TempDir()
+
+	writeFile(t, filepath.Join(dir, "client.log"), "active log\n")
+	writeFile(t, filepath.Join(dir, "other.log"), "unrelated\n")
+
+	timberjackRotated := "client-2026-05-21T10-30-45.000.log.gz"
+	writeGzFile(t, filepath.Join(dir, timberjackRotated), "timberjack rotated content\n")
+
+	logrotatePlain := "client.log.1"
+	writeFile(t, filepath.Join(dir, logrotatePlain), "logrotate plain content\n")
+
+	logrotateGz := "client.log.2.gz"
+	writeGzFile(t, filepath.Join(dir, logrotateGz), "logrotate gz content\n")
+
+	names := runAddRotatedLogFiles(t, dir, 10)
+
+	require.Contains(t, names, timberjackRotated, "timberjack rotated file should be in bundle")
+	require.Contains(t, names, logrotatePlain, "logrotate plain rotated file should be in bundle")
+	require.Contains(t, names, logrotateGz, "logrotate gzipped rotated file should be in bundle")
+	require.NotContains(t, names, "client.log", "active log should not be added by addRotatedLogFiles")
+	require.NotContains(t, names, "other.log", "unrelated files should not be in bundle")
+}
+
+// TestAddRotatedLogFiles_RespectsLogFileCount asserts that only the newest
+// logFileCount rotated files are bundled, ordered by mtime.
+func TestAddRotatedLogFiles_RespectsLogFileCount(t *testing.T) {
+	dir := t.TempDir()
+
+	oldest := filepath.Join(dir, "client.log.3")
+	middle := filepath.Join(dir, "client.log.2")
+	newest := filepath.Join(dir, "client.log.1")
+	writeFile(t, oldest, "old\n")
+	writeFile(t, middle, "mid\n")
+	writeFile(t, newest, "new\n")
+
+	now := time.Now()
+	require.NoError(t, os.Chtimes(oldest, now.Add(-2*time.Hour), now.Add(-2*time.Hour)))
+	require.NoError(t, os.Chtimes(middle, now.Add(-1*time.Hour), now.Add(-1*time.Hour)))
+	require.NoError(t, os.Chtimes(newest, now, now))
+
+	names := runAddRotatedLogFiles(t, dir, 2)
+
+	require.Contains(t, names, "client.log.1")
+	require.Contains(t, names, "client.log.2")
+	require.NotContains(t, names, "client.log.3", "oldest file should be dropped when logFileCount=2")
+}
+
+// runAddRotatedLogFiles calls addRotatedLogFiles against a fresh in-memory
+// zip writer and returns the set of entry names that ended up in the archive.
+func runAddRotatedLogFiles(t *testing.T, dir string, logFileCount uint32) map[string]struct{} {
+	t.Helper()
+
+	var buf bytes.Buffer
+	g := &BundleGenerator{
+		archive:      zip.NewWriter(&buf),
+		logFileCount: logFileCount,
+	}
+	g.addRotatedLogFiles(dir)
+	require.NoError(t, g.archive.Close())
+
+	zr, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
+	require.NoError(t, err)
+
+	names := make(map[string]struct{}, len(zr.File))
+	for _, f := range zr.File {
+		names[f.Name] = struct{}{}
+	}
+	return names
+}
+
+func writeFile(t *testing.T, path, content string) {
+	t.Helper()
+	require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
+}
+
+func writeGzFile(t *testing.T, path, content string) {
+	t.Helper()
+	var buf bytes.Buffer
+	gw := gzip.NewWriter(&buf)
+	_, err := io.WriteString(gw, content)
+	require.NoError(t, err)
+	require.NoError(t, gw.Close())
+	require.NoError(t, os.WriteFile(path, buf.Bytes(), 0o644))
+}

client/internal/engine.go

@@ -72,6 +72,7 @@ import (
 	sProto "github.com/netbirdio/netbird/shared/signal/proto"
 	"github.com/netbirdio/netbird/util"
 	"github.com/netbirdio/netbird/util/capture"
+	"github.com/netbirdio/netbird/version"
 )
 
 // PeerConnectionTimeoutMax is a timeout of an initial connection attempt to a remote peer.
@@ -1151,6 +1152,7 @@ func (e *Engine) handleBundle(params *mgmProto.BundleParameters) (*mgmProto.JobR
 		LogPath:        e.config.LogPath,
 		TempDir:        e.config.TempDir,
 		ClientMetrics:  e.clientMetrics,
+		DaemonVersion:  version.NetbirdVersion(),
 		RefreshStatus: func() {
 			e.RunHealthProbes(true)
 		},

client/proto/daemon.pb.go

@@ -2717,6 +2717,7 @@ type DebugBundleRequest struct {
 	SystemInfo    bool                   `protobuf:"varint,3,opt,name=systemInfo,proto3" json:"systemInfo,omitempty"`
 	UploadURL     string                 `protobuf:"bytes,4,opt,name=uploadURL,proto3" json:"uploadURL,omitempty"`
 	LogFileCount  uint32                 `protobuf:"varint,5,opt,name=logFileCount,proto3" json:"logFileCount,omitempty"`
+	CliVersion    string                 `protobuf:"bytes,6,opt,name=cliVersion,proto3" json:"cliVersion,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -2779,6 +2780,13 @@ func (x *DebugBundleRequest) GetLogFileCount() uint32 {
 	return 0
 }
 
+func (x *DebugBundleRequest) GetCliVersion() string {
+	if x != nil {
+		return x.CliVersion
+	}
+	return ""
+}
+
 type DebugBundleResponse struct {
 	state               protoimpl.MessageState `protogen:"open.v1"`
 	Path                string                 `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
@@ -6484,14 +6492,17 @@ const file_daemon_proto_rawDesc = "" +
 	"\x12translatedHostname\x18\x04 \x01(\tR\x12translatedHostname\x128\n" +
 	"\x0etranslatedPort\x18\x05 \x01(\v2\x10.daemon.PortInfoR\x0etranslatedPort\"G\n" +
 	"\x17ForwardingRulesResponse\x12,\n" +
-	"\x05rules\x18\x01 \x03(\v2\x16.daemon.ForwardingRuleR\x05rules\"\x94\x01\n" +
+	"\x05rules\x18\x01 \x03(\v2\x16.daemon.ForwardingRuleR\x05rules\"\xb4\x01\n" +
 	"\x12DebugBundleRequest\x12\x1c\n" +
 	"\tanonymize\x18\x01 \x01(\bR\tanonymize\x12\x1e\n" +
 	"\n" +
 	"systemInfo\x18\x03 \x01(\bR\n" +
 	"systemInfo\x12\x1c\n" +
 	"\tuploadURL\x18\x04 \x01(\tR\tuploadURL\x12\"\n" +
-	"\flogFileCount\x18\x05 \x01(\rR\flogFileCount\"}\n" +
+	"\flogFileCount\x18\x05 \x01(\rR\flogFileCount\x12\x1e\n" +
+	"\n" +
+	"cliVersion\x18\x06 \x01(\tR\n" +
+	"cliVersion\"}\n" +
 	"\x13DebugBundleResponse\x12\x12\n" +
 	"\x04path\x18\x01 \x01(\tR\x04path\x12 \n" +
 	"\vuploadedKey\x18\x02 \x01(\tR\vuploadedKey\x120\n" +

client/proto/daemon.proto

@@ -472,6 +472,7 @@ message DebugBundleRequest {
   bool systemInfo = 3;
   string uploadURL = 4;
   uint32 logFileCount = 5;
+  string cliVersion = 6;
 }
 
 message DebugBundleResponse {

client/proto/generate.sh

@@ -8,7 +8,7 @@ if ! which realpath >/dev/null 2>&1; then
 fi
 
 old_pwd=$(pwd)
-script_path=$(dirname $(realpath "$0"))
+script_path=$(dirname "$(realpath "$0")")
 cd "$script_path"
 go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.6
 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.6.1

client/server/debug.go

@@ -14,6 +14,7 @@ import (
 	"github.com/netbirdio/netbird/client/internal/debug"
 	"github.com/netbirdio/netbird/client/proto"
 	mgmProto "github.com/netbirdio/netbird/shared/management/proto"
+	"github.com/netbirdio/netbird/version"
 )
 
 // DebugBundle creates a debug bundle and returns the location.
@@ -67,6 +68,8 @@ func (s *Server) DebugBundle(_ context.Context, req *proto.DebugBundleRequest) (
 			CapturePath:    capturePath,
 			RefreshStatus:  refreshStatus,
 			ClientMetrics:  clientMetrics,
+			DaemonVersion:  version.NetbirdVersion(),
+			CliVersion:     req.CliVersion,
 		},
 		debug.BundleConfig{
 			Anonymize:         req.GetAnonymize(),

client/status/status.go

@@ -549,6 +549,16 @@ func (o *OutputOverview) GeneralSummary(showURL bool, showRelays bool, showNameS
 		goarm = fmt.Sprintf(" (ARMv%s)", os.Getenv("GOARM"))
 	}
 
+	daemonVersion := "N/A"
+	if o.DaemonVersion != "" {
+		daemonVersion = o.DaemonVersion
+	}
+
+	cliVersion := version.NetbirdVersion()
+	if o.CliVersion != "" {
+		cliVersion = o.CliVersion
+	}
+
 	wgPortString := "N/A"
 	if o.WgPort > 0 {
 		wgPortString = fmt.Sprintf("%d", o.WgPort)
@@ -575,8 +585,8 @@ func (o *OutputOverview) GeneralSummary(showURL bool, showRelays bool, showNameS
 			"%s"+
 			"Peers count: %s\n",
 		fmt.Sprintf("%s/%s%s", goos, goarch, goarm),
-		o.DaemonVersion,
-		version.NetbirdVersion(),
+		daemonVersion,
+		cliVersion,
 		o.ProfileName,
 		managementConnString,
 		signalConnString,

client/ui/debug.go

@@ -21,6 +21,7 @@ import (
 	"github.com/netbirdio/netbird/client/internal"
 	"github.com/netbirdio/netbird/client/proto"
 	uptypes "github.com/netbirdio/netbird/upload-server/types"
+	"github.com/netbirdio/netbird/version"
 )
 
 // Initial state for the debug collection
@@ -462,6 +463,7 @@ func (s *serviceClient) createDebugBundleFromCollection(
 	request := &proto.DebugBundleRequest{
 		Anonymize:  params.anonymize,
 		SystemInfo: params.systemInfo,
+		CliVersion: version.NetbirdVersion(),
 	}
 
 	if params.upload {
@@ -593,6 +595,7 @@ func (s *serviceClient) createDebugBundle(anonymize bool, systemInfo bool, uploa
 	request := &proto.DebugBundleRequest{
 		Anonymize:  anonymize,
 		SystemInfo: systemInfo,
+		CliVersion: version.NetbirdVersion(),
 	}
 
 	if uploadURL != "" {