[client] Prevent corruption from competing log rotation and improve debug bundle (#6214)

* Adds heuristic to detect an edge case on Linux where a system has configured logrotate as a separate service to rotate log files which would mangle our client log files. If we detect logrotate being configured for netbird, we disable our rotation.

* Adds new env var to disable log rotation: NB_LOG_DISABLE_ROTATION

* Adds compressed and plain logrotate files to debug bundle.

* Replaces lumberjack with timberjack (maintained fork with bug fixes and extra features).

* Clarifies which daemon version is running in the bundle stats.

* Change logging for client service status to console

util/log.go

@@ -1,15 +1,16 @@
 package util
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"slices"
 	"strconv"
 
+	"github.com/DeRuina/timberjack"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc/grpclog"
-	"gopkg.in/natefinch/lumberjack.v2"
 
 	"github.com/netbirdio/netbird/formatter"
 )
@@ -37,8 +38,7 @@ func InitLog(logLevel string, logs ...string) error {
 func InitLogger(logger *log.Logger, logLevel string, logs ...string) error {
 	level, err := log.ParseLevel(logLevel)
 	if err != nil {
-		logger.Errorf("Failed parsing log-level %s: %s", logLevel, err)
-		return err
+		return fmt.Errorf("failed parsing log-level %s: %w", logLevel, err)
 	}
 	var writers []io.Writer
 	logFmt := os.Getenv("NB_LOG_FORMAT")
@@ -59,7 +59,11 @@ func InitLogger(logger *log.Logger, logLevel string, logs ...string) error {
 		case "":
 			logger.Warnf("empty log path received: %#v", logPath)
 		default:
-			writers = append(writers, newRotatedOutput(logPath))
+			writer, err := setupLogFile(logPath, isRotationDisabled(logger))
+			if err != nil {
+				return fmt.Errorf("failed setting up log file: %s, %w", logPath, err)
+			}
+			writers = append(writers, writer)
 		}
 	}
 
@@ -94,17 +98,43 @@ func FindFirstLogPath(logs []string) string {
 	return ""
 }
 
+func isRotationDisabled(logger *log.Logger) bool {
+	v, _ := os.LookupEnv("NB_LOG_DISABLE_ROTATION")
+	disabled, _ := strconv.ParseBool(v)
+	if disabled {
+		logger.Warnf("log rotation is disabled by env flag")
+		return true
+	}
+	conflict, configPath := FindFirstLogrotateConflict()
+	if conflict {
+		logger.Warnf("log rotation conflict detected in: %#v, rotation is disabled", configPath)
+		return true
+	}
+	return false
+}
+
+func setupLogFile(logPath string, disableRotation bool) (io.Writer, error) {
+	if disableRotation {
+		file, err := os.OpenFile(logPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
+		if err != nil {
+			return nil, err
+		}
+		return file, nil
+	}
+	return newRotatedOutput(logPath), nil
+}
+
 func newRotatedOutput(logPath string) io.Writer {
 	maxLogSize := getLogMaxSize()
-	lumberjackLogger := &lumberjack.Logger{
+	timberjackLogger := &timberjack.Logger{
 		// Log file absolute path, os agnostic
-		Filename:   filepath.ToSlash(logPath),
-		MaxSize:    maxLogSize, // MB
-		MaxBackups: 10,
-		MaxAge:     30, // days
-		Compress:   true,
+		Filename:    filepath.ToSlash(logPath),
+		MaxSize:     maxLogSize, // MB
+		MaxBackups:  10,
+		MaxAge:      30, // days
+		Compression: "gzip",
 	}
-	return lumberjackLogger
+	return timberjackLogger
 }
 
 func setGRPCLibLogger(logger *log.Logger) {
@@ -127,7 +157,7 @@ func getLogMaxSize() int {
 	if sizeVar, ok := os.LookupEnv("NB_LOG_MAX_SIZE_MB"); ok {
 		size, err := strconv.ParseInt(sizeVar, 10, 64)
 		if err != nil {
-			log.Errorf("Failed parsing log-size %s: %s. Should be just an integer", sizeVar, err)
+			log.Errorf("failed parsing log-size %s: %s. Should be just an integer", sizeVar, err)
 			return defaultLogSize
 		}
 

util/log_test.go

@@ -0,0 +1,96 @@
+package util
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+// TestSetupLogFile_RotatesOnSize drives >MaxSize bytes through the writer
+// returned by setupLogFile and asserts a backup file appears.
+func TestSetupLogFile_RotatesOnSize(t *testing.T) {
+	t.Setenv("NB_LOG_MAX_SIZE_MB", "1")
+
+	dir := t.TempDir()
+	logPath := filepath.Join(dir, "netbird.log")
+
+	w, err := setupLogFile(logPath, false)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if c, ok := w.(io.Closer); ok {
+			_ = c.Close()
+		}
+	})
+
+	chunk := []byte(strings.Repeat("x", 64*1024) + "\n")
+	for range 20 {
+		_, err := w.Write(chunk)
+		require.NoError(t, err)
+	}
+
+	info, err := os.Stat(logPath)
+	require.NoError(t, err)
+	require.Less(t, info.Size(), int64(1<<20),
+		"active log should be < 1 MB after rotation, got %d", info.Size())
+
+	require.Eventually(t, func() bool {
+		entries, _ := os.ReadDir(dir)
+		for _, e := range entries {
+			name := e.Name()
+			if name == filepath.Base(logPath) {
+				continue
+			}
+			if strings.HasPrefix(name, "netbird-") && strings.HasSuffix(name, ".log.gz") {
+				return true
+			}
+		}
+		return false
+	}, 5*time.Second, 50*time.Millisecond, "expected a rotated backup file in %s", dir)
+}
+
+// TestSetupLogFile_RotationDisabled verifies that with rotation off, the file
+// grows past MaxSize and no backups are created.
+func TestSetupLogFile_RotationDisabled(t *testing.T) {
+	t.Setenv("NB_LOG_MAX_SIZE_MB", "1")
+
+	dir := t.TempDir()
+	logPath := filepath.Join(dir, "netbird.log")
+
+	w, err := setupLogFile(logPath, true)
+	require.NoError(t, err)
+
+	f, ok := w.(*os.File)
+	require.True(t, ok, "expected plain *os.File when rotation is disabled, got %T", w)
+	t.Cleanup(func() { _ = f.Close() })
+
+	chunk := []byte(strings.Repeat("y", 64*1024) + "\n")
+	for range 20 {
+		_, err := w.Write(chunk)
+		require.NoError(t, err)
+	}
+
+	info, err := os.Stat(logPath)
+	require.NoError(t, err)
+	require.GreaterOrEqual(t, info.Size(), int64(1<<20),
+		"file should exceed MaxSize when rotation is disabled, got %d", info.Size())
+
+	entries, err := os.ReadDir(dir)
+	require.NoError(t, err)
+	require.Len(t, entries, 1, "no backup files should exist when rotation is disabled, got %v", entries)
+}
+
+// TestIsRotationDisabled_EnvFlag covers the NB_LOG_DISABLE_ROTATION env path.
+// The logrotate-conflict branch is exercised separately on linux.
+func TestIsRotationDisabled_EnvFlag(t *testing.T) {
+	logger := log.New()
+	logger.SetOutput(io.Discard)
+
+	t.Setenv("NB_LOG_DISABLE_ROTATION", "true")
+	require.True(t, isRotationDisabled(logger))
+}

util/logrotate_linux.go

@@ -0,0 +1,93 @@
+//go:build linux
+
+package util
+
+import (
+	"bufio"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultLogrotateConfPath = "/etc/logrotate.conf"
+	defaultLogrotateConfDir  = "/etc/logrotate.d"
+	netbirdString            = "netbird"
+)
+
+// FindLogrotateConflicts scans the standard logrotate locations for
+// indications of conflict with netbird. It returns true and the config file
+// path if a conflict was found.
+func FindFirstLogrotateConflict() (bool, string) {
+	return findFirstLogrotateConflictIn(defaultLogrotateConfPath, defaultLogrotateConfDir)
+}
+
+func findFirstLogrotateConflictIn(confPath, confDir string) (bool, string) {
+	for _, f := range listLogrotateConfigs(confPath, confDir) {
+		present, err := scanLogrotateFile(f, netbirdString)
+		if err != nil {
+			if !errors.Is(err, fs.ErrNotExist) {
+				log.Debugf("scan %s: %v", f, err)
+			}
+			continue
+		}
+		if present {
+			return present, f
+		}
+	}
+	return false, ""
+}
+
+// listLogrotateConfigs returns all config files for logrotate.
+func listLogrotateConfigs(confPath, confDir string) []string {
+	files := []string{confPath}
+	entries, err := os.ReadDir(confDir)
+	if err != nil {
+		return files
+	}
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		files = append(files, filepath.Join(confDir, e.Name()))
+	}
+	return files
+}
+
+// scanLogrotateFile reads a config and reports if a non-comment line
+// contains the given substring.
+func scanLogrotateFile(path string, substring string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			log.Debugf("close %s: %v", path, err)
+		}
+	}()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(stripLogrotateComment(scanner.Text()))
+		if line == "" {
+			continue
+		}
+		if strings.Contains(line, substring) {
+			return true, nil
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return false, err
+	}
+	return false, nil
+}
+
+func stripLogrotateComment(line string) string {
+	before, _, _ := strings.Cut(line, "#")
+	return before
+}

util/logrotate_linux_test.go

@@ -0,0 +1,95 @@
+//go:build linux
+
+package util
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFindFirstLogrotateConflict(t *testing.T) {
+	t.Run("conflict in confDir", func(t *testing.T) {
+		confPath, confDir := newLogrotateLayout(t)
+		conflictPath := filepath.Join(confDir, "netbird")
+		writeLogrotateConfig(t, conflictPath, `/var/log/netbird/*.log {
+    daily
+    rotate 7
+}`)
+		writeLogrotateConfig(t, filepath.Join(confDir, "nginx"), `/var/log/nginx/*.log { daily }`)
+
+		got, path := findFirstLogrotateConflictIn(confPath, confDir)
+		require.True(t, got)
+		require.Equal(t, conflictPath, path)
+	})
+
+	t.Run("conflict in main conf file", func(t *testing.T) {
+		confPath, confDir := newLogrotateLayout(t)
+		writeLogrotateConfig(t, confPath, `weekly
+rotate 4
+include /etc/logrotate.d
+/var/log/netbird/client.log { rotate 5 }`)
+
+		got, path := findFirstLogrotateConflictIn(confPath, confDir)
+		require.True(t, got)
+		require.Equal(t, confPath, path)
+	})
+
+	t.Run("no conflict when netbird is absent", func(t *testing.T) {
+		confPath, confDir := newLogrotateLayout(t)
+		writeLogrotateConfig(t, filepath.Join(confDir, "nginx"), `/var/log/nginx/*.log { daily }`)
+		writeLogrotateConfig(t, filepath.Join(confDir, "syslog"), `/var/log/syslog { weekly }`)
+
+		got, path := findFirstLogrotateConflictIn(confPath, confDir)
+		require.False(t, got)
+		require.Empty(t, path)
+	})
+
+	t.Run("commented-out netbird line is ignored", func(t *testing.T) {
+		confPath, confDir := newLogrotateLayout(t)
+		writeLogrotateConfig(t, filepath.Join(confDir, "misc"), `# /var/log/netbird/*.log { daily }
+/var/log/other.log { weekly }`)
+
+		got, path := findFirstLogrotateConflictIn(confPath, confDir)
+		require.False(t, got)
+		require.Empty(t, path)
+	})
+
+	t.Run("subdirectories in confDir are ignored", func(t *testing.T) {
+		confPath, confDir := newLogrotateLayout(t)
+		sub := filepath.Join(confDir, "nested")
+		require.NoError(t, os.MkdirAll(sub, 0o755))
+		writeLogrotateConfig(t, filepath.Join(sub, "netbird"), `/var/log/netbird/*.log { daily }`)
+
+		got, path := findFirstLogrotateConflictIn(confPath, confDir)
+		require.False(t, got)
+		require.Empty(t, path)
+	})
+
+	t.Run("missing paths return no conflict", func(t *testing.T) {
+		dir := t.TempDir()
+		got, path := findFirstLogrotateConflictIn(
+			filepath.Join(dir, "does-not-exist.conf"),
+			filepath.Join(dir, "does-not-exist.d"),
+		)
+		require.False(t, got)
+		require.Empty(t, path)
+	})
+}
+
+// newLogrotateLayout creates a temp logrotate.conf path and logrotate.d dir,
+// returning their paths. The conf file itself is not created.
+func newLogrotateLayout(t *testing.T) (confPath, confDir string) {
+	t.Helper()
+	root := t.TempDir()
+	confDir = filepath.Join(root, "logrotate.d")
+	require.NoError(t, os.MkdirAll(confDir, 0o755))
+	return filepath.Join(root, "logrotate.conf"), confDir
+}
+
+func writeLogrotateConfig(t *testing.T, path, body string) {
+	t.Helper()
+	require.NoError(t, os.WriteFile(path, []byte(body), 0o644))
+}

util/logrotate_nonlinux.go

@@ -0,0 +1,10 @@
+//go:build !linux
+
+package util
+
+// FindLogrotateConflicts scans the standard logrotate locations for
+// indications of conflict with netbird. It will always return false for
+// non-linux devices.
+func FindFirstLogrotateConflict() (bool, string) {
+	return false, ""
+}