[management] Move reverse proxy access logs to a dedicated database (#8303)

2026-10-28 23:39:07 +01:00 · 2026-10-23 16:04:09 +03:00
parent 8397bed4af
commit 69f13e3733
18 changed files with 421 additions and 18 deletions
@@ -0,0 +1,92 @@
+package manager
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
+	"github.com/netbirdio/netbird/management/internals/shared/db"
+	"github.com/netbirdio/netbird/management/server/store"
+)
+
+const (
+	// StoreEngineEnv selects the access log store engine. Unset follows the main store engine.
+	StoreEngineEnv = "NB_ACCESS_LOG_STORE_ENGINE"
+	// PostgresDSNEnv is the access log Postgres DSN. Unset reuses the main store DSN.
+	PostgresDSNEnv = "NB_ACCESS_LOG_POSTGRES_DSN"
+	// SqliteFileEnv is the access log SQLite file, resolved against the data directory when relative.
+	SqliteFileEnv = "NB_ACCESS_LOG_SQLITE_FILE"
+	// MaxOpenConnsEnv sizes the access log connection pool. Unset applies NB_SQL_MAX_OPEN_CONNS.
+	MaxOpenConnsEnv = "NB_ACCESS_LOG_SQL_MAX_OPEN_CONNS"
+
+	sqliteFileName = "access_logs.db"
+)
+
+// OpenConn opens and migrates the access log database. Unless configured
+// otherwise, it follows mainEngine: SQLite uses its own file in dataDir and
+// Postgres reuses the main DSN.
+func OpenConn(ctx context.Context, dataDir string, mainEngine db.Engine) (*db.Conn, error) {
+	engine := mainEngine
+	if v := os.Getenv(StoreEngineEnv); v != "" {
+		engine = db.Engine(strings.ToLower(v))
+	}
+
+	conn, err := openConn(ctx, dataDir, engine)
+	if err != nil {
+		return nil, err
+	}
+	if err := conn.AutoMigrate(&accesslogs.AccessLogEntry{}); err != nil {
+		_ = conn.Close()
+		return nil, fmt.Errorf("migrate access log store: %w", err)
+	}
+	return conn, nil
+}
+
+func openConn(ctx context.Context, dataDir string, engine db.Engine) (*db.Conn, error) {
+	switch engine {
+	case db.SqliteStoreEngine:
+		log.WithContext(ctx).Info("using SQLite access log store engine")
+		file := sqliteFileName
+		if v := os.Getenv(SqliteFileEnv); v != "" {
+			file = v
+		}
+		return db.OpenSqliteFile(ctx, dataDir, file)
+	case db.PostgresStoreEngine:
+		log.WithContext(ctx).Info("using Postgres access log store engine")
+		dsn, err := postgresDSN()
+		if err != nil {
+			return nil, err
+		}
+		return db.OpenPostgresWithoutPool(ctx, dsn, connOptions(ctx)...)
+	default:
+		return nil, fmt.Errorf("unsupported access log store engine: %s", engine)
+	}
+}
+
+func postgresDSN() (string, error) {
+	if dsn := os.Getenv(PostgresDSNEnv); dsn != "" {
+		return dsn, nil
+	}
+	if dsn, ok := store.LookupPostgresDSN(); ok && dsn != "" {
+		return dsn, nil
+	}
+	return "", fmt.Errorf("neither %s nor %s is set", PostgresDSNEnv, store.PostgresDsnEnv)
+}
+
+func connOptions(ctx context.Context) []db.Option {
+	val := os.Getenv(MaxOpenConnsEnv)
+	if val == "" {
+		return nil
+	}
+	conns, err := strconv.Atoi(val)
+	if err != nil || conns <= 0 {
+		log.WithContext(ctx).Warnf("ignoring invalid %s value %q", MaxOpenConnsEnv, val)
+		return nil
+	}
+	return []db.Option{db.WithMaxOpenConns(conns)}
+}
@@ -0,0 +1,142 @@
+package manager
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+
+	"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
+	"github.com/netbirdio/netbird/management/internals/shared/db"
+	"github.com/netbirdio/netbird/management/server/store"
+	"github.com/netbirdio/netbird/management/server/testutil"
+)
+
+func unsetStoreEnv(t *testing.T) {
+	t.Helper()
+	for _, key := range []string{StoreEngineEnv, PostgresDSNEnv, SqliteFileEnv, MaxOpenConnsEnv, store.PostgresDsnEnv, store.PostgresDsnEnvLegacy} {
+		t.Setenv(key, "")
+		require.NoError(t, os.Unsetenv(key))
+	}
+}
+
+func openTestConn(t *testing.T, dataDir string, mainEngine db.Engine) *db.Conn {
+	t.Helper()
+	conn, err := OpenConn(context.Background(), dataDir, mainEngine)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = conn.Close() })
+	return conn
+}
+
+func assertStoresEntries(t *testing.T, conn *db.Conn) {
+	t.Helper()
+	ctx := context.Background()
+	repo := NewRepository(conn)
+	require.NoError(t, repo.Create(ctx, newEntry("entry", "acc", "GET", time.Minute)))
+
+	logs, total, err := repo.ListByAccount(ctx, db.LockingStrengthNone, "acc", accesslogs.AccessLogFilter{Page: 1, PageSize: 10})
+	require.NoError(t, err)
+	assert.EqualValues(t, 1, total, "the stored entry should be listed")
+	require.Len(t, logs, 1)
+	assert.Equal(t, "entry", logs[0].ID)
+}
+
+func TestOpenConn_SqliteUsesItsOwnFile(t *testing.T) {
+	unsetStoreEnv(t)
+	dataDir := t.TempDir()
+
+	conn := openTestConn(t, dataDir, db.SqliteStoreEngine)
+
+	assert.Equal(t, db.SqliteStoreEngine, conn.Engine())
+	assert.FileExists(t, filepath.Join(dataDir, sqliteFileName))
+	assert.NoFileExists(t, filepath.Join(dataDir, db.SqliteFileName), "the main store file must not be opened")
+	assertStoresEntries(t, conn)
+}
+
+func TestOpenConn_SqliteFileFromEnv(t *testing.T) {
+	unsetStoreEnv(t)
+	dataDir := t.TempDir()
+	t.Setenv(SqliteFileEnv, "custom_logs.db")
+
+	conn := openTestConn(t, dataDir, db.SqliteStoreEngine)
+
+	assert.FileExists(t, filepath.Join(dataDir, "custom_logs.db"))
+	assert.NoFileExists(t, filepath.Join(dataDir, sqliteFileName))
+	assertStoresEntries(t, conn)
+}
+
+func TestOpenConn_EngineFromEnvOverridesMainEngine(t *testing.T) {
+	unsetStoreEnv(t)
+	t.Setenv(StoreEngineEnv, "SQLite")
+
+	conn := openTestConn(t, t.TempDir(), db.PostgresStoreEngine)
+
+	assert.Equal(t, db.SqliteStoreEngine, conn.Engine(), "the configured engine should win over the main store engine")
+}
+
+func TestOpenConn_PostgresWithoutDSNFails(t *testing.T) {
+	unsetStoreEnv(t)
+
+	_, err := OpenConn(context.Background(), t.TempDir(), db.PostgresStoreEngine)
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), PostgresDSNEnv)
+}
+
+func TestOpenConn_UnsupportedEngineFails(t *testing.T) {
+	unsetStoreEnv(t)
+	t.Setenv(StoreEngineEnv, "mysql")
+
+	_, err := OpenConn(context.Background(), t.TempDir(), db.SqliteStoreEngine)
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "mysql")
+}
+
+func TestOpenConn_PostgresReusesMainDSNWithItsOwnPool(t *testing.T) {
+	if os.Getenv("NETBIRD_STORE_ENGINE") != string(db.PostgresStoreEngine) {
+		t.Skip("requires NETBIRD_STORE_ENGINE=postgres")
+	}
+	unsetStoreEnv(t)
+	cleanup, dsn, err := testutil.CreatePostgresTestContainer()
+	require.NoError(t, err)
+	t.Cleanup(cleanup)
+	t.Setenv(store.PostgresDsnEnv, dsn)
+	t.Setenv(MaxOpenConnsEnv, "3")
+
+	conn := openTestConn(t, t.TempDir(), db.PostgresStoreEngine)
+
+	assert.Equal(t, db.PostgresStoreEngine, conn.Engine())
+	assert.Nil(t, conn.Pool(nil), "access logs never take the pgx read paths, so no pgx pool should be opened")
+	sqlDB, err := conn.DB(nil).DB()
+	require.NoError(t, err)
+	assert.Equal(t, 3, sqlDB.Stats().MaxOpenConnections, "the pool should be sized by %s", MaxOpenConnsEnv)
+	assertStoresEntries(t, conn)
+}
+
+func TestConnOptions_SizesPoolFromEnv(t *testing.T) {
+	// Invalid values fall back to the store-wide setting instead of failing boot.
+	cases := map[string]int{"3": 3, "": 7, "zero": 7, "0": 7, "-1": 7}
+	for value, want := range cases {
+		t.Run(value, func(t *testing.T) {
+			t.Setenv("NB_SQL_MAX_OPEN_CONNS", "7")
+			t.Setenv(MaxOpenConnsEnv, value)
+
+			gormDB, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "pool.db")), db.GormConfig())
+			require.NoError(t, err)
+			conn, err := db.NewConn(context.Background(), gormDB, db.PostgresStoreEngine, nil, connOptions(context.Background())...)
+			require.NoError(t, err)
+			t.Cleanup(func() { _ = conn.Close() })
+
+			sqlDB, err := conn.DB(nil).DB()
+			require.NoError(t, err)
+			assert.Equal(t, want, sqlDB.Stats().MaxOpenConnections, "max open connections for %s=%q", MaxOpenConnsEnv, value)
+		})
+	}
+}
@@ -85,8 +85,8 @@ func applyFilters(query *gorm.DB, filter accesslogs.AccessLogFilter) *gorm.DB {
 	if filter.Search != nil {
 		searchPattern := "%" + *filter.Search + "%"
 		query = query.Where(
-			"id LIKE ? OR location_connection_ip LIKE ? OR host LIKE ? OR path LIKE ? OR CONCAT(host, path) LIKE ? OR user_id IN (SELECT id FROM users WHERE email LIKE ? OR name LIKE ?)",
-			searchPattern, searchPattern, searchPattern, searchPattern, searchPattern, searchPattern, searchPattern,
+			"id LIKE ? OR location_connection_ip LIKE ? OR host LIKE ? OR path LIKE ? OR CONCAT(host, path) LIKE ?",
+			searchPattern, searchPattern, searchPattern, searchPattern, searchPattern,
 		)
 	}

@@ -123,3 +123,37 @@ func TestSqlRepository_ListByAccount_StatusFilter(t *testing.T) {
 	assert.Equal(t, []string{"info", "notfound"}, listIDs("failed"))
 	assert.Equal(t, []string{"ok"}, listIDs("success"))
 }
+
+func TestSqlRepository_ListByAccount_Search(t *testing.T) {
+	repo, _ := newTestRepository(t)
+	ctx := context.Background()
+	api := newEntry("req-api", "acc", "GET", time.Hour)
+	api.Host = "api.example.com"
+	api.Path = "/v1/peers"
+	web := newEntry("req-web", "acc", "GET", 2*time.Hour)
+	web.Host = "web.example.com"
+	web.Path = "/login"
+	foreign := newEntry("foreign", "other", "GET", time.Hour)
+	foreign.Host = "api.example.com"
+	for _, entry := range []*accesslogs.AccessLogEntry{api, web, foreign} {
+		require.NoError(t, repo.Create(ctx, entry))
+	}
+
+	search := func(term string) []string {
+		logs, total, err := repo.ListByAccount(ctx, db.LockingStrengthNone, "acc", accesslogs.AccessLogFilter{Page: 1, PageSize: 10, Search: &term})
+		require.NoError(t, err)
+		require.EqualValues(t, len(logs), total)
+		ids := make([]string, 0, len(logs))
+		for _, entry := range logs {
+			ids = append(ids, entry.ID)
+		}
+		return ids
+	}
+
+	assert.Equal(t, []string{"req-api"}, search("api.example"), "search should match the host")
+	assert.Equal(t, []string{"req-web"}, search("/login"), "search should match the path")
+	assert.Equal(t, []string{"req-api"}, search("example.com/v1"), "search should match host and path joined")
+	assert.Equal(t, []string{"req-web"}, search("req-web"), "search should match the log ID")
+	assert.Equal(t, []string{"req-api", "req-web"}, search("example.com"), "search should list every match newest first")
+	assert.Empty(t, search("nothing-matches"))
+}