[management] Move reverse proxy access logs to a dedicated database (#8303)

management/internals/shared/db/conn.go

@@ -24,8 +24,8 @@ type TxMetrics interface {
 	CountTransactionDuration(duration time.Duration)
 }
 
-// Conn is the database connection shared by all repositories: one gorm handle,
-// the pgx pool of a Postgres deployment and the engine they talk to.
+// Conn is a database connection the repositories run on: one gorm handle, the
+// pgx pool of a Postgres deployment and the engine they talk to.
 type Conn struct {
 	db        *gorm.DB
 	pool      *pgxpool.Pool
@@ -34,15 +34,36 @@ type Conn struct {
 	metrics   TxMetrics
 }
 
+// Option tunes a Conn as it is opened.
+type Option func(*connOptions)
+
+type connOptions struct {
+	maxOpenConns int
+}
+
+// WithMaxOpenConns sizes the gorm connection pool of this Conn, taking
+// precedence over NB_SQL_MAX_OPEN_CONNS. A pgx pool keeps its PoolConfig size,
+// and SQLite always uses one connection.
+func WithMaxOpenConns(n int) Option {
+	return func(o *connOptions) {
+		o.maxOpenConns = n
+	}
+}
+
 // NewConn takes ownership of an open gorm handle and pool once it returns
 // without error, applying the connection limits and transaction timeout
-// configured through the environment.
-func NewConn(ctx context.Context, gormDB *gorm.DB, engine Engine, pool *pgxpool.Pool) (*Conn, error) {
+// configured through the environment and opts.
+func NewConn(ctx context.Context, gormDB *gorm.DB, engine Engine, pool *pgxpool.Pool, opts ...Option) (*Conn, error) {
 	sqlDB, err := gormDB.DB()
 	if err != nil {
 		return nil, err
 	}
 
+	var options connOptions
+	for _, opt := range opts {
+		opt(&options)
+	}
+
 	txTimeout := defaultTransactionTimeout
 	if v := os.Getenv("NB_STORE_TRANSACTION_TIMEOUT"); v != "" {
 		if parsed, err := time.ParseDuration(v); err == nil {
@@ -57,6 +78,9 @@ func NewConn(ctx context.Context, gormDB *gorm.DB, engine Engine, pool *pgxpool.
 	if connsConfigured {
 		conns = configuredConns
 	}
+	if options.maxOpenConns > 0 {
+		conns = options.maxOpenConns
+	}
 	if engine == SqliteStoreEngine {
 		if connsConfigured {
 			log.WithContext(ctx).Warnf("setting NB_SQL_MAX_OPEN_CONNS is not supported for sqlite, using default value 1")

management/internals/shared/db/conn_test.go

@@ -146,3 +146,22 @@ func TestNewConn_MaxOpenConnsFromEnv(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, 1, sqliteDB.Stats().MaxOpenConnections)
 }
+
+func TestNewConn_MaxOpenConnsOptionOverridesEnv(t *testing.T) {
+	t.Setenv("NB_SQL_MAX_OPEN_CONNS", "7")
+
+	open := func(engine Engine, opts ...Option) int {
+		gormDB, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "store.db")), GormConfig())
+		require.NoError(t, err)
+		conn, err := NewConn(context.Background(), gormDB, engine, nil, opts...)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = conn.Close() })
+		sqlDB, err := conn.DB(nil).DB()
+		require.NoError(t, err)
+		return sqlDB.Stats().MaxOpenConnections
+	}
+
+	assert.Equal(t, 3, open(PostgresStoreEngine, WithMaxOpenConns(3)), "option should take precedence over the environment")
+	assert.Equal(t, 7, open(PostgresStoreEngine, WithMaxOpenConns(0)), "a non-positive option should fall back to the environment")
+	assert.Equal(t, 1, open(SqliteStoreEngine, WithMaxOpenConns(3)), "sqlite should stay on a single connection")
+}

management/internals/shared/db/open.go

@@ -101,7 +101,7 @@ func OpenSqliteFile(ctx context.Context, dataDir, storeFile string) (*Conn, erro
 }
 
 // OpenPostgres opens a Postgres database through gorm and a pgx pool sized by pool.
-func OpenPostgres(ctx context.Context, dsn string, pool PoolConfig) (*Conn, error) {
+func OpenPostgres(ctx context.Context, dsn string, pool PoolConfig, opts ...Option) (*Conn, error) {
 	gormDB, err := gorm.Open(postgres.Open(dsn), GormConfig())
 	if err != nil {
 		return nil, err
@@ -111,7 +111,7 @@ func OpenPostgres(ctx context.Context, dsn string, pool PoolConfig) (*Conn, erro
 		closeGorm(gormDB)
 		return nil, err
 	}
-	conn, err := NewConn(ctx, gormDB, PostgresStoreEngine, pgxPool)
+	conn, err := NewConn(ctx, gormDB, PostgresStoreEngine, pgxPool, opts...)
 	if err != nil {
 		pgxPool.Close()
 		closeGorm(gormDB)
@@ -120,6 +120,21 @@ func OpenPostgres(ctx context.Context, dsn string, pool PoolConfig) (*Conn, erro
 	return conn, nil
 }
 
+// OpenPostgresWithoutPool opens a Postgres database through gorm alone, for
+// repositories that never take the pgx read paths.
+func OpenPostgresWithoutPool(ctx context.Context, dsn string, opts ...Option) (*Conn, error) {
+	gormDB, err := gorm.Open(postgres.Open(dsn), GormConfig())
+	if err != nil {
+		return nil, err
+	}
+	conn, err := NewConn(ctx, gormDB, PostgresStoreEngine, nil, opts...)
+	if err != nil {
+		closeGorm(gormDB)
+		return nil, err
+	}
+	return conn, nil
+}
+
 func newPgxPool(ctx context.Context, dsn string, cfg PoolConfig) (*pgxpool.Pool, error) {
 	config, err := pgxpool.ParseConfig(dsn)
 	if err != nil {