feat: include Francis data in Pocket ID backups (#1645)

Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Elias Schneider <login@eliasschneider.com>

backend/internal/bootstrap/actors_bootstrap.go

@@ -1,7 +1,7 @@
 package bootstrap
 
 import (
-	"database/sql"
+	"context"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -12,6 +12,7 @@ import (
 	"github.com/italypaleale/francis/builtin/ratelimit"
 	"github.com/italypaleale/francis/components"
 	"github.com/italypaleale/francis/components/postgres"
+	"github.com/italypaleale/francis/components/sqlite"
 	"github.com/italypaleale/francis/host/local"
 	"github.com/jackc/pgx/v5/pgxpool"
 	"gorm.io/gorm"
@@ -24,7 +25,6 @@ import (
 )
 
 type NewActorsOpts struct {
-	SQLite   *sql.DB
 	Postgres *pgxpool.Pool
 
 	EnvConfig   *common.EnvConfigSchema
@@ -116,14 +116,6 @@ func (o *NewActorsOpts) getPSK() ([]byte, error) {
 // It's meant for short-lived contexts such as CLI commands that need to persist actor state (for example, one-time access tokens) without running the full actor host.
 // The returned host must NOT be Run(): only direct state operations (Get/Set/Delete on state) are supported, and they require the actor state tables to already exist, which is the case whenever the server has run at least once against this database.
 func NewActorStateStore(o NewActorsOpts) (*local.Host, error) {
-	if o.Postgres == nil {
-		sqlDB, err := o.DB.DB()
-		if err != nil {
-			return nil, fmt.Errorf("failed to get *sql.DB connection from Gorm: %w", err)
-		}
-		o.SQLite = sqlDB
-	}
-
 	providerOpt, err := o.getProviderOption()
 	if err != nil {
 		return nil, err
@@ -156,10 +148,10 @@ func ActorsHostHealthCheckDeadline(haEnabled bool) time.Duration {
 	return 90 * time.Second
 }
 
-// ActorsProviderOptions builds the Francis provider options for the given database handles
-// The actor host and the cluster admin must use the same options so they address the same cluster
-// This is implemented separately and exported because the import method needs it too
-func ActorsProviderOptions(pg *pgxpool.Pool, sqliteDB *sql.DB) (components.ProviderOptions, error) {
+// ActorsProviderOptions builds the Francis provider options for the given database
+// The actor host, the cluster admin, and the backup provider must all use these so they address the same cluster
+// A Postgres deployment passes both handles, since the Gorm one wraps the same pool, and the pool is what the provider takes
+func ActorsProviderOptions(db *gorm.DB, pg *pgxpool.Pool) (components.ProviderOptions, error) {
 	// Log each provider operation, such as a lease renewal or an actor lookup, while debugging
 	// The statements those operations run are logged separately, by the instrumentation attached to the connection in ConnectDatabase
 	operationLog := components.OperationLogConfig{
@@ -167,26 +159,66 @@ func ActorsProviderOptions(pg *pgxpool.Pool, sqliteDB *sql.DB) (components.Provi
 	}
 
 	switch {
-	case pg != nil && sqliteDB != nil:
-		return nil, errors.New("cannot have both Postgres and SQLite connections")
 	case pg != nil:
 		return postgres.PostgresProviderOptions{
 			DB:           pg,
 			OperationLog: operationLog,
 		}, nil
-	case sqliteDB != nil:
+	case db != nil:
+		// The SQLite provider takes the raw connection, which only Gorm holds
+		sqliteDB, err := db.DB()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get *sql.DB connection from Gorm: %w", err)
+		}
 		return local.SQLiteProviderOptions{
 			DB:           sqliteDB,
 			OperationLog: operationLog,
 		}, nil
 	default:
-		return nil, errors.New("one of Postgres and SQLite must be set")
+		return nil, errors.New("one of the Postgres pool and the database connection must be set")
 	}
 }
 
+// NewActorsBackupProvider creates a Francis provider that talks to the same cluster as the actor host, without registering a host or joining the cluster
+// It's meant for backing up and restoring the actor host's own data
+// The caller owns the returned provider and must Close it: the database connection stays owned by the caller and is not closed.
+func NewActorsBackupProvider(ctx context.Context, providerOpts components.ProviderOptions) (components.ActorProvider, error) {
+	// The health check deadline must match the actor host's, since it decides when a host that stopped health-checking is considered gone, and a restore refuses to run while any host is still connected
+	// The remaining values are irrelevant here, because this provider never registers a host nor processes alarms
+	cfg := components.NewProviderConfig()
+	cfg.HostHealthCheckDeadline = ActorsHostHealthCheckDeadline(common.EnvConfig.HAEnabled)
+
+	log := slog.Default().With("scope", "actors-backup")
+
+	var (
+		provider components.ActorProvider
+		err      error
+	)
+	switch v := providerOpts.(type) {
+	case postgres.PostgresProviderOptions:
+		provider, err = postgres.NewPostgresProvider(log, v, cfg)
+	case sqlite.SQLiteProviderOptions:
+		provider, err = sqlite.NewSQLiteProvider(log, v, cfg)
+	default:
+		err = fmt.Errorf("unsupported provider options type: %T", providerOpts)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to create actor provider: %w", err)
+	}
+
+	// Init applies the provider's schema migrations, so this also works against a database the actor host has never run against
+	err = provider.Init(ctx)
+	if err != nil {
+		_ = provider.Close()
+		return nil, fmt.Errorf("failed to initialize actor provider: %w", err)
+	}
+
+	return provider, nil
+}
+
 // getProviderOption wraps the shared provider options in the host option the local host expects
 func (o *NewActorsOpts) getProviderOption() (local.HostOption, error) {
-	providerOpts, err := ActorsProviderOptions(o.Postgres, o.SQLite)
+	providerOpts, err := ActorsProviderOptions(o.DB, o.Postgres)
 	if err != nil {
 		return nil, err
 	}

backend/internal/bootstrap/actors_bootstrap_test.go

@@ -1,10 +1,14 @@
 package bootstrap
 
 import (
+	"bytes"
 	"encoding/hex"
+	"path/filepath"
 	"testing"
 
+	"github.com/libtnb/sqlite"
 	"github.com/stretchr/testify/require"
+	"gorm.io/gorm"
 
 	"github.com/pocket-id/pocket-id/backend/internal/common"
 )
@@ -26,3 +30,43 @@ func TestNewActorsOptsGetPSKUsesStableValue(t *testing.T) {
 	require.NoError(t, err)
 	require.Equalf(t, expected, actual, "actual result: %s", actual)
 }
+
+// TestNewActorsBackupProvider covers the provider the export and import use to back up and restore the actor host's data.
+// It builds the provider from the same options the actor host uses, so a mismatch between those options and the concrete provider would otherwise only surface at runtime, when an export or import is attempted.
+func TestNewActorsBackupProvider(t *testing.T) {
+	// Foreign keys must be enabled, which the provider validates on init and which the application enables on every connection
+	dbPath := filepath.Join(t.TempDir(), "pocket-id.db")
+	dsn := "file:" + dbPath + "?_txlock=immediate&_pragma=busy_timeout(2500)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)"
+	db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		sqlDB, dbErr := db.DB()
+		if dbErr == nil {
+			_ = sqlDB.Close()
+		}
+	})
+
+	providerOpts, err := ActorsProviderOptions(db, nil)
+	require.NoError(t, err)
+
+	provider, err := NewActorsBackupProvider(t.Context(), providerOpts)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = provider.Close()
+	})
+
+	// Init applies the actor host's schema migrations, so an export also works against a database the actor host has never run against
+	var tables int64
+	err = db.Raw(`SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name LIKE 'francis_%'`).Scan(&tables).Error
+	require.NoError(t, err)
+	require.Positive(t, tables, "the provider must create the actor host's own tables")
+
+	// Even an empty cluster produces a valid backup stream, which an import can restore
+	buf := &bytes.Buffer{}
+	err = provider.Backup(t.Context(), buf)
+	require.NoError(t, err)
+	require.Contains(t, buf.String(), "francis-backup")
+
+	err = provider.Restore(t.Context(), bytes.NewReader(buf.Bytes()))
+	require.NoError(t, err)
+}

backend/internal/bootstrap/bootstrap.go

@@ -92,12 +92,6 @@ func Bootstrap(ctx context.Context) error {
 		DB:          db,
 		FileStorage: fileStorage,
 	}
-	if pg == nil {
-		actorsOpts.SQLite, err = db.DB()
-		if err != nil {
-			return fmt.Errorf("failed to get *sql.DB connection from Gorm: %w", err)
-		}
-	}
 	actors, rateLimitServices, err := NewActors(actorsOpts)
 	if err != nil {
 		return fmt.Errorf("failed to initialize actors: %w", err)