feat: migrate app locks to Francis (#1624)

Co-authored-by: Elias Schneider <login@eliasschneider.com>
Co-authored-by: Kyle Mendell <kmendell@ofkm.us>

backend/internal/bootstrap/actors_bootstrap.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/italypaleale/francis/builtin/ratelimit"
+	"github.com/italypaleale/francis/components"
 	"github.com/italypaleale/francis/components/postgres"
 	"github.com/italypaleale/francis/host/local"
 	"github.com/jackc/pgx/v5/pgxpool"
@@ -43,22 +44,35 @@ func NewActors(o NewActorsOpts) (*local.Host, map[string]*ratelimit.RateLimitSer
 		return nil, nil, fmt.Errorf("failed to derive PSK: %w", err)
 	}
 
+	// Derive the cluster host limit from the HA setting
+	// With HA disabled the cluster is capped at a single replica
+	maxHosts := 1
+	if o.EnvConfig.HAEnabled {
+		// 0 = no cap
+		maxHosts = 0
+	}
+
 	// Options for the host
 	opts := []local.HostOption{
 		local.WithAddress(net.JoinHostPort(o.EnvConfig.ActorsHost, o.EnvConfig.ActorsPort)),
 		local.WithLogger(log.With("scope", "actor-host")),
 		local.WithRuntimePSKs(psk),
 		local.WithShutdownGracePeriod(10 * time.Second),
-		// TODO: Tweak these values once Pocket ID fully supports horizontal scaling.
-		// The relaxed intervals are appropriate for a single active host, but should be
-		// tuned for lower latency and better distribution across a multi-host cluster.
-		local.WithHostHealthCheckDeadline(90 * time.Second),
-		local.WithAlarmsPollInterval(5 * time.Minute),
-		local.WithAlarmsFetchAheadInterval(5 * time.Minute),
+		local.WithMaxHosts(maxHosts),
+		local.WithHostHealthCheckDeadline(ActorsHostHealthCheckDeadline(o.EnvConfig.HAEnabled)),
+	}
+
+	// With a single active host the relaxed alarm intervals reduce database load
+	// When HA is enabled they are dropped so Francis uses its tighter defaults, which distribute alarm work and fail over faster across multiple hosts
+	if !o.EnvConfig.HAEnabled {
+		opts = append(opts,
+			local.WithAlarmsPollInterval(5*time.Minute),
+			local.WithAlarmsFetchAheadInterval(5*time.Minute),
+		)
 	}
 
 	// Add the database connection
-	providerOpt, err := o.getProvider()
+	providerOpt, err := o.getProviderOption()
 	if err != nil {
 		return nil, nil, err
 	}
@@ -111,7 +125,7 @@ func NewActorStateStore(db *gorm.DB, pg *pgxpool.Pool) (*local.Host, error) {
 		opts.SQLite = sqlDB
 	}
 
-	providerOpt, err := opts.getProvider()
+	providerOpt, err := opts.getProviderOption()
 	if err != nil {
 		return nil, err
 	}
@@ -126,23 +140,53 @@ func NewActorStateStore(db *gorm.DB, pg *pgxpool.Pool) (*local.Host, error) {
 	)
 }
 
-func (o *NewActorsOpts) getProvider() (local.HostOption, error) {
+// ActorsHostHealthCheckDeadline returns the health-check deadline the actor host uses for the given HA setting
+// This is exported because the import method needs it too
+func ActorsHostHealthCheckDeadline(haEnabled bool) time.Duration {
+	if haEnabled {
+		return components.DefaultHostHealthCheckDeadline
+	}
+
+	// A single active host does not need aggressive health checks, so a longer deadline reduces database load
+	return 90 * time.Second
+}
+
+// ActorsProviderOptions builds the Francis provider options for the given database handles
+// The actor host and the cluster admin must use the same options so they address the same cluster
+// This is implemented separately and exported because the import method needs it too
+func ActorsProviderOptions(pg *pgxpool.Pool, sqliteDB *sql.DB) (components.ProviderOptions, error) {
 	switch {
-	case o.Postgres != nil && o.SQLite != nil:
+	case pg != nil && sqliteDB != nil:
 		return nil, errors.New("cannot have both Postgres and SQLite connections")
-	case o.Postgres != nil:
-		return local.WithPostgresProvider(postgres.PostgresProviderOptions{
-			DB: o.Postgres,
-		}), nil
-	case o.SQLite != nil:
-		return local.WithSQLiteProvider(local.SQLiteProviderOptions{
-			DB: o.SQLite,
-		}), nil
+	case pg != nil:
+		return postgres.PostgresProviderOptions{
+			DB: pg,
+		}, nil
+	case sqliteDB != nil:
+		return local.SQLiteProviderOptions{
+			DB: sqliteDB,
+		}, nil
 	default:
 		return nil, errors.New("one of Postgres and SQLite must be set")
 	}
 }
 
+// getProviderOption wraps the shared provider options in the host option the local host expects
+func (o *NewActorsOpts) getProviderOption() (local.HostOption, error) {
+	providerOpts, err := ActorsProviderOptions(o.Postgres, o.SQLite)
+	if err != nil {
+		return nil, err
+	}
+	switch v := providerOpts.(type) {
+	case postgres.PostgresProviderOptions:
+		return local.WithPostgresProvider(v), nil
+	case local.SQLiteProviderOptions:
+		return local.WithSQLiteProvider(v), nil
+	default:
+		return nil, fmt.Errorf("unsupported provider options type: %T", providerOpts)
+	}
+}
+
 func (o *NewActorsOpts) registerCronJobs(host *local.Host) (err error) {
 	// In test mode, we do not register anything
 	if common.EnvConfig.AppEnv == "test" {

backend/internal/bootstrap/bootstrap.go

@@ -9,6 +9,7 @@ import (
 
 	_ "github.com/golang-migrate/migrate/v4/source/file"
 
+	"github.com/italypaleale/francis/components"
 	"github.com/italypaleale/francis/host/local"
 	"github.com/italypaleale/go-kit/servicerunner"
 	"gorm.io/gorm"
@@ -16,7 +17,6 @@ import (
 	"github.com/pocket-id/pocket-id/backend/internal/common"
 	"github.com/pocket-id/pocket-id/backend/internal/instanceid"
 	"github.com/pocket-id/pocket-id/backend/internal/job"
-	"github.com/pocket-id/pocket-id/backend/internal/service"
 	"github.com/pocket-id/pocket-id/backend/internal/storage"
 )
 
@@ -43,7 +43,7 @@ func Bootstrap(ctx context.Context) error {
 	}
 	if pg != nil {
 		defer func() {
-			// Close the database connection pool only after the shutdown functions have run: some of them (e.g. releasing the application lock) still need to query the database.
+			// Close the database connection pool only after the shutdown functions have run: some of them (e.g. the actor host deregistering itself from the cluster) still need to query the database.
 			pg.Close()
 		}()
 	}
@@ -112,40 +112,19 @@ func Bootstrap(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to initialize services: %w", err)
 	}
-	services = append(services, svc.appLockService.RunRenewal)
 
 	// Migrate the pre-actor signup tokens into their actors, once the actor host is ready
 	services = append(services, actorsReady.Await(svc.userSignUpModule.RunSignupTokenMigration))
 
-	// Acquire the lock from the app lock service
-	waitUntil, err := svc.appLockService.Acquire(ctx, false)
-	if errors.Is(err, service.ErrLockUnavailable) {
-		return errors.New("it appears that there's already one instance of Pocket ID running; running multiple replicas of Pocket ID is currently not supported")
-	} else if err != nil {
-		return fmt.Errorf("failed to acquire application lock: %w", err)
-	}
-
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case <-time.After(time.Until(waitUntil)):
-	}
-
-	shutdowns.Add(func(shutdownCtx context.Context) error {
-		sErr := svc.appLockService.Release(shutdownCtx)
-		if sErr != nil {
-			return fmt.Errorf("failed to release application lock: %w", sErr)
-		}
-		return nil
-	})
-
 	// Register scheduled jobs, only in non-test mode
 	if common.EnvConfig.AppEnv != "test" {
 		err = registerScheduledJobs(ctx, db, svc, scheduler)
 		if err != nil {
 			return fmt.Errorf("failed to register scheduled jobs: %w", err)
 		}
-		services = append(services, scheduler.Run)
+
+		// The scheduler must wait on the actor host being ready, since jobs invoke actors
+		services = append(services, actorsReady.Await(scheduler.Run))
 	}
 
 	// Init the router
@@ -162,7 +141,10 @@ func Bootstrap(ctx context.Context) error {
 	// Run all background services
 	// This call blocks until the context is canceled
 	err = servicerunner.NewServiceRunner(services...).Run(ctx)
-	if err != nil {
+	if errors.Is(err, components.ErrClusterFull) {
+		// TODO: Once HA mode is supported, add a note about enabling it
+		return errors.New("it appears that there's already one instance of Pocket ID running - running multiple replicas is not (yet) supported")
+	} else if err != nil {
 		return fmt.Errorf("failed to run services: %w", err)
 	}
 

backend/internal/bootstrap/e2etest_router_bootstrap.go

@@ -17,7 +17,7 @@ import (
 func init() {
 	registerTestControllers = []func(apiGroup *gin.RouterGroup, db *gorm.DB, svc *services){
 		func(apiGroup *gin.RouterGroup, db *gorm.DB, svc *services) {
-			testService, err := service.NewTestService(db, svc.actors, svc.appConfigService, svc.jwtService, svc.ldapService, svc.appLockService, svc.fileStorage)
+			testService, err := service.NewTestService(db, svc.actors, svc.appConfigService, svc.jwtService, svc.ldapService, svc.fileStorage)
 			if err != nil {
 				slog.Error("Failed to initialize test service", slog.Any("error", err))
 				os.Exit(1)

backend/internal/bootstrap/services_bootstrap.go

@@ -38,7 +38,6 @@ type services struct {
 	ldapService        *service.LdapService
 	versionService     *service.VersionService
 	fileStorage        storage.FileStorage
-	appLockService     *service.AppLockService
 
 	apiKeyModule            *apikey.Module
 	deviceLoginModule       *devicelogin.Module
@@ -74,7 +73,6 @@ func initServices(
 
 	svc.fileStorage = fileStorage
 	svc.appImagesService = service.NewAppImagesService(imageExtensions, fileStorage)
-	svc.appLockService = service.NewAppLockService(db)
 
 	svc.emailModule, err = email.New(db)
 	if err != nil {