Merge pull request #3907 from fosrl/leader-election

Manage cert enabled with leader election

server/private/lib/certificates/index.ts

@@ -15,6 +15,23 @@ import logger from "@server/logger";
 import { privateConfig } from "#private/lib/config";
 import { acmeClientManager } from "./acme-client";
 import { jobScheduler } from "./scheduler";
+import { LeaderElection } from "#private/lib/leaderElection";
+
+// Only one instance sharing this Redis talks to the ACME server at a time.
+// Every instance runs the election; whichever holds the lease runs issuance
+// and renewal, and another takes over if it goes away.
+const acmeLeaderElection = new LeaderElection("acme-client", {
+    onElected: async () => {
+        logger.info("Starting certificate management server...");
+
+        // Initialize ACME client
+        await acmeClientManager.initialize();
+
+        // Start certificate issuance/renewal jobs
+        await jobScheduler.start();
+    },
+    onDemoted: () => jobScheduler.stopCertJobs()
+});
 
 export async function startCertificateManager() {
     const acmeConfig = privateConfig.getRawPrivateConfig().acme;
@@ -23,13 +40,7 @@ export async function startCertificateManager() {
         acmeConfig.cert_mode === "pangolin" &&
         acmeConfig.enable_acme_client
     ) {
-        logger.info("Starting certificate management server...");
-
-        // Initialize ACME client
-        await acmeClientManager.initialize();
-
-        // Start certificate issuance/renewal jobs
-        await jobScheduler.start();
+        acmeLeaderElection.start();
     }
 
     if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
@@ -41,5 +52,6 @@ export async function startCertificateManager() {
 }
 
 export async function stopCertificateManager() {
+    await acmeLeaderElection.stop();
     await jobScheduler.stop();
 }

server/private/lib/certificates/scheduler.ts

@@ -173,13 +173,21 @@ export class JobScheduler {
         logger.info("DNS validation job scheduler started successfully");
     }
 
-    async stop(): Promise<void> {
+    // Stops only issuance/renewal (e.g. when this instance stops being the
+    // ACME leader) and leaves the DNS jobs running. A batch already in
+    // flight finishes on its own; the per-domain cert locks keep it from
+    // overlapping with the new leader's work on the same certificate.
+    async stopCertJobs(): Promise<void> {
         if (this.certRunning) {
             logger.info("Stopping certificate job scheduler");
             this.certRunning = false;
             this.certIntervals.forEach((interval) => clearInterval(interval));
             this.certIntervals = [];
         }
+    }
+
+    async stop(): Promise<void> {
+        await this.stopCertJobs();
 
         if (this.dnsRunning) {
             logger.info("Stopping DNS validation job scheduler");

server/private/lib/leaderElection.ts

@@ -0,0 +1,259 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { v4 as uuidv4 } from "uuid";
+import redisManager from "#private/lib/redis";
+import { withTimeout } from "@server/lib/retry";
+import logger from "@server/logger";
+
+// How long a lease lasts without being renewed. If the leader dies, the
+// role sits empty for at most this long before a follower can take it.
+const LEASE_TTL_MS = 30_000;
+
+// How often the leader renews its lease and a follower tries to take it.
+const TICK_INTERVAL_MS = 10_000;
+
+// A leader that can't confirm its lease within this time steps down. Even
+// when the last successful renewal was just before a tick, it stops within
+// TICK_INTERVAL_MS + REDIS_TIMEOUT_MS (15s), well before the lease can run
+// out (LEASE_TTL_MS) and another instance can take it.
+const REDIS_TIMEOUT_MS = 5_000;
+
+const RENEW_SCRIPT = `
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+    return redis.call('PEXPIRE', KEYS[1], ARGV[2])
+else
+    return 0
+end
+`;
+
+const RELEASE_SCRIPT = `
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+    return redis.call('DEL', KEYS[1])
+else
+    return 0
+end
+`;
+
+type LeaderCallbacks = {
+    // Runs once each time this instance becomes leader. If it throws, the
+    // lease is given up and the next tick tries again (here or elsewhere).
+    onElected: () => Promise<void>;
+    // Runs when leadership is lost or given up. Must be safe to call even if
+    // onElected only partly finished.
+    onDemoted: () => Promise<void>;
+};
+
+/**
+ * Makes sure only one Pangolin instance sharing a Redis is running a given
+ * role at a time.
+ *
+ * Leadership is a Redis key, set with NX and a TTL, whose value is this
+ * instance's token. The leader renews it every tick and steps down as soon
+ * as a renewal fails or times out, so it has stopped before the lease
+ * can run out and another instance can take over.
+ *
+ * Talks to the Redis client directly rather than through lockManager:
+ * lockManager falls back to in-process locks while Redis is unreachable,
+ * which would let every instance become leader during an outage. Here an
+ * unreachable Redis means no instance is leader until it comes back.
+ *
+ * With Redis disabled, the deployment is assumed to be a single instance
+ * (Redis is required to run more than one), so this instance is always the
+ * leader.
+ */
+export class LeaderElection {
+    private readonly key: string;
+    private readonly callbacks: LeaderCallbacks;
+    private token: string | null = null;
+    private timer: NodeJS.Timeout | null = null;
+    private leader = false;
+    private running = false;
+
+    constructor(name: string, callbacks: LeaderCallbacks) {
+        this.key = `leader:${name}`;
+        this.callbacks = callbacks;
+    }
+
+    start(): void {
+        if (this.running) {
+            return;
+        }
+        this.running = true;
+
+        if (!redisManager.getWriteClient()) {
+            logger.info(
+                `Redis is disabled - assuming a single instance and taking ${this.key}`
+            );
+            void this.becomeLeader();
+            return;
+        }
+
+        logger.info(`Leader election started for ${this.key}`);
+        void this.tick();
+    }
+
+    async stop(): Promise<void> {
+        if (!this.running) {
+            return;
+        }
+        this.running = false;
+
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+
+        await this.stepDown("shutting down");
+    }
+
+    isLeader(): boolean {
+        return this.leader;
+    }
+
+    // Ticks are chained with setTimeout rather than setInterval so a slow
+    // acquire/renew can never overlap with the next one.
+    private async tick(): Promise<void> {
+        if (!this.running) {
+            return;
+        }
+
+        try {
+            if (this.leader) {
+                await this.renew();
+            } else {
+                await this.tryAcquire();
+            }
+        } catch (error) {
+            logger.error(`Leader election error for ${this.key}:`, error);
+            await this.stepDown("leader election error");
+        }
+
+        if (this.running) {
+            this.timer = setTimeout(() => void this.tick(), TICK_INTERVAL_MS);
+        }
+    }
+
+    private async tryAcquire(): Promise<void> {
+        const client = redisManager.getWriteClient()!;
+        const token = uuidv4();
+
+        const result = await withTimeout(
+            client.set(this.key, token, "PX", LEASE_TTL_MS, "NX"),
+            REDIS_TIMEOUT_MS,
+            `acquire ${this.key}`
+        );
+
+        if (result !== "OK") {
+            logger.debug(
+                `Not leader for ${this.key} - another instance holds the lease`
+            );
+            return;
+        }
+
+        this.token = token;
+
+        // stop() may have run while the SET was in flight.
+        if (!this.running) {
+            await this.release();
+            return;
+        }
+
+        await this.becomeLeader();
+    }
+
+    private async renew(): Promise<void> {
+        if (!this.token) {
+            throw new Error("leader has no lease token");
+        }
+
+        const client = redisManager.getWriteClient()!;
+        const renewed = (await withTimeout(
+            client.eval(
+                RENEW_SCRIPT,
+                1,
+                this.key,
+                this.token,
+                LEASE_TTL_MS.toString()
+            ),
+            REDIS_TIMEOUT_MS,
+            `renew ${this.key}`
+        )) as number;
+
+        if (renewed !== 1) {
+            throw new Error("lease is no longer held by this instance");
+        }
+    }
+
+    private async becomeLeader(): Promise<void> {
+        this.leader = true;
+        logger.info(`Elected leader for ${this.key}`);
+
+        try {
+            await this.callbacks.onElected();
+        } catch (error) {
+            logger.error(
+                `Failed to start ${this.key} after being elected leader:`,
+                error
+            );
+            // Without Redis there's nobody to hand over to and no retry loop,
+            // so keep the role, just as it was before leader election.
+            if (this.token) {
+                await this.stepDown("startup failed");
+            }
+        }
+    }
+
+    private async stepDown(reason: string): Promise<void> {
+        const wasLeader = this.leader;
+        this.leader = false;
+
+        if (wasLeader) {
+            logger.warn(`Stepping down as leader for ${this.key}: ${reason}`);
+            try {
+                await this.callbacks.onDemoted();
+            } catch (error) {
+                logger.error(
+                    `Error stopping ${this.key} after stepping down:`,
+                    error
+                );
+            }
+        }
+
+        // Only after onDemoted has stopped this instance from scheduling more
+        // work, so the next leader doesn't overlap with it.
+        await this.release();
+    }
+
+    // Best effort: if Redis is unreachable the lease just runs out after
+    // LEASE_TTL_MS instead.
+    private async release(): Promise<void> {
+        const token = this.token;
+        this.token = null;
+
+        const client = redisManager.getWriteClient();
+        if (!token || !client) {
+            return;
+        }
+
+        try {
+            await withTimeout(
+                client.eval(RELEASE_SCRIPT, 1, this.key, token),
+                REDIS_TIMEOUT_MS,
+                `release ${this.key}`
+            );
+        } catch (error) {
+            logger.warn(`Failed to release ${this.key}:`, error);
+        }
+    }
+}