From 3601ca17f176fec7235e338c55b56327b1c18e3d Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 14 Oct 2026 15:30:58 -0400 Subject: [PATCH] Manage cert enabled with leader election --- server/private/lib/certificates/index.ts | 26 +- server/private/lib/certificates/scheduler.ts | 10 +- server/private/lib/leaderElection.ts | 259 +++++++++++++++++++ 3 files changed, 287 insertions(+), 8 deletions(-) create mode 100644 server/private/lib/leaderElection.ts diff --git a/server/private/lib/certificates/index.ts b/server/private/lib/certificates/index.ts index 6097629a7..27dd24177 100644 --- a/server/private/lib/certificates/index.ts +++ b/server/private/lib/certificates/index.ts @@ -15,6 +15,23 @@ import logger from "@server/logger"; import { privateConfig } from "#private/lib/config"; import { acmeClientManager } from "./acme-client"; import { jobScheduler } from "./scheduler"; +import { LeaderElection } from "#private/lib/leaderElection"; + +// Only one instance sharing this Redis talks to the ACME server at a time. +// Every instance runs the election; whichever holds the lease runs issuance +// and renewal, and another takes over if it goes away. +const acmeLeaderElection = new LeaderElection("acme-client", { + onElected: async () => { + logger.info("Starting certificate management server..."); + + // Initialize ACME client + await acmeClientManager.initialize(); + + // Start certificate issuance/renewal jobs + await jobScheduler.start(); + }, + onDemoted: () => jobScheduler.stopCertJobs() +}); export async function startCertificateManager() { const acmeConfig = privateConfig.getRawPrivateConfig().acme; @@ -23,13 +40,7 @@ export async function startCertificateManager() { acmeConfig.cert_mode === "pangolin" && acmeConfig.enable_acme_client ) { - logger.info("Starting certificate management server..."); - - // Initialize ACME client - await acmeClientManager.initialize(); - - // Start certificate issuance/renewal jobs - await jobScheduler.start(); + acmeLeaderElection.start(); } if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) { @@ -41,5 +52,6 @@ export async function startCertificateManager() { } export async function stopCertificateManager() { + await acmeLeaderElection.stop(); await jobScheduler.stop(); } diff --git a/server/private/lib/certificates/scheduler.ts b/server/private/lib/certificates/scheduler.ts index c46463008..4bd52a6eb 100644 --- a/server/private/lib/certificates/scheduler.ts +++ b/server/private/lib/certificates/scheduler.ts @@ -173,13 +173,21 @@ export class JobScheduler { logger.info("DNS validation job scheduler started successfully"); } - async stop(): Promise { + // Stops only issuance/renewal (e.g. when this instance stops being the + // ACME leader) and leaves the DNS jobs running. A batch already in + // flight finishes on its own; the per-domain cert locks keep it from + // overlapping with the new leader's work on the same certificate. + async stopCertJobs(): Promise { if (this.certRunning) { logger.info("Stopping certificate job scheduler"); this.certRunning = false; this.certIntervals.forEach((interval) => clearInterval(interval)); this.certIntervals = []; } + } + + async stop(): Promise { + await this.stopCertJobs(); if (this.dnsRunning) { logger.info("Stopping DNS validation job scheduler"); diff --git a/server/private/lib/leaderElection.ts b/server/private/lib/leaderElection.ts new file mode 100644 index 000000000..27f850da4 --- /dev/null +++ b/server/private/lib/leaderElection.ts @@ -0,0 +1,259 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import { v4 as uuidv4 } from "uuid"; +import redisManager from "#private/lib/redis"; +import { withTimeout } from "@server/lib/retry"; +import logger from "@server/logger"; + +// How long a lease lasts without being renewed. If the leader dies, the +// role sits empty for at most this long before a follower can take it. +const LEASE_TTL_MS = 30_000; + +// How often the leader renews its lease and a follower tries to take it. +const TICK_INTERVAL_MS = 10_000; + +// A leader that can't confirm its lease within this time steps down. Even +// when the last successful renewal was just before a tick, it stops within +// TICK_INTERVAL_MS + REDIS_TIMEOUT_MS (15s), well before the lease can run +// out (LEASE_TTL_MS) and another instance can take it. +const REDIS_TIMEOUT_MS = 5_000; + +const RENEW_SCRIPT = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('PEXPIRE', KEYS[1], ARGV[2]) +else + return 0 +end +`; + +const RELEASE_SCRIPT = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +else + return 0 +end +`; + +type LeaderCallbacks = { + // Runs once each time this instance becomes leader. If it throws, the + // lease is given up and the next tick tries again (here or elsewhere). + onElected: () => Promise; + // Runs when leadership is lost or given up. Must be safe to call even if + // onElected only partly finished. + onDemoted: () => Promise; +}; + +/** + * Makes sure only one Pangolin instance sharing a Redis is running a given + * role at a time. + * + * Leadership is a Redis key, set with NX and a TTL, whose value is this + * instance's token. The leader renews it every tick and steps down as soon + * as a renewal fails or times out, so it has stopped before the lease + * can run out and another instance can take over. + * + * Talks to the Redis client directly rather than through lockManager: + * lockManager falls back to in-process locks while Redis is unreachable, + * which would let every instance become leader during an outage. Here an + * unreachable Redis means no instance is leader until it comes back. + * + * With Redis disabled, the deployment is assumed to be a single instance + * (Redis is required to run more than one), so this instance is always the + * leader. + */ +export class LeaderElection { + private readonly key: string; + private readonly callbacks: LeaderCallbacks; + private token: string | null = null; + private timer: NodeJS.Timeout | null = null; + private leader = false; + private running = false; + + constructor(name: string, callbacks: LeaderCallbacks) { + this.key = `leader:${name}`; + this.callbacks = callbacks; + } + + start(): void { + if (this.running) { + return; + } + this.running = true; + + if (!redisManager.getWriteClient()) { + logger.info( + `Redis is disabled - assuming a single instance and taking ${this.key}` + ); + void this.becomeLeader(); + return; + } + + logger.info(`Leader election started for ${this.key}`); + void this.tick(); + } + + async stop(): Promise { + if (!this.running) { + return; + } + this.running = false; + + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + + await this.stepDown("shutting down"); + } + + isLeader(): boolean { + return this.leader; + } + + // Ticks are chained with setTimeout rather than setInterval so a slow + // acquire/renew can never overlap with the next one. + private async tick(): Promise { + if (!this.running) { + return; + } + + try { + if (this.leader) { + await this.renew(); + } else { + await this.tryAcquire(); + } + } catch (error) { + logger.error(`Leader election error for ${this.key}:`, error); + await this.stepDown("leader election error"); + } + + if (this.running) { + this.timer = setTimeout(() => void this.tick(), TICK_INTERVAL_MS); + } + } + + private async tryAcquire(): Promise { + const client = redisManager.getWriteClient()!; + const token = uuidv4(); + + const result = await withTimeout( + client.set(this.key, token, "PX", LEASE_TTL_MS, "NX"), + REDIS_TIMEOUT_MS, + `acquire ${this.key}` + ); + + if (result !== "OK") { + logger.debug( + `Not leader for ${this.key} - another instance holds the lease` + ); + return; + } + + this.token = token; + + // stop() may have run while the SET was in flight. + if (!this.running) { + await this.release(); + return; + } + + await this.becomeLeader(); + } + + private async renew(): Promise { + if (!this.token) { + throw new Error("leader has no lease token"); + } + + const client = redisManager.getWriteClient()!; + const renewed = (await withTimeout( + client.eval( + RENEW_SCRIPT, + 1, + this.key, + this.token, + LEASE_TTL_MS.toString() + ), + REDIS_TIMEOUT_MS, + `renew ${this.key}` + )) as number; + + if (renewed !== 1) { + throw new Error("lease is no longer held by this instance"); + } + } + + private async becomeLeader(): Promise { + this.leader = true; + logger.info(`Elected leader for ${this.key}`); + + try { + await this.callbacks.onElected(); + } catch (error) { + logger.error( + `Failed to start ${this.key} after being elected leader:`, + error + ); + // Without Redis there's nobody to hand over to and no retry loop, + // so keep the role, just as it was before leader election. + if (this.token) { + await this.stepDown("startup failed"); + } + } + } + + private async stepDown(reason: string): Promise { + const wasLeader = this.leader; + this.leader = false; + + if (wasLeader) { + logger.warn(`Stepping down as leader for ${this.key}: ${reason}`); + try { + await this.callbacks.onDemoted(); + } catch (error) { + logger.error( + `Error stopping ${this.key} after stepping down:`, + error + ); + } + } + + // Only after onDemoted has stopped this instance from scheduling more + // work, so the next leader doesn't overlap with it. + await this.release(); + } + + // Best effort: if Redis is unreachable the lease just runs out after + // LEASE_TTL_MS instead. + private async release(): Promise { + const token = this.token; + this.token = null; + + const client = redisManager.getWriteClient(); + if (!token || !client) { + return; + } + + try { + await withTimeout( + client.eval(RELEASE_SCRIPT, 1, this.key, token), + REDIS_TIMEOUT_MS, + `release ${this.key}` + ); + } catch (error) { + logger.warn(`Failed to release ${this.key}:`, error); + } + } +}