Merge branch 'dev' into feat/redirect-tables

server/private/certificates.ts

@@ -0,0 +1,17 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+export {
+    startCertificateManager,
+    stopCertificateManager
+} from "./lib/certificates";

server/private/cleanup.ts

@@ -20,6 +20,8 @@ import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth"
 import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator";
 import { shutdownUsageRecorder } from "@server/lib/aiBudgetEnforcement";
 import { shutdownAiSessionLogger } from "@server/routers/aiGateway/logAiSession";
+import { stopDnsServer } from "./dns";
+import { stopCertificateManager } from "./certificates";
 
 async function cleanup() {
     await stopPingAccumulator();
@@ -31,6 +33,8 @@ async function cleanup() {
     await rateLimitService.cleanup();
     await wsCleanup();
     await logStreamingManager.shutdown();
+    await stopDnsServer();
+    await stopCertificateManager();
 
     process.exit(0);
 }

server/private/dns.ts

@@ -0,0 +1,37 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { AuthoritativeDNSServer } from "#private/lib/dns";
+import { privateConfig } from "#private/lib/config";
+
+let dnsServer: AuthoritativeDNSServer | undefined;
+
+export async function startDnsServer() {
+    const dnsConfig = privateConfig.getRawPrivateConfig().dns;
+    if (!dnsConfig || !dnsConfig.enabled) {
+        return;
+    }
+
+    dnsServer = new AuthoritativeDNSServer(dnsConfig.listen_port);
+
+    await dnsServer.start();
+}
+
+export async function stopDnsServer() {
+    if (!dnsServer) {
+        return;
+    }
+
+    await dnsServer.stop();
+    dnsServer = undefined;
+}

server/private/lib/cache.ts

@@ -11,17 +11,11 @@
  * This file is not licensed under the AGPLv3.
  */
 
-import NodeCache from "node-cache";
 import logger from "@server/logger";
-import { redisManager, regionalRedisManager } from "@server/private/lib/redis";
+import { createLocalCache } from "@server/lib/createLocalCache";
+import { redisManager, regionalRedisManager } from "#private/lib/redis";
 
-// Create local cache with maxKeys limit to prevent memory leaks
-// With ~10k requests/day and 5min TTL, 10k keys should be more than sufficient
-export const localCache = new NodeCache({
-    stdTTL: 3600,
-    checkperiod: 120,
-    maxKeys: 10000
-});
+export const localCache = createLocalCache();
 
 // Log cache statistics periodically for monitoring
 // setInterval(() => {
@@ -97,11 +91,11 @@ class AdaptiveCache {
                 const value = await redisManager.get(key);
 
                 if (value !== null) {
-                    logger.debug(`Cache hit in Redis: ${key}`);
+                    // logger.debug(`Cache hit in Redis: ${key}`);
                     return JSON.parse(value) as T;
                 }
 
-                logger.debug(`Cache miss in Redis: ${key}`);
+                // logger.debug(`Cache miss in Redis: ${key}`);
                 return undefined;
             } catch (error) {
                 logger.error(`Redis get error for key ${key}:`, error);
@@ -134,7 +128,7 @@ class AdaptiveCache {
                     const success = await redisManager.del(k);
                     if (success) {
                         deletedCount++;
-                        logger.debug(`Deleted key from Redis: ${k}`);
+                        // logger.debug(`Deleted key from Redis: ${k}`);
                     }
                 }
 
@@ -161,7 +155,7 @@ class AdaptiveCache {
             const success = localCache.del(k);
             if (success > 0) {
                 deletedCount++;
-                logger.debug(`Deleted key from local cache: ${k}`);
+                // logger.debug(`Deleted key from local cache: ${k}`);
             }
         }
 
@@ -229,7 +223,7 @@ class AdaptiveCache {
         }
 
         localCache.flushAll();
-        logger.debug("Flushed local cache");
+        // logger.debug("Flushed local cache");
     }
 
     /**
@@ -301,15 +295,11 @@ export default cache;
 
 /**
  * Regional adaptive cache backed by the in-cluster Redis instance.
- * Falls back to a local NodeCache when the regional Redis is unavailable.
+ * Falls back to a local LRU cache when the regional Redis is unavailable.
  * Use this for data that is regional in nature (e.g. status history) so
  * reads are served from the same cluster the user is hitting.
  */
-const regionalLocalCache = new NodeCache({
-    stdTTL: 3600,
-    checkperiod: 120,
-    maxKeys: 10000
-});
+const regionalLocalCache = createLocalCache();
 
 class RegionalAdaptiveCache {
     private useRedis(): boolean {
@@ -332,7 +322,7 @@ class RegionalAdaptiveCache {
                     redisTtl
                 );
                 if (success) {
-                    logger.debug(`[regional] Set key in Redis: ${key}`);
+                    // logger.debug(`[regional] Set key in Redis: ${key}`);
                     return true;
                 }
             } catch (error) {
@@ -353,10 +343,10 @@ class RegionalAdaptiveCache {
             try {
                 const value = await regionalRedisManager.get(key);
                 if (value !== null) {
-                    logger.debug(`[regional] Cache hit in Redis: ${key}`);
+                    // logger.debug(`[regional] Cache hit in Redis: ${key}`);
                     return JSON.parse(value) as T;
                 }
-                logger.debug(`[regional] Cache miss in Redis: ${key}`);
+                // logger.debug(`[regional] Cache miss in Redis: ${key}`);
                 return undefined;
             } catch (error) {
                 logger.error(
@@ -385,7 +375,7 @@ class RegionalAdaptiveCache {
                     const success = await regionalRedisManager.del(k);
                     if (success) {
                         deletedCount++;
-                        logger.debug(`[regional] Deleted key from Redis: ${k}`);
+                        // logger.debug(`[regional] Deleted key from Redis: ${k}`);
                     }
                 }
                 if (deletedCount === keys.length) return deletedCount;
@@ -400,7 +390,7 @@ class RegionalAdaptiveCache {
             const count = regionalLocalCache.del(k);
             if (count > 0) {
                 deletedCount++;
-                logger.debug(`[regional] Deleted key from local cache: ${k}`);
+                // logger.debug(`[regional] Deleted key from local cache: ${k}`);
             }
         }
         return deletedCount;

server/private/lib/certificates/acme-client.ts

@@ -0,0 +1,298 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import * as acme from "acme-client";
+import * as fs from "fs";
+import { eq } from "drizzle-orm/sql";
+import { privateConfig as config } from "#private/lib/config";
+import { DnsChallenge, db, dnsChallenge } from "@server/db";
+import { withRetry } from "@server/lib/retry";
+import logger from "@server/logger";
+import { acmeRateLimiter } from "./acmeRateLimiter";
+
+// acme-client's own retry/backoff logging (429 retries, 5xx retries, each
+// status-poll tick in waitForValidStatus) is a no-op by default - it only
+// activates via DEBUG=acme-client or this call, neither of which was wired
+// up. Without it, a cert silently retrying a Let's Encrypt rate limit for
+// several minutes is indistinguishable in our logs from one that's actually
+// hung, since our own logging only wraps the call, not what happens inside
+// it. Must run before any AcmeClient method is called.
+acme.setLogger((msg: string) => logger.info(`[acme-client] ${msg}`));
+
+// acme-client's axios retry wrapper treats any response-less request error
+// (timeout, connection reset, DNS blip reaching the ACME server) as
+// retryable, but once its internal retries are exhausted it falls through to
+// `validateStatus(response)` with `response` still undefined, throwing this
+// uninformative TypeError instead of the real network error.
+// https://github.com/publishlab/node-acme-client/blob/master/src/axios.js
+function isUnresponsiveAcmeError(error: unknown): boolean {
+    return (
+        error instanceof TypeError &&
+        error.message ===
+            "Cannot read properties of undefined (reading 'config')"
+    );
+}
+
+function normalizeAcmeError(error: unknown): Error {
+    if (isUnresponsiveAcmeError(error)) {
+        return new Error(
+            "ACME server did not respond after repeated attempts (network error reaching the ACME endpoint)",
+            { cause: error }
+        );
+    }
+    return error instanceof Error ? error : new Error(String(error));
+}
+
+export class AcmeClientManager {
+    private client: acme.Client | null = null;
+    private accountKey: string | null = null;
+
+    async initialize() {
+        try {
+            this.accountKey = await this.loadAccountKey();
+
+            this.client = new acme.Client({
+                directoryUrl: config.getRawConfig().acme!.acme_directory_url,
+                accountKey: this.accountKey
+            });
+
+            // Try to create account or get existing one
+            await this.client.createAccount({
+                termsOfServiceAgreed: true,
+                contact: [`mailto:${config.getRawConfig().acme!.contact_email}`]
+            });
+
+            logger.info("ACME client initialized successfully");
+        } catch (error) {
+            logger.error("Failed to initialize ACME client:", error);
+            throw error;
+        }
+    }
+
+    private async loadAccountKey(): Promise<string> {
+        const keyPath = config.getRawConfig().acme!.acme_account_key_path;
+
+        if (fs.existsSync(keyPath)) {
+            logger.info("Loading existing account key");
+            return fs.readFileSync(keyPath, "utf8");
+        } else {
+            logger.info("Generating new account key");
+            const privateKey = await acme.crypto.createPrivateKey();
+            const privateKeyString = privateKey.toString();
+            fs.writeFileSync(keyPath, privateKeyString);
+            return privateKeyString;
+        }
+    }
+
+    getClient(): acme.Client {
+        if (!this.client) {
+            throw new Error("ACME client not initialized");
+        }
+        return this.client;
+    }
+
+    async createOrder(domain: string, wildcard: boolean = false): Promise<any> {
+        const client = this.getClient();
+
+        const identifiers = wildcard
+            ? [
+                  { type: "dns", value: domain },
+                  { type: "dns", value: `*.${domain}` }
+              ]
+            : [{ type: "dns", value: domain }];
+
+        await acmeRateLimiter.acquire();
+        const order = await client.createOrder({
+            identifiers
+        });
+
+        if (wildcard) {
+            logger.info(`Created wildcard order for domain: ${domain}`);
+        } else {
+            logger.info(`Created order for domain: ${domain}`);
+        }
+        return order;
+    }
+
+    async getAuthorizations(order: any): Promise<any[]> {
+        const client = this.getClient();
+        await acmeRateLimiter.acquire();
+        return client.getAuthorizations(order);
+    }
+
+    async handleDnsChallenge(
+        dnsChallenges: {
+            authz: any;
+            challenge: any;
+        }[]
+    ): Promise<void> {
+        const client = this.getClient();
+
+        let challengeDomains: DnsChallenge[] = [];
+
+        for (const { authz, challenge } of dnsChallenges) {
+            const keyAuthorization =
+                await client.getChallengeKeyAuthorization(challenge);
+
+            // Extract the domain from authorization
+            const domain = authz.identifier.value;
+
+            // Store challenge in database for DNS server to pick up
+            challengeDomains = await withRetry(
+                () =>
+                    db
+                        .insert(dnsChallenge)
+                        .values({
+                            domain: domain,
+                            token: challenge.token,
+                            keyAuthorization,
+                            createdAt: Math.floor(Date.now() / 1000),
+                            expiresAt: Math.floor(
+                                (Date.now() +
+                                    config.getRawConfig().acme!
+                                        .challenge_ttl_ms) /
+                                    1000
+                            )
+                        })
+                        .returning(),
+                { label: `insert dnsChallenge for domain ${domain}` }
+            );
+
+            logger.info(
+                `DNS challenge stored for domain: ${domain} as token ${challenge.token} and keyAuthorization`
+            );
+        }
+
+        await new Promise((resolve) => setTimeout(resolve, 2000));
+
+        const failedDomains: string[] = [];
+
+        for (const { authz, challenge } of dnsChallenges) {
+            const domain = authz.identifier.value;
+            const challengeDomain = `_acme-challenge.${domain}`;
+
+            try {
+                // The ACME server occasionally has a transient network blip
+                // mid-sequence; retry the whole verify/complete/wait sequence
+                // rather than just the DNS challenge propagation wait, since
+                // these calls are safe to repeat against the ACME server.
+                await withRetry(
+                    async () => {
+                        // Verify challenge
+                        await acmeRateLimiter.acquire();
+                        await client.verifyChallenge(authz, challenge);
+
+                        // Complete challenge
+                        logger.info(
+                            `Completing challenge for domain: ${challengeDomain}`
+                        );
+                        await acmeRateLimiter.acquire();
+                        await client.completeChallenge(challenge);
+
+                        // Wait for validation
+                        logger.info(
+                            `Waiting for challenge to be validated for domain: ${challengeDomain}...`
+                        );
+                        await acmeRateLimiter.acquire();
+                        await client.waitForValidStatus(challenge);
+                    },
+                    {
+                        retries: 2,
+                        baseDelayMs: 5000,
+                        label: `ACME challenge completion for domain ${domain}`,
+                        // Only retry the known network-blip crash - a
+                        // genuine validation failure (e.g. challenge marked
+                        // "invalid" because the DNS record wasn't found) is
+                        // permanent and should fail immediately instead of
+                        // burning Let's Encrypt's per-hostname failed-
+                        // validation rate limit on retries that can't help.
+                        shouldRetry: isUnresponsiveAcmeError
+                    }
+                );
+
+                logger.info(`Challenge completed for domain: ${domain}`);
+            } catch (error) {
+                logger.error(
+                    `Failed to complete challenge for domain ${domain}:`,
+                    normalizeAcmeError(error)
+                );
+                failedDomains.push(domain);
+            }
+        }
+
+        for (const challengeDomain of challengeDomains) {
+            await this.removeDnsChallenge(challengeDomain.dnsChallengeId);
+            logger.info(
+                `Removed DNS challenge for domain: ${challengeDomain.domain}`
+            );
+        }
+
+        // A failed dns-01 challenge leaves the order stuck in "pending" -
+        // finalizing it would just fail with a confusing ACME error, so
+        // stop here and let the caller mark the certificate as failed.
+        if (failedDomains.length > 0) {
+            throw new Error(
+                `DNS-01 challenge validation failed for domain(s): ${failedDomains.join(", ")}`
+            );
+        }
+    }
+
+    async removeDnsChallenge(dnsChallengeId: number): Promise<void> {
+        try {
+            await withRetry(
+                () =>
+                    db
+                        .delete(dnsChallenge)
+                        .where(eq(dnsChallenge.dnsChallengeId, dnsChallengeId)),
+                { label: `delete dnsChallenge ${dnsChallengeId}` }
+            );
+        } catch (error) {
+            logger.error(
+                `Failed to clean up DNS challenge for id ${dnsChallengeId}:`,
+                error
+            );
+        }
+    }
+
+    async finalizeCertificate(
+        order: any,
+        domain: string,
+        wildcard: boolean = false
+    ): Promise<{ certificate: string; privateKey: string }> {
+        const client = this.getClient();
+
+        const altNames = wildcard ? [`*.${domain}`, domain] : [domain];
+
+        // Create CSR
+        const [privateKey, csr] = await acme.crypto.createCsr({
+            altNames
+        });
+
+        // Finalize order
+        await acmeRateLimiter.acquire();
+        const finalizedOrder = await client.finalizeOrder(order, csr);
+
+        // Get certificate
+        await acmeRateLimiter.acquire();
+        const certificate = await client.getCertificate(finalizedOrder);
+
+        logger.info(`Certificate obtained for domain: ${domain}`);
+
+        return {
+            certificate: certificate.toString(),
+            privateKey: privateKey.toString()
+        };
+    }
+}
+
+export const acmeClientManager = new AcmeClientManager();

server/private/lib/certificates/acmeRateLimiter.ts

@@ -0,0 +1,71 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { privateConfig as config } from "#private/lib/config";
+import logger from "@server/logger";
+import { redis } from "../redis";
+// Caps outgoing ACME API calls to a fixed budget per wall-clock second,
+// shared across all pops workers via Redis (mirrors the lockManager pattern
+// in @lib/lock) - a per-process limiter wouldn't be enough since multiple
+// workers issue certificates against the same Let's Encrypt account.
+const ACQUIRE_SCRIPT = `
+local key = KEYS[1]
+local limit = tonumber(ARGV[1])
+local current = redis.call('INCR', key)
+if current == 1 then
+    redis.call('PEXPIRE', key, 2000)
+end
+if current > limit then
+    return 0
+else
+    return 1
+end
+`;
+
+class AcmeRateLimiter {
+    async acquire(): Promise<void> {
+        const limit =
+            config.getRawConfig().acme?.acme_requests_per_second ?? 15;
+
+        for (;;) {
+            const bucket = Math.floor(Date.now() / 1000);
+            const key = `acme_rate_limit:${bucket}`;
+
+            let allowed: number;
+            try {
+                allowed = (await redis.eval(
+                    ACQUIRE_SCRIPT,
+                    1,
+                    key,
+                    limit.toString()
+                )) as number;
+            } catch (error) {
+                logger.error(
+                    "ACME rate limiter check failed, proceeding without throttling:",
+                    error
+                );
+                return;
+            }
+
+            if (allowed === 1) {
+                return;
+            }
+
+            // Budget for this second is spent - wait for the next window.
+            const waitMs = 1000 - (Date.now() % 1000) + 10;
+            await new Promise((resolve) => setTimeout(resolve, waitMs));
+        }
+    }
+}
+
+export const acmeRateLimiter = new AcmeRateLimiter();

server/private/lib/certificates/certificate-service.ts

@@ -0,0 +1,511 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { acmeClientManager } from "./acme-client";
+import { dnsValidator } from "./dns-validator";
+import { getTableColumns } from "drizzle-orm";
+import { eq, and, or, isNull, lt, asc } from "drizzle-orm/sql";
+import { config } from "@server/lib/config";
+import { db, certificates, domains, Certificate } from "@server/db";
+import { encrypt } from "@server/lib/crypto";
+import { withTimeout, withRetry } from "@server/lib/retry";
+import logger from "@server/logger";
+import { lockManager } from "../lock";
+import { pushCertUpdateToAffectedNewts } from "@server/lib/acmeCertSync";
+import crypto from "crypto";
+
+// Number of on-demand DNS validation attempts made right before a
+// certificate is (re)issued, to avoid burning Let's Encrypt rate limits on
+// domains whose DNS has drifted since they were last verified.
+const PRE_CERT_DNS_VALIDATION_ATTEMPTS = 3;
+
+// Hard ceiling on a single certificate's issuance/renewal flow. acme-client's
+// axios instance never sets a request timeout, so a stalled connection to
+// the ACME server hangs forever instead of erroring - and since
+// processPendingCertificates/processRenewalCandidates gate the *next* batch
+// on Promise.all(...) over the current one, one hung certificate would
+// otherwise stall every other domain permanently. Sized generously above the
+// legitimate worst case (acme-client's own bounded backoff is ~3.6min per
+// status-polling loop, and a wildcard cert's two identifiers plus order
+// finalization can chain a few of those) so this only fires on a genuine hang.
+const CERTIFICATE_ISSUANCE_TIMEOUT_MS = 20 * 60 * 1000;
+
+// "requested" is set the instant a cert starts processing and is never
+// queried anywhere else - processPendingCertificates only selects "pending"
+// and processRenewalCandidates only selects "valid". So if the *process*
+// dies mid-flight (OOM, node eviction, a rolling deploy) rather than just
+// hanging, the row is orphaned in "requested" permanently with nothing to
+// ever pick it back up, no matter how good the in-process timeouts are.
+// Threshold is set comfortably above CERTIFICATE_ISSUANCE_TIMEOUT_MS plus the
+// scheduler's own outer backstop so this never reclaims a cert that's still
+// genuinely being worked on.
+const STUCK_CERTIFICATE_THRESHOLD_MS = 40 * 60 * 1000;
+
+export class CertificateService {
+    // Runs at the top of every processPendingCertificates tick so an
+    // interrupted worker's leftovers always get put back in the queue
+    // instead of sitting invisible to every query forever.
+    private async reclaimStuckCertificates(): Promise<void> {
+        const staleBefore =
+            Math.floor(Date.now() / 1000) -
+            Math.floor(STUCK_CERTIFICATE_THRESHOLD_MS / 1000);
+
+        const reclaimed = await db
+            .update(certificates)
+            .set({
+                status: "pending",
+                errorMessage:
+                    'Reclaimed after being stuck in "requested" state - the worker processing it likely restarted or crashed',
+                updatedAt: Math.floor(Date.now() / 1000)
+            })
+            .where(
+                and(
+                    eq(certificates.status, "requested"),
+                    lt(certificates.updatedAt, staleBefore)
+                )
+            )
+            .returning({ domain: certificates.domain });
+
+        if (reclaimed.length > 0) {
+            logger.warn(
+                `Reclaimed ${reclaimed.length} certificate(s) stuck in "requested" state: ${reclaimed
+                    .map((c) => c.domain)
+                    .join(", ")}`
+            );
+        }
+    }
+
+    async processPendingCertificates(): Promise<void> {
+        logger.debug("Checking for pending certificates...");
+
+        await this.reclaimStuckCertificates();
+
+        const pendingCerts = await db
+            .select(getTableColumns(certificates))
+            .from(certificates)
+            .leftJoin(domains, eq(certificates.domainId, domains.domainId))
+            .where(
+                and(
+                    eq(certificates.status, "pending"),
+                    or(
+                        // Certs with no linked domain row (e.g. legacy certs
+                        // imported from acme.json) aren't gated on domain
+                        // verification since there's nothing to check.
+                        isNull(certificates.domainId),
+                        and(
+                            eq(domains.verified, true),
+                            eq(domains.failed, false)
+                        )
+                    )
+                )
+            )
+            .limit(10);
+
+        if (pendingCerts.length === 0) {
+            logger.debug("No pending certificates found");
+            return;
+        }
+
+        logger.info(`Found ${pendingCerts.length} pending certificates`);
+
+        // Process the batch concurrently so one domain stuck retrying a slow
+        // DNS-01 challenge (the ACME client's waitForValidStatus can spend
+        // minutes on a bad domain) doesn't stall the rest of the batch.
+        // processSingleCertificate catches its own errors and each cert uses
+        // an independent per-domain lock, so this is safe to parallelize.
+        await Promise.all(
+            pendingCerts.map((cert) => this.processSingleCertificate(cert))
+        );
+    }
+
+    async processRenewalCandidates(): Promise<void> {
+        logger.debug("Checking for certificates needing renewal...");
+
+        const now = Math.floor(Date.now() / 1000);
+
+        const renewalCandidates = await db
+            .select(getTableColumns(certificates))
+            .from(certificates)
+            .leftJoin(domains, eq(certificates.domainId, domains.domainId))
+            .where(
+                and(
+                    eq(certificates.status, "valid"),
+                    lt(certificates.expiresAt, now + 15 * 24 * 60 * 60), // 15 days from now
+                    or(
+                        // Certs with no linked domain row (e.g. legacy certs
+                        // imported from acme.json) aren't gated on domain
+                        // verification since there's nothing to check.
+                        isNull(certificates.domainId),
+                        and(
+                            eq(domains.verified, true),
+                            eq(domains.failed, false)
+                        )
+                    )
+                )
+            )
+            // Most urgent first, so already-expired certs aren't starved
+            // behind the limit by certs that still have weeks of runway.
+            .orderBy(asc(certificates.expiresAt))
+            .limit(50);
+
+        if (renewalCandidates.length === 0) {
+            logger.debug("No certificates need renewal");
+            return;
+        }
+
+        logger.info(
+            `Found ${renewalCandidates.length} certificates needing renewal`
+        );
+
+        for (const cert of renewalCandidates) {
+            if (cert.expiresAt !== null && cert.expiresAt < now) {
+                logger.warn(
+                    `Certificate for ${cert.domain} is marked "valid" but already expired at ${new Date(cert.expiresAt * 1000).toISOString()} (bad state) - renewing immediately`
+                );
+            }
+        }
+
+        // Process the batch concurrently - see processPendingCertificates for why.
+        await Promise.all(
+            renewalCandidates.map((cert) => this.renewCertificate(cert))
+        );
+    }
+
+    private async processSingleCertificate(cert: Certificate): Promise<void> {
+        const lockKey = `cert:${cert.domain}`;
+
+        const lockToken = await lockManager.acquireLock(lockKey);
+        if (!lockToken) {
+            logger.debug(
+                `Could not acquire lock for certificate: ${cert.domain}`
+            );
+            return;
+        }
+
+        try {
+            logger.info(`Processing certificate for domain: ${cert.domain}`);
+
+            // Update status to processing
+            await db
+                .update(certificates)
+                .set({
+                    status: "requested",
+                    updatedAt: Math.floor(Date.now() / 1000)
+                })
+                .where(eq(certificates.certId, cert.certId));
+            //
+
+            await withTimeout(
+                this.obtainCertificate(cert),
+                CERTIFICATE_ISSUANCE_TIMEOUT_MS,
+                `certificate issuance for ${cert.domain}`
+            );
+        } catch (error) {
+            logger.error(
+                `Failed to process certificate for ${cert.domain}:`,
+                error
+            );
+
+            await db
+                .update(certificates)
+                .set({
+                    status: "failed",
+                    errorMessage:
+                        error instanceof Error
+                            ? error.message
+                            : "Unknown error",
+                    updatedAt: Math.floor(Date.now() / 1000)
+                })
+                .where(eq(certificates.certId, cert.certId));
+        } finally {
+            await lockManager.releaseLock(lockKey, lockToken);
+        }
+    }
+
+    private async renewCertificate(cert: Certificate): Promise<void> {
+        const lockKey = `cert:${cert.domain}`;
+
+        const lockToken = await lockManager.acquireLock(lockKey);
+        if (!lockToken) {
+            logger.debug(
+                `Could not acquire lock for certificate renewal: ${cert.domain}`
+            );
+            return;
+        }
+
+        try {
+            logger.info(`Renewing certificate for domain: ${cert.domain}`);
+
+            // Update last renewal attempt
+            await db
+                .update(certificates)
+                .set({
+                    lastRenewalAttempt: Math.floor(Date.now() / 1000),
+                    updatedAt: Math.floor(Date.now() / 1000)
+                })
+                .where(eq(certificates.certId, cert.certId));
+
+            await withTimeout(
+                this.obtainCertificate(cert),
+                CERTIFICATE_ISSUANCE_TIMEOUT_MS,
+                `certificate renewal for ${cert.domain}`
+            );
+        } catch (error) {
+            logger.error(
+                `Failed to renew certificate for ${cert.domain}:`,
+                error
+            );
+
+            await db
+                .update(certificates)
+                .set({
+                    status: "failed",
+                    errorMessage:
+                        error instanceof Error
+                            ? error.message
+                            : "Unknown error",
+                    lastRenewalAttempt: Math.floor(Date.now() / 1000),
+                    updatedAt: Math.floor(Date.now() / 1000)
+                })
+                .where(eq(certificates.certId, cert.certId));
+        } finally {
+            await lockManager.releaseLock(lockKey, lockToken);
+        }
+    }
+
+    // Re-checks the domain's DNS records right before we spend a Let's
+    // Encrypt order on it, so drift that happened after the domain was
+    // originally verified doesn't burn ACME rate limits. Certs with no
+    // linked domain row (e.g. legacy/manually-managed certs) skip this and
+    // proceed as before, since there are no tracked DNS records to check.
+    private async verifyDomainBeforeIssuance(cert: Certificate): Promise<void> {
+        if (!cert.domainId) {
+            return;
+        }
+
+        const [domain] = await db
+            .select()
+            .from(domains)
+            .where(eq(domains.domainId, cert.domainId))
+            .limit(1);
+
+        if (!domain) {
+            return;
+        }
+
+        for (
+            let attempt = 1;
+            attempt <= PRE_CERT_DNS_VALIDATION_ATTEMPTS;
+            attempt++
+        ) {
+            // Offset `tries` so each attempt round-robins to a different
+            // privateConfigured DNS resolver instead of re-querying the same one.
+            const probe = { ...domain, tries: domain.tries + attempt - 1 };
+            if (
+                await dnsValidator.validateDomain(probe, {
+                    forceRecheck: true
+                })
+            ) {
+                await db
+                    .update(domains)
+                    .set({ verified: true, failed: false, errorMessage: null })
+                    .where(eq(domains.domainId, domain.domainId));
+                return;
+            }
+
+            logger.warn(
+                `Pre-certificate DNS check ${attempt}/${PRE_CERT_DNS_VALIDATION_ATTEMPTS} failed for domain ${domain.baseDomain} (cert: ${cert.domain})`
+            );
+        }
+
+        const errorMessage = `Domain failed DNS validation ${PRE_CERT_DNS_VALIDATION_ATTEMPTS} times before certificate issuance`;
+        await db
+            .update(domains)
+            .set({ verified: false, failed: true, errorMessage })
+            .where(eq(domains.domainId, domain.domainId));
+
+        throw new Error(errorMessage);
+    }
+
+    private async obtainCertificate(cert: Certificate): Promise<void> {
+        await this.verifyDomainBeforeIssuance(cert);
+
+        // Create order
+        const order = await acmeClientManager.createOrder(
+            cert.domain,
+            cert.wildcard || false
+        );
+
+        // Update with order ID
+        await withRetry(
+            () =>
+                db
+                    .update(certificates)
+                    .set({
+                        orderId: order.url,
+                        updatedAt: Math.floor(Date.now() / 1000)
+                    })
+                    .where(eq(certificates.certId, cert.certId)),
+            { label: `update orderId for certificate ${cert.domain}` }
+        );
+
+        // Get authorizations
+        const authorizations = await acmeClientManager.getAuthorizations(order);
+
+        // Aggregate all DNS-01 challenges
+        const dnsChallenges = authorizations.map((authz: any) => {
+            const dnsChallenge = authz.challenges.find(
+                (c: any) => c.type === "dns-01"
+            );
+            if (!dnsChallenge) {
+                throw new Error(
+                    `No DNS-01 challenge found for ${authz.identifier.value}`
+                );
+            }
+            return {
+                authz,
+                challenge: dnsChallenge
+            };
+        });
+
+        // Send all DNS-01 challenges in one request to handleDnsChallenge
+        await acmeClientManager.handleDnsChallenge(dnsChallenges);
+
+        // Finalize certificate
+        const { certificate, privateKey } =
+            await acmeClientManager.finalizeCertificate(
+                order,
+                cert.domain,
+                cert.wildcard || false
+            );
+
+        const encryptionKey = config.getRawConfig().server.secret;
+        if (!encryptionKey) {
+            throw new Error("Encryption key not provided");
+        }
+
+        // Encrypt certificate and private key
+        const encryptedCert = encrypt(certificate, encryptionKey);
+        const encryptedKey = encrypt(privateKey, encryptionKey);
+
+        // Parse certificate to get expiration date
+        const expiresAt = this.extractExpirationDate(certificate);
+
+        // Update database record. This persists the certificate we just
+        // obtained from the ACME server, so it's retried aggressively -
+        // losing this write means re-issuing the cert from scratch.
+        await withRetry(
+            () =>
+                db
+                    .update(certificates)
+                    .set({
+                        status: "valid",
+                        expiresAt: Math.floor(expiresAt.getTime() / 1000),
+                        renewalCount: (cert.renewalCount || 0) + 1,
+                        errorMessage: null,
+                        updatedAt: Math.floor(Date.now() / 1000),
+                        certFile: encryptedCert,
+                        keyFile: encryptedKey
+                    })
+                    .where(eq(certificates.certId, cert.certId)),
+            {
+                retries: 5,
+                label: `persist issued certificate for ${cert.domain}`
+            }
+        );
+
+        logger.info(
+            `Certificate successfully obtained/renewed for domain: ${cert.domain}`
+        );
+
+        await pushCertUpdateToAffectedNewts(
+            cert.domain,
+            cert.domainId ?? null,
+            certificate,
+            privateKey
+        );
+    }
+
+    private extractExpirationDate(certificate: string): Date {
+        try {
+            // Extract the certificate block
+            const pem = certificate
+                .replace(/-----BEGIN CERTIFICATE-----/g, "")
+                .replace(/-----END CERTIFICATE-----/g, "")
+                .replace(/\s+/g, "");
+            const der = Buffer.from(pem, "base64");
+
+            // Use Node.js crypto to parse the certificate
+            const x509 = new crypto.X509Certificate(der);
+            return new Date(x509.validTo);
+        } catch (error) {
+            logger.warn(
+                "Failed to parse certificate expiration date, using default",
+                error
+            );
+            // Default to 90 days from now (Let's Encrypt default)
+            return new Date(Date.now() + 90 * 24 * 60 * 60 * 1000);
+        }
+    }
+
+    async addCertificateRequest(domain: string): Promise<void> {
+        try {
+            await db.insert(certificates).values({
+                domain,
+                status: "pending",
+                createdAt: Math.floor(Date.now() / 1000),
+                updatedAt: Math.floor(Date.now() / 1000)
+            });
+            logger.info(`Certificate request added for domain: ${domain}`);
+        } catch (error) {
+            if (error instanceof Error && error.message.includes("unique")) {
+                logger.warn(
+                    `Certificate request already exists for domain: ${domain}`
+                );
+            } else {
+                throw error;
+            }
+        }
+    }
+
+    async getCertificateStatus(domain: string) {
+        const cert = await db
+            .select()
+            .from(certificates)
+            .where(eq(certificates.domain, domain))
+            .limit(1);
+
+        return cert[0] || null;
+    }
+
+    async cleanupExpiredChallenges(): Promise<void> {
+        try {
+            const result = await db
+                .delete(certificates)
+                .where(
+                    lt(certificates.expiresAt, Math.floor(Date.now() / 1000))
+                )
+                .returning();
+
+            if (result.length > 0) {
+                logger.info(
+                    `Cleaned up ${result.length} expired DNS challenges`
+                );
+            }
+        } catch (error) {
+            logger.error("Failed to cleanup expired challenges:", error);
+        }
+    }
+}
+
+export const certificateService = new CertificateService();

server/private/lib/certificates/dns-validator.ts

@@ -0,0 +1,334 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { eq, and, lt } from "drizzle-orm";
+import * as dns from "dns/promises";
+import { privateConfig as config } from "#private/lib/config";
+import { db, domains, DnsRecord, dnsRecords, Domain } from "@server/db";
+import logger from "@server/logger";
+import { lockManager } from "../lock";
+
+export const DNS_VALIDATOR_MAX_TRIES = 300;
+
+export class DNSValidator {
+    private static readonly MAX_TRIES = DNS_VALIDATOR_MAX_TRIES;
+
+    constructor() {}
+
+    async validateAll(): Promise<void> {
+        // Get all domains that are not yet verified and haven't exceeded max tries
+        const unverifiedDomains: Domain[] = await db
+            .select()
+            .from(domains)
+            .where(
+                and(
+                    eq(domains.verified, false),
+                    lt(domains.tries, DNSValidator.MAX_TRIES)
+                )
+            );
+
+        if (unverifiedDomains.length === 0) {
+            logger.debug("No unverified domains found for DNS validation");
+            return;
+        }
+
+        logger.info(`Validating ${unverifiedDomains.length} DNS records`);
+
+        for (const domain of unverifiedDomains) {
+            const lockKey = `dns:${domain.baseDomain}`;
+            const lockToken = await lockManager.acquireLock(lockKey);
+            if (!lockToken) {
+                logger.debug(
+                    `Could not acquire lock for DNS validation: ${domain.baseDomain}`
+                );
+                continue;
+            }
+            try {
+                const isValid = await this.validateDomain(domain);
+                if (isValid) {
+                    await db
+                        .update(domains)
+                        .set({
+                            verified: true,
+                            failed: false,
+                            tries: 0,
+                            errorMessage: null
+                        })
+                        .where(eq(domains.domainId, domain.domainId));
+                    logger.info(
+                        `Domain ${domain.baseDomain} validated successfully`
+                    );
+                } else {
+                    const newTries = domain.tries + 1;
+                    const shouldMarkAsFailed =
+                        newTries >= DNSValidator.MAX_TRIES;
+
+                    await db
+                        .update(domains)
+                        .set({
+                            tries: newTries,
+                            failed: shouldMarkAsFailed
+                        })
+                        .where(eq(domains.domainId, domain.domainId));
+
+                    if (shouldMarkAsFailed) {
+                        logger.warn(
+                            `Domain ${domain.baseDomain} exceeded maximum tries (${DNSValidator.MAX_TRIES}), marking as failed`
+                        );
+                    } else {
+                        logger.debug(
+                            `Domain ${domain.baseDomain} did not validate (attempt ${newTries}/${DNSValidator.MAX_TRIES})`
+                        );
+                    }
+                }
+            } catch (err) {
+                logger.warn(
+                    `Error validating domain ${domain.baseDomain}:`,
+                    err
+                );
+                // Increment tries even on error
+                const newTries = domain.tries + 1;
+                const shouldMarkAsFailed = newTries >= DNSValidator.MAX_TRIES;
+
+                await db
+                    .update(domains)
+                    .set({
+                        tries: newTries,
+                        failed: shouldMarkAsFailed
+                    })
+                    .where(eq(domains.domainId, domain.domainId));
+            } finally {
+                await lockManager.releaseLock(lockKey, lockToken);
+            }
+        }
+    }
+
+    async validateDomain(
+        domain: Domain,
+        opts: { forceRecheck?: boolean } = {}
+    ): Promise<boolean> {
+        const { forceRecheck = false } = opts;
+        const resolver = new dns.Resolver();
+        const servers = config.getRawConfig().acme?.dns_resolvers;
+        if (!servers || servers.length === 0) {
+            throw new Error("No DNS resolvers configured");
+        }
+        const dnsServer = servers[domain.tries % servers.length]!;
+        resolver.setServers([dnsServer]);
+        logger.debug(
+            `Using DNS server ${dnsServer} for domain ${domain.baseDomain} (try ${domain.tries})`
+        );
+
+        // Get all DNS records for this domain
+        const records: DnsRecord[] = await db
+            .select()
+            .from(dnsRecords)
+            .where(eq(dnsRecords.domainId, domain.domainId));
+
+        if (records.length === 0) {
+            logger.warn(`No DNS records found for domain ${domain.baseDomain}`);
+            return false;
+        }
+
+        if (!forceRecheck && records.every((r) => r.verified)) {
+            logger.info(
+                `All DNS records already verified for domain ${domain.baseDomain}`
+            );
+            return true;
+        }
+
+        logger.info(
+            `Validating ${records.length} DNS records for domain ${domain.baseDomain}`
+        );
+
+        // Collect the full set of expected NS values for this domain so we can
+        // detect extra records that are present in DNS but not in our DB.
+        const expectedNsValues = new Set<string>(
+            records.filter((r) => r.recordType === "NS").map((r) => r.value)
+        );
+
+        // Cache resolved NS records across iterations — there will be 3 NS
+        // records in the DB and we don't need to hit the upstream server 3 times.
+        let previousNs: string[] | null = null;
+
+        for (const record of records) {
+            // Skip already verified records, unless a live recheck was requested
+            if (record.verified && !forceRecheck) {
+                continue;
+            }
+
+            let isValid = false;
+
+            try {
+                if (record.recordType === "NS") {
+                    let nsRecords: string[] | null = previousNs;
+                    if (!nsRecords) {
+                        nsRecords = await resolver.resolveNs(
+                            record.baseDomain || domain.baseDomain
+                        );
+                    }
+                    logger.info(
+                        `NS records for ${
+                            record.baseDomain || domain.baseDomain
+                        }:`,
+                        nsRecords
+                    );
+
+                    // Check if this expected NS value is present in the live records.
+                    // A stale/legacy expected value (e.g. left over from a
+                    // nameserver rebrand) is also accepted as long as the live
+                    // records resolve to some other known-valid nameserver —
+                    // the specific literal hostname stored per-domain isn't
+                    // meaningful once it's a recognized alias.
+                    isValid = nsRecords.some((ns) => ns === record.value);
+
+                    previousNs = nsRecords;
+                } else if (record.recordType === "CNAME") {
+                    const cnameRecords = await resolver.resolveCname(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    logger.info(
+                        `CNAME records for ${
+                            record.baseDomain || domain.baseDomain
+                        }:`,
+                        cnameRecords
+                    );
+
+                    // Check if the CNAME record matches the expected value
+                    isValid =
+                        cnameRecords.length === 1 &&
+                        cnameRecords[0] === record.value;
+                } else if (record.recordType === "TXT") {
+                    const txtRecords = await resolver.resolveTxt(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    logger.info(
+                        `TXT records for ${
+                            record.baseDomain || domain.baseDomain
+                        }:`,
+                        txtRecords
+                    );
+
+                    // TXT records come as an array of arrays, flatten and check
+                    const flatTxtRecords = txtRecords.flat();
+                    isValid = flatTxtRecords.includes(record.value);
+                } else if (record.recordType === "A") {
+                    const aRecords = await resolver.resolve4(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    logger.info(
+                        `A records for ${
+                            record.baseDomain || domain.baseDomain
+                        }:`,
+                        aRecords
+                    );
+
+                    // Check if the A record matches the expected value
+                    isValid = aRecords.includes(record.value);
+                } else {
+                    logger.warn(
+                        `Unsupported record type: ${record.recordType}`
+                    );
+                    continue;
+                }
+            } catch (error) {
+                isValid = false;
+                logger.debug(
+                    `Did not resolve ${record.recordType} record for ${
+                        record.baseDomain || domain.baseDomain
+                    }:`,
+                    error
+                );
+            }
+
+            // Update the individual record verification status. Runs for
+            // both a mismatched value and a failed/thrown DNS lookup, so a
+            // previously-verified record that stops resolving (e.g. NXDOMAIN
+            // after NS delegation is dropped) gets downgraded instead of
+            // leaving stale `verified: true` state behind.
+            if (isValid) {
+                await db
+                    .update(dnsRecords)
+                    .set({ verified: true })
+                    .where(eq(dnsRecords.id, record.id));
+                logger.info(
+                    `DNS record ${record.id} (${record.recordType}) for ${
+                        record.baseDomain || domain.baseDomain
+                    } verified successfully`
+                );
+            } else {
+                if (record.verified) {
+                    await db
+                        .update(dnsRecords)
+                        .set({ verified: false })
+                        .where(eq(dnsRecords.id, record.id));
+                }
+                logger.debug(
+                    `DNS record ${record.id} (${record.recordType}) for ${
+                        record.baseDomain || domain.baseDomain
+                    } does not match expected value: ${record.value}`
+                );
+            }
+        }
+
+        // --- Extra NS record check ---
+        // If we resolved NS records during this pass, verify that the live DNS
+        // has no nameservers beyond the ones we expect.  Individual records may
+        // already be marked verified above, but we must block full domain
+        // verification until the extra records are removed.
+        if (previousNs !== null && expectedNsValues.size > 0) {
+            const extraNsRecords = previousNs.filter(
+                (ns) => !expectedNsValues.has(ns)
+            );
+
+            if (extraNsRecords.length > 0) {
+                const errorMessage = `Extra NS records found that are not expected: ${extraNsRecords.join(", ")}. Remove these nameservers to complete domain verification.`;
+
+                await db
+                    .update(domains)
+                    .set({ errorMessage })
+                    .where(eq(domains.domainId, domain.domainId));
+
+                logger.warn(
+                    `Domain ${domain.baseDomain} has extra NS records that prevent verification: ${extraNsRecords.join(", ")}`
+                );
+
+                return false;
+            }
+
+            // No extras — clear any stale error that was previously written
+            await db
+                .update(domains)
+                .set({ errorMessage: null })
+                .where(eq(domains.domainId, domain.domainId));
+        }
+
+        // Check if all records are now verified
+        const updatedRecords: DnsRecord[] = await db
+            .select()
+            .from(dnsRecords)
+            .where(eq(dnsRecords.domainId, domain.domainId));
+
+        const allRecordsVerified = updatedRecords.every((r) => r.verified);
+
+        logger.info(
+            `Domain ${domain.baseDomain}: ${
+                updatedRecords.filter((r) => r.verified).length
+            }/${updatedRecords.length} records verified`
+        );
+
+        return allRecordsVerified;
+    }
+}
+
+export const dnsValidator = new DNSValidator();

server/private/lib/certificates/domain-reverifier.ts

@@ -0,0 +1,233 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { eq, and, or, isNull, lt } from "drizzle-orm";
+import * as dns from "dns/promises";
+import { DNS_VALIDATOR_MAX_TRIES } from "./dns-validator";
+import { db, domains, DnsRecord, dnsRecords, Domain } from "@server/db";
+import logger from "@server/logger";
+import { lockManager } from "../lock";
+import { privateConfig as config } from "#private/lib/config";
+
+// Module-level counter so successive domains in a batch round-robin across servers.
+let serverIndex = 0;
+
+export class DomainReverifier {
+    async reverifyAll(): Promise<void> {
+        const certConfig = config.getRawConfig().acme;
+        if (!certConfig) {
+            logger.debug(
+                "No certificate config — skipping domain reverification"
+            );
+            return;
+        }
+
+        const windowMs = certConfig.domain_reverification_window_ms;
+        const batchSize = certConfig.domain_reverification_batch_size;
+        const windowSecs = Math.floor(windowMs / 1000);
+        const cutoff = Math.floor(Date.now() / 1000) - windowSecs;
+
+        const domainsToCheck: Domain[] = await db
+            .select()
+            .from(domains)
+            .where(
+                and(
+                    eq(domains.verified, true),
+                    or(
+                        isNull(domains.lastCheckedAt),
+                        lt(domains.lastCheckedAt, cutoff)
+                    )
+                )
+            )
+            .limit(batchSize);
+
+        if (domainsToCheck.length === 0) {
+            logger.debug("No verified domains due for reverification");
+            return;
+        }
+
+        logger.info(`Reverifying ${domainsToCheck.length} domains`);
+
+        for (const domain of domainsToCheck) {
+            const lockKey = `dns-reverify:${domain.baseDomain}`;
+            const lockToken = await lockManager.acquireLock(lockKey);
+            if (!lockToken) {
+                logger.debug(
+                    `Could not acquire lock for domain reverification: ${domain.baseDomain}`
+                );
+                continue;
+            }
+
+            try {
+                await this.reverifyDomain(domain, certConfig.dns_resolvers);
+            } catch (err) {
+                logger.warn(
+                    `Unexpected error reverifying domain ${domain.baseDomain}:`,
+                    err
+                );
+                // Still stamp lastCheckedAt so we don't hammer a broken domain every run.
+                await db
+                    .update(domains)
+                    .set({ lastCheckedAt: Math.floor(Date.now() / 1000) })
+                    .where(eq(domains.domainId, domain.domainId));
+            } finally {
+                await lockManager.releaseLock(lockKey, lockToken);
+            }
+        }
+    }
+
+    private async reverifyDomain(
+        domain: Domain,
+        servers: string[]
+    ): Promise<void> {
+        if (!servers || servers.length === 0) {
+            throw new Error("No DNS resolvers configured");
+        }
+
+        // Round-robin across servers; advance the global counter so the next
+        // domain in the same batch gets a different server.
+        const dnsServer = servers[serverIndex % servers.length]!;
+        serverIndex++;
+
+        const resolver = new dns.Resolver();
+        resolver.setServers([dnsServer]);
+
+        logger.debug(
+            `Reverifying domain ${domain.baseDomain} using DNS server ${dnsServer}`
+        );
+
+        const records: DnsRecord[] = await db
+            .select()
+            .from(dnsRecords)
+            .where(eq(dnsRecords.domainId, domain.domainId));
+
+        if (records.length === 0) {
+            logger.warn(
+                `No DNS records found for domain ${domain.baseDomain} during reverification — marking failed`
+            );
+            await this.markFailed(
+                domain.domainId,
+                "No DNS records found during periodic reverification"
+            );
+            return;
+        }
+
+        const expectedNsValues = new Set<string>(
+            records.filter((r) => r.recordType === "NS").map((r) => r.value)
+        );
+
+        let allValid = true;
+        let errorMessage: string | null = null;
+        let resolvedNs: string[] | null = null;
+
+        for (const record of records) {
+            let isValid = false;
+
+            try {
+                if (record.recordType === "NS") {
+                    if (!resolvedNs) {
+                        resolvedNs = await resolver.resolveNs(
+                            record.baseDomain || domain.baseDomain
+                        );
+                    }
+                    isValid = resolvedNs.some((ns) => ns === record.value);
+                } else if (record.recordType === "CNAME") {
+                    const cnameRecords = await resolver.resolveCname(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    isValid =
+                        cnameRecords.length === 1 &&
+                        cnameRecords[0] === record.value;
+                } else if (record.recordType === "TXT") {
+                    const txtRecords = await resolver.resolveTxt(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    isValid = txtRecords.flat().includes(record.value);
+                } else if (record.recordType === "A") {
+                    const aRecords = await resolver.resolve4(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    isValid = aRecords.includes(record.value);
+                } else {
+                    logger.warn(
+                        `Unsupported record type ${record.recordType} during reverification of ${domain.baseDomain}`
+                    );
+                    continue;
+                }
+            } catch (err) {
+                logger.debug(
+                    `DNS lookup failed for ${record.recordType} record on ${record.baseDomain || domain.baseDomain}:`,
+                    err
+                );
+                isValid = false;
+            }
+
+            if (!isValid) {
+                allValid = false;
+                errorMessage = `${record.recordType} record for ${record.baseDomain || domain.baseDomain} no longer resolves to expected value "${record.value}"`;
+                break;
+            }
+        }
+
+        // Check for extra NS records beyond what we expect.
+        if (allValid && resolvedNs !== null && expectedNsValues.size > 0) {
+            const extraNs = resolvedNs.filter(
+                (ns) => !expectedNsValues.has(ns)
+            );
+            if (extraNs.length > 0) {
+                allValid = false;
+                errorMessage = `Extra NS records found: ${extraNs.join(", ")}. Remove these nameservers.`;
+            }
+        }
+
+        const now = Math.floor(Date.now() / 1000);
+
+        if (allValid) {
+            await db
+                .update(domains)
+                .set({ lastCheckedAt: now, errorMessage: null })
+                .where(eq(domains.domainId, domain.domainId));
+            logger.debug(
+                `Domain ${domain.baseDomain} passed periodic reverification`
+            );
+        } else {
+            await this.markFailed(domain.domainId, errorMessage);
+            logger.warn(
+                `Domain ${domain.baseDomain} failed periodic reverification: ${errorMessage}`
+            );
+        }
+    }
+
+    private async markFailed(
+        domainId: string,
+        errorMessage: string | null
+    ): Promise<void> {
+        await db
+            .update(domains)
+            .set({
+                verified: false,
+                failed: true,
+                // Three below MAX_TRIES: keeps the domain out of the DNS
+                // validator's immediate retry loop, while still leaving it
+                // eligible (tries < MAX_TRIES) for a few more validation
+                // passes instead of being excluded forever once tries hits
+                // MAX_TRIES.
+                tries: DNS_VALIDATOR_MAX_TRIES - 3,
+                lastCheckedAt: Math.floor(Date.now() / 1000),
+                errorMessage
+            })
+            .where(eq(domains.domainId, domainId));
+    }
+}
+
+export const domainReverifier = new DomainReverifier();

server/private/lib/certificates/index.ts

@@ -0,0 +1,45 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import logger from "@server/logger";
+import { privateConfig } from "#private/lib/config";
+import { acmeClientManager } from "./acme-client";
+import { jobScheduler } from "./scheduler";
+
+export async function startCertificateManager() {
+    const acmeConfig = privateConfig.getRawPrivateConfig().acme;
+    if (
+        acmeConfig &&
+        acmeConfig.cert_mode === "pangolin" &&
+        acmeConfig.enable_acme_client
+    ) {
+        logger.info("Starting certificate management server...");
+
+        // Initialize ACME client
+        await acmeClientManager.initialize();
+
+        // Start certificate issuance/renewal jobs
+        await jobScheduler.start();
+    }
+
+    if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+        // DNS record validation/reverification doesn't require certs, so it
+        // runs whenever Pangolin is acting as the authoritative DNS server,
+        // independent of the cert manager above.
+        await jobScheduler.startDnsJobs();
+    }
+}
+
+export async function stopCertificateManager() {
+    await jobScheduler.stop();
+}

server/private/lib/certificates/scheduler.ts

@@ -0,0 +1,197 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { withTimeout } from "@server/lib/retry";
+import logger from "@server/logger";
+import { certificateService } from "./certificate-service";
+import { privateConfig as config } from "#private/lib/config";
+import { dnsValidator } from "./dns-validator";
+import { domainReverifier } from "./domain-reverifier";
+import license from "#private/license/license";
+
+// Backstop for runExclusive: no single job's own internal timeouts (e.g.
+// certificate-service's per-cert issuance timeout) are relied on here. This
+// is the last line of defense - if *anything* inside a job hangs with no
+// error (a stalled Redis/DB call, a future code path that forgets to bound
+// itself, etc.), state.active must still reset so the next tick can run.
+// Without it, one hung run permanently skips every future tick for that job,
+// since runExclusive only clears state.active after the job promise settles.
+const RUN_EXCLUSIVE_TIMEOUT_MS = 30 * 60 * 1000;
+
+export class JobScheduler {
+    private certIntervals: NodeJS.Timeout[] = [];
+    private dnsIntervals: NodeJS.Timeout[] = [];
+    private certRunning = false;
+    private dnsRunning = false;
+
+    // Guards against a slow batch (e.g. 10 certs whose DNS challenges take a
+    // while) still being processed when the next interval tick fires -
+    // without this, overlapping ticks would each pull their own batch of up
+    // to 10 pending/renewal certs and process them concurrently instead of
+    // waiting for the prior batch to finish.
+    private runExclusive(
+        job: () => Promise<void>,
+        state: { active: boolean },
+        label: string
+    ): () => Promise<void> {
+        return async () => {
+            if (!(await license.hasTier(["personal", "tier2", "enterprise"]))) {
+                logger.debug(
+                    `Skipping ${label} tick - requires a tier2 license`
+                );
+                return;
+            }
+            if (state.active) {
+                logger.debug(
+                    `Skipping ${label} tick - previous run still in progress`
+                );
+                return;
+            }
+            state.active = true;
+            try {
+                await withTimeout(job(), RUN_EXCLUSIVE_TIMEOUT_MS, label);
+            } catch (error) {
+                logger.error(`Error in ${label}:`, error);
+            } finally {
+                state.active = false;
+            }
+        };
+    }
+
+    // Certificate issuance/renewal - requires an ACME client, so this is
+    // only started when Pangolin is actually managing certs.
+    async start(): Promise<void> {
+        if (this.certRunning) {
+            logger.warn("Certificate job scheduler is already running");
+            return;
+        }
+
+        this.certRunning = true;
+        logger.info("Starting certificate job scheduler");
+
+        const newCertState = { active: false };
+        const renewalState = { active: false };
+
+        const runNewCertCheck = this.runExclusive(
+            () => certificateService.processPendingCertificates(),
+            newCertState,
+            "processing pending certificates"
+        );
+        const runRenewalCheck = this.runExclusive(
+            () => certificateService.processRenewalCandidates(),
+            renewalState,
+            "processing renewal candidates"
+        );
+
+        // Schedule new certificate processing
+        const newCertInterval = setInterval(
+            runNewCertCheck,
+            config.getRawConfig().acme!.new_cert_check_interval_ms
+        );
+
+        // Schedule renewal processing (every 24 hours)
+        const renewalInterval = setInterval(
+            runRenewalCheck,
+            config.getRawConfig().acme!.renewal_check_interval_ms
+        );
+
+        this.certIntervals.push(newCertInterval, renewalInterval);
+
+        // Run initial checks
+        setTimeout(async () => {
+            try {
+                await runNewCertCheck();
+                // await runRenewalCheck();
+            } catch (error) {
+                logger.error("Error in initial certificate processing:", error);
+            }
+        }, 1000); // Wait 1 second after startup
+
+        logger.info("Certificate job scheduler started successfully");
+    }
+
+    // DNS record validation/reverification - doesn't touch certs at all, so
+    // this runs independently whenever Pangolin is acting as the
+    // authoritative DNS server, regardless of cert_mode.
+    async startDnsJobs(): Promise<void> {
+        if (this.dnsRunning) {
+            logger.warn("DNS validation job scheduler is already running");
+            return;
+        }
+
+        this.dnsRunning = true;
+        logger.info("Starting DNS validation job scheduler");
+
+        const dnsValidationState = { active: false };
+        const reverifyState = { active: false };
+
+        const runDnsValidation = this.runExclusive(
+            () => dnsValidator.validateAll(),
+            dnsValidationState,
+            "validating DNS records"
+        );
+        const runReverify = this.runExclusive(
+            () => domainReverifier.reverifyAll(),
+            reverifyState,
+            "reverifying domains"
+        );
+
+        // Schedule DNS validation
+        const dnsValidationInterval = setInterval(
+            runDnsValidation,
+            config.getRawConfig().acme?.dns_check_interval_ms ?? 60000
+        );
+
+        // Schedule periodic reverification of already-verified domains
+        const reverifyInterval = setInterval(
+            runReverify,
+            config.getRawConfig().acme?.domain_reverification_interval_ms ??
+                3600000
+        );
+
+        this.dnsIntervals.push(dnsValidationInterval, reverifyInterval);
+
+        // Run an initial validation pass shortly after startup
+        setTimeout(async () => {
+            try {
+                await runDnsValidation();
+            } catch (error) {
+                logger.error("Error in initial DNS validation:", error);
+            }
+        }, 1000);
+
+        logger.info("DNS validation job scheduler started successfully");
+    }
+
+    async stop(): Promise<void> {
+        if (this.certRunning) {
+            logger.info("Stopping certificate job scheduler");
+            this.certRunning = false;
+            this.certIntervals.forEach((interval) => clearInterval(interval));
+            this.certIntervals = [];
+        }
+
+        if (this.dnsRunning) {
+            logger.info("Stopping DNS validation job scheduler");
+            this.dnsRunning = false;
+            this.dnsIntervals.forEach((interval) => clearInterval(interval));
+            this.dnsIntervals = [];
+        }
+    }
+
+    isRunning(): boolean {
+        return this.certRunning || this.dnsRunning;
+    }
+}
+
+export const jobScheduler = new JobScheduler();

server/private/lib/config.ts

@@ -146,12 +146,20 @@ export class PrivateConfig {
             process.env.USE_PANGOLIN_DNS =
                 this.rawPrivateConfig.flags.use_pangolin_dns.toString();
         }
+
+        if (this.rawPrivateConfig.acme?.cert_mode) {
+            process.env.CERT_MODE = this.rawPrivateConfig.acme.cert_mode;
+        }
     }
 
     public getRawPrivateConfig() {
         return this.rawPrivateConfig;
     }
 
+    public getRawConfig() {
+        return this.getRawPrivateConfig();
+    }
+
     // `flags.enable_acme_cert_sync`, `flags.disable_private_http_placeholder`,
     // and `acme` used to live in the private config file. They now live in
     // the public config file. If an operator still has them set in the

server/private/lib/dns/generateDomains.ts

@@ -0,0 +1,47 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { build } from "@server/build";
+import privateConfig from "#private/lib/config";
+
+export function createCname(domainId: string, baseDomain: string) {
+    if (!privateConfig.getRawPrivateConfig().dns?.cname_extension) {
+        throw new Error("CNAME extension not configured");
+    }
+
+    let cnameRecords = [
+        {
+            value: `${domainId}.${privateConfig.getRawPrivateConfig().dns?.cname_extension}`,
+            baseDomain: baseDomain
+        },
+        {
+            value: `_acme-challenge.${domainId}.${privateConfig.getRawPrivateConfig().dns?.cname_extension}`,
+            baseDomain: `_acme-challenge.${baseDomain}`
+        }
+    ];
+
+    return cnameRecords;
+}
+
+export function createNs() {
+    if (!privateConfig.getRawPrivateConfig().dns?.nameserver_name) {
+        throw new Error("Nameservers not configured");
+    }
+
+    const nsRecords = [
+        privateConfig.getRawPrivateConfig().dns?.nameserver_name,
+        ...(privateConfig.getRawPrivateConfig().dns?.alternate_nameservers ||
+            [])
+    ] as string[];
+    return nsRecords;
+}

server/private/lib/dns/index.ts

@@ -0,0 +1,14 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+export * from "./server";

server/private/lib/exitNodes/exitNodeComms.ts

@@ -18,6 +18,7 @@ import { eq } from "drizzle-orm";
 import { sendToClient } from "#private/routers/ws";
 import privateConfig from "#private/lib/config";
 import config from "@server/lib/config";
+import { hasExitNodeCheckedIn } from "@server/lib/exitNodes";
 
 interface ExitNodeRequest {
     remoteType?: string;
@@ -138,13 +139,19 @@ export async function sendToExitNode(
 
             return response.data;
         } catch (error) {
-            if (axios.isAxiosError(error)) {
-                logger.error(
-                    `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${hostname} (status: ${error.response?.status}): ${error.message}`
-                );
+            const message = axios.isAxiosError(error)
+                ? `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${hostname} (status: ${error.response?.status}): ${error.message}`
+                : `Error making ${method} request for exit node at ${hostname}: ${error}`;
+
+            // The exit node (gerbil) may still be starting up and not yet
+            // reachable. Until it has checked in at least once, log this at a
+            // lower level since it's expected; once it has checked in, a
+            // connection failure is a real problem.
+            if (hasExitNodeCheckedIn(exitNode.exitNodeId)) {
+                logger.error(message);
             } else {
-                logger.error(
-                    `Error making ${method} request for exit node at ${hostname}: ${error}`
+                logger.warn(
+                    `${message} (exit node has not checked in yet since startup, this is expected briefly)`
                 );
             }
         }

server/private/lib/readConfigFile.ts

@@ -48,7 +48,6 @@ export const privateConfigSchema = z
                     .optional()
                     .transform(getEnvOrYaml("FOSSORIAL_API_KEY"))
             })
-            .optional()
             .prefault({}),
         redis: z
             .object({
@@ -95,6 +94,70 @@ export const privateConfigSchema = z
                     .optional()
             })
             .optional(),
+        dns: z
+            .object({
+                enabled: z.boolean().optional().default(false),
+                listen_port: z.number().int().positive().optional().default(53),
+                nameserver_name: z.string(),
+                cname_extension: z.string(),
+                site_extension: z.string().optional(),
+                cname_alternate_extensions: z
+                    .array(z.string())
+                    .optional()
+                    .default([]),
+                alternate_nameservers: z
+                    .array(z.string())
+                    .optional()
+                    .default([]),
+                rate_limit: z
+                    .object({
+                        enabled: z.boolean().optional().default(true),
+                        window_ms: z
+                            .number()
+                            .int()
+                            .min(1000)
+                            .max(600000)
+                            .optional()
+                            .default(60000),
+                        max_requests: z
+                            .number()
+                            .int()
+                            .min(50)
+                            .max(100000)
+                            .optional()
+                            .default(1200),
+                        max_requests_per_query_type: z
+                            .number()
+                            .int()
+                            .min(10)
+                            .max(50000)
+                            .optional()
+                            .default(600)
+                    })
+                    .default({
+                        enabled: true,
+                        window_ms: 60000,
+                        max_requests: 1200,
+                        max_requests_per_query_type: 600
+                    }),
+                static_records: z
+                    .array(
+                        z.object({
+                            domain: z.string(),
+                            type: z.enum(["TXT", "CNAME", "A", "NS"]),
+                            value: z.string(),
+                            ttl: z
+                                .number()
+                                .int()
+                                .positive()
+                                .optional()
+                                .default(300)
+                        })
+                    )
+                    .optional()
+                    .default([])
+            })
+            .optional(),
         gerbil: z
             .object({
                 local_exit_node_reachable_at: z
@@ -102,7 +165,6 @@ export const privateConfigSchema = z
                     .optional()
                     .default("http://gerbil:3004")
             })
-            .optional()
             .prefault({}),
         flags: z
             .object({
@@ -123,17 +185,87 @@ export const privateConfigSchema = z
                 // (server/private/lib/config.ts).
                 disable_private_http_placeholder: z.boolean().optional()
             })
-            .optional()
             .prefault({}),
-        // @deprecated Moved to the public config file as `acme`
-        // (server/lib/readConfigFile.ts). Kept here only so existing private
-        // config files keep parsing; any value set here is migrated into the
-        // public config at startup by PrivateConfig (server/private/lib/config.ts).
         acme: z
             .object({
+                cert_mode: z
+                    .enum(["traefik", "pangolin"])
+                    .optional()
+                    .default("traefik"),
+                enable_acme_client: z.boolean().optional().default(false),
+                // @deprecated Moved to the public config file
+                // (server/lib/readConfigFile.ts). Kept here only so existing private
+                // config files keep parsing; any value set here is migrated into the
+                // public config at startup by PrivateConfig (server/private/lib/config.ts).
                 acme_json_path: z.string().optional(),
+                // @deprecated Moved to the public config file
+                // (server/lib/readConfigFile.ts). Kept here only so existing private
+                // config files keep parsing; any value set here is migrated into the
+                // public config at startup by PrivateConfig (server/private/lib/config.ts).
                 acme_http_endpoint: z.string().optional(),
-                sync_interval_ms: z.number().optional()
+                // @deprecated Moved to the public config file
+                // (server/lib/readConfigFile.ts). Kept here only so existing private
+                // config files keep parsing; any value set here is migrated into the
+                // public config at startup by PrivateConfig (server/private/lib/config.ts).
+                sync_interval_ms: z.number().optional(),
+                acme_directory_url: z
+                    .string()
+                    .url()
+                    .default("https://acme-v02.api.letsencrypt.org/directory"),
+                contact_email: z.string().email().optional(),
+                acme_account_key_path: z
+                    .string()
+                    .default("./config/account.key"),
+                challenge_ttl_ms: z.number().int().positive().default(300000),
+                renewal_check_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(3600000),
+                new_cert_check_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(5000),
+                // Kept safely under Let's Encrypt's ~20 req/s limit since this
+                // budget is shared across all pops workers and only covers the
+                // request-issuing calls we make directly (not every request
+                // acme-client makes internally, e.g. while polling for
+                // challenge/order status).
+                acme_requests_per_second: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(15),
+                dns_check_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(60000),
+                domain_reverification_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(3600000), // 1 hour — how often to run the reverification pass
+                domain_reverification_window_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(259200000), // 72 hours — how old checkedAt must be before rechecking
+                domain_reverification_batch_size: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(20), // max domains to recheck per pass
+                dns_resolvers: z
+                    .array(z.string())
+                    .optional()
+                    .default([
+                        "8.8.8.8",
+                        "1.1.1.1",
+                        "9.9.9.9",
+                        "208.67.222.222"
+                    ])
             })
             .optional(),
         branding: z

server/private/lib/traefik/getTraefikConfig.ts

@@ -55,6 +55,7 @@ import {
     getValidCertificatesForDomains
 } from "@server/lib/certificates";
 import { build } from "@server/build";
+import license from "#private/license/license";
 import regionalCache from "#private/lib/cache";
 import { ResourceWithTargets } from "@server/lib/traefik/types";
 import { buildWildcardTls } from "@server/lib/traefik/certResolver";
@@ -491,8 +492,15 @@ export async function getTraefikConfig(
         { depth: null }
     );
 
+    // Pangolin-managed DNS-01/ACME cert mode requires either a tier1
+    // license (self-hosted) or a saas build - otherwise fall back to
+    // Traefik's own cert resolvers (buildWildcardTls) throughout.
+    const pangolinCertModeEnabled =
+        privateConfig.getRawPrivateConfig().acme?.cert_mode == "pangolin" &&
+        (await license.hasTier(["personal", "tier2", "enterprise"]));
+
     let validCerts: CertificateResult[] = [];
-    if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+    if (pangolinCertModeEnabled) {
         // create a list of all domains to get certs for
         const domains = new Set<string>();
         for (const resource of resourcesMap.values()) {
@@ -629,7 +637,7 @@ export async function getTraefikConfig(
             );
 
             let tls = {};
-            if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+            if (!pangolinCertModeEnabled) {
                 tls = buildWildcardTls({
                     fullDomain,
                     hasSubdomain: !!resource.subdomain,
@@ -650,6 +658,30 @@ export async function getTraefikConfig(
                 }
             }
 
+            config_output.http.services![serviceName] = {
+                loadBalancer: {
+                    servers: buildHttpLoadBalancerServers(targets),
+                    ...(resource.stickySession
+                        ? buildStickySessionCookie(resource.ssl)
+                        : {})
+                }
+            };
+
+            if (resource.tlsServerName) {
+                if (!config_output.http.serversTransports) {
+                    config_output.http.serversTransports = {};
+                }
+                config_output.http.serversTransports![transportName] = {
+                    serverName: resource.tlsServerName,
+                    //unfortunately the following needs to be set. traefik doesn't merge the default serverTransport settings
+                    // if defined in the static config and here. if not set, self-signed certs won't work
+                    insecureSkipVerify: true
+                };
+                config_output.http.services![
+                    serviceName
+                ].loadBalancer.serversTransport = transportName;
+            }
+
             if (resource.ssl) {
                 config_output.http.routers![routerName + "-redirect"] = {
                     entryPoints: [
@@ -814,31 +846,6 @@ export async function getTraefikConfig(
                 priority: priority,
                 ...(resource.ssl ? { tls } : {})
             };
-
-            config_output.http.services![serviceName] = {
-                loadBalancer: {
-                    servers: buildHttpLoadBalancerServers(targets),
-                    ...(resource.stickySession
-                        ? buildStickySessionCookie(resource.ssl)
-                        : {})
-                }
-            };
-
-            // Add the serversTransport if TLS server name is provided
-            if (resource.tlsServerName) {
-                if (!config_output.http.serversTransports) {
-                    config_output.http.serversTransports = {};
-                }
-                config_output.http.serversTransports![transportName] = {
-                    serverName: resource.tlsServerName,
-                    //unfortunately the following needs to be set. traefik doesn't merge the default serverTransport settings
-                    // if defined in the static config and here. if not set, self-signed certs won't work
-                    insecureSkipVerify: true
-                };
-                config_output.http.services![
-                    serviceName
-                ].loadBalancer.serversTransport = transportName;
-            }
         } else if (resource.mode == "tcp" || resource.mode == "udp") {
             // Non-HTTP (TCP/UDP) configuration
             if (!resource.enableProxy) {
@@ -923,9 +930,7 @@ export async function getTraefikConfig(
                 domainCertResolver,
                 preferWildcardCert
             }) => {
-                if (
-                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
-                ) {
+                if (!pangolinCertModeEnabled) {
                     return buildWildcardTls({
                         fullDomain,
                         hasSubdomain,
@@ -966,9 +971,7 @@ export async function getTraefikConfig(
             maintenancePageUiUrl,
             redirectHttpsMiddlewareName,
             resolveTls: (fullDomain) => {
-                if (
-                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
-                ) {
+                if (!pangolinCertModeEnabled) {
                     // siteResource aliases don't have a per-domain cert
                     // resolver stored, so always fall back to the global
                     // defaults.
@@ -1059,7 +1062,7 @@ export async function getTraefikConfig(
             const rule = buildHostRule(fullDomain, ir.wildcard);
 
             let tls: any = {};
-            if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+            if (!pangolinCertModeEnabled) {
                 tls = buildWildcardTls({
                     fullDomain,
                     hasSubdomain: !!ir.subdomain,
@@ -1139,9 +1142,7 @@ export async function getTraefikConfig(
                 const rule = `Host(\`${fullDomain}\`) && ClientIP(\`${exitNode.address}\`)`; // restrict to coming from the exit node ip range that the client is connected to
 
                 let tls: any = {};
-                if (
-                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
-                ) {
+                if (!pangolinCertModeEnabled) {
                     // siteResource aliases don't have a per-domain cert
                     // resolver stored, so always fall back to the global
                     // defaults.
@@ -1215,7 +1216,7 @@ export async function getTraefikConfig(
             .where(eq(exitNodes.exitNodeId, exitNodeId));
 
         let validCertsLoginPages: CertificateResult[] = [];
-        if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+        if (pangolinCertModeEnabled) {
             // create a list of all domains to get certs for
             const domains = new Set<string>();
             for (const lp of exitNodeLoginPages) {
@@ -1260,9 +1261,7 @@ export async function getTraefikConfig(
                 }
 
                 const tls = {};
-                if (
-                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
-                ) {
+                if (!pangolinCertModeEnabled) {
                     // TODO: we need to add the wildcard logic here too
                 } else {
                     // find a cert that matches the full domain, if not continue

server/private/license/license.ts

@@ -14,7 +14,7 @@
 import { db, HostMeta, sites, users } from "@server/db";
 import { hostMeta, licenseKey } from "@server/db";
 import logger from "@server/logger";
-import NodeCache from "node-cache";
+import { createLocalCache } from "@server/lib/createLocalCache";
 import { validateJWT } from "./licenseJwt";
 import { count, eq } from "drizzle-orm";
 import moment from "moment";
@@ -26,6 +26,7 @@ import {
     LicenseStatus
 } from "@server/license/license";
 import { setHostMeta } from "@server/lib/hostMeta";
+import { build } from "@server/build";
 
 type ActivateLicenseKeyAPIResponse = {
     data: {
@@ -65,8 +66,8 @@ export class License {
     private validationServerUrl = `${this.serverBaseUrl}/api/v1/license/enterprise/validate`;
     private activationServerUrl = `${this.serverBaseUrl}/api/v1/license/enterprise/activate`;
 
-    private statusCache = new NodeCache();
-    private licenseKeyCache = new NodeCache();
+    private statusCache = createLocalCache();
+    private licenseKeyCache = createLocalCache();
 
     private statusKey = "status";
     private serverSecret!: string;
@@ -119,6 +120,9 @@ LQIDAQAB
     }
 
     public async isUnlocked(): Promise<boolean> {
+        if (build == "saas") {
+            return true;
+        }
         const status = await this.check();
         if (status.isHostLicensed) {
             if (status.isLicenseValid) {
@@ -128,6 +132,20 @@ LQIDAQAB
         return false;
     }
 
+    public async hasTier(tier: LicenseKeyTier[]): Promise<boolean> {
+        if (build == "saas") {
+            return true;
+        }
+        const status = await this.check();
+        if (status.isHostLicensed && status.isLicenseValid) {
+            return (
+                status.tier !== undefined &&
+                tier.includes(status.tier as LicenseKeyTier)
+            );
+        }
+        return false;
+    }
+
     public async check(): Promise<LicenseStatus> {
         // If a check is already in progress, return the last known status
         if (this.checkInProgress) {
@@ -135,8 +153,7 @@ LQIDAQAB
                 "License check already in progress, returning last known status"
             );
             const lastStatus = this.statusCache.get(this.statusKey) as
-                | LicenseStatus
-                | undefined;
+                LicenseStatus | undefined;
             if (lastStatus) {
                 return lastStatus;
             }
@@ -179,7 +196,7 @@ LQIDAQAB
                 status.isHostLicensed = false;
                 // Invalidate all and set new cache (empty)
                 this.licenseKeyCache.flushAll();
-                this.statusCache.set(this.statusKey, status);
+                this.statusCache.set(this.statusKey, status, 0);
                 return status;
             }
 
@@ -389,7 +406,7 @@ LQIDAQAB
             // Invalidate old cache and set new cache
             this.licenseKeyCache.flushAll();
             for (const [key, value] of newCache.entries()) {
-                this.licenseKeyCache.set<LicenseKeyCache>(key, value);
+                this.licenseKeyCache.set(key, value, 0);
             }
         } catch (error) {
             logger.error("Error checking license status:");
@@ -398,7 +415,7 @@ LQIDAQAB
             this.checkInProgress = false;
         }
 
-        this.statusCache.set(this.statusKey, status);
+        this.statusCache.set(this.statusKey, status, 0);
         return status;
     }
 

server/private/routers/alertRule/testAlertRule.ts

@@ -14,7 +14,7 @@
 import { getRandomItemInArray } from "@app/lib/getRandomItemInArray";
 import response from "@server/lib/response";
 import logger from "@server/logger";
-import { processTestAlerts } from "@server/private/lib/alerts/processTestAlerts";
+import { processTestAlerts } from "#private/lib/alerts/processTestAlerts";
 import { type AlertAction } from "@server/routers/alertRule/types";
 import HttpCode from "@server/types/HttpCode";
 import { NextFunction, Request, Response } from "express";

server/private/routers/alertRule/updateAlertRule.ts

@@ -33,8 +33,11 @@ import { OpenAPITags, registry } from "@server/openApi";
 import { and, eq } from "drizzle-orm";
 import { encrypt } from "@server/lib/crypto";
 import config from "@server/lib/config";
-import { HC_EVENT_TYPES, SITE_EVENT_TYPES, RESOURCE_EVENT_TYPES } from "./createAlertRule";
-import { invalidateAllRemoteExitNodeSessions } from "@server/private/auth/sessions/remoteExitNode";
+import {
+    HC_EVENT_TYPES,
+    SITE_EVENT_TYPES,
+    RESOURCE_EVENT_TYPES
+} from "./createAlertRule";
 
 const paramsSchema = z
     .object({
@@ -85,35 +88,57 @@ const bodySchema = z
         const isHcEvent = (HC_EVENT_TYPES as readonly string[]).includes(
             val.eventType
         );
-        const isResourceEvent = (RESOURCE_EVENT_TYPES as readonly string[]).includes(
-            val.eventType
-        );
+        const isResourceEvent = (
+            RESOURCE_EVENT_TYPES as readonly string[]
+        ).includes(val.eventType);
 
-        if (isSiteEvent && val.siteIds !== undefined && val.siteIds.length === 0 && !val.allSites) {
+        if (
+            isSiteEvent &&
+            val.siteIds !== undefined &&
+            val.siteIds.length === 0 &&
+            !val.allSites
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "At least one siteId is required for site event types when allSites is false",
+                message:
+                    "At least one siteId is required for site event types when allSites is false",
                 path: ["siteIds"]
             });
         }
 
-        if (isHcEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length === 0 && !val.allHealthChecks) {
+        if (
+            isHcEvent &&
+            val.healthCheckIds !== undefined &&
+            val.healthCheckIds.length === 0 &&
+            !val.allHealthChecks
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "At least one healthCheckId is required for health check event types when allHealthChecks is false",
+                message:
+                    "At least one healthCheckId is required for health check event types when allHealthChecks is false",
                 path: ["healthCheckIds"]
             });
         }
 
-        if (isResourceEvent && val.resourceIds !== undefined && val.resourceIds.length === 0 && !val.allResources) {
+        if (
+            isResourceEvent &&
+            val.resourceIds !== undefined &&
+            val.resourceIds.length === 0 &&
+            !val.allResources
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "At least one resourceId is required for resource event types when allResources is false",
+                message:
+                    "At least one resourceId is required for resource event types when allResources is false",
                 path: ["resourceIds"]
             });
         }
 
-        if (isSiteEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length > 0) {
+        if (
+            isSiteEvent &&
+            val.healthCheckIds !== undefined &&
+            val.healthCheckIds.length > 0
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
                 message: "healthCheckIds must not be set for site event types",
@@ -129,7 +154,11 @@ const bodySchema = z
             });
         }
 
-        if (isResourceEvent && val.siteIds !== undefined && val.siteIds.length > 0) {
+        if (
+            isResourceEvent &&
+            val.siteIds !== undefined &&
+            val.siteIds.length > 0
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
                 message: "siteIds must not be set for resource event types",
@@ -137,10 +166,15 @@ const bodySchema = z
             });
         }
 
-        if (isResourceEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length > 0) {
+        if (
+            isResourceEvent &&
+            val.healthCheckIds !== undefined &&
+            val.healthCheckIds.length > 0
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "healthCheckIds must not be set for resource event types",
+                message:
+                    "healthCheckIds must not be set for resource event types",
                 path: ["healthCheckIds"]
             });
         }
@@ -153,7 +187,6 @@ const UpdateAlertRuleResponseDataSchema = z.object({
     alertRuleId: z.number()
 });
 
-
 registry.registerPath({
     method: "post",
     path: "/org/{orgId}/alert-rule/{alertRuleId}",
@@ -174,7 +207,9 @@ registry.registerPath({
             description: "Successful response",
             content: {
                 "application/json": {
-                    schema: createApiResponseSchema(UpdateAlertRuleResponseDataSchema)
+                    schema: createApiResponseSchema(
+                        UpdateAlertRuleResponseDataSchema
+                    )
                 }
             }
         }
@@ -250,9 +285,11 @@ export async function updateAlertRule(
         if (name !== undefined) updateData.name = name;
         if (eventType !== undefined) updateData.eventType = eventType;
         if (enabled !== undefined) updateData.enabled = enabled;
-        if (cooldownSeconds !== undefined) updateData.cooldownSeconds = cooldownSeconds;
+        if (cooldownSeconds !== undefined)
+            updateData.cooldownSeconds = cooldownSeconds;
         if (allSites !== undefined) updateData.allSites = allSites;
-        if (allHealthChecks !== undefined) updateData.allHealthChecks = allHealthChecks;
+        if (allHealthChecks !== undefined)
+            updateData.allHealthChecks = allHealthChecks;
         if (allResources !== undefined) updateData.allResources = allResources;
 
         await db
@@ -273,7 +310,11 @@ export async function updateAlertRule(
 
             // Only insert junction rows when allSites is not true
             const effectiveAllSites = allSites ?? false;
-            if (!effectiveAllSites && siteIds !== undefined && siteIds.length > 0) {
+            if (
+                !effectiveAllSites &&
+                siteIds !== undefined &&
+                siteIds.length > 0
+            ) {
                 await db.insert(alertSites).values(
                     siteIds.map((siteId) => ({
                         alertRuleId,
@@ -290,7 +331,11 @@ export async function updateAlertRule(
                 .where(eq(alertHealthChecks.alertRuleId, alertRuleId));
 
             const effectiveAllHealthChecks = allHealthChecks ?? false;
-            if (!effectiveAllHealthChecks && healthCheckIds !== undefined && healthCheckIds.length > 0) {
+            if (
+                !effectiveAllHealthChecks &&
+                healthCheckIds !== undefined &&
+                healthCheckIds.length > 0
+            ) {
                 await db.insert(alertHealthChecks).values(
                     healthCheckIds.map((healthCheckId) => ({
                         alertRuleId,
@@ -307,7 +352,11 @@ export async function updateAlertRule(
                 .where(eq(alertResources.alertRuleId, alertRuleId));
 
             const effectiveAllResources = allResources ?? false;
-            if (!effectiveAllResources && resourceIds !== undefined && resourceIds.length > 0) {
+            if (
+                !effectiveAllResources &&
+                resourceIds !== undefined &&
+                resourceIds.length > 0
+            ) {
                 await db.insert(alertResources).values(
                     resourceIds.map((resourceId) => ({
                         alertRuleId,
@@ -392,7 +441,10 @@ export async function updateAlertRule(
                     webhookActions.map((wa) => ({
                         alertRuleId,
                         webhookUrl: wa.webhookUrl,
-                        config: wa.config != null ? encrypt(wa.config, serverSecret) : null,
+                        config:
+                            wa.config != null
+                                ? encrypt(wa.config, serverSecret)
+                                : null,
                         enabled: wa.enabled
                     }))
                 );

server/private/routers/auditLogs/queryAccessAuditLog.ts

@@ -54,7 +54,6 @@ export const queryAccessAuditLogsQuery = z.object({
             error: "timeEnd must be a valid ISO date string"
         })
         .transform((val) => Math.floor(new Date(val).getTime() / 1000))
-        .optional()
         .prefault(() => new Date().toISOString())
         .openapi({
             type: "string",

server/private/routers/auditLogs/queryActionAuditLog.ts

@@ -47,7 +47,6 @@ export const queryActionAuditLogsQuery = z.object({
             error: "timeEnd must be a valid ISO date string"
         })
         .transform((val) => Math.floor(new Date(val).getTime() / 1000))
-        .optional()
         .prefault(() => new Date().toISOString())
         .openapi({
             type: "string",

server/private/routers/auditLogs/queryConnectionAuditLog.ts

@@ -55,7 +55,6 @@ export const queryConnectionAuditLogsQuery = z.object({
             error: "timeEnd must be a valid ISO date string"
         })
         .transform((val) => Math.floor(new Date(val).getTime() / 1000))
-        .optional()
         .prefault(() => new Date().toISOString())
         .openapi({
             type: "string",

server/private/routers/billing/hooks/handleSubscriptionCreated.ts

@@ -222,15 +222,16 @@ export async function handleSubscriptionCreated(
 
                 let numUsers: number;
                 let numSites: number;
+                let tier = "enterprise";
 
-                if (subscriptionPriceId === priceSet[LicenseId.SMALL_LICENSE]) {
+                if (subscriptionPriceId === priceSet[LicenseId.TIER1]) {
                     numUsers = 25;
                     numSites = 25;
-                } else if (
-                    subscriptionPriceId === priceSet[LicenseId.BIG_LICENSE]
-                ) {
+                    tier = "tier1";
+                } else if (subscriptionPriceId === priceSet[LicenseId.TIER2]) {
                     numUsers = 50;
                     numSites = 100;
+                    tier = "tier2";
                 } else {
                     logger.error(
                         `Unknown price ID ${subscriptionPriceId} for subscription ${subscription.id}`
@@ -242,6 +243,14 @@ export async function handleSubscriptionCreated(
                     `License type determined: ${numUsers} users, ${numSites} sites for subscription ${subscription.id}`
                 );
 
+                // Grace period of 5 days added on top of the current billing
+                // period end (usually ~1 year out) before the license expires
+                const currentPeriodEnd =
+                    fullSubscription.items.data[0]?.current_period_end;
+                const expiresAt =
+                    (currentPeriodEnd ?? subscription.created) +
+                    5 * 24 * 60 * 60;
+
                 const response = await fetch(
                     `${privateConfig.getRawPrivateConfig().server.fossorial_api}/api/v1/license-internal/enterprise/paid-for`,
                     {
@@ -256,7 +265,9 @@ export async function handleSubscriptionCreated(
                             licenseId: parseInt(licenseId),
                             paidFor: true,
                             users: numUsers,
-                            sites: numSites
+                            sites: numSites,
+                            tier: tier,
+                            expiresAt: expiresAt
                         })
                     }
                 );
@@ -265,6 +276,13 @@ export async function handleSubscriptionCreated(
 
                 logger.debug(`Fossorial API response: ${JSON.stringify(data)}`);
 
+                if (!response.ok || !data.success) {
+                    logger.error(
+                        `Fossorial API returned ${response.status} when setting paid-for for orgId ${customer.orgId} and subscription ID ${subscription.id}: ${JSON.stringify(data)}`
+                    );
+                    return;
+                }
+
                 if (customer.email) {
                     logger.debug(
                         `Sending license key email to ${customer.email} for subscription ${subscription.id}`

server/private/routers/billing/hooks/handleSubscriptionDeleted.ts

@@ -125,9 +125,7 @@ export async function handleSubscriptionDeleted(
                 `Handling license subscription deletion for orgId ${customer.orgId} and subscription ID ${subscription.id}`
             );
             try {
-                // WARNING:
-                // this invalidates ALL OF THE ENTERPRISE LICENSES for this orgId
-                await fetch(
+                const invalidateResponse = await fetch(
                     `${privateConfig.getRawPrivateConfig().server.fossorial_api}/api/v1/license-internal/enterprise/invalidate`,
                     {
                         method: "POST",
@@ -139,9 +137,18 @@ export async function handleSubscriptionDeleted(
                         },
                         body: JSON.stringify({
                             orgId: customer.orgId,
+                            licenseKeyId: parseInt(
+                                subscription.metadata.licenseKeyId
+                            )
                         })
                     }
                 );
+
+                if (!invalidateResponse.ok) {
+                    logger.error(
+                        `Fossorial API returned ${invalidateResponse.status} when invalidating license for orgId ${customer.orgId} and subscription ID ${subscription.id}: ${await invalidateResponse.text()}`
+                    );
+                }
             } catch (error) {
                 logger.error(
                     `Error notifying Fossorial API of license subscription deletion for orgId ${customer.orgId} and subscription ID ${subscription.id}:`,

server/private/routers/billing/hooks/handleSubscriptionUpdated.ts

@@ -23,7 +23,10 @@ import {
 } from "@server/db";
 import { eq, and } from "drizzle-orm";
 import logger from "@server/logger";
-import { getFeatureIdByMetricId, getFeatureIdByPriceId } from "@server/lib/billing/features";
+import {
+    getFeatureIdByMetricId,
+    getFeatureIdByPriceId
+} from "@server/lib/billing/features";
 import stripe from "#private/lib/stripe";
 import { handleSubscriptionLifesycle } from "../subscriptionLifecycle";
 import { getSubType, SubscriptionType } from "./getSubType";
@@ -66,7 +69,8 @@ export async function handleSubscriptionUpdated(
             .limit(1);
 
         const type = getSubType(fullSubscription);
-        const previousType = existingSubscription.type as SubscriptionType | null;
+        const previousType =
+            existingSubscription.type as SubscriptionType | null;
 
         // If the subscription has been manually overridden, we lock the
         // status down so Stripe webhooks can no longer change it.
@@ -100,7 +104,11 @@ export async function handleSubscriptionUpdated(
             logger.info(
                 `Tier change detected for org ${customer.orgId}: ${previousType} -> ${type}`
             );
-            await handleTierChange(customer.orgId, type, previousType ?? undefined);
+            await handleTierChange(
+                customer.orgId,
+                type,
+                previousType ?? undefined
+            );
         }
 
         // Upsert subscription items
@@ -113,7 +121,8 @@ export async function handleSubscriptionUpdated(
 
             const itemsToUpsert = fullSubscription.items.data.map((item) => {
                 // Try to get featureId from price
-                let featureId: string | null = getFeatureIdByPriceId(item.price.id) || null;
+                let featureId: string | null =
+                    getFeatureIdByPriceId(item.price.id) || null;
 
                 // If no match, try to preserve existing featureId
                 if (!featureId) {
@@ -302,14 +311,20 @@ export async function handleSubscriptionUpdated(
                     logger.info(
                         `Subscription ${subscription.id} for org ${customer.orgId} is ${effectiveStatus}, disabling paid features`
                     );
-                    await handleTierChange(customer.orgId, null, previousType ?? undefined);
+                    await handleTierChange(
+                        customer.orgId,
+                        null,
+                        previousType ?? undefined
+                    );
                 }
             } else if (type === "license") {
-                if (effectiveStatus === "canceled" || effectiveStatus == "unpaid" || effectiveStatus == "incomplete_expired") {
+                if (
+                    effectiveStatus === "canceled" ||
+                    effectiveStatus == "unpaid" ||
+                    effectiveStatus == "incomplete_expired"
+                ) {
                     try {
-                        // WARNING:
-                        // this invalidates ALL OF THE ENTERPRISE LICENSES for this orgId
-                        await fetch(
+                        const invalidateResponse = await fetch(
                             `${privateConfig.getRawPrivateConfig().server.fossorial_api}/api/v1/license-internal/enterprise/invalidate`,
                             {
                                 method: "POST",
@@ -320,16 +335,95 @@ export async function handleSubscriptionUpdated(
                                     "Content-Type": "application/json"
                                 },
                                 body: JSON.stringify({
-                                    orgId: customer.orgId
+                                    orgId: customer.orgId,
+                                    licenseKeyId: parseInt(
+                                        subscription.metadata.licenseKeyId
+                                    )
                                 })
                             }
                         );
+
+                        if (!invalidateResponse.ok) {
+                            logger.error(
+                                `Fossorial API returned ${invalidateResponse.status} when invalidating license for orgId ${customer.orgId} and subscription ID ${subscription.id}: ${await invalidateResponse.text()}`
+                            );
+                        }
                     } catch (error) {
                         logger.error(
                             `Error notifying Fossorial API of license subscription deletion for orgId ${customer.orgId} and subscription ID ${subscription.id}:`,
                             error
                         );
                     }
+                } else if (effectiveStatus === "active" && previousAttributes) {
+                    // Detect a successful renewal: the billing period rolled
+                    // forward (the invoice was paid and the new period began
+                    // right where the previous one ended).
+                    const currentItem = fullSubscription.items.data[0];
+                    const prevItems = previousAttributes.items?.data;
+                    const prevItem = Array.isArray(prevItems)
+                        ? prevItems.find(
+                              (pi: any) => pi.id === currentItem?.id
+                          )
+                        : undefined;
+
+                    const renewed =
+                        currentItem &&
+                        prevItem?.current_period_end &&
+                        currentItem.current_period_start ===
+                            prevItem.current_period_end &&
+                        currentItem.current_period_start >
+                            prevItem.current_period_start;
+
+                    if (renewed) {
+                        const licenseKeyId =
+                            subscription.metadata.licenseKeyId;
+
+                        if (!licenseKeyId) {
+                            logger.error(
+                                `No licenseKeyId in metadata for subscription ${subscription.id}, cannot extend license.`
+                            );
+                        } else {
+                            // Grace period of 5 days added on top of the new
+                            // billing period end (usually ~1 year out)
+                            const expiresAt =
+                                currentItem.current_period_end +
+                                5 * 24 * 60 * 60;
+
+                            try {
+                                const extendResponse = await fetch(
+                                    `${privateConfig.getRawPrivateConfig().server.fossorial_api}/api/v1/license-internal/enterprise/extend`,
+                                    {
+                                        method: "POST",
+                                        headers: {
+                                            "api-key":
+                                                privateConfig.getRawPrivateConfig()
+                                                    .server.fossorial_api_key!,
+                                            "Content-Type": "application/json"
+                                        },
+                                        body: JSON.stringify({
+                                            licenseId: parseInt(licenseKeyId),
+                                            expiresAt: expiresAt
+                                        })
+                                    }
+                                );
+
+                                if (!extendResponse.ok) {
+                                    logger.error(
+                                        `Fossorial API returned ${extendResponse.status} when extending license ${licenseKeyId} for subscription ${subscription.id}: ${await extendResponse.text()}`
+                                    );
+                                } else {
+                                    logger.info(
+                                        `Extended license ${licenseKeyId} for subscription ${subscription.id} to expire at ${expiresAt}.`
+                                    );
+                                }
+                            } catch (error) {
+                                logger.error(
+                                    `Error notifying Fossorial API of license renewal for subscription ${subscription.id}:`,
+                                    error
+                                );
+                            }
+                        }
+                    }
                 }
             }
         }

server/private/routers/generatedLicense/clearInstanceName.ts

@@ -31,7 +31,9 @@ export async function clearInstanceName(
     next: NextFunction
 ): Promise<any> {
     try {
-        const parsedParams = clearInstanceNameParamsSchema.safeParse(req.params);
+        const parsedParams = clearInstanceNameParamsSchema.safeParse(
+            req.params
+        );
         if (!parsedParams.success) {
             return next(
                 createHttpError(
@@ -63,7 +65,8 @@ export async function clearInstanceName(
             return next(
                 createHttpError(
                     data.status || HttpCode.BAD_REQUEST,
-                    data.message || "Failed to clear instance name from Fossorial API"
+                    data.message ||
+                        "Failed to clear server ID from Fossorial API"
                 )
             );
         }
@@ -72,7 +75,7 @@ export async function clearInstanceName(
             data: null,
             success: true,
             error: false,
-            message: "Instance name cleared successfully",
+            message: "Server ID cleared successfully",
             status: HttpCode.OK
         });
     } catch (error) {
@@ -80,8 +83,8 @@ export async function clearInstanceName(
         return next(
             createHttpError(
                 HttpCode.INTERNAL_SERVER_ERROR,
-                "An error occurred while clearing the instance name."
+                "An error occurred while clearing the server ID."
             )
         );
     }
-}
+}

server/private/routers/generatedLicense/generateNewEnterpriseLicense.ts

@@ -64,14 +64,11 @@ export async function generateNewEnterpriseLicense(
 
         const licenseData = req.body;
 
-        if (
-            licenseData.tier != "big_license" &&
-            licenseData.tier != "small_license"
-        ) {
+        if (licenseData.tier != "tier2" && licenseData.tier != "tier1") {
             return next(
                 createHttpError(
                     HttpCode.BAD_REQUEST,
-                    "Invalid tier specified. Must be either 'big_license' or 'small_license'."
+                    "Invalid tier specified. Must be either 'tier2' or 'tier1'."
                 )
             );
         }
@@ -99,6 +96,8 @@ export async function generateNewEnterpriseLicense(
             );
         }
 
+        const licenseKeyValue = apiResponse?.data?.licenseKey?.licenseKey;
+
         // check if we already have a customer for this org
         const [customer] = await db
             .select()
@@ -118,9 +117,7 @@ export async function generateNewEnterpriseLicense(
         }
 
         const tier =
-            licenseData.tier === "big_license"
-                ? LicenseId.BIG_LICENSE
-                : LicenseId.SMALL_LICENSE;
+            licenseData.tier === "tier2" ? LicenseId.TIER2 : LicenseId.TIER1;
         const tierPrice = getLicensePriceSet()[tier];
 
         const session = await stripe!.checkout.sessions.create({
@@ -134,6 +131,16 @@ export async function generateNewEnterpriseLicense(
             ], // Start with the standard feature set that matches the free limits
             customer: customer.customerId,
             mode: "subscription",
+            subscription_data: {
+                description: licenseKeyValue
+                    ? `License ${licenseKeyValue}`
+                    : `License key ID ${keyId}`,
+                metadata: {
+                    licenseKeyId: keyId.toString(),
+                    licenseKey: licenseKeyValue ?? "",
+                    tier: licenseData.tier
+                }
+            },
             allow_promotion_codes: true,
             success_url: `${config.getRawConfig().app.dashboard_url}/${orgId}/settings/license?success=true&session_id={CHECKOUT_SESSION_ID}`,
             cancel_url: `${config.getRawConfig().app.dashboard_url}/${orgId}/settings/license?canceled=true`

server/private/routers/gerbil/createExitNode.ts

@@ -12,8 +12,8 @@
  */
 
 import { db, ExitNode, exitNodes } from "@server/db";
-import { getUniqueExitNodeEndpointName } from "@server/db/names";
 import config from "@server/lib/config";
+import privateConfig from "#private/lib/config";
 import { getNextAvailableSubnet } from "@server/lib/exitNodes";
 import logger from "@server/logger";
 import { eq } from "drizzle-orm";
@@ -45,6 +45,8 @@ export async function createExitNode(
                 .values({
                     publicKey,
                     endpoint: config.getRawConfig().gerbil.base_endpoint,
+                    region:
+                        privateConfig.getRawPrivateConfig().app.region || null,
                     address,
                     listenPort,
                     online: true,

server/private/routers/remoteExitNode/getRemoteExitNodeToken.ts

@@ -26,7 +26,7 @@ import {
     validateRemoteExitNodeSessionToken,
     EXPIRES
 } from "#private/auth/sessions/remoteExitNode";
-import { getOrCreateCachedToken } from "@server/private/lib/tokenCache";
+import { getOrCreateCachedToken } from "#private/lib/tokenCache";
 import { verifyPassword } from "@server/auth/password";
 import logger from "@server/logger";
 import config from "@server/lib/config";