Merge branch 'dev' into feat/redirect-tables

server/certificates.ts

@@ -0,0 +1,8 @@
+export async function startCertificateManager() {
+    // No-op: ACME certificate generation/management is only available in
+    // builds that include the private/enterprise feature set.
+}
+
+export async function stopCertificateManager() {
+    // No-op counterpart to startCertificateManager.
+}

server/db/pg/driver.ts

@@ -1,5 +1,6 @@
 import { drizzle as DrizzlePostgres } from "drizzle-orm/node-postgres";
 import { readConfigFile } from "@server/lib/readConfigFile";
+import { readEnvOrFile } from "@server/lib/getEnvOrYaml";
 import { withReplicas } from "drizzle-orm/pg-core";
 import { createPool } from "./poolConfig";
 
@@ -7,17 +8,20 @@ function createDb() {
     const config = readConfigFile();
 
     // check the environment variables for postgres config first before the config file
-    if (process.env.POSTGRES_CONNECTION_STRING) {
+    const envConnectionString = readEnvOrFile("POSTGRES_CONNECTION_STRING");
+    if (envConnectionString) {
         config.postgres = {
-            connection_string: process.env.POSTGRES_CONNECTION_STRING
+            connection_string: envConnectionString
         };
-        if (process.env.POSTGRES_REPLICA_CONNECTION_STRINGS) {
-            const replicas =
-                process.env.POSTGRES_REPLICA_CONNECTION_STRINGS.split(",").map(
-                    (conn) => ({
-                        connection_string: conn.trim()
-                    })
-                );
+        const replicaConnectionStrings = readEnvOrFile(
+            "POSTGRES_REPLICA_CONNECTION_STRINGS"
+        );
+        if (replicaConnectionStrings) {
+            const replicas = replicaConnectionStrings
+                .split(",")
+                .map((conn) => ({
+                    connection_string: conn.trim()
+                }));
             config.postgres.replicas = replicas;
         }
     }

server/db/pg/logsDriver.ts

@@ -1,5 +1,6 @@
 import { drizzle as DrizzlePostgres } from "drizzle-orm/node-postgres";
 import { readConfigFile } from "@server/lib/readConfigFile";
+import { readEnvOrFile } from "@server/lib/getEnvOrYaml";
 import { withReplicas } from "drizzle-orm/pg-core";
 import { build } from "@server/build";
 import { db as mainDb } from "./driver";
@@ -17,7 +18,7 @@ function createLogsDb() {
     const logsConfig = config.postgres_logs;
 
     // Check environment variable first
-    let connectionString = process.env.POSTGRES_LOGS_CONNECTION_STRING;
+    let connectionString = readEnvOrFile("POSTGRES_LOGS_CONNECTION_STRING");
     let replicaConnections: Array<{ connection_string: string }> = [];
 
     if (!connectionString && logsConfig) {
@@ -26,13 +27,15 @@ function createLogsDb() {
     }
 
     // If POSTGRES_LOGS_REPLICA_CONNECTION_STRINGS is set, use it
-    if (process.env.POSTGRES_LOGS_REPLICA_CONNECTION_STRINGS) {
-        replicaConnections =
-            process.env.POSTGRES_LOGS_REPLICA_CONNECTION_STRINGS.split(",").map(
-                (conn) => ({
-                    connection_string: conn.trim()
-                })
-            );
+    const replicaConnectionStrings = readEnvOrFile(
+        "POSTGRES_LOGS_REPLICA_CONNECTION_STRINGS"
+    );
+    if (replicaConnectionStrings) {
+        replicaConnections = replicaConnectionStrings
+            .split(",")
+            .map((conn) => ({
+                connection_string: conn.trim()
+            }));
     }
 
     // If no logs database is configured, fall back to main database

server/db/pg/schema/schema.ts

@@ -714,6 +714,8 @@ export const newts = pgTable(
         secretHash: varchar("secretHash").notNull(),
         dateCreated: varchar("dateCreated").notNull(),
         version: varchar("version"),
+        agent: varchar("agent"), // either newt or cli
+        agentVersion: varchar("agentVersion"),
         siteId: integer("siteId").references(() => sites.siteId, {
             onDelete: "cascade"
         })

server/db/sqlite/driver.ts

@@ -6,6 +6,7 @@ import fs from "fs";
 import { APP_PATH } from "@server/lib/consts";
 import { existsSync, mkdirSync } from "fs";
 import logger from "@server/logger";
+import { readEnvOrFile } from "@server/lib/getEnvOrYaml";
 
 export const location = path.join(APP_PATH, "db", "db.sqlite");
 export const exists = checkFileExists(location);
@@ -19,7 +20,7 @@ function createDb() {
             : undefined;
     const sqlite = new Database(location, { verbose });
 
-    if (process.env.ENABLE_SQLITE_WAL_MODE == "true") {
+    if (readEnvOrFile("ENABLE_SQLITE_WAL_MODE") == "true") {
         // Enable WAL mode — allows concurrent readers + single writer, preventing
         // contention across subsystems (verifySession, Traefik, audit, ping).
         // NOTE: journal_mode persists in the DB file once set; unsetting this

server/db/sqlite/schema/schema.ts

@@ -737,6 +737,8 @@ export const newts = sqliteTable(
         secretHash: text("secretHash").notNull(),
         dateCreated: text("dateCreated").notNull(),
         version: text("version"),
+        agent: text("agent"), // either newt or cli
+        agentVersion: text("agentVersion"),
         siteId: integer("siteId").references(() => sites.siteId, {
             onDelete: "cascade"
         })

server/dns.ts

@@ -0,0 +1,8 @@
+export async function startDnsServer() {
+    // No-op: the authoritative DNS server is only available in builds
+    // that include the private/enterprise feature set.
+}
+
+export async function stopDnsServer() {
+    // No-op counterpart to startDnsServer.
+}

server/emails/templates/EnterpriseEditionKeyGenerated.tsx

@@ -93,9 +93,8 @@ export const EnterpriseEditionKeyGenerated = ({
                         </EmailSection>
 
                         <EmailText>
-                            If you need to purchase additional license keys or
-                            modify your existing license, please reach out to
-                            our support team at{" "}
+                            For any questions or concerns regarding your license
+                            or billing, please reach out to support at{" "}
                             <a
                                 href="mailto:support@pangolin.net"
                                 className="text-primary font-medium"

server/index.ts

@@ -25,6 +25,8 @@ import { setHostMeta } from "@server/lib/hostMeta";
 import { TraefikConfigManager } from "@server/lib/traefik/TraefikConfigManager";
 import { initCleanup } from "#dynamic/cleanup";
 import { startSchedulers } from "#dynamic/startSchedulers";
+import { startDnsServer } from "#dynamic/dns";
+import { startCertificateManager } from "#dynamic/certificates";
 import license from "#dynamic/license/license";
 import { fetchServerIp } from "@server/lib/serverIpService";
 import { initAiModelCatalog } from "@server/lib/aiModelCatalog";
@@ -45,6 +47,10 @@ async function startServers() {
 
     startSchedulers();
 
+    await startDnsServer();
+
+    await startCertificateManager();
+
     // Start all servers
     const apiServer = createApiServer();
     const internalServer = createInternalServer();

server/lib/backupFileName.test.ts

@@ -0,0 +1,101 @@
+import { formatBackupTimestamp } from "./backupFileName";
+import { assertEquals } from "@test/assert";
+
+// Local-time constructors are used throughout, matching formatBackupTimestamp,
+// so these cases do not depend on the machine's timezone.
+
+function testMonthIsOneIndexed() {
+    console.log("Running month indexing tests...");
+
+    // The case from the report: a backup taken on 12 September 2026 was named
+    // db_2026-8-12_... because Date#getMonth is zero-indexed.
+    {
+        const result = formatBackupTimestamp(new Date(2026, 8, 12, 20, 35, 56));
+        assertEquals(
+            result,
+            "2026-09-12_20-35-56",
+            "September must render as 09, not 8"
+        );
+    }
+
+    // The other reported name, db_2026-0-23_..., was a January backup.
+    {
+        const result = formatBackupTimestamp(new Date(2026, 0, 23, 20, 25, 49));
+        assertEquals(
+            result,
+            "2026-01-23_20-25-49",
+            "January must render as 01, not 0"
+        );
+    }
+
+    {
+        const result = formatBackupTimestamp(new Date(2026, 11, 31, 23, 59, 59));
+        assertEquals(
+            result,
+            "2026-12-31_23-59-59",
+            "December must render as 12"
+        );
+    }
+}
+
+function testEveryFieldIsZeroPadded() {
+    console.log("Running zero padding tests...");
+
+    // db_2026-8-12_20-36-2 in the report: a single-digit second was not padded.
+    {
+        const result = formatBackupTimestamp(new Date(2026, 8, 12, 20, 36, 2));
+        assertEquals(
+            result,
+            "2026-09-12_20-36-02",
+            "Single-digit seconds must be padded"
+        );
+    }
+
+    {
+        const result = formatBackupTimestamp(new Date(2026, 0, 1, 0, 0, 0));
+        assertEquals(
+            result,
+            "2026-01-01_00-00-00",
+            "Midnight on the first of the month must pad every field"
+        );
+    }
+}
+
+function testNamesSortChronologically() {
+    console.log("Running sort order tests...");
+
+    // Zero padding means a plain lexicographic sort of the backups directory
+    // lists the backups in the order they were taken.
+    const taken = [
+        new Date(2026, 8, 12, 20, 36, 2),
+        new Date(2026, 0, 23, 20, 25, 49),
+        new Date(2026, 8, 12, 20, 35, 56),
+        new Date(2026, 11, 31, 23, 59, 59)
+    ];
+
+    const sorted = taken
+        .map((date) => formatBackupTimestamp(date))
+        .sort();
+
+    assertEquals(
+        sorted.join(","),
+        [
+            "2026-01-23_20-25-49",
+            "2026-09-12_20-35-56",
+            "2026-09-12_20-36-02",
+            "2026-12-31_23-59-59"
+        ].join(","),
+        "Backup names must sort into the order the backups were taken"
+    );
+}
+
+// Run all tests
+try {
+    testMonthIsOneIndexed();
+    testEveryFieldIsZeroPadded();
+    testNamesSortChronologically();
+    console.log("All tests passed successfully!");
+} catch (error) {
+    console.error("Test failed:", error);
+    process.exit(1);
+}

server/lib/backupFileName.ts

@@ -0,0 +1,28 @@
+/**
+ * Builds the timestamp segment of a database backup file name.
+ *
+ * `Date#getMonth` is zero-indexed, so building this inline produced names like
+ * `db_2026-8-12_...` for a backup taken on 12 September 2026. Every field is
+ * also zero-padded, which keeps the names unambiguous and makes them sort
+ * lexicographically in the order they were taken.
+ *
+ * @param date The moment the backup is being taken. Defaults to now.
+ * @returns A timestamp of the form `YYYY-MM-DD_HH-MM-SS`.
+ */
+export function formatBackupTimestamp(date: Date = new Date()): string {
+    const pad = (value: number): string => String(value).padStart(2, "0");
+
+    const datePart = [
+        date.getFullYear(),
+        pad(date.getMonth() + 1),
+        pad(date.getDate())
+    ].join("-");
+
+    const timePart = [
+        pad(date.getHours()),
+        pad(date.getMinutes()),
+        pad(date.getSeconds())
+    ].join("-");
+
+    return `${datePart}_${timePart}`;
+}

server/lib/billing/licenses.ts

@@ -1,6 +1,6 @@
 export enum LicenseId {
-    SMALL_LICENSE = "small_license",
-    BIG_LICENSE = "big_license"
+    TIER1 = "tier1",
+    TIER2 = "tier2"
 }
 
 export type LicensePriceSet = {
@@ -9,15 +9,15 @@ export type LicensePriceSet = {
 
 export const licensePriceSet: LicensePriceSet = {
     // Free license matches the freeLimitSet
-    [LicenseId.SMALL_LICENSE]: "price_1TMJzmD3Ee2Ir7Wm05NlGImT",
-    [LicenseId.BIG_LICENSE]: "price_1TMJzzD3Ee2Ir7WmzJw9TerS"
+    [LicenseId.TIER1]: "price_1TMJzmD3Ee2Ir7Wm05NlGImT",
+    [LicenseId.TIER2]: "price_1TMJzzD3Ee2Ir7WmzJw9TerS"
 };
 
 export const licensePriceSetSandbox: LicensePriceSet = {
     // Free license matches the freeLimitSet
     // when matching license the keys closer to 0 index are matched first so list the licenses in descending order of value
-    [LicenseId.SMALL_LICENSE]: "price_1SxDwuDCpkOb237Bz0yTiOgN",
-    [LicenseId.BIG_LICENSE]: "price_1SxDy0DCpkOb237BWJxrxYkl"
+    [LicenseId.TIER1]: "price_1SxDwuDCpkOb237Bz0yTiOgN",
+    [LicenseId.TIER2]: "price_1SxDy0DCpkOb237BWJxrxYkl"
 };
 
 export function getLicensePriceSet(

server/lib/cache.ts

@@ -1,13 +1,7 @@
-import NodeCache from "node-cache";
 import logger from "@server/logger";
+import { createLocalCache } from "@server/lib/createLocalCache";
 
-// Create local cache with maxKeys limit to prevent memory leaks
-// With ~10k requests/day and 5min TTL, 10k keys should be more than sufficient
-export const localCache = new NodeCache({
-    stdTTL: 3600,
-    checkperiod: 120,
-    maxKeys: 10000
-});
+export const localCache = createLocalCache();
 
 // Log cache statistics periodically for monitoring
 // setInterval(() => {

server/lib/consts.ts

@@ -2,7 +2,7 @@ import path from "path";
 import { fileURLToPath } from "url";
 
 // This is a placeholder value replaced by the build process
-export const APP_VERSION = "1.22.0";
+export const APP_VERSION = "1.23.0";
 
 export const __FILENAME = fileURLToPath(import.meta.url);
 export const __DIRNAME = path.dirname(__FILENAME);

server/lib/createLocalCache.ts

@@ -0,0 +1,82 @@
+import { LRUCache } from "lru-cache";
+
+const DEFAULT_MAX_KEYS = 10000;
+const DEFAULT_TTL_SECONDS = 3600;
+
+export type LocalCache = {
+    get<T>(key: string): T | undefined;
+    set(key: string, value: unknown, ttlSeconds?: number): boolean;
+    del(key: string | string[]): number;
+    has(key: string): boolean;
+    keys(): string[];
+    flushAll(): void;
+    getStats(): { keys: number };
+    getTtl(key: string): number | undefined;
+};
+
+export function createLocalCache(
+    max = DEFAULT_MAX_KEYS,
+    ttlSeconds = DEFAULT_TTL_SECONDS
+): LocalCache {
+    const lru = new LRUCache<string, {}>({
+        max,
+        ttl: ttlSeconds * 1000,
+        updateAgeOnGet: false
+    });
+
+    return {
+        get<T>(key: string): T | undefined {
+            return lru.get(key) as T | undefined;
+        },
+
+        set(key: string, value: unknown, ttlSeconds?: number): boolean {
+            const stored = value as {};
+            if (ttlSeconds === undefined) {
+                lru.set(key, stored);
+            } else if (ttlSeconds === 0) {
+                lru.set(key, stored, { ttl: 0 });
+            } else {
+                lru.set(key, stored, { ttl: ttlSeconds * 1000 });
+            }
+            return true;
+        },
+
+        del(key: string | string[]): number {
+            const keys = Array.isArray(key) ? key : [key];
+            let deleted = 0;
+            for (const k of keys) {
+                if (lru.delete(k)) {
+                    deleted++;
+                }
+            }
+            return deleted;
+        },
+
+        has(key: string): boolean {
+            return lru.has(key);
+        },
+
+        keys(): string[] {
+            return [...lru.keys()];
+        },
+
+        flushAll(): void {
+            lru.clear();
+        },
+
+        getStats(): { keys: number } {
+            return { keys: lru.size };
+        },
+
+        getTtl(key: string): number | undefined {
+            if (!lru.has(key)) {
+                return undefined;
+            }
+            const remaining = lru.getRemainingTTL(key);
+            if (!Number.isFinite(remaining)) {
+                return 0;
+            }
+            return Date.now() + remaining;
+        }
+    };
+}

server/lib/dns/generateDomains.ts

@@ -0,0 +1,10 @@
+export function createCname(
+    domainId: string,
+    baseDomain: string
+): { baseDomain: string; value: string }[] {
+    throw new Error("Creating CNAME records is not supported in this build");
+}
+
+export function createNs(): string[] {
+    throw new Error("Creating NS records is not supported in this build");
+}

server/lib/exitNodes/exitNodeCheckIn.ts

@@ -0,0 +1,14 @@
+// Tracks, per process lifetime, whether a given exit node has ever checked in
+// (called /gerbil/get-config) since this Pangolin instance started. This lets
+// callers distinguish "gerbil hasn't come up yet" (expected briefly after a
+// restart, since gerbil depends on pangolin's container starting first) from
+// "gerbil was reachable and now isn't" (a real problem worth an error log).
+const checkedInExitNodeIds = new Set<number>();
+
+export function markExitNodeCheckedIn(exitNodeId: number): void {
+    checkedInExitNodeIds.add(exitNodeId);
+}
+
+export function hasExitNodeCheckedIn(exitNodeId: number): boolean {
+    return checkedInExitNodeIds.has(exitNodeId);
+}

server/lib/exitNodes/exitNodeComms.ts

@@ -1,6 +1,7 @@
 import axios from "axios";
 import logger from "@server/logger";
 import { ExitNode } from "@server/db";
+import { hasExitNodeCheckedIn } from "./exitNodeCheckIn";
 
 interface ExitNodeRequest {
     remoteType?: string;
@@ -72,13 +73,19 @@ export async function sendToExitNode(
 
         return response.data;
     } catch (error) {
-        if (axios.isAxiosError(error)) {
-            logger.error(
-                `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${exitNode.reachableAt} (status: ${error.response?.status}): ${error.message}`
-            );
+        const message = axios.isAxiosError(error)
+            ? `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${exitNode.reachableAt} (status: ${error.response?.status}): ${error.message}`
+            : `Error making ${method} request for exit node at ${exitNode.reachableAt}: ${error}`;
+
+        // The exit node (gerbil) may still be starting up and not yet
+        // reachable. Until it has checked in at least once, log this at a
+        // lower level since it's expected; once it has checked in, a
+        // connection failure is a real problem.
+        if (hasExitNodeCheckedIn(exitNode.exitNodeId)) {
+            logger.error(message);
         } else {
-            logger.error(
-                `Error making ${method} request for exit node at ${exitNode.reachableAt}: ${error}`
+            logger.warn(
+                `${message} (exit node has not checked in yet since startup, this is expected briefly)`
             );
         }
         throw error;

server/lib/exitNodes/index.ts

@@ -1,5 +1,6 @@
 export * from "./exitNodes";
 export * from "./exitNodeComms";
+export * from "./exitNodeCheckIn";
 export * from "./subnet";
 export * from "./getCurrentExitNodeId";
 export * from "./calculateExitNodeWeight";

server/lib/getEnvOrYaml.ts

@@ -1,3 +1,37 @@
-export const getEnvOrYaml = (envVar: string) => (valFromYaml: any) => {
-    return process.env[envVar] ?? valFromYaml;
+import fs from "fs";
+
+// Resolves an environment variable, also honoring a `<envVar>_FILE` variant
+// that points to a file whose (trimmed) contents should be used as the
+// value. This is the common convention for consuming Docker/Swarm secrets
+// (e.g. mounted at /run/secrets/...) without putting the raw value in the
+// container's environment.
+export const readEnvOrFile = (envVar: string): string | undefined => {
+    const fileEnvVar = `${envVar}_FILE`;
+    const filePath = process.env[fileEnvVar];
+
+    if (filePath) {
+        if (process.env[envVar]) {
+            throw new Error(
+                `Both ${envVar} and ${fileEnvVar} are set. Please set only one.`
+            );
+        }
+
+        try {
+            return fs.readFileSync(filePath, "utf8").trim();
+        } catch (error) {
+            throw new Error(
+                `Failed to read ${fileEnvVar} (${filePath}): ${
+                    error instanceof Error ? error.message : error
+                }`
+            );
+        }
+    }
+
+    return process.env[envVar];
 };
+
+export const getEnvOrYaml =
+    (envVar: string) =>
+    (valFromYaml: string | undefined): string | undefined => {
+        return readEnvOrFile(envVar) ?? valFromYaml;
+    };

server/lib/readConfigFile.ts

@@ -3,7 +3,7 @@ import * as yaml from "js-yaml";
 import { configFilePath1, configFilePath2 } from "./consts";
 import { z } from "zod";
 import stoi from "./stoi";
-import { getEnvOrYaml } from "./getEnvOrYaml";
+import { getEnvOrYaml, readEnvOrFile } from "./getEnvOrYaml";
 
 const portSchema = z.number().positive().gt(0).lte(65535);
 
@@ -26,14 +26,12 @@ export const configSchema = z
                     .object({
                         anonymous_usage: z.boolean().optional().default(true)
                     })
-                    .optional()
                     .prefault({}),
                 notifications: z
                     .object({
                         product_updates: z.boolean().optional().default(true),
                         new_releases: z.boolean().optional().default(true)
                     })
-                    .optional()
                     .prefault({})
             })
             .optional()
@@ -109,7 +107,6 @@ export const configSchema = z
                         id: z.string().optional().default("P-Access-Token-Id"),
                         token: z.string().optional().default("P-Access-Token")
                     })
-                    .optional()
                     .prefault({}),
                 remote_headers: z
                     .object({
@@ -126,7 +123,6 @@ export const configSchema = z
                         name: z.string().optional().default("Remote-Name"),
                         role: z.string().optional().default("Remote-Role")
                     })
-                    .optional()
                     .prefault({}),
                 resource_session_request_param: z
                     .string()
@@ -164,14 +160,17 @@ export const configSchema = z
                     .boolean()
                     .optional()
                     .default(false)
-                    .transform((val) =>
-                        process.env.ENABLE_AI_GATEWAY_CLIENT_IP_HEADER !==
-                        undefined
-                            ? process.env.ENABLE_AI_GATEWAY_CLIENT_IP_HEADER ===
-                              "true"
-                            : val
-                    ),
-                secret: z.string().pipe(z.string().min(8)).optional(),
+                    .transform((val) => {
+                        const envVal = readEnvOrFile(
+                            "ENABLE_AI_GATEWAY_CLIENT_IP_HEADER"
+                        );
+                        return envVal !== undefined ? envVal === "true" : val;
+                    }),
+                secret: z
+                    .string()
+                    .pipe(z.string().min(8))
+                    .optional()
+                    .transform(getEnvOrYaml("SERVER_SECRET")),
                 maxmind_db_path: z.string().optional(),
                 maxmind_asn_path: z.string().optional()
             })
@@ -202,7 +201,8 @@ export const configSchema = z
                 dashboard_session_length_hours: 720,
                 resource_session_length_hours: 720,
                 trust_proxy: 1,
-                enable_ai_gateway_client_ip_header: false
+                enable_ai_gateway_client_ip_header: false,
+                secret: undefined
             }),
         postgres: z
             .object({
@@ -238,7 +238,6 @@ export const configSchema = z
                             .default(5000),
                         jit_mode: z.boolean().default(true)
                     })
-                    .optional()
                     .prefault({})
             })
             .optional(),
@@ -278,7 +277,6 @@ export const configSchema = z
                             .optional()
                             .default(5000)
                     })
-                    .optional()
                     .prefault({})
             })
             .optional(),
@@ -325,10 +323,8 @@ export const configSchema = z
                             .optional()
                             .default(50)
                     })
-                    .optional()
                     .prefault({})
             })
-            .optional()
             .prefault({}),
         gerbil: z
             .object({
@@ -357,7 +353,6 @@ export const configSchema = z
                     .optional()
                     .default(30)
             })
-            .optional()
             .prefault({}),
         orgs: z
             .object({
@@ -391,7 +386,6 @@ export const configSchema = z
                             .optional()
                             .default(500)
                     })
-                    .optional()
                     .prefault({}),
                 auth: z
                     .object({
@@ -408,10 +402,8 @@ export const configSchema = z
                             .optional()
                             .default(500)
                     })
-                    .optional()
                     .prefault({})
             })
-            .optional()
             .prefault({}),
         email: z
             .object({
@@ -489,27 +481,8 @@ export const configSchema = z
                             .optional()
                             .default(12)
                     })
-                    .optional()
                     .prefault({})
             })
-            .optional()
-            .prefault({}),
-        dns: z
-            .object({
-                nameservers: z
-                    .array(z.string().optional().optional())
-                    .optional()
-                    .default([
-                        "ns1.pangolin.net",
-                        "ns2.pangolin.net",
-                        "ns3.pangolin.net"
-                    ]),
-                cname_extension: z
-                    .string()
-                    .optional()
-                    .default("cname.pangolin.net")
-            })
-            .optional()
             .prefault({})
     })
     .refine(
@@ -530,10 +503,7 @@ export const configSchema = z
     )
     .refine(
         (data) => {
-            // If hybrid is not defined, server secret must be defined. If its not defined already then pull it from env
-            if (data.server?.secret === undefined) {
-                data.server.secret = process.env.SERVER_SECRET;
-            }
+            // If hybrid is not defined, server secret must be defined
             return (
                 data.server?.secret !== undefined &&
                 data.server.secret.length > 0

server/lib/retry.ts

@@ -0,0 +1,72 @@
+import logger from "@server/logger";
+
+export async function withRetry<T>(
+    fn: () => Promise<T>,
+    options: {
+        retries?: number;
+        baseDelayMs?: number;
+        label?: string;
+        // Called with each caught error to decide whether it's worth
+        // retrying. Defaults to retrying everything (existing behavior) -
+        // pass this to exclude errors that are known to be permanent (e.g.
+        // an upstream rate limit or validation rejection) rather than
+        // transient, so they fail fast instead of wasting retry attempts.
+        shouldRetry?: (error: unknown) => boolean;
+    } = {}
+): Promise<T> {
+    const {
+        retries = 3,
+        baseDelayMs = 250,
+        label = "operation",
+        shouldRetry = () => true
+    } = options;
+
+    let attempt = 0;
+    while (true) {
+        try {
+            return await fn();
+        } catch (error) {
+            attempt++;
+            if (attempt > retries || !shouldRetry(error)) {
+                throw error;
+            }
+
+            // Exponential backoff with jitter so retries don't all land at once.
+            const delay =
+                baseDelayMs * 2 ** (attempt - 1) * (0.5 + Math.random());
+
+            logger.warn(
+                `${label} failed (attempt ${attempt}/${retries + 1}), retrying in ${delay.toFixed(0)}ms`,
+                error
+            );
+
+            await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+    }
+}
+
+// Bounds an operation that has no timeout of its own (e.g. acme-client's
+// axios instance never sets one, so a stalled TCP connection to the ACME
+// server hangs forever instead of erroring). Without this, a single hung
+// call can leave its caller's promise permanently unsettled - fatal for
+// code that gates future work on that promise resolving, like the
+// scheduler's runExclusive() waiting on a batch's Promise.all.
+export async function withTimeout<T>(
+    promise: Promise<T>,
+    ms: number,
+    label = "operation"
+): Promise<T> {
+    let timer: NodeJS.Timeout;
+    const timeout = new Promise<never>((_, reject) => {
+        timer = setTimeout(
+            () => reject(new Error(`${label} timed out after ${ms}ms`)),
+            ms
+        );
+    });
+
+    try {
+        return await Promise.race([promise, timeout]);
+    } finally {
+        clearTimeout(timer!);
+    }
+}

server/lib/traefik/TraefikConfigManager.ts

@@ -6,11 +6,15 @@ import * as yaml from "js-yaml";
 import axios from "axios";
 import { db, exitNodes } from "@server/db";
 import { eq } from "drizzle-orm";
-import { getCurrentExitNodeId } from "@server/lib/exitNodes";
+import {
+    getCurrentExitNodeId,
+    hasExitNodeCheckedIn
+} from "@server/lib/exitNodes";
 import { getTraefikConfig } from "#dynamic/lib/traefik";
 import { getValidCertificatesForDomains } from "@server/lib/certificates";
 import { sendToExitNode } from "#dynamic/lib/exitNodes";
 import { build } from "@server/build";
+import license from "#dynamic/license/license";
 
 export class TraefikConfigManager {
     private intervalId: NodeJS.Timeout | null = null;
@@ -341,10 +345,6 @@ export class TraefikConfigManager {
 
             const { domains, traefikConfig } = getTraefikConfig;
 
-            // Add static domains from config
-            // const staticDomains = [config.getRawConfig().app.dashboard_url];
-            // staticDomains.forEach((domain) => domains.add(domain));
-
             // Log if domains changed
             if (
                 this.lastActiveDomains.size !== domains.size ||
@@ -358,7 +358,11 @@ export class TraefikConfigManager {
                 this.lastActiveDomains = new Set(domains);
             }
 
-            if (process.env.USE_PANGOLIN_DNS === "true" && build != "oss") {
+            if (
+                process.env.CERT_MODE === "pangolin" &&
+                build != "oss" &&
+                (await license.hasTier(["personal", "tier2", "enterprise"]))
+            ) {
                 // Scan current local certificate state
                 this.lastLocalCertificateState =
                     await this.scanLocalCertificateState();
@@ -439,13 +443,13 @@ export class TraefikConfigManager {
                     // Always ensure all existing certificates (including wildcards) are in the config
                     await this.updateDynamicConfigFromLocalCerts(domains);
                 } else {
-                    const timeSinceLastFetch = this.lastCertificateFetch
-                        ? Math.round(
-                              (Date.now() -
-                                  this.lastCertificateFetch.getTime()) /
-                                  (1000 * 60)
-                          )
-                        : 0;
+                    // const timeSinceLastFetch = this.lastCertificateFetch
+                    //     ? Math.round(
+                    //           (Date.now() -
+                    //               this.lastCertificateFetch.getTime()) /
+                    //               (1000 * 60)
+                    //       )
+                    //     : 0;
 
                     // logger.debug(
                     //     `Skipping certificate fetch - no changes detected and within 24-hour window (last fetch: ${timeSinceLastFetch} minutes ago)`
@@ -466,32 +470,51 @@ export class TraefikConfigManager {
             await this.writeTraefikDynamicConfig(traefikConfig);
 
             // Send domains to SNI proxy
+            let exitNodeForSni: typeof exitNodes.$inferSelect | undefined;
             try {
-                let exitNode;
                 if (config.getRawConfig().gerbil.exit_node_name) {
                     const exitNodeName =
                         config.getRawConfig().gerbil.exit_node_name!;
-                    [exitNode] = await db
+                    [exitNodeForSni] = await db
                         .select()
                         .from(exitNodes)
                         .where(eq(exitNodes.name, exitNodeName))
                         .limit(1);
                 } else {
-                    [exitNode] = await db.select().from(exitNodes).limit(1);
+                    [exitNodeForSni] = await db
+                        .select()
+                        .from(exitNodes)
+                        .limit(1);
                 }
-                if (exitNode) {
-                    await sendToExitNode(exitNode, {
+                if (exitNodeForSni) {
+                    await sendToExitNode(exitNodeForSni, {
                         localPath: "/update-local-snis",
                         method: "POST",
-                        data: { fullDomains: Array.from(domains) }
+                        data: {
+                            fullDomains: [
+                                ...Array.from(domains),
+                                ...config.getRawConfig().traefik.static_domains
+                            ]
+                        }
                     });
                 } else {
-                    logger.error(
+                    logger.warn(
                         "No exit node found. Has gerbil registered yet?"
                     );
                 }
             } catch (err) {
-                logger.error("Failed to post domains to SNI proxy:", err);
+                // sendToExitNode already logs the underlying connection
+                // error at the appropriate level (warn before the exit node
+                // has checked in since startup, error after), so avoid
+                // double-logging it as an error here.
+                if (
+                    exitNodeForSni &&
+                    !hasExitNodeCheckedIn(exitNodeForSni.exitNodeId)
+                ) {
+                    logger.warn("Failed to post domains to SNI proxy:", err);
+                } else {
+                    logger.error("Failed to post domains to SNI proxy:", err);
+                }
             }
 
             // Update active domains tracking
@@ -699,10 +722,9 @@ export class TraefikConfigManager {
         }
         if (shouldWrite) {
             try {
-                fs.writeFileSync(
+                this.atomicWriteFileSync(
                     traefikDynamicConfigPath,
-                    yaml.dump(traefikConfig, { noRefs: true }),
-                    "utf8"
+                    yaml.dump(traefikConfig, { noRefs: true })
                 );
                 logger.info("Traefik dynamic config updated");
             } catch (err) {
@@ -804,7 +826,7 @@ export class TraefikConfigManager {
         // Only write the config if it has changed
         const newConfigYaml = yaml.dump(dynamicConfig, { noRefs: true });
         if (newConfigYaml !== originalConfigYaml) {
-            fs.writeFileSync(dynamicConfigPath, newConfigYaml, "utf8");
+            this.atomicWriteFileSync(dynamicConfigPath, newConfigYaml);
             logger.info("Dynamic cert config updated from local certificates");
         }
     }
@@ -882,26 +904,23 @@ export class TraefikConfigManager {
                         `Processing certificate for domain: ${cert.domain}`
                     );
 
-                    fs.writeFileSync(certPath, cert.certFile, "utf8");
-                    fs.writeFileSync(keyPath, cert.keyFile, "utf8");
-
-                    // Set appropriate permissions (readable by owner only for key file)
-                    fs.chmodSync(certPath, 0o644);
-                    fs.chmodSync(keyPath, 0o600);
+                    // Write atomically (temp file + rename) so Traefik's
+                    // file watcher never observes a partially written
+                    // cert/key and fails with "failed to find any PEM data".
+                    this.atomicWriteFileSync(certPath, cert.certFile, 0o644);
+                    this.atomicWriteFileSync(keyPath, cert.keyFile, 0o600);
 
                     // Write/update .last_update file with current timestamp
-                    fs.writeFileSync(
+                    this.atomicWriteFileSync(
                         lastUpdatePath,
-                        new Date().toISOString(),
-                        "utf8"
+                        new Date().toISOString()
                     );
 
                     // Check if this is a wildcard certificate and store it
                     const wildcardPath = path.join(domainDir, ".wildcard");
-                    fs.writeFileSync(
+                    this.atomicWriteFileSync(
                         wildcardPath,
-                        cert.wildcard ? "true" : "false",
-                        "utf8"
+                        cert.wildcard ? "true" : "false"
                     );
 
                     logger.info(
@@ -913,10 +932,9 @@ export class TraefikConfigManager {
                 // even if the cert content didn't change
                 if (cert.expiresAt) {
                     const expiresAtPath = path.join(domainDir, ".expires_at");
-                    fs.writeFileSync(
+                    this.atomicWriteFileSync(
                         expiresAtPath,
-                        cert.expiresAt.toString(),
-                        "utf8"
+                        cert.expiresAt.toString()
                     );
                 }
 
@@ -952,7 +970,7 @@ export class TraefikConfigManager {
         // Only write the config if it has changed
         const newConfigYaml = yaml.dump(dynamicConfig, { noRefs: true });
         if (newConfigYaml !== originalConfigYaml) {
-            fs.writeFileSync(dynamicConfigPath, newConfigYaml, "utf8");
+            this.atomicWriteFileSync(dynamicConfigPath, newConfigYaml);
             logger.info("Dynamic cert config updated");
         }
     }
@@ -1123,10 +1141,9 @@ export class TraefikConfigManager {
 
             if (configChanged) {
                 try {
-                    fs.writeFileSync(
+                    this.atomicWriteFileSync(
                         dynamicConfigPath,
-                        yaml.dump(dynamicConfig, { noRefs: true }),
-                        "utf8"
+                        yaml.dump(dynamicConfig, { noRefs: true })
                     );
                     logger.info("Dynamic config updated after cleanup");
                 } catch (err) {
@@ -1153,6 +1170,36 @@ export class TraefikConfigManager {
         }
     }
 
+    /**
+     * Write a file atomically by writing to a temp file in the same
+     * directory and renaming it into place. This avoids Traefik (which
+     * watches these files/directories) picking up a partially written
+     * file and failing to parse it (e.g. "failed to find any PEM data").
+     */
+    private atomicWriteFileSync(
+        filePath: string,
+        data: string,
+        mode?: number
+    ): void {
+        const dir = path.dirname(filePath);
+        const tmpPath = path.join(
+            dir,
+            `.${path.basename(filePath)}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`
+        );
+        try {
+            fs.writeFileSync(tmpPath, data, "utf8");
+            if (mode !== undefined) {
+                fs.chmodSync(tmpPath, mode);
+            }
+            fs.renameSync(tmpPath, filePath);
+        } catch (error) {
+            try {
+                fs.rmSync(tmpPath, { force: true });
+            } catch {}
+            throw error;
+        }
+    }
+
     /**
      * Check if file exists
      */

server/lib/traefik/rule.ts

@@ -1,7 +1,5 @@
-/**
- * Build the Host()/HostRegexp() Traefik rule for a resource's domain.
- * Wildcard resources match any single subdomain via HostRegexp.
- */
+// Build the Host()/HostRegexp() Traefik rule for a resource's domain.
+// Wildcard resources match any single subdomain via HostRegexp.
 export function buildHostRule(
     fullDomain: string,
     wildcard?: boolean | null
@@ -14,10 +12,8 @@ export function buildHostRule(
     return `Host(\`${fullDomain}\`)`;
 }
 
-/**
- * Append a path-matching clause to a Traefik rule based on the resource's
- * configured path and pathMatchType.
- */
+// Append a path-matching clause to a Traefik rule based on the resource's
+// configured path and pathMatchType.
 export function appendPathMatch(
     rule: string,
     path: string | null | undefined,
@@ -82,10 +78,8 @@ export function matchesPath(
     return true;
 }
 
-/**
- * Compute the router priority for a resource, favoring an explicit override
- * and otherwise deriving it from the path match specificity.
- */
+// Compute the router priority for a resource, favoring an explicit override
+// and otherwise deriving it from the path match specificity.
 export function computeRoutePriority(
     priority: number | null | undefined,
     path: string | null | undefined,

server/license/license.ts

@@ -4,7 +4,7 @@ import { setHostMeta } from "@server/lib/hostMeta";
 const keyTypes = ["host"] as const;
 export type LicenseKeyType = (typeof keyTypes)[number];
 
-const keyTiers = ["personal", "enterprise"] as const;
+const keyTiers = ["personal", "enterprise", "tier1", "tier2"] as const;
 export type LicenseKeyTier = (typeof keyTiers)[number];
 
 export type LicenseStatus = {
@@ -33,7 +33,7 @@ export type LicenseKeyCache = {
 export class License {
     private serverSecret!: string;
 
-    constructor(private hostMeta: HostMeta) { }
+    constructor(private hostMeta: HostMeta) {}
 
     public async check(): Promise<LicenseStatus> {
         return {
@@ -50,6 +50,10 @@ export class License {
     public async isUnlocked() {
         return false;
     }
+
+    public async hasTier(tier: LicenseKeyTier[]) {
+        return false;
+    }
 }
 
 await setHostMeta();

server/private/certificates.ts

@@ -0,0 +1,17 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+export {
+    startCertificateManager,
+    stopCertificateManager
+} from "./lib/certificates";

server/private/cleanup.ts

@@ -20,6 +20,8 @@ import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth"
 import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator";
 import { shutdownUsageRecorder } from "@server/lib/aiBudgetEnforcement";
 import { shutdownAiSessionLogger } from "@server/routers/aiGateway/logAiSession";
+import { stopDnsServer } from "./dns";
+import { stopCertificateManager } from "./certificates";
 
 async function cleanup() {
     await stopPingAccumulator();
@@ -31,6 +33,8 @@ async function cleanup() {
     await rateLimitService.cleanup();
     await wsCleanup();
     await logStreamingManager.shutdown();
+    await stopDnsServer();
+    await stopCertificateManager();
 
     process.exit(0);
 }

server/private/dns.ts

@@ -0,0 +1,37 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { AuthoritativeDNSServer } from "#private/lib/dns";
+import { privateConfig } from "#private/lib/config";
+
+let dnsServer: AuthoritativeDNSServer | undefined;
+
+export async function startDnsServer() {
+    const dnsConfig = privateConfig.getRawPrivateConfig().dns;
+    if (!dnsConfig || !dnsConfig.enabled) {
+        return;
+    }
+
+    dnsServer = new AuthoritativeDNSServer(dnsConfig.listen_port);
+
+    await dnsServer.start();
+}
+
+export async function stopDnsServer() {
+    if (!dnsServer) {
+        return;
+    }
+
+    await dnsServer.stop();
+    dnsServer = undefined;
+}

server/private/lib/cache.ts

@@ -11,17 +11,11 @@
  * This file is not licensed under the AGPLv3.
  */
 
-import NodeCache from "node-cache";
 import logger from "@server/logger";
-import { redisManager, regionalRedisManager } from "@server/private/lib/redis";
+import { createLocalCache } from "@server/lib/createLocalCache";
+import { redisManager, regionalRedisManager } from "#private/lib/redis";
 
-// Create local cache with maxKeys limit to prevent memory leaks
-// With ~10k requests/day and 5min TTL, 10k keys should be more than sufficient
-export const localCache = new NodeCache({
-    stdTTL: 3600,
-    checkperiod: 120,
-    maxKeys: 10000
-});
+export const localCache = createLocalCache();
 
 // Log cache statistics periodically for monitoring
 // setInterval(() => {
@@ -97,11 +91,11 @@ class AdaptiveCache {
                 const value = await redisManager.get(key);
 
                 if (value !== null) {
-                    logger.debug(`Cache hit in Redis: ${key}`);
+                    // logger.debug(`Cache hit in Redis: ${key}`);
                     return JSON.parse(value) as T;
                 }
 
-                logger.debug(`Cache miss in Redis: ${key}`);
+                // logger.debug(`Cache miss in Redis: ${key}`);
                 return undefined;
             } catch (error) {
                 logger.error(`Redis get error for key ${key}:`, error);
@@ -134,7 +128,7 @@ class AdaptiveCache {
                     const success = await redisManager.del(k);
                     if (success) {
                         deletedCount++;
-                        logger.debug(`Deleted key from Redis: ${k}`);
+                        // logger.debug(`Deleted key from Redis: ${k}`);
                     }
                 }
 
@@ -161,7 +155,7 @@ class AdaptiveCache {
             const success = localCache.del(k);
             if (success > 0) {
                 deletedCount++;
-                logger.debug(`Deleted key from local cache: ${k}`);
+                // logger.debug(`Deleted key from local cache: ${k}`);
             }
         }
 
@@ -229,7 +223,7 @@ class AdaptiveCache {
         }
 
         localCache.flushAll();
-        logger.debug("Flushed local cache");
+        // logger.debug("Flushed local cache");
     }
 
     /**
@@ -301,15 +295,11 @@ export default cache;
 
 /**
  * Regional adaptive cache backed by the in-cluster Redis instance.
- * Falls back to a local NodeCache when the regional Redis is unavailable.
+ * Falls back to a local LRU cache when the regional Redis is unavailable.
  * Use this for data that is regional in nature (e.g. status history) so
  * reads are served from the same cluster the user is hitting.
  */
-const regionalLocalCache = new NodeCache({
-    stdTTL: 3600,
-    checkperiod: 120,
-    maxKeys: 10000
-});
+const regionalLocalCache = createLocalCache();
 
 class RegionalAdaptiveCache {
     private useRedis(): boolean {
@@ -332,7 +322,7 @@ class RegionalAdaptiveCache {
                     redisTtl
                 );
                 if (success) {
-                    logger.debug(`[regional] Set key in Redis: ${key}`);
+                    // logger.debug(`[regional] Set key in Redis: ${key}`);
                     return true;
                 }
             } catch (error) {
@@ -353,10 +343,10 @@ class RegionalAdaptiveCache {
             try {
                 const value = await regionalRedisManager.get(key);
                 if (value !== null) {
-                    logger.debug(`[regional] Cache hit in Redis: ${key}`);
+                    // logger.debug(`[regional] Cache hit in Redis: ${key}`);
                     return JSON.parse(value) as T;
                 }
-                logger.debug(`[regional] Cache miss in Redis: ${key}`);
+                // logger.debug(`[regional] Cache miss in Redis: ${key}`);
                 return undefined;
             } catch (error) {
                 logger.error(
@@ -385,7 +375,7 @@ class RegionalAdaptiveCache {
                     const success = await regionalRedisManager.del(k);
                     if (success) {
                         deletedCount++;
-                        logger.debug(`[regional] Deleted key from Redis: ${k}`);
+                        // logger.debug(`[regional] Deleted key from Redis: ${k}`);
                     }
                 }
                 if (deletedCount === keys.length) return deletedCount;
@@ -400,7 +390,7 @@ class RegionalAdaptiveCache {
             const count = regionalLocalCache.del(k);
             if (count > 0) {
                 deletedCount++;
-                logger.debug(`[regional] Deleted key from local cache: ${k}`);
+                // logger.debug(`[regional] Deleted key from local cache: ${k}`);
             }
         }
         return deletedCount;

server/private/lib/certificates/acme-client.ts

@@ -0,0 +1,298 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import * as acme from "acme-client";
+import * as fs from "fs";
+import { eq } from "drizzle-orm/sql";
+import { privateConfig as config } from "#private/lib/config";
+import { DnsChallenge, db, dnsChallenge } from "@server/db";
+import { withRetry } from "@server/lib/retry";
+import logger from "@server/logger";
+import { acmeRateLimiter } from "./acmeRateLimiter";
+
+// acme-client's own retry/backoff logging (429 retries, 5xx retries, each
+// status-poll tick in waitForValidStatus) is a no-op by default - it only
+// activates via DEBUG=acme-client or this call, neither of which was wired
+// up. Without it, a cert silently retrying a Let's Encrypt rate limit for
+// several minutes is indistinguishable in our logs from one that's actually
+// hung, since our own logging only wraps the call, not what happens inside
+// it. Must run before any AcmeClient method is called.
+acme.setLogger((msg: string) => logger.info(`[acme-client] ${msg}`));
+
+// acme-client's axios retry wrapper treats any response-less request error
+// (timeout, connection reset, DNS blip reaching the ACME server) as
+// retryable, but once its internal retries are exhausted it falls through to
+// `validateStatus(response)` with `response` still undefined, throwing this
+// uninformative TypeError instead of the real network error.
+// https://github.com/publishlab/node-acme-client/blob/master/src/axios.js
+function isUnresponsiveAcmeError(error: unknown): boolean {
+    return (
+        error instanceof TypeError &&
+        error.message ===
+            "Cannot read properties of undefined (reading 'config')"
+    );
+}
+
+function normalizeAcmeError(error: unknown): Error {
+    if (isUnresponsiveAcmeError(error)) {
+        return new Error(
+            "ACME server did not respond after repeated attempts (network error reaching the ACME endpoint)",
+            { cause: error }
+        );
+    }
+    return error instanceof Error ? error : new Error(String(error));
+}
+
+export class AcmeClientManager {
+    private client: acme.Client | null = null;
+    private accountKey: string | null = null;
+
+    async initialize() {
+        try {
+            this.accountKey = await this.loadAccountKey();
+
+            this.client = new acme.Client({
+                directoryUrl: config.getRawConfig().acme!.acme_directory_url,
+                accountKey: this.accountKey
+            });
+
+            // Try to create account or get existing one
+            await this.client.createAccount({
+                termsOfServiceAgreed: true,
+                contact: [`mailto:${config.getRawConfig().acme!.contact_email}`]
+            });
+
+            logger.info("ACME client initialized successfully");
+        } catch (error) {
+            logger.error("Failed to initialize ACME client:", error);
+            throw error;
+        }
+    }
+
+    private async loadAccountKey(): Promise<string> {
+        const keyPath = config.getRawConfig().acme!.acme_account_key_path;
+
+        if (fs.existsSync(keyPath)) {
+            logger.info("Loading existing account key");
+            return fs.readFileSync(keyPath, "utf8");
+        } else {
+            logger.info("Generating new account key");
+            const privateKey = await acme.crypto.createPrivateKey();
+            const privateKeyString = privateKey.toString();
+            fs.writeFileSync(keyPath, privateKeyString);
+            return privateKeyString;
+        }
+    }
+
+    getClient(): acme.Client {
+        if (!this.client) {
+            throw new Error("ACME client not initialized");
+        }
+        return this.client;
+    }
+
+    async createOrder(domain: string, wildcard: boolean = false): Promise<any> {
+        const client = this.getClient();
+
+        const identifiers = wildcard
+            ? [
+                  { type: "dns", value: domain },
+                  { type: "dns", value: `*.${domain}` }
+              ]
+            : [{ type: "dns", value: domain }];
+
+        await acmeRateLimiter.acquire();
+        const order = await client.createOrder({
+            identifiers
+        });
+
+        if (wildcard) {
+            logger.info(`Created wildcard order for domain: ${domain}`);
+        } else {
+            logger.info(`Created order for domain: ${domain}`);
+        }
+        return order;
+    }
+
+    async getAuthorizations(order: any): Promise<any[]> {
+        const client = this.getClient();
+        await acmeRateLimiter.acquire();
+        return client.getAuthorizations(order);
+    }
+
+    async handleDnsChallenge(
+        dnsChallenges: {
+            authz: any;
+            challenge: any;
+        }[]
+    ): Promise<void> {
+        const client = this.getClient();
+
+        let challengeDomains: DnsChallenge[] = [];
+
+        for (const { authz, challenge } of dnsChallenges) {
+            const keyAuthorization =
+                await client.getChallengeKeyAuthorization(challenge);
+
+            // Extract the domain from authorization
+            const domain = authz.identifier.value;
+
+            // Store challenge in database for DNS server to pick up
+            challengeDomains = await withRetry(
+                () =>
+                    db
+                        .insert(dnsChallenge)
+                        .values({
+                            domain: domain,
+                            token: challenge.token,
+                            keyAuthorization,
+                            createdAt: Math.floor(Date.now() / 1000),
+                            expiresAt: Math.floor(
+                                (Date.now() +
+                                    config.getRawConfig().acme!
+                                        .challenge_ttl_ms) /
+                                    1000
+                            )
+                        })
+                        .returning(),
+                { label: `insert dnsChallenge for domain ${domain}` }
+            );
+
+            logger.info(
+                `DNS challenge stored for domain: ${domain} as token ${challenge.token} and keyAuthorization`
+            );
+        }
+
+        await new Promise((resolve) => setTimeout(resolve, 2000));
+
+        const failedDomains: string[] = [];
+
+        for (const { authz, challenge } of dnsChallenges) {
+            const domain = authz.identifier.value;
+            const challengeDomain = `_acme-challenge.${domain}`;
+
+            try {
+                // The ACME server occasionally has a transient network blip
+                // mid-sequence; retry the whole verify/complete/wait sequence
+                // rather than just the DNS challenge propagation wait, since
+                // these calls are safe to repeat against the ACME server.
+                await withRetry(
+                    async () => {
+                        // Verify challenge
+                        await acmeRateLimiter.acquire();
+                        await client.verifyChallenge(authz, challenge);
+
+                        // Complete challenge
+                        logger.info(
+                            `Completing challenge for domain: ${challengeDomain}`
+                        );
+                        await acmeRateLimiter.acquire();
+                        await client.completeChallenge(challenge);
+
+                        // Wait for validation
+                        logger.info(
+                            `Waiting for challenge to be validated for domain: ${challengeDomain}...`
+                        );
+                        await acmeRateLimiter.acquire();
+                        await client.waitForValidStatus(challenge);
+                    },
+                    {
+                        retries: 2,
+                        baseDelayMs: 5000,
+                        label: `ACME challenge completion for domain ${domain}`,
+                        // Only retry the known network-blip crash - a
+                        // genuine validation failure (e.g. challenge marked
+                        // "invalid" because the DNS record wasn't found) is
+                        // permanent and should fail immediately instead of
+                        // burning Let's Encrypt's per-hostname failed-
+                        // validation rate limit on retries that can't help.
+                        shouldRetry: isUnresponsiveAcmeError
+                    }
+                );
+
+                logger.info(`Challenge completed for domain: ${domain}`);
+            } catch (error) {
+                logger.error(
+                    `Failed to complete challenge for domain ${domain}:`,
+                    normalizeAcmeError(error)
+                );
+                failedDomains.push(domain);
+            }
+        }
+
+        for (const challengeDomain of challengeDomains) {
+            await this.removeDnsChallenge(challengeDomain.dnsChallengeId);
+            logger.info(
+                `Removed DNS challenge for domain: ${challengeDomain.domain}`
+            );
+        }
+
+        // A failed dns-01 challenge leaves the order stuck in "pending" -
+        // finalizing it would just fail with a confusing ACME error, so
+        // stop here and let the caller mark the certificate as failed.
+        if (failedDomains.length > 0) {
+            throw new Error(
+                `DNS-01 challenge validation failed for domain(s): ${failedDomains.join(", ")}`
+            );
+        }
+    }
+
+    async removeDnsChallenge(dnsChallengeId: number): Promise<void> {
+        try {
+            await withRetry(
+                () =>
+                    db
+                        .delete(dnsChallenge)
+                        .where(eq(dnsChallenge.dnsChallengeId, dnsChallengeId)),
+                { label: `delete dnsChallenge ${dnsChallengeId}` }
+            );
+        } catch (error) {
+            logger.error(
+                `Failed to clean up DNS challenge for id ${dnsChallengeId}:`,
+                error
+            );
+        }
+    }
+
+    async finalizeCertificate(
+        order: any,
+        domain: string,
+        wildcard: boolean = false
+    ): Promise<{ certificate: string; privateKey: string }> {
+        const client = this.getClient();
+
+        const altNames = wildcard ? [`*.${domain}`, domain] : [domain];
+
+        // Create CSR
+        const [privateKey, csr] = await acme.crypto.createCsr({
+            altNames
+        });
+
+        // Finalize order
+        await acmeRateLimiter.acquire();
+        const finalizedOrder = await client.finalizeOrder(order, csr);
+
+        // Get certificate
+        await acmeRateLimiter.acquire();
+        const certificate = await client.getCertificate(finalizedOrder);
+
+        logger.info(`Certificate obtained for domain: ${domain}`);
+
+        return {
+            certificate: certificate.toString(),
+            privateKey: privateKey.toString()
+        };
+    }
+}
+
+export const acmeClientManager = new AcmeClientManager();

server/private/lib/certificates/acmeRateLimiter.ts

@@ -0,0 +1,71 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { privateConfig as config } from "#private/lib/config";
+import logger from "@server/logger";
+import { redis } from "../redis";
+// Caps outgoing ACME API calls to a fixed budget per wall-clock second,
+// shared across all pops workers via Redis (mirrors the lockManager pattern
+// in @lib/lock) - a per-process limiter wouldn't be enough since multiple
+// workers issue certificates against the same Let's Encrypt account.
+const ACQUIRE_SCRIPT = `
+local key = KEYS[1]
+local limit = tonumber(ARGV[1])
+local current = redis.call('INCR', key)
+if current == 1 then
+    redis.call('PEXPIRE', key, 2000)
+end
+if current > limit then
+    return 0
+else
+    return 1
+end
+`;
+
+class AcmeRateLimiter {
+    async acquire(): Promise<void> {
+        const limit =
+            config.getRawConfig().acme?.acme_requests_per_second ?? 15;
+
+        for (;;) {
+            const bucket = Math.floor(Date.now() / 1000);
+            const key = `acme_rate_limit:${bucket}`;
+
+            let allowed: number;
+            try {
+                allowed = (await redis.eval(
+                    ACQUIRE_SCRIPT,
+                    1,
+                    key,
+                    limit.toString()
+                )) as number;
+            } catch (error) {
+                logger.error(
+                    "ACME rate limiter check failed, proceeding without throttling:",
+                    error
+                );
+                return;
+            }
+
+            if (allowed === 1) {
+                return;
+            }
+
+            // Budget for this second is spent - wait for the next window.
+            const waitMs = 1000 - (Date.now() % 1000) + 10;
+            await new Promise((resolve) => setTimeout(resolve, waitMs));
+        }
+    }
+}
+
+export const acmeRateLimiter = new AcmeRateLimiter();

server/private/lib/certificates/certificate-service.ts

@@ -0,0 +1,511 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { acmeClientManager } from "./acme-client";
+import { dnsValidator } from "./dns-validator";
+import { getTableColumns } from "drizzle-orm";
+import { eq, and, or, isNull, lt, asc } from "drizzle-orm/sql";
+import { config } from "@server/lib/config";
+import { db, certificates, domains, Certificate } from "@server/db";
+import { encrypt } from "@server/lib/crypto";
+import { withTimeout, withRetry } from "@server/lib/retry";
+import logger from "@server/logger";
+import { lockManager } from "../lock";
+import { pushCertUpdateToAffectedNewts } from "@server/lib/acmeCertSync";
+import crypto from "crypto";
+
+// Number of on-demand DNS validation attempts made right before a
+// certificate is (re)issued, to avoid burning Let's Encrypt rate limits on
+// domains whose DNS has drifted since they were last verified.
+const PRE_CERT_DNS_VALIDATION_ATTEMPTS = 3;
+
+// Hard ceiling on a single certificate's issuance/renewal flow. acme-client's
+// axios instance never sets a request timeout, so a stalled connection to
+// the ACME server hangs forever instead of erroring - and since
+// processPendingCertificates/processRenewalCandidates gate the *next* batch
+// on Promise.all(...) over the current one, one hung certificate would
+// otherwise stall every other domain permanently. Sized generously above the
+// legitimate worst case (acme-client's own bounded backoff is ~3.6min per
+// status-polling loop, and a wildcard cert's two identifiers plus order
+// finalization can chain a few of those) so this only fires on a genuine hang.
+const CERTIFICATE_ISSUANCE_TIMEOUT_MS = 20 * 60 * 1000;
+
+// "requested" is set the instant a cert starts processing and is never
+// queried anywhere else - processPendingCertificates only selects "pending"
+// and processRenewalCandidates only selects "valid". So if the *process*
+// dies mid-flight (OOM, node eviction, a rolling deploy) rather than just
+// hanging, the row is orphaned in "requested" permanently with nothing to
+// ever pick it back up, no matter how good the in-process timeouts are.
+// Threshold is set comfortably above CERTIFICATE_ISSUANCE_TIMEOUT_MS plus the
+// scheduler's own outer backstop so this never reclaims a cert that's still
+// genuinely being worked on.
+const STUCK_CERTIFICATE_THRESHOLD_MS = 40 * 60 * 1000;
+
+export class CertificateService {
+    // Runs at the top of every processPendingCertificates tick so an
+    // interrupted worker's leftovers always get put back in the queue
+    // instead of sitting invisible to every query forever.
+    private async reclaimStuckCertificates(): Promise<void> {
+        const staleBefore =
+            Math.floor(Date.now() / 1000) -
+            Math.floor(STUCK_CERTIFICATE_THRESHOLD_MS / 1000);
+
+        const reclaimed = await db
+            .update(certificates)
+            .set({
+                status: "pending",
+                errorMessage:
+                    'Reclaimed after being stuck in "requested" state - the worker processing it likely restarted or crashed',
+                updatedAt: Math.floor(Date.now() / 1000)
+            })
+            .where(
+                and(
+                    eq(certificates.status, "requested"),
+                    lt(certificates.updatedAt, staleBefore)
+                )
+            )
+            .returning({ domain: certificates.domain });
+
+        if (reclaimed.length > 0) {
+            logger.warn(
+                `Reclaimed ${reclaimed.length} certificate(s) stuck in "requested" state: ${reclaimed
+                    .map((c) => c.domain)
+                    .join(", ")}`
+            );
+        }
+    }
+
+    async processPendingCertificates(): Promise<void> {
+        logger.debug("Checking for pending certificates...");
+
+        await this.reclaimStuckCertificates();
+
+        const pendingCerts = await db
+            .select(getTableColumns(certificates))
+            .from(certificates)
+            .leftJoin(domains, eq(certificates.domainId, domains.domainId))
+            .where(
+                and(
+                    eq(certificates.status, "pending"),
+                    or(
+                        // Certs with no linked domain row (e.g. legacy certs
+                        // imported from acme.json) aren't gated on domain
+                        // verification since there's nothing to check.
+                        isNull(certificates.domainId),
+                        and(
+                            eq(domains.verified, true),
+                            eq(domains.failed, false)
+                        )
+                    )
+                )
+            )
+            .limit(10);
+
+        if (pendingCerts.length === 0) {
+            logger.debug("No pending certificates found");
+            return;
+        }
+
+        logger.info(`Found ${pendingCerts.length} pending certificates`);
+
+        // Process the batch concurrently so one domain stuck retrying a slow
+        // DNS-01 challenge (the ACME client's waitForValidStatus can spend
+        // minutes on a bad domain) doesn't stall the rest of the batch.
+        // processSingleCertificate catches its own errors and each cert uses
+        // an independent per-domain lock, so this is safe to parallelize.
+        await Promise.all(
+            pendingCerts.map((cert) => this.processSingleCertificate(cert))
+        );
+    }
+
+    async processRenewalCandidates(): Promise<void> {
+        logger.debug("Checking for certificates needing renewal...");
+
+        const now = Math.floor(Date.now() / 1000);
+
+        const renewalCandidates = await db
+            .select(getTableColumns(certificates))
+            .from(certificates)
+            .leftJoin(domains, eq(certificates.domainId, domains.domainId))
+            .where(
+                and(
+                    eq(certificates.status, "valid"),
+                    lt(certificates.expiresAt, now + 15 * 24 * 60 * 60), // 15 days from now
+                    or(
+                        // Certs with no linked domain row (e.g. legacy certs
+                        // imported from acme.json) aren't gated on domain
+                        // verification since there's nothing to check.
+                        isNull(certificates.domainId),
+                        and(
+                            eq(domains.verified, true),
+                            eq(domains.failed, false)
+                        )
+                    )
+                )
+            )
+            // Most urgent first, so already-expired certs aren't starved
+            // behind the limit by certs that still have weeks of runway.
+            .orderBy(asc(certificates.expiresAt))
+            .limit(50);
+
+        if (renewalCandidates.length === 0) {
+            logger.debug("No certificates need renewal");
+            return;
+        }
+
+        logger.info(
+            `Found ${renewalCandidates.length} certificates needing renewal`
+        );
+
+        for (const cert of renewalCandidates) {
+            if (cert.expiresAt !== null && cert.expiresAt < now) {
+                logger.warn(
+                    `Certificate for ${cert.domain} is marked "valid" but already expired at ${new Date(cert.expiresAt * 1000).toISOString()} (bad state) - renewing immediately`
+                );
+            }
+        }
+
+        // Process the batch concurrently - see processPendingCertificates for why.
+        await Promise.all(
+            renewalCandidates.map((cert) => this.renewCertificate(cert))
+        );
+    }
+
+    private async processSingleCertificate(cert: Certificate): Promise<void> {
+        const lockKey = `cert:${cert.domain}`;
+
+        const lockToken = await lockManager.acquireLock(lockKey);
+        if (!lockToken) {
+            logger.debug(
+                `Could not acquire lock for certificate: ${cert.domain}`
+            );
+            return;
+        }
+
+        try {
+            logger.info(`Processing certificate for domain: ${cert.domain}`);
+
+            // Update status to processing
+            await db
+                .update(certificates)
+                .set({
+                    status: "requested",
+                    updatedAt: Math.floor(Date.now() / 1000)
+                })
+                .where(eq(certificates.certId, cert.certId));
+            //
+
+            await withTimeout(
+                this.obtainCertificate(cert),
+                CERTIFICATE_ISSUANCE_TIMEOUT_MS,
+                `certificate issuance for ${cert.domain}`
+            );
+        } catch (error) {
+            logger.error(
+                `Failed to process certificate for ${cert.domain}:`,
+                error
+            );
+
+            await db
+                .update(certificates)
+                .set({
+                    status: "failed",
+                    errorMessage:
+                        error instanceof Error
+                            ? error.message
+                            : "Unknown error",
+                    updatedAt: Math.floor(Date.now() / 1000)
+                })
+                .where(eq(certificates.certId, cert.certId));
+        } finally {
+            await lockManager.releaseLock(lockKey, lockToken);
+        }
+    }
+
+    private async renewCertificate(cert: Certificate): Promise<void> {
+        const lockKey = `cert:${cert.domain}`;
+
+        const lockToken = await lockManager.acquireLock(lockKey);
+        if (!lockToken) {
+            logger.debug(
+                `Could not acquire lock for certificate renewal: ${cert.domain}`
+            );
+            return;
+        }
+
+        try {
+            logger.info(`Renewing certificate for domain: ${cert.domain}`);
+
+            // Update last renewal attempt
+            await db
+                .update(certificates)
+                .set({
+                    lastRenewalAttempt: Math.floor(Date.now() / 1000),
+                    updatedAt: Math.floor(Date.now() / 1000)
+                })
+                .where(eq(certificates.certId, cert.certId));
+
+            await withTimeout(
+                this.obtainCertificate(cert),
+                CERTIFICATE_ISSUANCE_TIMEOUT_MS,
+                `certificate renewal for ${cert.domain}`
+            );
+        } catch (error) {
+            logger.error(
+                `Failed to renew certificate for ${cert.domain}:`,
+                error
+            );
+
+            await db
+                .update(certificates)
+                .set({
+                    status: "failed",
+                    errorMessage:
+                        error instanceof Error
+                            ? error.message
+                            : "Unknown error",
+                    lastRenewalAttempt: Math.floor(Date.now() / 1000),
+                    updatedAt: Math.floor(Date.now() / 1000)
+                })
+                .where(eq(certificates.certId, cert.certId));
+        } finally {
+            await lockManager.releaseLock(lockKey, lockToken);
+        }
+    }
+
+    // Re-checks the domain's DNS records right before we spend a Let's
+    // Encrypt order on it, so drift that happened after the domain was
+    // originally verified doesn't burn ACME rate limits. Certs with no
+    // linked domain row (e.g. legacy/manually-managed certs) skip this and
+    // proceed as before, since there are no tracked DNS records to check.
+    private async verifyDomainBeforeIssuance(cert: Certificate): Promise<void> {
+        if (!cert.domainId) {
+            return;
+        }
+
+        const [domain] = await db
+            .select()
+            .from(domains)
+            .where(eq(domains.domainId, cert.domainId))
+            .limit(1);
+
+        if (!domain) {
+            return;
+        }
+
+        for (
+            let attempt = 1;
+            attempt <= PRE_CERT_DNS_VALIDATION_ATTEMPTS;
+            attempt++
+        ) {
+            // Offset `tries` so each attempt round-robins to a different
+            // privateConfigured DNS resolver instead of re-querying the same one.
+            const probe = { ...domain, tries: domain.tries + attempt - 1 };
+            if (
+                await dnsValidator.validateDomain(probe, {
+                    forceRecheck: true
+                })
+            ) {
+                await db
+                    .update(domains)
+                    .set({ verified: true, failed: false, errorMessage: null })
+                    .where(eq(domains.domainId, domain.domainId));
+                return;
+            }
+
+            logger.warn(
+                `Pre-certificate DNS check ${attempt}/${PRE_CERT_DNS_VALIDATION_ATTEMPTS} failed for domain ${domain.baseDomain} (cert: ${cert.domain})`
+            );
+        }
+
+        const errorMessage = `Domain failed DNS validation ${PRE_CERT_DNS_VALIDATION_ATTEMPTS} times before certificate issuance`;
+        await db
+            .update(domains)
+            .set({ verified: false, failed: true, errorMessage })
+            .where(eq(domains.domainId, domain.domainId));
+
+        throw new Error(errorMessage);
+    }
+
+    private async obtainCertificate(cert: Certificate): Promise<void> {
+        await this.verifyDomainBeforeIssuance(cert);
+
+        // Create order
+        const order = await acmeClientManager.createOrder(
+            cert.domain,
+            cert.wildcard || false
+        );
+
+        // Update with order ID
+        await withRetry(
+            () =>
+                db
+                    .update(certificates)
+                    .set({
+                        orderId: order.url,
+                        updatedAt: Math.floor(Date.now() / 1000)
+                    })
+                    .where(eq(certificates.certId, cert.certId)),
+            { label: `update orderId for certificate ${cert.domain}` }
+        );
+
+        // Get authorizations
+        const authorizations = await acmeClientManager.getAuthorizations(order);
+
+        // Aggregate all DNS-01 challenges
+        const dnsChallenges = authorizations.map((authz: any) => {
+            const dnsChallenge = authz.challenges.find(
+                (c: any) => c.type === "dns-01"
+            );
+            if (!dnsChallenge) {
+                throw new Error(
+                    `No DNS-01 challenge found for ${authz.identifier.value}`
+                );
+            }
+            return {
+                authz,
+                challenge: dnsChallenge
+            };
+        });
+
+        // Send all DNS-01 challenges in one request to handleDnsChallenge
+        await acmeClientManager.handleDnsChallenge(dnsChallenges);
+
+        // Finalize certificate
+        const { certificate, privateKey } =
+            await acmeClientManager.finalizeCertificate(
+                order,
+                cert.domain,
+                cert.wildcard || false
+            );
+
+        const encryptionKey = config.getRawConfig().server.secret;
+        if (!encryptionKey) {
+            throw new Error("Encryption key not provided");
+        }
+
+        // Encrypt certificate and private key
+        const encryptedCert = encrypt(certificate, encryptionKey);
+        const encryptedKey = encrypt(privateKey, encryptionKey);
+
+        // Parse certificate to get expiration date
+        const expiresAt = this.extractExpirationDate(certificate);
+
+        // Update database record. This persists the certificate we just
+        // obtained from the ACME server, so it's retried aggressively -
+        // losing this write means re-issuing the cert from scratch.
+        await withRetry(
+            () =>
+                db
+                    .update(certificates)
+                    .set({
+                        status: "valid",
+                        expiresAt: Math.floor(expiresAt.getTime() / 1000),
+                        renewalCount: (cert.renewalCount || 0) + 1,
+                        errorMessage: null,
+                        updatedAt: Math.floor(Date.now() / 1000),
+                        certFile: encryptedCert,
+                        keyFile: encryptedKey
+                    })
+                    .where(eq(certificates.certId, cert.certId)),
+            {
+                retries: 5,
+                label: `persist issued certificate for ${cert.domain}`
+            }
+        );
+
+        logger.info(
+            `Certificate successfully obtained/renewed for domain: ${cert.domain}`
+        );
+
+        await pushCertUpdateToAffectedNewts(
+            cert.domain,
+            cert.domainId ?? null,
+            certificate,
+            privateKey
+        );
+    }
+
+    private extractExpirationDate(certificate: string): Date {
+        try {
+            // Extract the certificate block
+            const pem = certificate
+                .replace(/-----BEGIN CERTIFICATE-----/g, "")
+                .replace(/-----END CERTIFICATE-----/g, "")
+                .replace(/\s+/g, "");
+            const der = Buffer.from(pem, "base64");
+
+            // Use Node.js crypto to parse the certificate
+            const x509 = new crypto.X509Certificate(der);
+            return new Date(x509.validTo);
+        } catch (error) {
+            logger.warn(
+                "Failed to parse certificate expiration date, using default",
+                error
+            );
+            // Default to 90 days from now (Let's Encrypt default)
+            return new Date(Date.now() + 90 * 24 * 60 * 60 * 1000);
+        }
+    }
+
+    async addCertificateRequest(domain: string): Promise<void> {
+        try {
+            await db.insert(certificates).values({
+                domain,
+                status: "pending",
+                createdAt: Math.floor(Date.now() / 1000),
+                updatedAt: Math.floor(Date.now() / 1000)
+            });
+            logger.info(`Certificate request added for domain: ${domain}`);
+        } catch (error) {
+            if (error instanceof Error && error.message.includes("unique")) {
+                logger.warn(
+                    `Certificate request already exists for domain: ${domain}`
+                );
+            } else {
+                throw error;
+            }
+        }
+    }
+
+    async getCertificateStatus(domain: string) {
+        const cert = await db
+            .select()
+            .from(certificates)
+            .where(eq(certificates.domain, domain))
+            .limit(1);
+
+        return cert[0] || null;
+    }
+
+    async cleanupExpiredChallenges(): Promise<void> {
+        try {
+            const result = await db
+                .delete(certificates)
+                .where(
+                    lt(certificates.expiresAt, Math.floor(Date.now() / 1000))
+                )
+                .returning();
+
+            if (result.length > 0) {
+                logger.info(
+                    `Cleaned up ${result.length} expired DNS challenges`
+                );
+            }
+        } catch (error) {
+            logger.error("Failed to cleanup expired challenges:", error);
+        }
+    }
+}
+
+export const certificateService = new CertificateService();

server/private/lib/certificates/dns-validator.ts

@@ -0,0 +1,334 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { eq, and, lt } from "drizzle-orm";
+import * as dns from "dns/promises";
+import { privateConfig as config } from "#private/lib/config";
+import { db, domains, DnsRecord, dnsRecords, Domain } from "@server/db";
+import logger from "@server/logger";
+import { lockManager } from "../lock";
+
+export const DNS_VALIDATOR_MAX_TRIES = 300;
+
+export class DNSValidator {
+    private static readonly MAX_TRIES = DNS_VALIDATOR_MAX_TRIES;
+
+    constructor() {}
+
+    async validateAll(): Promise<void> {
+        // Get all domains that are not yet verified and haven't exceeded max tries
+        const unverifiedDomains: Domain[] = await db
+            .select()
+            .from(domains)
+            .where(
+                and(
+                    eq(domains.verified, false),
+                    lt(domains.tries, DNSValidator.MAX_TRIES)
+                )
+            );
+
+        if (unverifiedDomains.length === 0) {
+            logger.debug("No unverified domains found for DNS validation");
+            return;
+        }
+
+        logger.info(`Validating ${unverifiedDomains.length} DNS records`);
+
+        for (const domain of unverifiedDomains) {
+            const lockKey = `dns:${domain.baseDomain}`;
+            const lockToken = await lockManager.acquireLock(lockKey);
+            if (!lockToken) {
+                logger.debug(
+                    `Could not acquire lock for DNS validation: ${domain.baseDomain}`
+                );
+                continue;
+            }
+            try {
+                const isValid = await this.validateDomain(domain);
+                if (isValid) {
+                    await db
+                        .update(domains)
+                        .set({
+                            verified: true,
+                            failed: false,
+                            tries: 0,
+                            errorMessage: null
+                        })
+                        .where(eq(domains.domainId, domain.domainId));
+                    logger.info(
+                        `Domain ${domain.baseDomain} validated successfully`
+                    );
+                } else {
+                    const newTries = domain.tries + 1;
+                    const shouldMarkAsFailed =
+                        newTries >= DNSValidator.MAX_TRIES;
+
+                    await db
+                        .update(domains)
+                        .set({
+                            tries: newTries,
+                            failed: shouldMarkAsFailed
+                        })
+                        .where(eq(domains.domainId, domain.domainId));
+
+                    if (shouldMarkAsFailed) {
+                        logger.warn(
+                            `Domain ${domain.baseDomain} exceeded maximum tries (${DNSValidator.MAX_TRIES}), marking as failed`
+                        );
+                    } else {
+                        logger.debug(
+                            `Domain ${domain.baseDomain} did not validate (attempt ${newTries}/${DNSValidator.MAX_TRIES})`
+                        );
+                    }
+                }
+            } catch (err) {
+                logger.warn(
+                    `Error validating domain ${domain.baseDomain}:`,
+                    err
+                );
+                // Increment tries even on error
+                const newTries = domain.tries + 1;
+                const shouldMarkAsFailed = newTries >= DNSValidator.MAX_TRIES;
+
+                await db
+                    .update(domains)
+                    .set({
+                        tries: newTries,
+                        failed: shouldMarkAsFailed
+                    })
+                    .where(eq(domains.domainId, domain.domainId));
+            } finally {
+                await lockManager.releaseLock(lockKey, lockToken);
+            }
+        }
+    }
+
+    async validateDomain(
+        domain: Domain,
+        opts: { forceRecheck?: boolean } = {}
+    ): Promise<boolean> {
+        const { forceRecheck = false } = opts;
+        const resolver = new dns.Resolver();
+        const servers = config.getRawConfig().acme?.dns_resolvers;
+        if (!servers || servers.length === 0) {
+            throw new Error("No DNS resolvers configured");
+        }
+        const dnsServer = servers[domain.tries % servers.length]!;
+        resolver.setServers([dnsServer]);
+        logger.debug(
+            `Using DNS server ${dnsServer} for domain ${domain.baseDomain} (try ${domain.tries})`
+        );
+
+        // Get all DNS records for this domain
+        const records: DnsRecord[] = await db
+            .select()
+            .from(dnsRecords)
+            .where(eq(dnsRecords.domainId, domain.domainId));
+
+        if (records.length === 0) {
+            logger.warn(`No DNS records found for domain ${domain.baseDomain}`);
+            return false;
+        }
+
+        if (!forceRecheck && records.every((r) => r.verified)) {
+            logger.info(
+                `All DNS records already verified for domain ${domain.baseDomain}`
+            );
+            return true;
+        }
+
+        logger.info(
+            `Validating ${records.length} DNS records for domain ${domain.baseDomain}`
+        );
+
+        // Collect the full set of expected NS values for this domain so we can
+        // detect extra records that are present in DNS but not in our DB.
+        const expectedNsValues = new Set<string>(
+            records.filter((r) => r.recordType === "NS").map((r) => r.value)
+        );
+
+        // Cache resolved NS records across iterations — there will be 3 NS
+        // records in the DB and we don't need to hit the upstream server 3 times.
+        let previousNs: string[] | null = null;
+
+        for (const record of records) {
+            // Skip already verified records, unless a live recheck was requested
+            if (record.verified && !forceRecheck) {
+                continue;
+            }
+
+            let isValid = false;
+
+            try {
+                if (record.recordType === "NS") {
+                    let nsRecords: string[] | null = previousNs;
+                    if (!nsRecords) {
+                        nsRecords = await resolver.resolveNs(
+                            record.baseDomain || domain.baseDomain
+                        );
+                    }
+                    logger.info(
+                        `NS records for ${
+                            record.baseDomain || domain.baseDomain
+                        }:`,
+                        nsRecords
+                    );
+
+                    // Check if this expected NS value is present in the live records.
+                    // A stale/legacy expected value (e.g. left over from a
+                    // nameserver rebrand) is also accepted as long as the live
+                    // records resolve to some other known-valid nameserver —
+                    // the specific literal hostname stored per-domain isn't
+                    // meaningful once it's a recognized alias.
+                    isValid = nsRecords.some((ns) => ns === record.value);
+
+                    previousNs = nsRecords;
+                } else if (record.recordType === "CNAME") {
+                    const cnameRecords = await resolver.resolveCname(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    logger.info(
+                        `CNAME records for ${
+                            record.baseDomain || domain.baseDomain
+                        }:`,
+                        cnameRecords
+                    );
+
+                    // Check if the CNAME record matches the expected value
+                    isValid =
+                        cnameRecords.length === 1 &&
+                        cnameRecords[0] === record.value;
+                } else if (record.recordType === "TXT") {
+                    const txtRecords = await resolver.resolveTxt(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    logger.info(
+                        `TXT records for ${
+                            record.baseDomain || domain.baseDomain
+                        }:`,
+                        txtRecords
+                    );
+
+                    // TXT records come as an array of arrays, flatten and check
+                    const flatTxtRecords = txtRecords.flat();
+                    isValid = flatTxtRecords.includes(record.value);
+                } else if (record.recordType === "A") {
+                    const aRecords = await resolver.resolve4(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    logger.info(
+                        `A records for ${
+                            record.baseDomain || domain.baseDomain
+                        }:`,
+                        aRecords
+                    );
+
+                    // Check if the A record matches the expected value
+                    isValid = aRecords.includes(record.value);
+                } else {
+                    logger.warn(
+                        `Unsupported record type: ${record.recordType}`
+                    );
+                    continue;
+                }
+            } catch (error) {
+                isValid = false;
+                logger.debug(
+                    `Did not resolve ${record.recordType} record for ${
+                        record.baseDomain || domain.baseDomain
+                    }:`,
+                    error
+                );
+            }
+
+            // Update the individual record verification status. Runs for
+            // both a mismatched value and a failed/thrown DNS lookup, so a
+            // previously-verified record that stops resolving (e.g. NXDOMAIN
+            // after NS delegation is dropped) gets downgraded instead of
+            // leaving stale `verified: true` state behind.
+            if (isValid) {
+                await db
+                    .update(dnsRecords)
+                    .set({ verified: true })
+                    .where(eq(dnsRecords.id, record.id));
+                logger.info(
+                    `DNS record ${record.id} (${record.recordType}) for ${
+                        record.baseDomain || domain.baseDomain
+                    } verified successfully`
+                );
+            } else {
+                if (record.verified) {
+                    await db
+                        .update(dnsRecords)
+                        .set({ verified: false })
+                        .where(eq(dnsRecords.id, record.id));
+                }
+                logger.debug(
+                    `DNS record ${record.id} (${record.recordType}) for ${
+                        record.baseDomain || domain.baseDomain
+                    } does not match expected value: ${record.value}`
+                );
+            }
+        }
+
+        // --- Extra NS record check ---
+        // If we resolved NS records during this pass, verify that the live DNS
+        // has no nameservers beyond the ones we expect.  Individual records may
+        // already be marked verified above, but we must block full domain
+        // verification until the extra records are removed.
+        if (previousNs !== null && expectedNsValues.size > 0) {
+            const extraNsRecords = previousNs.filter(
+                (ns) => !expectedNsValues.has(ns)
+            );
+
+            if (extraNsRecords.length > 0) {
+                const errorMessage = `Extra NS records found that are not expected: ${extraNsRecords.join(", ")}. Remove these nameservers to complete domain verification.`;
+
+                await db
+                    .update(domains)
+                    .set({ errorMessage })
+                    .where(eq(domains.domainId, domain.domainId));
+
+                logger.warn(
+                    `Domain ${domain.baseDomain} has extra NS records that prevent verification: ${extraNsRecords.join(", ")}`
+                );
+
+                return false;
+            }
+
+            // No extras — clear any stale error that was previously written
+            await db
+                .update(domains)
+                .set({ errorMessage: null })
+                .where(eq(domains.domainId, domain.domainId));
+        }
+
+        // Check if all records are now verified
+        const updatedRecords: DnsRecord[] = await db
+            .select()
+            .from(dnsRecords)
+            .where(eq(dnsRecords.domainId, domain.domainId));
+
+        const allRecordsVerified = updatedRecords.every((r) => r.verified);
+
+        logger.info(
+            `Domain ${domain.baseDomain}: ${
+                updatedRecords.filter((r) => r.verified).length
+            }/${updatedRecords.length} records verified`
+        );
+
+        return allRecordsVerified;
+    }
+}
+
+export const dnsValidator = new DNSValidator();

server/private/lib/certificates/domain-reverifier.ts

@@ -0,0 +1,233 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { eq, and, or, isNull, lt } from "drizzle-orm";
+import * as dns from "dns/promises";
+import { DNS_VALIDATOR_MAX_TRIES } from "./dns-validator";
+import { db, domains, DnsRecord, dnsRecords, Domain } from "@server/db";
+import logger from "@server/logger";
+import { lockManager } from "../lock";
+import { privateConfig as config } from "#private/lib/config";
+
+// Module-level counter so successive domains in a batch round-robin across servers.
+let serverIndex = 0;
+
+export class DomainReverifier {
+    async reverifyAll(): Promise<void> {
+        const certConfig = config.getRawConfig().acme;
+        if (!certConfig) {
+            logger.debug(
+                "No certificate config — skipping domain reverification"
+            );
+            return;
+        }
+
+        const windowMs = certConfig.domain_reverification_window_ms;
+        const batchSize = certConfig.domain_reverification_batch_size;
+        const windowSecs = Math.floor(windowMs / 1000);
+        const cutoff = Math.floor(Date.now() / 1000) - windowSecs;
+
+        const domainsToCheck: Domain[] = await db
+            .select()
+            .from(domains)
+            .where(
+                and(
+                    eq(domains.verified, true),
+                    or(
+                        isNull(domains.lastCheckedAt),
+                        lt(domains.lastCheckedAt, cutoff)
+                    )
+                )
+            )
+            .limit(batchSize);
+
+        if (domainsToCheck.length === 0) {
+            logger.debug("No verified domains due for reverification");
+            return;
+        }
+
+        logger.info(`Reverifying ${domainsToCheck.length} domains`);
+
+        for (const domain of domainsToCheck) {
+            const lockKey = `dns-reverify:${domain.baseDomain}`;
+            const lockToken = await lockManager.acquireLock(lockKey);
+            if (!lockToken) {
+                logger.debug(
+                    `Could not acquire lock for domain reverification: ${domain.baseDomain}`
+                );
+                continue;
+            }
+
+            try {
+                await this.reverifyDomain(domain, certConfig.dns_resolvers);
+            } catch (err) {
+                logger.warn(
+                    `Unexpected error reverifying domain ${domain.baseDomain}:`,
+                    err
+                );
+                // Still stamp lastCheckedAt so we don't hammer a broken domain every run.
+                await db
+                    .update(domains)
+                    .set({ lastCheckedAt: Math.floor(Date.now() / 1000) })
+                    .where(eq(domains.domainId, domain.domainId));
+            } finally {
+                await lockManager.releaseLock(lockKey, lockToken);
+            }
+        }
+    }
+
+    private async reverifyDomain(
+        domain: Domain,
+        servers: string[]
+    ): Promise<void> {
+        if (!servers || servers.length === 0) {
+            throw new Error("No DNS resolvers configured");
+        }
+
+        // Round-robin across servers; advance the global counter so the next
+        // domain in the same batch gets a different server.
+        const dnsServer = servers[serverIndex % servers.length]!;
+        serverIndex++;
+
+        const resolver = new dns.Resolver();
+        resolver.setServers([dnsServer]);
+
+        logger.debug(
+            `Reverifying domain ${domain.baseDomain} using DNS server ${dnsServer}`
+        );
+
+        const records: DnsRecord[] = await db
+            .select()
+            .from(dnsRecords)
+            .where(eq(dnsRecords.domainId, domain.domainId));
+
+        if (records.length === 0) {
+            logger.warn(
+                `No DNS records found for domain ${domain.baseDomain} during reverification — marking failed`
+            );
+            await this.markFailed(
+                domain.domainId,
+                "No DNS records found during periodic reverification"
+            );
+            return;
+        }
+
+        const expectedNsValues = new Set<string>(
+            records.filter((r) => r.recordType === "NS").map((r) => r.value)
+        );
+
+        let allValid = true;
+        let errorMessage: string | null = null;
+        let resolvedNs: string[] | null = null;
+
+        for (const record of records) {
+            let isValid = false;
+
+            try {
+                if (record.recordType === "NS") {
+                    if (!resolvedNs) {
+                        resolvedNs = await resolver.resolveNs(
+                            record.baseDomain || domain.baseDomain
+                        );
+                    }
+                    isValid = resolvedNs.some((ns) => ns === record.value);
+                } else if (record.recordType === "CNAME") {
+                    const cnameRecords = await resolver.resolveCname(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    isValid =
+                        cnameRecords.length === 1 &&
+                        cnameRecords[0] === record.value;
+                } else if (record.recordType === "TXT") {
+                    const txtRecords = await resolver.resolveTxt(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    isValid = txtRecords.flat().includes(record.value);
+                } else if (record.recordType === "A") {
+                    const aRecords = await resolver.resolve4(
+                        record.baseDomain || domain.baseDomain
+                    );
+                    isValid = aRecords.includes(record.value);
+                } else {
+                    logger.warn(
+                        `Unsupported record type ${record.recordType} during reverification of ${domain.baseDomain}`
+                    );
+                    continue;
+                }
+            } catch (err) {
+                logger.debug(
+                    `DNS lookup failed for ${record.recordType} record on ${record.baseDomain || domain.baseDomain}:`,
+                    err
+                );
+                isValid = false;
+            }
+
+            if (!isValid) {
+                allValid = false;
+                errorMessage = `${record.recordType} record for ${record.baseDomain || domain.baseDomain} no longer resolves to expected value "${record.value}"`;
+                break;
+            }
+        }
+
+        // Check for extra NS records beyond what we expect.
+        if (allValid && resolvedNs !== null && expectedNsValues.size > 0) {
+            const extraNs = resolvedNs.filter(
+                (ns) => !expectedNsValues.has(ns)
+            );
+            if (extraNs.length > 0) {
+                allValid = false;
+                errorMessage = `Extra NS records found: ${extraNs.join(", ")}. Remove these nameservers.`;
+            }
+        }
+
+        const now = Math.floor(Date.now() / 1000);
+
+        if (allValid) {
+            await db
+                .update(domains)
+                .set({ lastCheckedAt: now, errorMessage: null })
+                .where(eq(domains.domainId, domain.domainId));
+            logger.debug(
+                `Domain ${domain.baseDomain} passed periodic reverification`
+            );
+        } else {
+            await this.markFailed(domain.domainId, errorMessage);
+            logger.warn(
+                `Domain ${domain.baseDomain} failed periodic reverification: ${errorMessage}`
+            );
+        }
+    }
+
+    private async markFailed(
+        domainId: string,
+        errorMessage: string | null
+    ): Promise<void> {
+        await db
+            .update(domains)
+            .set({
+                verified: false,
+                failed: true,
+                // Three below MAX_TRIES: keeps the domain out of the DNS
+                // validator's immediate retry loop, while still leaving it
+                // eligible (tries < MAX_TRIES) for a few more validation
+                // passes instead of being excluded forever once tries hits
+                // MAX_TRIES.
+                tries: DNS_VALIDATOR_MAX_TRIES - 3,
+                lastCheckedAt: Math.floor(Date.now() / 1000),
+                errorMessage
+            })
+            .where(eq(domains.domainId, domainId));
+    }
+}
+
+export const domainReverifier = new DomainReverifier();

server/private/lib/certificates/index.ts

@@ -0,0 +1,45 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import logger from "@server/logger";
+import { privateConfig } from "#private/lib/config";
+import { acmeClientManager } from "./acme-client";
+import { jobScheduler } from "./scheduler";
+
+export async function startCertificateManager() {
+    const acmeConfig = privateConfig.getRawPrivateConfig().acme;
+    if (
+        acmeConfig &&
+        acmeConfig.cert_mode === "pangolin" &&
+        acmeConfig.enable_acme_client
+    ) {
+        logger.info("Starting certificate management server...");
+
+        // Initialize ACME client
+        await acmeClientManager.initialize();
+
+        // Start certificate issuance/renewal jobs
+        await jobScheduler.start();
+    }
+
+    if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+        // DNS record validation/reverification doesn't require certs, so it
+        // runs whenever Pangolin is acting as the authoritative DNS server,
+        // independent of the cert manager above.
+        await jobScheduler.startDnsJobs();
+    }
+}
+
+export async function stopCertificateManager() {
+    await jobScheduler.stop();
+}

server/private/lib/certificates/scheduler.ts

@@ -0,0 +1,197 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { withTimeout } from "@server/lib/retry";
+import logger from "@server/logger";
+import { certificateService } from "./certificate-service";
+import { privateConfig as config } from "#private/lib/config";
+import { dnsValidator } from "./dns-validator";
+import { domainReverifier } from "./domain-reverifier";
+import license from "#private/license/license";
+
+// Backstop for runExclusive: no single job's own internal timeouts (e.g.
+// certificate-service's per-cert issuance timeout) are relied on here. This
+// is the last line of defense - if *anything* inside a job hangs with no
+// error (a stalled Redis/DB call, a future code path that forgets to bound
+// itself, etc.), state.active must still reset so the next tick can run.
+// Without it, one hung run permanently skips every future tick for that job,
+// since runExclusive only clears state.active after the job promise settles.
+const RUN_EXCLUSIVE_TIMEOUT_MS = 30 * 60 * 1000;
+
+export class JobScheduler {
+    private certIntervals: NodeJS.Timeout[] = [];
+    private dnsIntervals: NodeJS.Timeout[] = [];
+    private certRunning = false;
+    private dnsRunning = false;
+
+    // Guards against a slow batch (e.g. 10 certs whose DNS challenges take a
+    // while) still being processed when the next interval tick fires -
+    // without this, overlapping ticks would each pull their own batch of up
+    // to 10 pending/renewal certs and process them concurrently instead of
+    // waiting for the prior batch to finish.
+    private runExclusive(
+        job: () => Promise<void>,
+        state: { active: boolean },
+        label: string
+    ): () => Promise<void> {
+        return async () => {
+            if (!(await license.hasTier(["personal", "tier2", "enterprise"]))) {
+                logger.debug(
+                    `Skipping ${label} tick - requires a tier2 license`
+                );
+                return;
+            }
+            if (state.active) {
+                logger.debug(
+                    `Skipping ${label} tick - previous run still in progress`
+                );
+                return;
+            }
+            state.active = true;
+            try {
+                await withTimeout(job(), RUN_EXCLUSIVE_TIMEOUT_MS, label);
+            } catch (error) {
+                logger.error(`Error in ${label}:`, error);
+            } finally {
+                state.active = false;
+            }
+        };
+    }
+
+    // Certificate issuance/renewal - requires an ACME client, so this is
+    // only started when Pangolin is actually managing certs.
+    async start(): Promise<void> {
+        if (this.certRunning) {
+            logger.warn("Certificate job scheduler is already running");
+            return;
+        }
+
+        this.certRunning = true;
+        logger.info("Starting certificate job scheduler");
+
+        const newCertState = { active: false };
+        const renewalState = { active: false };
+
+        const runNewCertCheck = this.runExclusive(
+            () => certificateService.processPendingCertificates(),
+            newCertState,
+            "processing pending certificates"
+        );
+        const runRenewalCheck = this.runExclusive(
+            () => certificateService.processRenewalCandidates(),
+            renewalState,
+            "processing renewal candidates"
+        );
+
+        // Schedule new certificate processing
+        const newCertInterval = setInterval(
+            runNewCertCheck,
+            config.getRawConfig().acme!.new_cert_check_interval_ms
+        );
+
+        // Schedule renewal processing (every 24 hours)
+        const renewalInterval = setInterval(
+            runRenewalCheck,
+            config.getRawConfig().acme!.renewal_check_interval_ms
+        );
+
+        this.certIntervals.push(newCertInterval, renewalInterval);
+
+        // Run initial checks
+        setTimeout(async () => {
+            try {
+                await runNewCertCheck();
+                // await runRenewalCheck();
+            } catch (error) {
+                logger.error("Error in initial certificate processing:", error);
+            }
+        }, 1000); // Wait 1 second after startup
+
+        logger.info("Certificate job scheduler started successfully");
+    }
+
+    // DNS record validation/reverification - doesn't touch certs at all, so
+    // this runs independently whenever Pangolin is acting as the
+    // authoritative DNS server, regardless of cert_mode.
+    async startDnsJobs(): Promise<void> {
+        if (this.dnsRunning) {
+            logger.warn("DNS validation job scheduler is already running");
+            return;
+        }
+
+        this.dnsRunning = true;
+        logger.info("Starting DNS validation job scheduler");
+
+        const dnsValidationState = { active: false };
+        const reverifyState = { active: false };
+
+        const runDnsValidation = this.runExclusive(
+            () => dnsValidator.validateAll(),
+            dnsValidationState,
+            "validating DNS records"
+        );
+        const runReverify = this.runExclusive(
+            () => domainReverifier.reverifyAll(),
+            reverifyState,
+            "reverifying domains"
+        );
+
+        // Schedule DNS validation
+        const dnsValidationInterval = setInterval(
+            runDnsValidation,
+            config.getRawConfig().acme?.dns_check_interval_ms ?? 60000
+        );
+
+        // Schedule periodic reverification of already-verified domains
+        const reverifyInterval = setInterval(
+            runReverify,
+            config.getRawConfig().acme?.domain_reverification_interval_ms ??
+                3600000
+        );
+
+        this.dnsIntervals.push(dnsValidationInterval, reverifyInterval);
+
+        // Run an initial validation pass shortly after startup
+        setTimeout(async () => {
+            try {
+                await runDnsValidation();
+            } catch (error) {
+                logger.error("Error in initial DNS validation:", error);
+            }
+        }, 1000);
+
+        logger.info("DNS validation job scheduler started successfully");
+    }
+
+    async stop(): Promise<void> {
+        if (this.certRunning) {
+            logger.info("Stopping certificate job scheduler");
+            this.certRunning = false;
+            this.certIntervals.forEach((interval) => clearInterval(interval));
+            this.certIntervals = [];
+        }
+
+        if (this.dnsRunning) {
+            logger.info("Stopping DNS validation job scheduler");
+            this.dnsRunning = false;
+            this.dnsIntervals.forEach((interval) => clearInterval(interval));
+            this.dnsIntervals = [];
+        }
+    }
+
+    isRunning(): boolean {
+        return this.certRunning || this.dnsRunning;
+    }
+}
+
+export const jobScheduler = new JobScheduler();

server/private/lib/config.ts

@@ -146,12 +146,20 @@ export class PrivateConfig {
             process.env.USE_PANGOLIN_DNS =
                 this.rawPrivateConfig.flags.use_pangolin_dns.toString();
         }
+
+        if (this.rawPrivateConfig.acme?.cert_mode) {
+            process.env.CERT_MODE = this.rawPrivateConfig.acme.cert_mode;
+        }
     }
 
     public getRawPrivateConfig() {
         return this.rawPrivateConfig;
     }
 
+    public getRawConfig() {
+        return this.getRawPrivateConfig();
+    }
+
     // `flags.enable_acme_cert_sync`, `flags.disable_private_http_placeholder`,
     // and `acme` used to live in the private config file. They now live in
     // the public config file. If an operator still has them set in the

server/private/lib/dns/generateDomains.ts

@@ -0,0 +1,47 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { build } from "@server/build";
+import privateConfig from "#private/lib/config";
+
+export function createCname(domainId: string, baseDomain: string) {
+    if (!privateConfig.getRawPrivateConfig().dns?.cname_extension) {
+        throw new Error("CNAME extension not configured");
+    }
+
+    let cnameRecords = [
+        {
+            value: `${domainId}.${privateConfig.getRawPrivateConfig().dns?.cname_extension}`,
+            baseDomain: baseDomain
+        },
+        {
+            value: `_acme-challenge.${domainId}.${privateConfig.getRawPrivateConfig().dns?.cname_extension}`,
+            baseDomain: `_acme-challenge.${baseDomain}`
+        }
+    ];
+
+    return cnameRecords;
+}
+
+export function createNs() {
+    if (!privateConfig.getRawPrivateConfig().dns?.nameserver_name) {
+        throw new Error("Nameservers not configured");
+    }
+
+    const nsRecords = [
+        privateConfig.getRawPrivateConfig().dns?.nameserver_name,
+        ...(privateConfig.getRawPrivateConfig().dns?.alternate_nameservers ||
+            [])
+    ] as string[];
+    return nsRecords;
+}

server/private/lib/dns/index.ts

@@ -0,0 +1,14 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+export * from "./server";

server/private/lib/exitNodes/exitNodeComms.ts

@@ -18,6 +18,7 @@ import { eq } from "drizzle-orm";
 import { sendToClient } from "#private/routers/ws";
 import privateConfig from "#private/lib/config";
 import config from "@server/lib/config";
+import { hasExitNodeCheckedIn } from "@server/lib/exitNodes";
 
 interface ExitNodeRequest {
     remoteType?: string;
@@ -138,13 +139,19 @@ export async function sendToExitNode(
 
             return response.data;
         } catch (error) {
-            if (axios.isAxiosError(error)) {
-                logger.error(
-                    `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${hostname} (status: ${error.response?.status}): ${error.message}`
-                );
+            const message = axios.isAxiosError(error)
+                ? `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${hostname} (status: ${error.response?.status}): ${error.message}`
+                : `Error making ${method} request for exit node at ${hostname}: ${error}`;
+
+            // The exit node (gerbil) may still be starting up and not yet
+            // reachable. Until it has checked in at least once, log this at a
+            // lower level since it's expected; once it has checked in, a
+            // connection failure is a real problem.
+            if (hasExitNodeCheckedIn(exitNode.exitNodeId)) {
+                logger.error(message);
             } else {
-                logger.error(
-                    `Error making ${method} request for exit node at ${hostname}: ${error}`
+                logger.warn(
+                    `${message} (exit node has not checked in yet since startup, this is expected briefly)`
                 );
             }
         }

server/private/lib/readConfigFile.ts

@@ -48,7 +48,6 @@ export const privateConfigSchema = z
                     .optional()
                     .transform(getEnvOrYaml("FOSSORIAL_API_KEY"))
             })
-            .optional()
             .prefault({}),
         redis: z
             .object({
@@ -95,6 +94,70 @@ export const privateConfigSchema = z
                     .optional()
             })
             .optional(),
+        dns: z
+            .object({
+                enabled: z.boolean().optional().default(false),
+                listen_port: z.number().int().positive().optional().default(53),
+                nameserver_name: z.string(),
+                cname_extension: z.string(),
+                site_extension: z.string().optional(),
+                cname_alternate_extensions: z
+                    .array(z.string())
+                    .optional()
+                    .default([]),
+                alternate_nameservers: z
+                    .array(z.string())
+                    .optional()
+                    .default([]),
+                rate_limit: z
+                    .object({
+                        enabled: z.boolean().optional().default(true),
+                        window_ms: z
+                            .number()
+                            .int()
+                            .min(1000)
+                            .max(600000)
+                            .optional()
+                            .default(60000),
+                        max_requests: z
+                            .number()
+                            .int()
+                            .min(50)
+                            .max(100000)
+                            .optional()
+                            .default(1200),
+                        max_requests_per_query_type: z
+                            .number()
+                            .int()
+                            .min(10)
+                            .max(50000)
+                            .optional()
+                            .default(600)
+                    })
+                    .default({
+                        enabled: true,
+                        window_ms: 60000,
+                        max_requests: 1200,
+                        max_requests_per_query_type: 600
+                    }),
+                static_records: z
+                    .array(
+                        z.object({
+                            domain: z.string(),
+                            type: z.enum(["TXT", "CNAME", "A", "NS"]),
+                            value: z.string(),
+                            ttl: z
+                                .number()
+                                .int()
+                                .positive()
+                                .optional()
+                                .default(300)
+                        })
+                    )
+                    .optional()
+                    .default([])
+            })
+            .optional(),
         gerbil: z
             .object({
                 local_exit_node_reachable_at: z
@@ -102,7 +165,6 @@ export const privateConfigSchema = z
                     .optional()
                     .default("http://gerbil:3004")
             })
-            .optional()
             .prefault({}),
         flags: z
             .object({
@@ -123,17 +185,87 @@ export const privateConfigSchema = z
                 // (server/private/lib/config.ts).
                 disable_private_http_placeholder: z.boolean().optional()
             })
-            .optional()
             .prefault({}),
-        // @deprecated Moved to the public config file as `acme`
-        // (server/lib/readConfigFile.ts). Kept here only so existing private
-        // config files keep parsing; any value set here is migrated into the
-        // public config at startup by PrivateConfig (server/private/lib/config.ts).
         acme: z
             .object({
+                cert_mode: z
+                    .enum(["traefik", "pangolin"])
+                    .optional()
+                    .default("traefik"),
+                enable_acme_client: z.boolean().optional().default(false),
+                // @deprecated Moved to the public config file
+                // (server/lib/readConfigFile.ts). Kept here only so existing private
+                // config files keep parsing; any value set here is migrated into the
+                // public config at startup by PrivateConfig (server/private/lib/config.ts).
                 acme_json_path: z.string().optional(),
+                // @deprecated Moved to the public config file
+                // (server/lib/readConfigFile.ts). Kept here only so existing private
+                // config files keep parsing; any value set here is migrated into the
+                // public config at startup by PrivateConfig (server/private/lib/config.ts).
                 acme_http_endpoint: z.string().optional(),
-                sync_interval_ms: z.number().optional()
+                // @deprecated Moved to the public config file
+                // (server/lib/readConfigFile.ts). Kept here only so existing private
+                // config files keep parsing; any value set here is migrated into the
+                // public config at startup by PrivateConfig (server/private/lib/config.ts).
+                sync_interval_ms: z.number().optional(),
+                acme_directory_url: z
+                    .string()
+                    .url()
+                    .default("https://acme-v02.api.letsencrypt.org/directory"),
+                contact_email: z.string().email().optional(),
+                acme_account_key_path: z
+                    .string()
+                    .default("./config/account.key"),
+                challenge_ttl_ms: z.number().int().positive().default(300000),
+                renewal_check_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(3600000),
+                new_cert_check_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(5000),
+                // Kept safely under Let's Encrypt's ~20 req/s limit since this
+                // budget is shared across all pops workers and only covers the
+                // request-issuing calls we make directly (not every request
+                // acme-client makes internally, e.g. while polling for
+                // challenge/order status).
+                acme_requests_per_second: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(15),
+                dns_check_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(60000),
+                domain_reverification_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(3600000), // 1 hour — how often to run the reverification pass
+                domain_reverification_window_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(259200000), // 72 hours — how old checkedAt must be before rechecking
+                domain_reverification_batch_size: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(20), // max domains to recheck per pass
+                dns_resolvers: z
+                    .array(z.string())
+                    .optional()
+                    .default([
+                        "8.8.8.8",
+                        "1.1.1.1",
+                        "9.9.9.9",
+                        "208.67.222.222"
+                    ])
             })
             .optional(),
         branding: z

server/private/lib/traefik/getTraefikConfig.ts

@@ -55,6 +55,7 @@ import {
     getValidCertificatesForDomains
 } from "@server/lib/certificates";
 import { build } from "@server/build";
+import license from "#private/license/license";
 import regionalCache from "#private/lib/cache";
 import { ResourceWithTargets } from "@server/lib/traefik/types";
 import { buildWildcardTls } from "@server/lib/traefik/certResolver";
@@ -491,8 +492,15 @@ export async function getTraefikConfig(
         { depth: null }
     );
 
+    // Pangolin-managed DNS-01/ACME cert mode requires either a tier1
+    // license (self-hosted) or a saas build - otherwise fall back to
+    // Traefik's own cert resolvers (buildWildcardTls) throughout.
+    const pangolinCertModeEnabled =
+        privateConfig.getRawPrivateConfig().acme?.cert_mode == "pangolin" &&
+        (await license.hasTier(["personal", "tier2", "enterprise"]));
+
     let validCerts: CertificateResult[] = [];
-    if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+    if (pangolinCertModeEnabled) {
         // create a list of all domains to get certs for
         const domains = new Set<string>();
         for (const resource of resourcesMap.values()) {
@@ -629,7 +637,7 @@ export async function getTraefikConfig(
             );
 
             let tls = {};
-            if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+            if (!pangolinCertModeEnabled) {
                 tls = buildWildcardTls({
                     fullDomain,
                     hasSubdomain: !!resource.subdomain,
@@ -650,6 +658,30 @@ export async function getTraefikConfig(
                 }
             }
 
+            config_output.http.services![serviceName] = {
+                loadBalancer: {
+                    servers: buildHttpLoadBalancerServers(targets),
+                    ...(resource.stickySession
+                        ? buildStickySessionCookie(resource.ssl)
+                        : {})
+                }
+            };
+
+            if (resource.tlsServerName) {
+                if (!config_output.http.serversTransports) {
+                    config_output.http.serversTransports = {};
+                }
+                config_output.http.serversTransports![transportName] = {
+                    serverName: resource.tlsServerName,
+                    //unfortunately the following needs to be set. traefik doesn't merge the default serverTransport settings
+                    // if defined in the static config and here. if not set, self-signed certs won't work
+                    insecureSkipVerify: true
+                };
+                config_output.http.services![
+                    serviceName
+                ].loadBalancer.serversTransport = transportName;
+            }
+
             if (resource.ssl) {
                 config_output.http.routers![routerName + "-redirect"] = {
                     entryPoints: [
@@ -814,31 +846,6 @@ export async function getTraefikConfig(
                 priority: priority,
                 ...(resource.ssl ? { tls } : {})
             };
-
-            config_output.http.services![serviceName] = {
-                loadBalancer: {
-                    servers: buildHttpLoadBalancerServers(targets),
-                    ...(resource.stickySession
-                        ? buildStickySessionCookie(resource.ssl)
-                        : {})
-                }
-            };
-
-            // Add the serversTransport if TLS server name is provided
-            if (resource.tlsServerName) {
-                if (!config_output.http.serversTransports) {
-                    config_output.http.serversTransports = {};
-                }
-                config_output.http.serversTransports![transportName] = {
-                    serverName: resource.tlsServerName,
-                    //unfortunately the following needs to be set. traefik doesn't merge the default serverTransport settings
-                    // if defined in the static config and here. if not set, self-signed certs won't work
-                    insecureSkipVerify: true
-                };
-                config_output.http.services![
-                    serviceName
-                ].loadBalancer.serversTransport = transportName;
-            }
         } else if (resource.mode == "tcp" || resource.mode == "udp") {
             // Non-HTTP (TCP/UDP) configuration
             if (!resource.enableProxy) {
@@ -923,9 +930,7 @@ export async function getTraefikConfig(
                 domainCertResolver,
                 preferWildcardCert
             }) => {
-                if (
-                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
-                ) {
+                if (!pangolinCertModeEnabled) {
                     return buildWildcardTls({
                         fullDomain,
                         hasSubdomain,
@@ -966,9 +971,7 @@ export async function getTraefikConfig(
             maintenancePageUiUrl,
             redirectHttpsMiddlewareName,
             resolveTls: (fullDomain) => {
-                if (
-                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
-                ) {
+                if (!pangolinCertModeEnabled) {
                     // siteResource aliases don't have a per-domain cert
                     // resolver stored, so always fall back to the global
                     // defaults.
@@ -1059,7 +1062,7 @@ export async function getTraefikConfig(
             const rule = buildHostRule(fullDomain, ir.wildcard);
 
             let tls: any = {};
-            if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+            if (!pangolinCertModeEnabled) {
                 tls = buildWildcardTls({
                     fullDomain,
                     hasSubdomain: !!ir.subdomain,
@@ -1139,9 +1142,7 @@ export async function getTraefikConfig(
                 const rule = `Host(\`${fullDomain}\`) && ClientIP(\`${exitNode.address}\`)`; // restrict to coming from the exit node ip range that the client is connected to
 
                 let tls: any = {};
-                if (
-                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
-                ) {
+                if (!pangolinCertModeEnabled) {
                     // siteResource aliases don't have a per-domain cert
                     // resolver stored, so always fall back to the global
                     // defaults.
@@ -1215,7 +1216,7 @@ export async function getTraefikConfig(
             .where(eq(exitNodes.exitNodeId, exitNodeId));
 
         let validCertsLoginPages: CertificateResult[] = [];
-        if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
+        if (pangolinCertModeEnabled) {
             // create a list of all domains to get certs for
             const domains = new Set<string>();
             for (const lp of exitNodeLoginPages) {
@@ -1260,9 +1261,7 @@ export async function getTraefikConfig(
                 }
 
                 const tls = {};
-                if (
-                    !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
-                ) {
+                if (!pangolinCertModeEnabled) {
                     // TODO: we need to add the wildcard logic here too
                 } else {
                     // find a cert that matches the full domain, if not continue

server/private/license/license.ts

@@ -14,7 +14,7 @@
 import { db, HostMeta, sites, users } from "@server/db";
 import { hostMeta, licenseKey } from "@server/db";
 import logger from "@server/logger";
-import NodeCache from "node-cache";
+import { createLocalCache } from "@server/lib/createLocalCache";
 import { validateJWT } from "./licenseJwt";
 import { count, eq } from "drizzle-orm";
 import moment from "moment";
@@ -26,6 +26,7 @@ import {
     LicenseStatus
 } from "@server/license/license";
 import { setHostMeta } from "@server/lib/hostMeta";
+import { build } from "@server/build";
 
 type ActivateLicenseKeyAPIResponse = {
     data: {
@@ -65,8 +66,8 @@ export class License {
     private validationServerUrl = `${this.serverBaseUrl}/api/v1/license/enterprise/validate`;
     private activationServerUrl = `${this.serverBaseUrl}/api/v1/license/enterprise/activate`;
 
-    private statusCache = new NodeCache();
-    private licenseKeyCache = new NodeCache();
+    private statusCache = createLocalCache();
+    private licenseKeyCache = createLocalCache();
 
     private statusKey = "status";
     private serverSecret!: string;
@@ -119,6 +120,9 @@ LQIDAQAB
     }
 
     public async isUnlocked(): Promise<boolean> {
+        if (build == "saas") {
+            return true;
+        }
         const status = await this.check();
         if (status.isHostLicensed) {
             if (status.isLicenseValid) {
@@ -128,6 +132,20 @@ LQIDAQAB
         return false;
     }
 
+    public async hasTier(tier: LicenseKeyTier[]): Promise<boolean> {
+        if (build == "saas") {
+            return true;
+        }
+        const status = await this.check();
+        if (status.isHostLicensed && status.isLicenseValid) {
+            return (
+                status.tier !== undefined &&
+                tier.includes(status.tier as LicenseKeyTier)
+            );
+        }
+        return false;
+    }
+
     public async check(): Promise<LicenseStatus> {
         // If a check is already in progress, return the last known status
         if (this.checkInProgress) {
@@ -135,8 +153,7 @@ LQIDAQAB
                 "License check already in progress, returning last known status"
             );
             const lastStatus = this.statusCache.get(this.statusKey) as
-                | LicenseStatus
-                | undefined;
+                LicenseStatus | undefined;
             if (lastStatus) {
                 return lastStatus;
             }
@@ -179,7 +196,7 @@ LQIDAQAB
                 status.isHostLicensed = false;
                 // Invalidate all and set new cache (empty)
                 this.licenseKeyCache.flushAll();
-                this.statusCache.set(this.statusKey, status);
+                this.statusCache.set(this.statusKey, status, 0);
                 return status;
             }
 
@@ -389,7 +406,7 @@ LQIDAQAB
             // Invalidate old cache and set new cache
             this.licenseKeyCache.flushAll();
             for (const [key, value] of newCache.entries()) {
-                this.licenseKeyCache.set<LicenseKeyCache>(key, value);
+                this.licenseKeyCache.set(key, value, 0);
             }
         } catch (error) {
             logger.error("Error checking license status:");
@@ -398,7 +415,7 @@ LQIDAQAB
             this.checkInProgress = false;
         }
 
-        this.statusCache.set(this.statusKey, status);
+        this.statusCache.set(this.statusKey, status, 0);
         return status;
     }
 

server/private/routers/alertRule/testAlertRule.ts

@@ -14,7 +14,7 @@
 import { getRandomItemInArray } from "@app/lib/getRandomItemInArray";
 import response from "@server/lib/response";
 import logger from "@server/logger";
-import { processTestAlerts } from "@server/private/lib/alerts/processTestAlerts";
+import { processTestAlerts } from "#private/lib/alerts/processTestAlerts";
 import { type AlertAction } from "@server/routers/alertRule/types";
 import HttpCode from "@server/types/HttpCode";
 import { NextFunction, Request, Response } from "express";

server/private/routers/alertRule/updateAlertRule.ts

@@ -33,8 +33,11 @@ import { OpenAPITags, registry } from "@server/openApi";
 import { and, eq } from "drizzle-orm";
 import { encrypt } from "@server/lib/crypto";
 import config from "@server/lib/config";
-import { HC_EVENT_TYPES, SITE_EVENT_TYPES, RESOURCE_EVENT_TYPES } from "./createAlertRule";
-import { invalidateAllRemoteExitNodeSessions } from "@server/private/auth/sessions/remoteExitNode";
+import {
+    HC_EVENT_TYPES,
+    SITE_EVENT_TYPES,
+    RESOURCE_EVENT_TYPES
+} from "./createAlertRule";
 
 const paramsSchema = z
     .object({
@@ -85,35 +88,57 @@ const bodySchema = z
         const isHcEvent = (HC_EVENT_TYPES as readonly string[]).includes(
             val.eventType
         );
-        const isResourceEvent = (RESOURCE_EVENT_TYPES as readonly string[]).includes(
-            val.eventType
-        );
+        const isResourceEvent = (
+            RESOURCE_EVENT_TYPES as readonly string[]
+        ).includes(val.eventType);
 
-        if (isSiteEvent && val.siteIds !== undefined && val.siteIds.length === 0 && !val.allSites) {
+        if (
+            isSiteEvent &&
+            val.siteIds !== undefined &&
+            val.siteIds.length === 0 &&
+            !val.allSites
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "At least one siteId is required for site event types when allSites is false",
+                message:
+                    "At least one siteId is required for site event types when allSites is false",
                 path: ["siteIds"]
             });
         }
 
-        if (isHcEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length === 0 && !val.allHealthChecks) {
+        if (
+            isHcEvent &&
+            val.healthCheckIds !== undefined &&
+            val.healthCheckIds.length === 0 &&
+            !val.allHealthChecks
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "At least one healthCheckId is required for health check event types when allHealthChecks is false",
+                message:
+                    "At least one healthCheckId is required for health check event types when allHealthChecks is false",
                 path: ["healthCheckIds"]
             });
         }
 
-        if (isResourceEvent && val.resourceIds !== undefined && val.resourceIds.length === 0 && !val.allResources) {
+        if (
+            isResourceEvent &&
+            val.resourceIds !== undefined &&
+            val.resourceIds.length === 0 &&
+            !val.allResources
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "At least one resourceId is required for resource event types when allResources is false",
+                message:
+                    "At least one resourceId is required for resource event types when allResources is false",
                 path: ["resourceIds"]
             });
         }
 
-        if (isSiteEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length > 0) {
+        if (
+            isSiteEvent &&
+            val.healthCheckIds !== undefined &&
+            val.healthCheckIds.length > 0
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
                 message: "healthCheckIds must not be set for site event types",
@@ -129,7 +154,11 @@ const bodySchema = z
             });
         }
 
-        if (isResourceEvent && val.siteIds !== undefined && val.siteIds.length > 0) {
+        if (
+            isResourceEvent &&
+            val.siteIds !== undefined &&
+            val.siteIds.length > 0
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
                 message: "siteIds must not be set for resource event types",
@@ -137,10 +166,15 @@ const bodySchema = z
             });
         }
 
-        if (isResourceEvent && val.healthCheckIds !== undefined && val.healthCheckIds.length > 0) {
+        if (
+            isResourceEvent &&
+            val.healthCheckIds !== undefined &&
+            val.healthCheckIds.length > 0
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "healthCheckIds must not be set for resource event types",
+                message:
+                    "healthCheckIds must not be set for resource event types",
                 path: ["healthCheckIds"]
             });
         }
@@ -153,7 +187,6 @@ const UpdateAlertRuleResponseDataSchema = z.object({
     alertRuleId: z.number()
 });
 
-
 registry.registerPath({
     method: "post",
     path: "/org/{orgId}/alert-rule/{alertRuleId}",
@@ -174,7 +207,9 @@ registry.registerPath({
             description: "Successful response",
             content: {
                 "application/json": {
-                    schema: createApiResponseSchema(UpdateAlertRuleResponseDataSchema)
+                    schema: createApiResponseSchema(
+                        UpdateAlertRuleResponseDataSchema
+                    )
                 }
             }
         }
@@ -250,9 +285,11 @@ export async function updateAlertRule(
         if (name !== undefined) updateData.name = name;
         if (eventType !== undefined) updateData.eventType = eventType;
         if (enabled !== undefined) updateData.enabled = enabled;
-        if (cooldownSeconds !== undefined) updateData.cooldownSeconds = cooldownSeconds;
+        if (cooldownSeconds !== undefined)
+            updateData.cooldownSeconds = cooldownSeconds;
         if (allSites !== undefined) updateData.allSites = allSites;
-        if (allHealthChecks !== undefined) updateData.allHealthChecks = allHealthChecks;
+        if (allHealthChecks !== undefined)
+            updateData.allHealthChecks = allHealthChecks;
         if (allResources !== undefined) updateData.allResources = allResources;
 
         await db
@@ -273,7 +310,11 @@ export async function updateAlertRule(
 
             // Only insert junction rows when allSites is not true
             const effectiveAllSites = allSites ?? false;
-            if (!effectiveAllSites && siteIds !== undefined && siteIds.length > 0) {
+            if (
+                !effectiveAllSites &&
+                siteIds !== undefined &&
+                siteIds.length > 0
+            ) {
                 await db.insert(alertSites).values(
                     siteIds.map((siteId) => ({
                         alertRuleId,
@@ -290,7 +331,11 @@ export async function updateAlertRule(
                 .where(eq(alertHealthChecks.alertRuleId, alertRuleId));
 
             const effectiveAllHealthChecks = allHealthChecks ?? false;
-            if (!effectiveAllHealthChecks && healthCheckIds !== undefined && healthCheckIds.length > 0) {
+            if (
+                !effectiveAllHealthChecks &&
+                healthCheckIds !== undefined &&
+                healthCheckIds.length > 0
+            ) {
                 await db.insert(alertHealthChecks).values(
                     healthCheckIds.map((healthCheckId) => ({
                         alertRuleId,
@@ -307,7 +352,11 @@ export async function updateAlertRule(
                 .where(eq(alertResources.alertRuleId, alertRuleId));
 
             const effectiveAllResources = allResources ?? false;
-            if (!effectiveAllResources && resourceIds !== undefined && resourceIds.length > 0) {
+            if (
+                !effectiveAllResources &&
+                resourceIds !== undefined &&
+                resourceIds.length > 0
+            ) {
                 await db.insert(alertResources).values(
                     resourceIds.map((resourceId) => ({
                         alertRuleId,
@@ -392,7 +441,10 @@ export async function updateAlertRule(
                     webhookActions.map((wa) => ({
                         alertRuleId,
                         webhookUrl: wa.webhookUrl,
-                        config: wa.config != null ? encrypt(wa.config, serverSecret) : null,
+                        config:
+                            wa.config != null
+                                ? encrypt(wa.config, serverSecret)
+                                : null,
                         enabled: wa.enabled
                     }))
                 );

server/private/routers/auditLogs/queryAccessAuditLog.ts

@@ -54,7 +54,6 @@ export const queryAccessAuditLogsQuery = z.object({
             error: "timeEnd must be a valid ISO date string"
         })
         .transform((val) => Math.floor(new Date(val).getTime() / 1000))
-        .optional()
         .prefault(() => new Date().toISOString())
         .openapi({
             type: "string",

server/private/routers/auditLogs/queryActionAuditLog.ts

@@ -47,7 +47,6 @@ export const queryActionAuditLogsQuery = z.object({
             error: "timeEnd must be a valid ISO date string"
         })
         .transform((val) => Math.floor(new Date(val).getTime() / 1000))
-        .optional()
         .prefault(() => new Date().toISOString())
         .openapi({
             type: "string",

server/private/routers/auditLogs/queryConnectionAuditLog.ts

@@ -55,7 +55,6 @@ export const queryConnectionAuditLogsQuery = z.object({
             error: "timeEnd must be a valid ISO date string"
         })
         .transform((val) => Math.floor(new Date(val).getTime() / 1000))
-        .optional()
         .prefault(() => new Date().toISOString())
         .openapi({
             type: "string",

server/private/routers/billing/hooks/handleSubscriptionCreated.ts

@@ -222,15 +222,16 @@ export async function handleSubscriptionCreated(
 
                 let numUsers: number;
                 let numSites: number;
+                let tier = "enterprise";
 
-                if (subscriptionPriceId === priceSet[LicenseId.SMALL_LICENSE]) {
+                if (subscriptionPriceId === priceSet[LicenseId.TIER1]) {
                     numUsers = 25;
                     numSites = 25;
-                } else if (
-                    subscriptionPriceId === priceSet[LicenseId.BIG_LICENSE]
-                ) {
+                    tier = "tier1";
+                } else if (subscriptionPriceId === priceSet[LicenseId.TIER2]) {
                     numUsers = 50;
                     numSites = 100;
+                    tier = "tier2";
                 } else {
                     logger.error(
                         `Unknown price ID ${subscriptionPriceId} for subscription ${subscription.id}`
@@ -242,6 +243,14 @@ export async function handleSubscriptionCreated(
                     `License type determined: ${numUsers} users, ${numSites} sites for subscription ${subscription.id}`
                 );
 
+                // Grace period of 5 days added on top of the current billing
+                // period end (usually ~1 year out) before the license expires
+                const currentPeriodEnd =
+                    fullSubscription.items.data[0]?.current_period_end;
+                const expiresAt =
+                    (currentPeriodEnd ?? subscription.created) +
+                    5 * 24 * 60 * 60;
+
                 const response = await fetch(
                     `${privateConfig.getRawPrivateConfig().server.fossorial_api}/api/v1/license-internal/enterprise/paid-for`,
                     {
@@ -256,7 +265,9 @@ export async function handleSubscriptionCreated(
                             licenseId: parseInt(licenseId),
                             paidFor: true,
                             users: numUsers,
-                            sites: numSites
+                            sites: numSites,
+                            tier: tier,
+                            expiresAt: expiresAt
                         })
                     }
                 );
@@ -265,6 +276,13 @@ export async function handleSubscriptionCreated(
 
                 logger.debug(`Fossorial API response: ${JSON.stringify(data)}`);
 
+                if (!response.ok || !data.success) {
+                    logger.error(
+                        `Fossorial API returned ${response.status} when setting paid-for for orgId ${customer.orgId} and subscription ID ${subscription.id}: ${JSON.stringify(data)}`
+                    );
+                    return;
+                }
+
                 if (customer.email) {
                     logger.debug(
                         `Sending license key email to ${customer.email} for subscription ${subscription.id}`

server/private/routers/billing/hooks/handleSubscriptionDeleted.ts

@@ -125,9 +125,7 @@ export async function handleSubscriptionDeleted(
                 `Handling license subscription deletion for orgId ${customer.orgId} and subscription ID ${subscription.id}`
             );
             try {
-                // WARNING:
-                // this invalidates ALL OF THE ENTERPRISE LICENSES for this orgId
-                await fetch(
+                const invalidateResponse = await fetch(
                     `${privateConfig.getRawPrivateConfig().server.fossorial_api}/api/v1/license-internal/enterprise/invalidate`,
                     {
                         method: "POST",
@@ -139,9 +137,18 @@ export async function handleSubscriptionDeleted(
                         },
                         body: JSON.stringify({
                             orgId: customer.orgId,
+                            licenseKeyId: parseInt(
+                                subscription.metadata.licenseKeyId
+                            )
                         })
                     }
                 );
+
+                if (!invalidateResponse.ok) {
+                    logger.error(
+                        `Fossorial API returned ${invalidateResponse.status} when invalidating license for orgId ${customer.orgId} and subscription ID ${subscription.id}: ${await invalidateResponse.text()}`
+                    );
+                }
             } catch (error) {
                 logger.error(
                     `Error notifying Fossorial API of license subscription deletion for orgId ${customer.orgId} and subscription ID ${subscription.id}:`,

server/private/routers/billing/hooks/handleSubscriptionUpdated.ts

@@ -23,7 +23,10 @@ import {
 } from "@server/db";
 import { eq, and } from "drizzle-orm";
 import logger from "@server/logger";
-import { getFeatureIdByMetricId, getFeatureIdByPriceId } from "@server/lib/billing/features";
+import {
+    getFeatureIdByMetricId,
+    getFeatureIdByPriceId
+} from "@server/lib/billing/features";
 import stripe from "#private/lib/stripe";
 import { handleSubscriptionLifesycle } from "../subscriptionLifecycle";
 import { getSubType, SubscriptionType } from "./getSubType";
@@ -66,7 +69,8 @@ export async function handleSubscriptionUpdated(
             .limit(1);
 
         const type = getSubType(fullSubscription);
-        const previousType = existingSubscription.type as SubscriptionType | null;
+        const previousType =
+            existingSubscription.type as SubscriptionType | null;
 
         // If the subscription has been manually overridden, we lock the
         // status down so Stripe webhooks can no longer change it.
@@ -100,7 +104,11 @@ export async function handleSubscriptionUpdated(
             logger.info(
                 `Tier change detected for org ${customer.orgId}: ${previousType} -> ${type}`
             );
-            await handleTierChange(customer.orgId, type, previousType ?? undefined);
+            await handleTierChange(
+                customer.orgId,
+                type,
+                previousType ?? undefined
+            );
         }
 
         // Upsert subscription items
@@ -113,7 +121,8 @@ export async function handleSubscriptionUpdated(
 
             const itemsToUpsert = fullSubscription.items.data.map((item) => {
                 // Try to get featureId from price
-                let featureId: string | null = getFeatureIdByPriceId(item.price.id) || null;
+                let featureId: string | null =
+                    getFeatureIdByPriceId(item.price.id) || null;
 
                 // If no match, try to preserve existing featureId
                 if (!featureId) {
@@ -302,14 +311,20 @@ export async function handleSubscriptionUpdated(
                     logger.info(
                         `Subscription ${subscription.id} for org ${customer.orgId} is ${effectiveStatus}, disabling paid features`
                     );
-                    await handleTierChange(customer.orgId, null, previousType ?? undefined);
+                    await handleTierChange(
+                        customer.orgId,
+                        null,
+                        previousType ?? undefined
+                    );
                 }
             } else if (type === "license") {
-                if (effectiveStatus === "canceled" || effectiveStatus == "unpaid" || effectiveStatus == "incomplete_expired") {
+                if (
+                    effectiveStatus === "canceled" ||
+                    effectiveStatus == "unpaid" ||
+                    effectiveStatus == "incomplete_expired"
+                ) {
                     try {
-                        // WARNING:
-                        // this invalidates ALL OF THE ENTERPRISE LICENSES for this orgId
-                        await fetch(
+                        const invalidateResponse = await fetch(
                             `${privateConfig.getRawPrivateConfig().server.fossorial_api}/api/v1/license-internal/enterprise/invalidate`,
                             {
                                 method: "POST",
@@ -320,16 +335,95 @@ export async function handleSubscriptionUpdated(
                                     "Content-Type": "application/json"
                                 },
                                 body: JSON.stringify({
-                                    orgId: customer.orgId
+                                    orgId: customer.orgId,
+                                    licenseKeyId: parseInt(
+                                        subscription.metadata.licenseKeyId
+                                    )
                                 })
                             }
                         );
+
+                        if (!invalidateResponse.ok) {
+                            logger.error(
+                                `Fossorial API returned ${invalidateResponse.status} when invalidating license for orgId ${customer.orgId} and subscription ID ${subscription.id}: ${await invalidateResponse.text()}`
+                            );
+                        }
                     } catch (error) {
                         logger.error(
                             `Error notifying Fossorial API of license subscription deletion for orgId ${customer.orgId} and subscription ID ${subscription.id}:`,
                             error
                         );
                     }
+                } else if (effectiveStatus === "active" && previousAttributes) {
+                    // Detect a successful renewal: the billing period rolled
+                    // forward (the invoice was paid and the new period began
+                    // right where the previous one ended).
+                    const currentItem = fullSubscription.items.data[0];
+                    const prevItems = previousAttributes.items?.data;
+                    const prevItem = Array.isArray(prevItems)
+                        ? prevItems.find(
+                              (pi: any) => pi.id === currentItem?.id
+                          )
+                        : undefined;
+
+                    const renewed =
+                        currentItem &&
+                        prevItem?.current_period_end &&
+                        currentItem.current_period_start ===
+                            prevItem.current_period_end &&
+                        currentItem.current_period_start >
+                            prevItem.current_period_start;
+
+                    if (renewed) {
+                        const licenseKeyId =
+                            subscription.metadata.licenseKeyId;
+
+                        if (!licenseKeyId) {
+                            logger.error(
+                                `No licenseKeyId in metadata for subscription ${subscription.id}, cannot extend license.`
+                            );
+                        } else {
+                            // Grace period of 5 days added on top of the new
+                            // billing period end (usually ~1 year out)
+                            const expiresAt =
+                                currentItem.current_period_end +
+                                5 * 24 * 60 * 60;
+
+                            try {
+                                const extendResponse = await fetch(
+                                    `${privateConfig.getRawPrivateConfig().server.fossorial_api}/api/v1/license-internal/enterprise/extend`,
+                                    {
+                                        method: "POST",
+                                        headers: {
+                                            "api-key":
+                                                privateConfig.getRawPrivateConfig()
+                                                    .server.fossorial_api_key!,
+                                            "Content-Type": "application/json"
+                                        },
+                                        body: JSON.stringify({
+                                            licenseId: parseInt(licenseKeyId),
+                                            expiresAt: expiresAt
+                                        })
+                                    }
+                                );
+
+                                if (!extendResponse.ok) {
+                                    logger.error(
+                                        `Fossorial API returned ${extendResponse.status} when extending license ${licenseKeyId} for subscription ${subscription.id}: ${await extendResponse.text()}`
+                                    );
+                                } else {
+                                    logger.info(
+                                        `Extended license ${licenseKeyId} for subscription ${subscription.id} to expire at ${expiresAt}.`
+                                    );
+                                }
+                            } catch (error) {
+                                logger.error(
+                                    `Error notifying Fossorial API of license renewal for subscription ${subscription.id}:`,
+                                    error
+                                );
+                            }
+                        }
+                    }
                 }
             }
         }

server/private/routers/generatedLicense/clearInstanceName.ts

@@ -31,7 +31,9 @@ export async function clearInstanceName(
     next: NextFunction
 ): Promise<any> {
     try {
-        const parsedParams = clearInstanceNameParamsSchema.safeParse(req.params);
+        const parsedParams = clearInstanceNameParamsSchema.safeParse(
+            req.params
+        );
         if (!parsedParams.success) {
             return next(
                 createHttpError(
@@ -63,7 +65,8 @@ export async function clearInstanceName(
             return next(
                 createHttpError(
                     data.status || HttpCode.BAD_REQUEST,
-                    data.message || "Failed to clear instance name from Fossorial API"
+                    data.message ||
+                        "Failed to clear server ID from Fossorial API"
                 )
             );
         }
@@ -72,7 +75,7 @@ export async function clearInstanceName(
             data: null,
             success: true,
             error: false,
-            message: "Instance name cleared successfully",
+            message: "Server ID cleared successfully",
             status: HttpCode.OK
         });
     } catch (error) {
@@ -80,8 +83,8 @@ export async function clearInstanceName(
         return next(
             createHttpError(
                 HttpCode.INTERNAL_SERVER_ERROR,
-                "An error occurred while clearing the instance name."
+                "An error occurred while clearing the server ID."
             )
         );
     }
-}
+}

server/private/routers/generatedLicense/generateNewEnterpriseLicense.ts

@@ -64,14 +64,11 @@ export async function generateNewEnterpriseLicense(
 
         const licenseData = req.body;
 
-        if (
-            licenseData.tier != "big_license" &&
-            licenseData.tier != "small_license"
-        ) {
+        if (licenseData.tier != "tier2" && licenseData.tier != "tier1") {
             return next(
                 createHttpError(
                     HttpCode.BAD_REQUEST,
-                    "Invalid tier specified. Must be either 'big_license' or 'small_license'."
+                    "Invalid tier specified. Must be either 'tier2' or 'tier1'."
                 )
             );
         }
@@ -99,6 +96,8 @@ export async function generateNewEnterpriseLicense(
             );
         }
 
+        const licenseKeyValue = apiResponse?.data?.licenseKey?.licenseKey;
+
         // check if we already have a customer for this org
         const [customer] = await db
             .select()
@@ -118,9 +117,7 @@ export async function generateNewEnterpriseLicense(
         }
 
         const tier =
-            licenseData.tier === "big_license"
-                ? LicenseId.BIG_LICENSE
-                : LicenseId.SMALL_LICENSE;
+            licenseData.tier === "tier2" ? LicenseId.TIER2 : LicenseId.TIER1;
         const tierPrice = getLicensePriceSet()[tier];
 
         const session = await stripe!.checkout.sessions.create({
@@ -134,6 +131,16 @@ export async function generateNewEnterpriseLicense(
             ], // Start with the standard feature set that matches the free limits
             customer: customer.customerId,
             mode: "subscription",
+            subscription_data: {
+                description: licenseKeyValue
+                    ? `License ${licenseKeyValue}`
+                    : `License key ID ${keyId}`,
+                metadata: {
+                    licenseKeyId: keyId.toString(),
+                    licenseKey: licenseKeyValue ?? "",
+                    tier: licenseData.tier
+                }
+            },
             allow_promotion_codes: true,
             success_url: `${config.getRawConfig().app.dashboard_url}/${orgId}/settings/license?success=true&session_id={CHECKOUT_SESSION_ID}`,
             cancel_url: `${config.getRawConfig().app.dashboard_url}/${orgId}/settings/license?canceled=true`

server/private/routers/gerbil/createExitNode.ts

@@ -12,8 +12,8 @@
  */
 
 import { db, ExitNode, exitNodes } from "@server/db";
-import { getUniqueExitNodeEndpointName } from "@server/db/names";
 import config from "@server/lib/config";
+import privateConfig from "#private/lib/config";
 import { getNextAvailableSubnet } from "@server/lib/exitNodes";
 import logger from "@server/logger";
 import { eq } from "drizzle-orm";
@@ -45,6 +45,8 @@ export async function createExitNode(
                 .values({
                     publicKey,
                     endpoint: config.getRawConfig().gerbil.base_endpoint,
+                    region:
+                        privateConfig.getRawPrivateConfig().app.region || null,
                     address,
                     listenPort,
                     online: true,

server/private/routers/remoteExitNode/getRemoteExitNodeToken.ts

@@ -26,7 +26,7 @@ import {
     validateRemoteExitNodeSessionToken,
     EXPIRES
 } from "#private/auth/sessions/remoteExitNode";
-import { getOrCreateCachedToken } from "@server/private/lib/tokenCache";
+import { getOrCreateCachedToken } from "#private/lib/tokenCache";
 import { verifyPassword } from "@server/auth/password";
 import logger from "@server/logger";
 import config from "@server/lib/config";

server/routers/auditLogs/queryAiSessionLog.ts

@@ -47,7 +47,6 @@ export const queryAiSessionLogsQuery = z.strictObject({
             error: "timeEnd must be a valid ISO date string"
         })
         .transform((val) => Math.floor(new Date(val).getTime() / 1000))
-        .optional()
         .prefault(() => new Date().toISOString())
         .openapi({
             type: "string",

server/routers/auditLogs/queryRequestAnalytics.ts

@@ -20,7 +20,6 @@ const queryAccessAuditLogsQuery = z.object({
             error: "timeStart must be a valid ISO date string"
         })
         .transform((val) => Math.floor(new Date(val).getTime() / 1000))
-        .optional()
         .prefault(() => getSevenDaysAgo().toISOString())
         .openapi({
             type: "string",
@@ -34,7 +33,6 @@ const queryAccessAuditLogsQuery = z.object({
             error: "timeEnd must be a valid ISO date string"
         })
         .transform((val) => Math.floor(new Date(val).getTime() / 1000))
-        .optional()
         .prefault(() => new Date().toISOString())
         .openapi({
             type: "string",

server/routers/auditLogs/queryRequestAuditLog.ts

@@ -41,7 +41,6 @@ export const queryAccessAuditLogsQuery = z.strictObject({
             error: "timeEnd must be a valid ISO date string"
         })
         .transform((val) => Math.floor(new Date(val).getTime() / 1000))
-        .optional()
         .prefault(() => new Date().toISOString())
         .openapi({
             type: "string",

server/routers/auth/securityKey.ts

@@ -17,7 +17,7 @@ import {
 import type {
     GenerateRegistrationOptionsOpts,
     GenerateAuthenticationOptionsOpts,
-    AuthenticatorTransportFuture
+    AuthenticatorTransport
 } from "@simplewebauthn/server";
 import { isoBase64URL } from "@simplewebauthn/server/helpers";
 import config from "@server/lib/config";
@@ -221,7 +221,7 @@ export async function startRegistration(
         const excludeCredentials = existingSecurityKeys.map((key) => ({
             id: key.credentialId,
             transports: key.transports
-                ? (JSON.parse(key.transports) as AuthenticatorTransportFuture[])
+                ? (JSON.parse(key.transports) as AuthenticatorTransport[])
                 : undefined
         }));
 
@@ -571,7 +571,7 @@ export async function startAuthentication(
                 transports: key.transports
                     ? (JSON.parse(
                           key.transports
-                      ) as AuthenticatorTransportFuture[])
+                      ) as AuthenticatorTransport[])
                     : undefined
             }));
         }
@@ -702,7 +702,7 @@ export async function verifyAuthentication(
                 transports: securityKey.transports
                     ? (JSON.parse(
                           securityKey.transports
-                      ) as AuthenticatorTransportFuture[])
+                      ) as AuthenticatorTransport[])
                     : undefined
             },
             requireUserVerification: false

server/routers/badger/verifySession.ts

@@ -721,7 +721,9 @@ export async function verifyResourceSession(
                 );
 
                 resourceSession = result?.resourceSession;
-                localCache.set(sessionCacheKey, resourceSession, 5);
+                if (resourceSession) {
+                    localCache.set(sessionCacheKey, resourceSession, 5);
+                }
             }
 
             if (resourceSession?.isRequestToken) {
@@ -1216,7 +1218,9 @@ async function allowAccessToken(
                 resource.resourceId
             );
             resourceSession = result?.resourceSession;
-            localCache.set(sessionCacheKey, resourceSession, 5);
+            if (resourceSession) {
+                localCache.set(sessionCacheKey, resourceSession, 5);
+            }
         }
 
         if (

server/routers/domain/createOrgDomain.ts

@@ -21,6 +21,7 @@ import { LimitId } from "@server/lib/billing";
 import { isSecondLevelDomain, isValidDomain } from "@server/lib/validators";
 import { build } from "@server/build";
 import config from "@server/lib/config";
+import { createNs, createCname } from "#dynamic/lib/dns/generateDomains";
 
 const paramsSchema = z.strictObject({
     orgId: z.string()
@@ -283,8 +284,7 @@ export async function createOrgDomain(
 
             // TODO: This needs to be cross region and not hardcoded
             if (type === "ns") {
-                nsRecords = config.getRawConfig().dns.nameservers as string[];
-
+                nsRecords = createNs();
                 // Save NS records to database
                 for (const nsValue of nsRecords) {
                     recordsToInsert.push({
@@ -296,16 +296,7 @@ export async function createOrgDomain(
                     });
                 }
             } else if (type === "cname") {
-                cnameRecords = [
-                    {
-                        value: `${domainId}.${config.getRawConfig().dns.cname_extension}`,
-                        baseDomain: baseDomain
-                    },
-                    {
-                        value: `_acme-challenge.${domainId}.${config.getRawConfig().dns.cname_extension}`,
-                        baseDomain: `_acme-challenge.${baseDomain}`
-                    }
-                ];
+                cnameRecords = createCname(domainId, baseDomain);
 
                 // Save CNAME records to database
                 for (const cnameRecord of cnameRecords) {

server/routers/external.ts

@@ -88,6 +88,12 @@ authenticated.get("/org/checkId", org.checkId);
 authenticated.put("/org", getUserOrgs, org.createOrg);
 
 authenticated.get("/orgs", verifyUserIsServerAdmin, org.listOrgs);
+authenticated.get("/admin/orgs", verifyUserIsServerAdmin, org.adminListOrgs);
+authenticated.delete(
+    "/admin/org/:orgId",
+    verifyUserIsServerAdmin,
+    org.adminDeleteOrg
+);
 authenticated.get("/user/:userId/orgs", verifyIsLoggedInUser, org.listUserOrgs);
 
 authenticated.get(
@@ -1379,6 +1385,12 @@ if (build !== "saas") {
         user.adminGeneratePasswordResetCode
     );
 
+    authenticated.post(
+        "/user/:userId/server-admin",
+        verifyUserIsServerAdmin,
+        user.adminSetServerAdmin
+    );
+
     authenticated.delete(
         "/user/:userId",
         verifyUserIsServerAdmin,

server/routers/gerbil/getConfig.ts

@@ -10,6 +10,7 @@ import config from "@server/lib/config";
 import { fromError } from "zod-validation-error";
 import { getAllowedIps } from "../target/helpers";
 import { createExitNode } from "#dynamic/routers/gerbil/createExitNode";
+import { markExitNodeCheckedIn } from "@server/lib/exitNodes";
 
 // Define Zod schema for request validation
 const getConfigSchema = z.object({
@@ -65,6 +66,8 @@ export async function getConfig(
             );
         }
 
+        markExitNodeCheckedIn(exitNode.exitNodeId);
+
         const configResponse = await generateGerbilConfig(exitNode);
 
         logger.debug("Sending config: ", configResponse);

server/routers/launcher/launcherResourceAccess.ts

@@ -522,6 +522,7 @@ async function fetchLabelsForResources(
 type SiteGroupRow = {
     siteId: number;
     name: string;
+    niceId: string;
     type: string;
     online: boolean;
     itemCount: number;
@@ -556,6 +557,7 @@ async function listSiteGroups(
             .select({
                 siteId: sites.siteId,
                 name: sites.name,
+                niceId: sites.niceId,
                 type: sites.type,
                 online: sites.online,
                 itemCount: countDistinct(resources.resourceId)
@@ -576,7 +578,13 @@ async function listSiteGroups(
 
         const publicRows = await publicQuery
             .where(and(...publicConditions))
-            .groupBy(sites.siteId, sites.name, sites.type, sites.online);
+            .groupBy(
+                sites.siteId,
+                sites.name,
+                sites.niceId,
+                sites.type,
+                sites.online
+            );
 
         for (const row of publicRows) {
             const existing = siteCountMap.get(row.siteId);
@@ -586,6 +594,7 @@ async function listSiteGroups(
                 siteCountMap.set(row.siteId, {
                     siteId: row.siteId,
                     name: row.name,
+                    niceId: row.niceId,
                     type: row.type,
                     online: row.online,
                     itemCount: Number(row.itemCount)
@@ -612,6 +621,7 @@ async function listSiteGroups(
             .select({
                 siteId: sites.siteId,
                 name: sites.name,
+                niceId: sites.niceId,
                 type: sites.type,
                 online: sites.online,
                 itemCount: countDistinct(siteResources.siteResourceId)
@@ -638,7 +648,13 @@ async function listSiteGroups(
 
         const siteRows = await siteResourceQuery
             .where(and(...siteConditions))
-            .groupBy(sites.siteId, sites.name, sites.type, sites.online);
+            .groupBy(
+                sites.siteId,
+                sites.name,
+                sites.niceId,
+                sites.type,
+                sites.online
+            );
 
         for (const row of siteRows) {
             const existing = siteCountMap.get(row.siteId);
@@ -648,6 +664,7 @@ async function listSiteGroups(
                 siteCountMap.set(row.siteId, {
                     siteId: row.siteId,
                     name: row.name,
+                    niceId: row.niceId,
                     type: row.type,
                     online: row.online,
                     itemCount: Number(row.itemCount)
@@ -1061,6 +1078,43 @@ export async function listLauncherGroupsForUser(
     };
 }
 
+function toLauncherSiteInfo(row: {
+    siteId: number | null;
+    siteName: string | null;
+    siteNiceId: string | null;
+    siteType: string | null;
+    siteOnline: boolean | null;
+}): LauncherSiteInfo | null {
+    if (
+        row.siteId == null ||
+        row.siteName == null ||
+        row.siteNiceId == null ||
+        row.siteType == null
+    ) {
+        return null;
+    }
+    return {
+        siteId: row.siteId,
+        name: row.siteName,
+        niceId: row.siteNiceId,
+        type: row.siteType,
+        online: row.siteOnline ?? undefined
+    };
+}
+
+function pickPrimarySite(
+    sites: LauncherSiteInfo[],
+    siteIdFilter?: number
+): LauncherSiteInfo | undefined {
+    if (sites.length === 0) {
+        return undefined;
+    }
+    if (siteIdFilter != null) {
+        return sites.find((site) => site.siteId === siteIdFilter) ?? sites[0];
+    }
+    return sites[0];
+}
+
 async function mapPublicResources(
     orgId: string,
     resourceIds: number[],
@@ -1084,6 +1138,7 @@ async function mapPublicResources(
             enabled: resources.enabled,
             siteId: sites.siteId,
             siteName: sites.name,
+            siteNiceId: sites.niceId,
             siteType: sites.type,
             siteOnline: sites.online,
             exitNodeEndpoint: exitNodes.endpoint
@@ -1097,56 +1152,65 @@ async function mapPublicResources(
                 inArray(resources.resourceId, resourceIds),
                 eq(resources.orgId, orgId),
                 eq(resources.enabled, true),
-                eq(resources.status, "approved"),
-                siteIdFilter != null
-                    ? eq(sites.siteId, siteIdFilter)
-                    : undefined
+                eq(resources.status, "approved")
             )
         );
 
-    const seen = new Set<string>();
-    const result: LauncherResource[] = [];
+    const byKey = new Map<string, LauncherResource>();
+    const siteIdsByKey = new Map<string, Set<number>>();
 
     for (const row of rows) {
         const key = `public:${row.resourceId}`;
-        if (seen.has(key)) {
+        let item = byKey.get(key);
+
+        if (!item) {
+            const access = formatPublicResourceAccess({
+                mode: row.mode,
+                fullDomain: row.fullDomain,
+                ssl: row.ssl,
+                proxyPort: row.proxyPort,
+                wildcard: row.wildcard,
+                exitNodeEndpoint: row.exitNodeEndpoint
+            });
+
+            item = {
+                launcherResourceKey: key,
+                resourceType: "public",
+                resourceId: row.resourceId,
+                niceId: row.niceId,
+                name: row.name,
+                ...access,
+                iconUrl: null,
+                enabled: row.enabled,
+                mode: row.mode,
+                labels: labelMaps.byResourceId.get(row.resourceId) ?? [],
+                sites: []
+            };
+            byKey.set(key, item);
+            siteIdsByKey.set(key, new Set());
+        }
+
+        const site = toLauncherSiteInfo(row);
+        if (!site) {
             continue;
         }
-        seen.add(key);
 
-        const access = formatPublicResourceAccess({
-            mode: row.mode,
-            fullDomain: row.fullDomain,
-            ssl: row.ssl,
-            proxyPort: row.proxyPort,
-            wildcard: row.wildcard,
-            exitNodeEndpoint: row.exitNodeEndpoint
-        });
-
-        result.push({
-            launcherResourceKey: key,
-            resourceType: "public",
-            resourceId: row.resourceId,
-            niceId: row.niceId,
-            name: row.name,
-            ...access,
-            iconUrl: null,
-            enabled: row.enabled,
-            mode: row.mode,
-            labels: labelMaps.byResourceId.get(row.resourceId) ?? [],
-            site:
-                row.siteId != null
-                    ? {
-                          siteId: row.siteId,
-                          name: row.siteName!,
-                          type: row.siteType!,
-                          online: row.siteOnline ?? undefined
-                      }
-                    : undefined
-        });
+        const seenSiteIds = siteIdsByKey.get(key)!;
+        if (seenSiteIds.has(site.siteId)) {
+            continue;
+        }
+        seenSiteIds.add(site.siteId);
+        item.sites.push(site);
     }
 
-    return result;
+    for (const item of byKey.values()) {
+        item.sites.sort((a, b) =>
+            a.name.localeCompare(b.name, undefined, { sensitivity: "base" })
+        );
+        item.site = pickPrimarySite(item.sites, siteIdFilter);
+    }
+
+    return Array.from(byKey.values());
 }
 
 async function mapSiteResources(
@@ -1175,6 +1239,7 @@ async function mapSiteResources(
             enabled: siteResources.enabled,
             siteId: sites.siteId,
             siteName: sites.name,
+            siteNiceId: sites.niceId,
             siteType: sites.type,
             siteOnline: sites.online
         })
@@ -1189,59 +1254,69 @@ async function mapSiteResources(
                 inArray(siteResources.siteResourceId, siteResourceIds),
                 eq(siteResources.orgId, orgId),
                 eq(siteResources.enabled, true),
-                eq(siteResources.status, "approved"),
-                siteIdFilter != null
-                    ? eq(sites.siteId, siteIdFilter)
-                    : undefined
+                eq(siteResources.status, "approved")
             )
         );
 
-    const seen = new Set<string>();
-    const result: LauncherResource[] = [];
+    const byKey = new Map<string, LauncherResource>();
+    const siteIdsByKey = new Map<string, Set<number>>();
 
     for (const row of rows) {
         const key = `site:${row.siteResourceId}`;
-        if (seen.has(key)) {
+        let item = byKey.get(key);
+
+        if (!item) {
+            const access = formatSiteResourceAccess({
+                mode: row.mode,
+                destination: row.destination,
+                destinationPort: row.destinationPort,
+                scheme: row.scheme,
+                ssl: row.ssl,
+                fullDomain: row.fullDomain,
+                alias: row.alias,
+                aliasAddress: row.aliasAddress
+            });
+
+            item = {
+                launcherResourceKey: key,
+                resourceType: "site",
+                resourceId: row.siteResourceId,
+                siteResourceId: row.siteResourceId,
+                niceId: row.niceId,
+                name: row.name,
+                ...access,
+                iconUrl: null,
+                enabled: row.enabled,
+                mode: row.mode,
+                labels:
+                    labelMaps.bySiteResourceId.get(row.siteResourceId) ?? [],
+                sites: []
+            };
+            byKey.set(key, item);
+            siteIdsByKey.set(key, new Set());
+        }
+
+        const site = toLauncherSiteInfo(row);
+        if (!site) {
             continue;
         }
-        seen.add(key);
 
-        const access = formatSiteResourceAccess({
-            mode: row.mode,
-            destination: row.destination,
-            destinationPort: row.destinationPort,
-            scheme: row.scheme,
-            ssl: row.ssl,
-            fullDomain: row.fullDomain,
-            alias: row.alias,
-            aliasAddress: row.aliasAddress
-        });
-
-        result.push({
-            launcherResourceKey: key,
-            resourceType: "site",
-            resourceId: row.siteResourceId,
-            siteResourceId: row.siteResourceId,
-            niceId: row.niceId,
-            name: row.name,
-            ...access,
-            iconUrl: null,
-            enabled: row.enabled,
-            mode: row.mode,
-            labels: labelMaps.bySiteResourceId.get(row.siteResourceId) ?? [],
-            site:
-                row.siteId != null
-                    ? {
-                          siteId: row.siteId,
-                          name: row.siteName!,
-                          type: row.siteType!,
-                          online: row.siteOnline ?? undefined
-                      }
-                    : undefined
-        });
+        const seenSiteIds = siteIdsByKey.get(key)!;
+        if (seenSiteIds.has(site.siteId)) {
+            continue;
+        }
+        seenSiteIds.add(site.siteId);
+        item.sites.push(site);
     }
 
-    return result;
+    for (const item of byKey.values()) {
+        item.sites.sort((a, b) =>
+            a.name.localeCompare(b.name, undefined, { sensitivity: "base" })
+        );
+        item.site = pickPrimarySite(item.sites, siteIdFilter);
+    }
+
+    return Array.from(byKey.values());
 }
 
 function filterResourcesBySite(
@@ -1252,13 +1327,17 @@ function filterResourcesBySite(
         return items.filter((item) => item.mode === "inference");
     }
     if (groupKey === LAUNCHER_NO_SITE_GROUP_KEY) {
-        return items.filter((item) => !item.site && item.mode !== "inference");
+        return items.filter(
+            (item) => item.sites.length === 0 && item.mode !== "inference"
+        );
     }
     const siteId = Number.parseInt(groupKey, 10);
     if (!Number.isFinite(siteId)) {
         return items;
     }
-    return items.filter((item) => item.site?.siteId === siteId);
+    return items.filter((item) =>
+        item.sites.some((site) => site.siteId === siteId)
+    );
 }
 
 function filterResourcesByLabel(
@@ -1499,6 +1578,7 @@ async function collectAccessibleSites(
             .select({
                 siteId: sites.siteId,
                 name: sites.name,
+                niceId: sites.niceId,
                 type: sites.type,
                 online: sites.online,
                 itemCount: countDistinct(resources.resourceId)
@@ -1507,7 +1587,13 @@ async function collectAccessibleSites(
             .innerJoin(resources, eq(targets.resourceId, resources.resourceId))
             .innerJoin(sites, eq(targets.siteId, sites.siteId))
             .where(and(...publicConditions))
-            .groupBy(sites.siteId, sites.name, sites.type, sites.online);
+            .groupBy(
+                sites.siteId,
+                sites.name,
+                sites.niceId,
+                sites.type,
+                sites.online
+            );
 
         for (const row of publicRows) {
             const existing = siteCountMap.get(row.siteId);
@@ -1517,6 +1603,7 @@ async function collectAccessibleSites(
                 siteCountMap.set(row.siteId, {
                     siteId: row.siteId,
                     name: row.name,
+                    niceId: row.niceId,
                     type: row.type,
                     online: row.online,
                     itemCount: Number(row.itemCount)
@@ -1540,6 +1627,7 @@ async function collectAccessibleSites(
             .select({
                 siteId: sites.siteId,
                 name: sites.name,
+                niceId: sites.niceId,
                 type: sites.type,
                 online: sites.online,
                 itemCount: countDistinct(siteResources.siteResourceId)
@@ -1551,7 +1639,13 @@ async function collectAccessibleSites(
             )
             .innerJoin(sites, eq(siteNetworks.siteId, sites.siteId))
             .where(and(...siteConditions))
-            .groupBy(sites.siteId, sites.name, sites.type, sites.online);
+            .groupBy(
+                sites.siteId,
+                sites.name,
+                sites.niceId,
+                sites.type,
+                sites.online
+            );
 
         for (const row of siteRows) {
             const existing = siteCountMap.get(row.siteId);
@@ -1561,6 +1655,7 @@ async function collectAccessibleSites(
                 siteCountMap.set(row.siteId, {
                     siteId: row.siteId,
                     name: row.name,
+                    niceId: row.niceId,
                     type: row.type,
                     online: row.online,
                     itemCount: Number(row.itemCount)
@@ -1675,6 +1770,7 @@ export async function listAccessibleLauncherSitesForUser(
         .map((row) => ({
             siteId: row.siteId,
             name: row.name,
+            niceId: row.niceId,
             type: row.type,
             online: row.online
         }))

server/routers/launcher/types.ts

@@ -32,6 +32,7 @@ export type LauncherLabel = {
 export type LauncherSiteInfo = {
     siteId: number;
     name: string;
+    niceId: string;
     type: string;
     online?: boolean;
 };
@@ -51,6 +52,7 @@ export type LauncherResource = {
     mode: string;
     labels: LauncherLabel[];
     site?: LauncherSiteInfo;
+    sites: LauncherSiteInfo[];
 };
 
 export type LauncherGroup = {
@@ -184,8 +186,7 @@ export function parseIdListParam(value: string | undefined): number[] {
 export const DEFAULT_LAUNCHER_VIEW_ID = "default" as const;
 
 export type LauncherViewSelection =
-    | { type: "default" }
-    | { type: "saved"; viewId: number };
+    { type: "default" } | { type: "saved"; viewId: number };
 
 export type LauncherScaleCapabilities = {
     allowSiteGrouping: boolean;

server/routers/newt/error.ts

@@ -0,0 +1,24 @@
+import { sendToClient } from "#dynamic/routers/ws";
+
+// Error codes for registration failures
+export const NewtErrorCodes = {
+    NO_AVAILABLE_SUBNET: {
+        code: "NO_AVAILABLE_SUBNET",
+        message:
+            "No available subnet could be assigned to this site on its exit node. Please contact your administrator to increase the available address space for this exit node's subnet."
+    }
+} as const;
+
+// Helper function to send registration error
+export async function sendNewtError(
+    error: (typeof NewtErrorCodes)[keyof typeof NewtErrorCodes],
+    newtId: string
+) {
+    sendToClient(newtId, {
+        type: "newt/error",
+        data: {
+            code: error.code,
+            message: error.message
+        }
+    });
+}

server/routers/newt/getNewtVersion.ts

@@ -13,31 +13,40 @@ import logger from "@server/logger";
 import { regionalCache as cache } from "#dynamic/lib/cache";
 import config from "@server/lib/config";
 
-// Stale-while-revalidate in-memory fallback for the releases API.
 type ReleaseInfo = {
     version: string;
     // binary filename -> sha256 hex (sourced from asset `digest` field in GitHub API)
     assetDigests: Record<string, string>;
 };
-let staleReleaseInfo: ReleaseInfo | null = null;
+
+// Cache key holding the last known good release info. It never expires, so
+// it keeps serving if GitHub is unreachable, even across restarts/nodes.
+const RELEASE_INFO_KEY = "cache:releaseInfo";
+// Short-lived marker controlling how often we re-check GitHub. While it's
+// missing (expired, or a previous attempt failed) every request retries.
+const RELEASE_INFO_FRESH_KEY = "cache:releaseInfoFresh";
+const RELEASE_INFO_REFRESH_SECONDS = 3600;
 
 /**
  * Fetches the latest stable newt release from GitHub and returns the version
  * tag together with a map of asset-name → sha256 hex digest.
- * Results are cached for one hour; stale data is returned on failure.
+ * The last successful result is cached indefinitely and re-checked hourly;
+ * on failure the last known good data keeps being served and every
+ * subsequent request retries GitHub until it succeeds again.
  */
-async function getLatestReleaseInfo(): Promise<ReleaseInfo | null> {
-    try {
-        const cached = await cache.get<ReleaseInfo>("cache:newtReleaseInfo");
-        if (cached) {
-            return cached;
-        }
+async function getLatestReleaseInfo(repo: string): Promise<ReleaseInfo | null> {
+    const stored = await cache.get<ReleaseInfo>(`${RELEASE_INFO_KEY}:${repo}`);
+    const isFresh = await cache.has(`${RELEASE_INFO_FRESH_KEY}:${repo}`);
+    if (stored && isFresh) {
+        return stored;
+    }
 
+    try {
         const controller = new AbortController();
         const timeoutId = setTimeout(() => controller.abort(), 5000);
 
         const fetchResponse = await fetch(
-            "https://api.github.com/repos/fosrl/newt/releases",
+            `https://api.github.com/repos/fosrl/${repo}/releases`,
             { signal: controller.signal }
         );
 
@@ -47,13 +56,13 @@ async function getLatestReleaseInfo(): Promise<ReleaseInfo | null> {
             logger.warn(
                 `Failed to fetch Newt releases from GitHub: ${fetchResponse.status} ${fetchResponse.statusText}`
             );
-            return staleReleaseInfo;
+            return stored ?? null;
         }
 
         let releases: any[] = await fetchResponse.json();
         if (!Array.isArray(releases) || releases.length === 0) {
-            logger.warn("No releases found for Newt repository");
-            return staleReleaseInfo;
+            logger.warn("No releases found for repository");
+            return stored ?? null;
         }
 
         const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
@@ -81,8 +90,8 @@ async function getLatestReleaseInfo(): Promise<ReleaseInfo | null> {
         });
 
         if (releases.length === 0) {
-            logger.warn("No stable releases found for Newt repository");
-            return staleReleaseInfo;
+            logger.warn("No stable releases found for repository");
+            return stored ?? null;
         }
 
         const latest = releases[0];
@@ -106,8 +115,12 @@ async function getLatestReleaseInfo(): Promise<ReleaseInfo | null> {
         }
 
         const info: ReleaseInfo = { version, assetDigests };
-        staleReleaseInfo = info;
-        await cache.set("cache:newtReleaseInfo", info, 3600);
+        await cache.set(RELEASE_INFO_KEY, info, 0);
+        await cache.set(
+            RELEASE_INFO_FRESH_KEY,
+            true,
+            RELEASE_INFO_REFRESH_SECONDS
+        );
         return info;
     } catch (error: any) {
         if (error.name === "AbortError") {
@@ -118,14 +131,15 @@ async function getLatestReleaseInfo(): Promise<ReleaseInfo | null> {
                 error.message || error
             );
         }
-        return staleReleaseInfo;
+        return stored ?? null;
     }
 }
 
 const bodySchema = z.object({
     newtId: z.string(),
     secret: z.string(),
-    platform: z.string() // e.g. "linux_amd64", "darwin_arm64"
+    platform: z.string(), // e.g. "linux_amd64", "darwin_arm64"
+    agent: z.string().optional().default("newt")
 });
 
 export type GetNewtVersionBody = z.infer<typeof bodySchema>;
@@ -153,7 +167,7 @@ export async function getNewtVersion(
         );
     }
 
-    const { newtId, secret, platform } = parsedBody.data;
+    const { newtId, secret, platform, agent } = parsedBody.data;
 
     try {
         // Verify newt credentials
@@ -258,9 +272,13 @@ export async function getNewtVersion(
         }
 
         // Fetch latest release info (version + asset digests) in one API call.
-        const releaseInfo = await getLatestReleaseInfo();
+        const releaseInfoNewt = await getLatestReleaseInfo("newt");
+        let releaseInfoCli: ReleaseInfo | undefined | null;
+        if (agent == "cli") {
+            releaseInfoCli = await getLatestReleaseInfo("cli");
+        }
 
-        if (!releaseInfo) {
+        if (!releaseInfoNewt || (agent == "cli" && !releaseInfoCli)) {
             return next(
                 createHttpError(
                     HttpCode.INTERNAL_SERVER_ERROR,
@@ -269,18 +287,25 @@ export async function getNewtVersion(
             );
         }
 
-        const latestVersion = releaseInfo.version;
+        const latestVersion = releaseInfoNewt.version;
 
         // Binary name follows the get-newt.sh convention: newt_<platform>[.exe]
-        const binaryName = platform.includes("windows")
+        const binaryNameNewt = platform.includes("windows")
             ? `newt_${platform}.exe`
             : `newt_${platform}`;
 
-        const downloadUrl = `https://github.com/fosrl/newt/releases/download/${latestVersion}/${binaryName}`;
+        const binaryNameCli = platform.includes("windows")
+            ? `pangolin-cli_${platform}.exe`
+            : `pangolin-cli_${platform}`;
+
+        const downloadUrl = `https://github.com/fosrl/${agent}/releases/download/${agent == "cli" ? releaseInfoCli?.version : releaseInfoNewt.version}/${agent == "cli" ? binaryNameCli : binaryNameNewt}`;
 
         // Look up the SHA256 digest for this specific binary from the GitHub
         // release asset metadata (the `digest` field, format "sha256:<hex>").
-        const sha256 = releaseInfo.assetDigests[binaryName] ?? "";
+        const sha256 =
+            releaseInfoNewt.assetDigests[
+                agent == "cli" ? binaryNameCli : binaryNameNewt
+            ] ?? "";
 
         // Determine whether the newt that's asking is already up to date.
         // We store the current version on the newt row when it registers.
@@ -300,8 +325,8 @@ export async function getNewtVersion(
 
         return response<GetNewtVersionResponse>(res, {
             data: {
-                latestVersion,
-                currentIsLatest,
+                latestVersion, // this will always be the newt version
+                currentIsLatest, // this will always be based on the newt version
                 downloadUrl,
                 sha256
             },

server/routers/newt/handleNewtRegisterMessage.ts

@@ -14,6 +14,7 @@ import { getUniqueSubnetForExitNode } from "@server/lib/exitNodes";
 import { fetchContainers } from "./dockerSocket";
 import { buildTargetConfigurationForNewtClient } from "./buildConfiguration";
 import { canCompress } from "@server/lib/clientVersionChecks";
+import { NewtErrorCodes, sendNewtError } from "./error";
 
 export const handleNewtRegisterMessage: MessageHandler = async (context) => {
     const { message, client, sendToClient } = context;
@@ -37,6 +38,8 @@ export const handleNewtRegisterMessage: MessageHandler = async (context) => {
         publicKey,
         pingResults,
         newtVersion,
+        agent,
+        agentVersion,
         backwardsCompatible,
         chainId
     } = message.data;
@@ -114,6 +117,7 @@ export const handleNewtRegisterMessage: MessageHandler = async (context) => {
             logger.error(
                 `No available subnets found for the new exit node id ${exitNodeId} and site id ${siteId}`
             );
+            sendNewtError(NewtErrorCodes.NO_AVAILABLE_SUBNET, newt.newtId);
             return;
         }
 
@@ -169,22 +173,21 @@ export const handleNewtRegisterMessage: MessageHandler = async (context) => {
         logger.error(`Failed to add peer to exit node: ${error}`);
     }
 
-    if (newtVersion && newtVersion !== newt.version) {
+    if (
+        newtVersion !== newt.version ||
+        agent !== newt.agent ||
+        agentVersion !== newt.agentVersion
+    ) {
         // update the newt version in the database
         await db
             .update(newts)
             .set({
-                version: newtVersion as string
-            })
-            .where(eq(newts.newtId, newt.newtId));
-    }
-
-    if (newtVersion && newtVersion !== newt.version) {
-        // update the newt version in the database
-        await db
-            .update(newts)
-            .set({
-                version: newtVersion as string
+                version: newtVersion as string,
+                agent: agent,
+                agentVersion:
+                    !agentVersion && agent == "newt"
+                        ? newtVersion
+                        : agentVersion
             })
             .where(eq(newts.newtId, newt.newtId));
     }

server/routers/newt/handleRequestLogMessage.ts

@@ -1,16 +1,3 @@
-/*
- * This file is part of a proprietary work.
- *
- * Copyright (c) 2025-2026 Fossorial, Inc.
- * All rights reserved.
- *
- * This file is licensed under the Fossorial Commercial License.
- * You may not use this file except in compliance with the License.
- * Unauthorized use, copying, modification, or distribution is strictly prohibited.
- *
- * This file is not licensed under the AGPLv3.
- */
-
 import { db } from "@server/db";
 import { MessageHandler } from "@server/routers/ws";
 import { sites, Newt, orgs, clients, clientSitesAssociationsCache, users } from "@server/db";

server/routers/olm/error.ts

@@ -94,6 +94,11 @@ export const OlmErrorCodes = {
     HOLEPUNCH_MISSING: {
         code: "HOLEPUNCH_MISSING",
         message: `Unable to coordinate client P2P connection. Please ensure your client can reach the server on UDP port ${udpPort} and try registering again.`
+    },
+    NO_AVAILABLE_SUBNET: {
+        code: "NO_AVAILABLE_SUBNET",
+        message:
+            "No available subnet could be assigned to this client on the selected exit node. Please contact your administrator to increase the available address space for this exit node's subnet."
     }
 } as const;
 

server/routers/olm/handleOlmRegisterMessage.ts

@@ -347,6 +347,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => {
                 `[handleOlmRegisterMessage] No available subnets found for exit node id ${exitNodeId} and client id ${client.clientId}`,
                 { orgId: client.orgId, clientId: client.clientId }
             );
+            sendOlmError(OlmErrorCodes.NO_AVAILABLE_SUBNET, olm.olmId);
             return;
         }
 

server/routers/org/adminDeleteOrg.ts

@@ -0,0 +1,98 @@
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+import { deleteOrgById, sendTerminationMessages } from "@server/lib/deleteOrg";
+import { db, orgs } from "@server/db";
+import { eq } from "drizzle-orm";
+
+const adminDeleteOrgSchema = z.strictObject({
+    orgId: z.string()
+});
+
+export type AdminDeleteOrgResponse = {};
+
+registry.registerPath({
+    method: "delete",
+    path: "/admin/org/{orgId}",
+    description: "Delete any organization in the system (server admin).",
+    tags: [OpenAPITags.Org],
+    request: {
+        params: adminDeleteOrgSchema
+    },
+    responses: {
+        200: {
+            description: "Successful response",
+            content: {
+                "application/json": {
+                    schema: z.object({
+                        data: z.record(z.string(), z.any()).nullable(),
+                        success: z.boolean(),
+                        error: z.boolean(),
+                        message: z.string(),
+                        status: z.number()
+                    })
+                }
+            }
+        }
+    }
+});
+
+export async function adminDeleteOrg(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = adminDeleteOrgSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+        const { orgId } = parsedParams.data;
+
+        const [org] = await db
+            .select()
+            .from(orgs)
+            .where(eq(orgs.orgId, orgId))
+            .limit(1);
+
+        if (!org) {
+            return next(
+                createHttpError(
+                    HttpCode.NOT_FOUND,
+                    `Organization with ID ${orgId} not found`
+                )
+            );
+        }
+
+        const result = await deleteOrgById(orgId);
+        sendTerminationMessages(result);
+        return response(res, {
+            data: null,
+            success: true,
+            error: false,
+            message: "Organization deleted successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        if (createHttpError.isHttpError(error)) {
+            return next(error);
+        }
+        logger.error(error);
+        return next(
+            createHttpError(
+                HttpCode.INTERNAL_SERVER_ERROR,
+                "An error occurred..."
+            )
+        );
+    }
+}

server/routers/org/adminListOrgs.ts

@@ -0,0 +1,241 @@
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import { db, users } from "@server/db";
+import { orgs, resources, sites, userOrgs } from "@server/db";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import { and, asc, desc, eq, like, or, sql, type SQL } from "drizzle-orm";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+import { createApiResponseSchema } from "@server/lib/openapi/createApiResponseSchema";
+import type { PaginatedResponse } from "@server/types/Pagination";
+
+const adminListOrgsSchema = z.strictObject({
+    pageSize: z.coerce
+        .number<string>()
+        .int()
+        .positive()
+        .optional()
+        .catch(20)
+        .default(20)
+        .openapi({
+            type: "integer",
+            default: 20,
+            description: "Number of items per page"
+        }),
+    page: z.coerce
+        .number<string>()
+        .int()
+        .positive()
+        .optional()
+        .catch(1)
+        .default(1)
+        .openapi({
+            type: "integer",
+            default: 1,
+            description: "Page number to retrieve"
+        }),
+    query: z.string().optional(),
+    sort_by: z
+        .enum(["name", "createdAt"])
+        .optional()
+        .catch(undefined)
+        .openapi({
+            type: "string",
+            enum: ["name", "createdAt"],
+            description: "Field to sort by"
+        }),
+    order: z
+        .enum(["asc", "desc"])
+        .optional()
+        .default("asc")
+        .catch("asc")
+        .openapi({
+            type: "string",
+            enum: ["asc", "desc"],
+            default: "asc",
+            description: "Sort order"
+        })
+});
+
+export type AdminOrgRow = {
+    orgId: string;
+    name: string;
+    subnet: string | null;
+    utilitySubnet: string | null;
+    createdAt: string | null;
+    userCount: number;
+    siteCount: number;
+    resourceCount: number;
+    owner: {
+        userId: string;
+        username: string;
+    } | null;
+};
+
+export type AdminListOrgsResponse = PaginatedResponse<{
+    orgs: AdminOrgRow[];
+}>;
+
+const AdminListOrgsResponseDataSchema = z.object({
+    orgs: z.array(
+        z.object({
+            orgId: z.string(),
+            name: z.string(),
+            subnet: z.string().nullable(),
+            createdAt: z.string().nullable(),
+            userCount: z.number(),
+            siteCount: z.number(),
+            resourceCount: z.number()
+        })
+    ),
+    pagination: z.object({
+        total: z.number(),
+        page: z.number(),
+        pageSize: z.number()
+    })
+});
+
+registry.registerPath({
+    method: "get",
+    path: "/admin/orgs",
+    description:
+        "List all organizations in the system with usage counts (server admin).",
+    tags: [OpenAPITags.Org],
+    request: {
+        query: adminListOrgsSchema
+    },
+    responses: {
+        200: {
+            description: "Successful response",
+            content: {
+                "application/json": {
+                    schema: createApiResponseSchema(
+                        AdminListOrgsResponseDataSchema
+                    )
+                }
+            }
+        }
+    }
+});
+
+export async function adminListOrgs(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedQuery = adminListOrgsSchema.safeParse(req.query);
+        if (!parsedQuery.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedQuery.error)
+                )
+            );
+        }
+
+        const { pageSize, page, query, sort_by, order } = parsedQuery.data;
+
+        let conditions: (SQL<unknown> | undefined)[] = [];
+        if (query) {
+            const q = "%" + query.toLowerCase() + "%";
+            conditions.push(
+                or(
+                    like(sql`LOWER(${orgs.name})`, q),
+                    like(sql`LOWER(${orgs.orgId})`, q),
+                    like(sql`LOWER(${orgs.subnet})`, q)
+                )
+            );
+        }
+
+        const sortColumns = {
+            name: orgs.name,
+            createdAt: orgs.createdAt
+        } as const;
+
+        const orderBy = sort_by
+            ? order === "asc"
+                ? asc(sortColumns[sort_by])
+                : desc(sortColumns[sort_by])
+            : asc(orgs.name);
+
+        // Drizzle renders bare column references in the select list without their
+        // table prefix, which would make a correlated subquery compare a column to
+        // itself, so the outer `orgs` side is qualified explicitly.
+        const orgIdRef = sql`${sql.identifier("orgs")}.${sql.identifier("orgId")}`;
+
+        const [countRows, rows] = await Promise.all([
+            db
+                .select({ count: sql<number>`count(*)` })
+                .from(orgs)
+                .where(and(...conditions)),
+            db
+                .selectDistinct({
+                    orgId: orgs.orgId,
+                    name: orgs.name,
+                    subnet: orgs.subnet,
+                    utilitySubnet: orgs.utilitySubnet,
+                    createdAt: orgs.createdAt,
+                    userCount: sql<number>`(
+                        SELECT COUNT(*)
+                        FROM ${userOrgs}
+                        WHERE ${userOrgs.orgId} = ${orgIdRef}
+                    )`.as("userCount"),
+                    siteCount: sql<number>`(
+                        SELECT COUNT(*)
+                        FROM ${sites}
+                        WHERE ${sites.orgId} = ${orgIdRef}
+                    )`.as("siteCount"),
+                    resourceCount: sql<number>`(
+                        SELECT COUNT(*)
+                        FROM ${resources}
+                        WHERE ${resources.orgId} = ${orgIdRef}
+                    )`.as("resourceCount"),
+                    owner: {
+                        userId: users.userId,
+                        username: users.username
+                    }
+                })
+                .from(orgs)
+                .where(and(...conditions, eq(userOrgs.isOwner, true)))
+                .leftJoin(userOrgs, eq(userOrgs.orgId, orgs.orgId))
+                .leftJoin(users, eq(userOrgs.userId, users.userId))
+                .limit(pageSize)
+                .offset(pageSize * (page - 1))
+                .orderBy(orderBy)
+        ]);
+
+        const totalCount = Number(countRows[0]?.count ?? 0);
+
+        return response<AdminListOrgsResponse>(res, {
+            data: {
+                orgs: rows.map((row) => ({
+                    ...row,
+                    userCount: Number(row.userCount ?? 0),
+                    siteCount: Number(row.siteCount ?? 0),
+                    resourceCount: Number(row.resourceCount ?? 0)
+                })),
+                pagination: {
+                    total: totalCount,
+                    page,
+                    pageSize
+                }
+            },
+            success: true,
+            error: false,
+            message: "Organizations retrieved successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(
+                HttpCode.INTERNAL_SERVER_ERROR,
+                "An error occurred..."
+            )
+        );
+    }
+}

server/routers/org/index.ts

@@ -9,3 +9,5 @@ export * from "./listOrgs";
 export * from "./pickOrgDefaults";
 export * from "./checkOrgUserAccess";
 export * from "./resetOrgBandwidth";
+export * from "./adminListOrgs";
+export * from "./adminDeleteOrg";

server/routers/site/getSite.ts

@@ -48,6 +48,8 @@ type SiteQueryRow = NonNullable<Awaited<ReturnType<typeof query>>>;
 export type GetSiteResponse = SiteQueryRow["sites"] & {
     newtId: string | null;
     newtVersion: string | null;
+    agent: string | null;
+    agentVersion: string | null;
     countryCode: string | null;
 };
 
@@ -137,6 +139,8 @@ export async function getSite(
             ...site.sites,
             newtId: site.newt ? site.newt.newtId : null,
             newtVersion: site.newt?.version ?? null,
+            agent: site.newt?.agent ?? null,
+            agentVersion: site.newt?.agentVersion ?? null,
             countryCode: site.sites.endpoint
                 ? ((await getCountryCodeForIp(site.sites.endpoint)) ?? null)
                 : null

server/routers/site/listSites.ts

@@ -133,6 +133,8 @@ function querySitesBase() {
             online: sites.online,
             address: sites.address,
             newtVersion: newts.version,
+            agent: newts.agent,
+            agentVersion: newts.agentVersion,
             exitNodeId: sites.exitNodeId,
             exitNodeName: exitNodes.name,
             exitNodeEndpoint: exitNodes.endpoint,

server/routers/target/handleHealthcheckStatusMessage.ts

@@ -43,7 +43,7 @@ export const handleHealthcheckStatusMessage: MessageHandler = async (
     const { message, client: c } = context;
     const newt = c as Newt;
 
-    logger.info("Handling healthcheck status message");
+    logger.debug("Handling healthcheck status message");
 
     if (!newt) {
         logger.warn("Newt not found");

server/routers/user/adminListUsers.ts

@@ -4,7 +4,7 @@ import { db, idp, users } from "@server/db";
 import response from "@server/lib/response";
 import HttpCode from "@server/types/HttpCode";
 import createHttpError from "http-errors";
-import { and, asc, desc, eq, like, or, sql } from "drizzle-orm";
+import { and, asc, desc, eq, like, or, sql, type SQL } from "drizzle-orm";
 import logger from "@server/logger";
 import { fromZodError } from "zod-validation-error";
 import { OpenAPITags, registry } from "@server/openApi";
@@ -88,6 +88,15 @@ const listUsersSchema = z.strictObject({
             type: "boolean",
             description:
                 "Filter by 2FA state matching: enabled if twoFactorEnabled or twoFactorSetupRequested"
+        }),
+    server_admin: z
+        .enum(["true", "false"])
+        .transform((v) => v === "true")
+        .optional()
+        .catch(undefined)
+        .openapi({
+            type: "boolean",
+            description: "Filter by server admin status"
         })
 });
 
@@ -177,7 +186,8 @@ export async function adminListUsers(
             sort_by,
             order,
             idp_id,
-            two_factor: twoFactorFilter
+            two_factor: twoFactorFilter,
+            server_admin: serverAdminFilter
         } = parsedQuery.data;
 
         if (typeof idp_id === "number") {
@@ -196,7 +206,7 @@ export async function adminListUsers(
             }
         }
 
-        const conditions = [eq(users.serverAdmin, false)];
+        const conditions: Array<SQL<unknown> | undefined> = [];
 
         if (query) {
             const q = "%" + query.toLowerCase() + "%";
@@ -233,6 +243,10 @@ export async function adminListUsers(
             }
         }
 
+        if (typeof serverAdminFilter === "boolean") {
+            conditions.push(eq(users.serverAdmin, serverAdminFilter));
+        }
+
         const whereClause = and(...conditions);
 
         const countQuery = db.$count(

server/routers/user/adminSetServerAdmin.ts

@@ -0,0 +1,151 @@
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import { db, users } from "@server/db";
+import { eq } from "drizzle-orm";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+import { createApiResponseSchema } from "@server/lib/openapi/createApiResponseSchema";
+
+const setServerAdminParamsSchema = z.strictObject({
+    userId: z.string()
+});
+
+const setServerAdminBodySchema = z.strictObject({
+    serverAdmin: z.boolean()
+});
+
+export type AdminSetServerAdminResponse = {
+    userId: string;
+    serverAdmin: boolean;
+};
+
+const AdminSetServerAdminResponseDataSchema = z.object({
+    userId: z.string(),
+    serverAdmin: z.boolean()
+});
+
+registry.registerPath({
+    method: "post",
+    path: "/user/{userId}/server-admin",
+    description:
+        "Promote or demote a user's server admin status (server admin).",
+    tags: [OpenAPITags.User],
+    request: {
+        params: setServerAdminParamsSchema,
+        body: {
+            content: {
+                "application/json": {
+                    schema: setServerAdminBodySchema
+                }
+            }
+        }
+    },
+    responses: {
+        200: {
+            description: "Successful response",
+            content: {
+                "application/json": {
+                    schema: createApiResponseSchema(
+                        AdminSetServerAdminResponseDataSchema
+                    )
+                }
+            }
+        }
+    }
+});
+
+export async function adminSetServerAdmin(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = setServerAdminParamsSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const parsedBody = setServerAdminBodySchema.safeParse(req.body);
+        if (!parsedBody.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedBody.error).toString()
+                )
+            );
+        }
+
+        const { userId } = parsedParams.data;
+        const { serverAdmin } = parsedBody.data;
+
+        const [existingUser] = await db
+            .select({
+                userId: users.userId,
+                serverAdmin: users.serverAdmin,
+                type: users.type
+            })
+            .from(users)
+            .where(eq(users.userId, userId))
+            .limit(1);
+
+        if (!existingUser) {
+            return next(createHttpError(HttpCode.NOT_FOUND, "User not found"));
+        }
+
+        if (existingUser.type !== "internal") {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    "Server admin status can only be changed for internal users"
+                )
+            );
+        }
+
+        if (!serverAdmin && req.user?.userId === userId) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    "You cannot remove your own server admin status"
+                )
+            );
+        }
+
+        if (existingUser.serverAdmin !== serverAdmin) {
+            logger.info(
+                `${serverAdmin ? "Promoting" : "Demoting"} user ${userId} ${serverAdmin ? "to" : "from"} server admin (by ${req.user?.userId})`
+            );
+
+            await db
+                .update(users)
+                .set({ serverAdmin })
+                .where(eq(users.userId, userId));
+        }
+
+        return response<AdminSetServerAdminResponse>(res, {
+            data: {
+                userId: existingUser.userId,
+                serverAdmin
+            },
+            success: true,
+            error: false,
+            message: serverAdmin
+                ? "User promoted to server admin successfully"
+                : "User demoted from server admin successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
+        );
+    }
+}

server/routers/user/index.ts

@@ -11,6 +11,7 @@ export * from "./adminListUsers";
 export * from "./adminRemoveUser";
 export * from "./adminGetUser";
 export * from "./adminGeneratePasswordResetCode";
+export * from "./adminSetServerAdmin";
 export * from "./listInvitations";
 export * from "./removeInvitation";
 export * from "./createOrgUser";

server/routers/user/listUsers.ts

@@ -95,22 +95,34 @@ const listUsersSchema = z.strictObject({
                 'Filter by identity provider id, or "internal" for internal users'
         }),
     role_id: z
-        .preprocess((val) => {
-            if (val === undefined || val === null || val === "") {
-                return undefined;
-            }
-            const raw = Array.isArray(val) ? val : [val];
-            const nums = raw
-                .map((v) =>
-                    typeof v === "string" ? parseInt(v, 10) : Number(v)
-                )
-                .filter((n) => Number.isInteger(n) && n > 0);
-            const unique = [...new Set(nums)];
-            return unique.length ? unique : undefined;
-        }, z.array(z.number().int().positive()).optional())
+        .preprocess(
+            (val) => {
+                if (val === undefined || val === null || val === "") {
+                    return undefined;
+                }
+                const raw = Array.isArray(val) ? val : [val];
+                const includeOwner = raw.some((v) => v === "owner");
+                const nums = raw
+                    .map((v) =>
+                        typeof v === "string" ? parseInt(v, 10) : Number(v)
+                    )
+                    .filter((n) => Number.isInteger(n) && n > 0);
+                const unique = [...new Set(nums)];
+                if (!unique.length && !includeOwner) {
+                    return undefined;
+                }
+                return { roleIds: unique, includeOwner };
+            },
+            z
+                .object({
+                    roleIds: z.array(z.number().int().positive()),
+                    includeOwner: z.boolean()
+                })
+                .optional()
+        )
         .openapi({
             description:
-                "Filter users who have any of these role ids in the organization (repeat query param)"
+                'Filter users who have any of these role ids in the organization, or "owner" for organization owners (repeat query param)'
         })
 });
 
@@ -193,7 +205,8 @@ export async function listUsers(
         }
         const { page, pageSize, sort_by, order, query, idp_id, role_id } =
             parsedQuery.data;
-        const roleIds = role_id ?? [];
+        const roleIds = role_id?.roleIds ?? [];
+        const includeOwner = role_id?.includeOwner ?? false;
 
         const parsedParams = listUsersParamsSchema.safeParse(req.params);
         if (!parsedParams.success) {
@@ -267,21 +280,35 @@ export async function listUsers(
             conditions.push(eq(users.idpId, idp_id));
         }
 
-        if (roleIds.length > 0) {
-            conditions.push(
-                exists(
-                    db
-                        .select()
-                        .from(userOrgRoles)
-                        .where(
-                            and(
-                                eq(userOrgRoles.userId, users.userId),
-                                eq(userOrgRoles.orgId, orgId),
-                                inArray(userOrgRoles.roleId, roleIds)
+        if (roleIds.length > 0 || includeOwner) {
+            const roleFilterParts = [];
+
+            if (includeOwner) {
+                roleFilterParts.push(eq(userOrgs.isOwner, true));
+            }
+
+            if (roleIds.length > 0) {
+                roleFilterParts.push(
+                    exists(
+                        db
+                            .select()
+                            .from(userOrgRoles)
+                            .where(
+                                and(
+                                    eq(userOrgRoles.userId, users.userId),
+                                    eq(userOrgRoles.orgId, orgId),
+                                    inArray(userOrgRoles.roleId, roleIds)
+                                )
                             )
-                        )
-                )
-            );
+                    )
+                );
+            }
+
+            if (roleFilterParts.length === 1) {
+                conditions.push(roleFilterParts[0]);
+            } else if (roleFilterParts.length > 1) {
+                conditions.push(or(...roleFilterParts));
+            }
         }
 
         const countQuery = db.$count(

server/setup/migrationsPg.ts

@@ -29,6 +29,7 @@ import m20 from "./scriptsPg/1.19.0";
 import m21 from "./scriptsPg/1.20.0";
 import m22 from "./scriptsPg/1.21.0";
 import m23 from "./scriptsPg/1.22.0";
+import m24 from "./scriptsPg/1.23.0";
 
 // THIS CANNOT IMPORT ANYTHING FROM THE SERVER
 // EXCEPT FOR THE DATABASE AND THE SCHEMA
@@ -57,7 +58,8 @@ const migrations = [
     { version: "1.19.0", run: m20 },
     { version: "1.20.0", run: m21 },
     { version: "1.21.0", run: m22 },
-    { version: "1.22.0", run: m23 }
+    { version: "1.22.0", run: m23 },
+    { version: "1.23.0", run: m24 }
     // Add new migrations here as they are created
 ] as {
     version: string;
@@ -66,6 +68,12 @@ const migrations = [
 
 await run();
 
+// The pg Pool is created with allowExitOnIdle: false (see poolConfig.ts) so
+// its sockets keep the event loop alive even when idle. Without an explicit
+// exit here, this one-shot script would hang until the pool's
+// idleTimeoutMillis elapses before the process could terminate.
+process.exit(0);
+
 async function run() {
     // run the migrations
     await runMigrations();

server/setup/migrationsSqlite.ts

@@ -5,6 +5,7 @@ import path from "path";
 import semver from "semver";
 import { versionMigrations } from "../db/sqlite";
 import { __DIRNAME, APP_PATH, APP_VERSION } from "@server/lib/consts";
+import { formatBackupTimestamp } from "@server/lib/backupFileName";
 import { SqliteError } from "better-sqlite3";
 import fs from "fs";
 import { build } from "@server/build";
@@ -48,6 +49,7 @@ import m42 from "./scriptsSqlite/1.19.1";
 import m43 from "./scriptsSqlite/1.20.0";
 import m44 from "./scriptsSqlite/1.21.0";
 import m45 from "./scriptsSqlite/1.22.0";
+import m46 from "./scriptsSqlite/1.23.0";
 
 // THIS CANNOT IMPORT ANYTHING FROM THE SERVER
 // EXCEPT FOR THE DATABASE AND THE SCHEMA
@@ -93,7 +95,8 @@ const migrations = [
     { version: "1.19.1", run: m42 },
     { version: "1.20.0", run: m43 },
     { version: "1.21.0", run: m44 },
-    { version: "1.22.0", run: m45 }
+    { version: "1.22.0", run: m45 },
+    { version: "1.23.0", run: m46 }
     // Add new migrations here as they are created
 ] as const;
 
@@ -119,7 +122,7 @@ function backupDb() {
     // copy the db.sqlite file to backups
     // add the date to the filename
     const date = new Date();
-    const dateString = `${date.getFullYear()}-${date.getMonth()}-${date.getDate()}_${date.getHours()}-${date.getMinutes()}-${date.getSeconds()}`;
+    const dateString = formatBackupTimestamp(date);
     const dbPath = path.join(dbDir, "db.sqlite");
     const backupPath = path.join(backupsDir, `db_${dateString}.sqlite`);
     fs.copyFileSync(dbPath, backupPath);

server/setup/scriptsPg/1.23.0.ts

@@ -0,0 +1,30 @@
+import { db } from "@server/db/pg/driver";
+import { sql } from "drizzle-orm";
+
+const version = "1.23.0";
+
+export default async function migration() {
+    console.log(`Running setup script ${version}...`);
+
+    try {
+        await db.execute(sql`BEGIN`);
+
+        await db.execute(sql`
+            ALTER TABLE "newt" ADD COLUMN IF NOT EXISTS "agent" varchar;
+        `);
+
+        await db.execute(sql`
+            ALTER TABLE "newt" ADD COLUMN IF NOT EXISTS "agentVersion" varchar;
+        `);
+
+        await db.execute(sql`COMMIT`);
+        console.log("Migrated database");
+    } catch (e) {
+        await db.execute(sql`ROLLBACK`);
+        console.log("Unable to migrate database");
+        console.log(e);
+        throw e;
+    }
+
+    console.log(`${version} migration complete`);
+}

server/setup/scriptsSqlite/1.0.0-beta9.ts

@@ -10,6 +10,7 @@ import {
     users
 } from "../../db/sqlite";
 import { APP_PATH, configFilePath1, configFilePath2 } from "@server/lib/consts";
+import { formatBackupTimestamp } from "@server/lib/backupFileName";
 import { eq, sql } from "drizzle-orm";
 import fs from "fs";
 import * as yaml from "js-yaml";
@@ -34,7 +35,7 @@ export default async function migration() {
     // copy the db.sqlite file to backups
     // add the date to the filename
     const date = new Date();
-    const dateString = `${date.getFullYear()}-${date.getMonth()}-${date.getDate()}_${date.getHours()}-${date.getMinutes()}-${date.getSeconds()}`;
+    const dateString = formatBackupTimestamp(date);
     const dbPath = path.join(dbDir, "db.sqlite");
     const backupPath = path.join(backupsDir, `db_${dateString}.sqlite`);
     fs.copyFileSync(dbPath, backupPath);

server/setup/scriptsSqlite/1.23.0.ts

@@ -0,0 +1,39 @@
+import { APP_PATH } from "@server/lib/consts";
+import Database from "better-sqlite3";
+import path from "path";
+
+const version = "1.23.0";
+
+export default async function migration() {
+    console.log(`Running setup script ${version}...`);
+
+    const location = path.join(APP_PATH, "db", "db.sqlite");
+    const db = new Database(location);
+
+    try {
+        db.pragma("foreign_keys = OFF");
+
+        db.transaction(() => {
+            db.prepare(
+                `
+            ALTER TABLE 'newt' ADD COLUMN 'agent' text;
+                `
+            ).run();
+
+            db.prepare(
+                `
+            ALTER TABLE 'newt' ADD COLUMN 'agentVersion' text;
+                `
+            ).run();
+        })();
+
+        db.pragma("foreign_keys = ON");
+
+        console.log("Migrated database");
+    } catch (e) {
+        console.log("Failed to migrate db:", e);
+        throw e;
+    }
+
+    console.log(`${version} migration complete`);
+}