Merge branch 'private-http-ha' into alerting-rules

server/private/lib/acmeCertSync.ts

@@ -0,0 +1,478 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import fs from "fs";
+import crypto from "crypto";
+import {
+    certificates,
+    clients,
+    clientSiteResourcesAssociationsCache,
+    db,
+    domains,
+    newts,
+    siteNetworks,
+    SiteResource,
+    siteResources
+} from "@server/db";
+import { and, eq } from "drizzle-orm";
+import { encrypt, decrypt } from "@server/lib/crypto";
+import logger from "@server/logger";
+import privateConfig from "#private/lib/config";
+import config from "@server/lib/config";
+import {
+    generateSubnetProxyTargetV2,
+    SubnetProxyTargetV2
+} from "@server/lib/ip";
+import { updateTargets } from "@server/routers/client/targets";
+import cache from "#private/lib/cache";
+import { build } from "@server/build";
+
+interface AcmeCert {
+    domain: { main: string; sans?: string[] };
+    certificate: string;
+    key: string;
+    Store: string;
+}
+
+interface AcmeJson {
+    [resolver: string]: {
+        Certificates: AcmeCert[];
+    };
+}
+
+async function pushCertUpdateToAffectedNewts(
+    domain: string,
+    domainId: string | null,
+    oldCertPem: string | null,
+    oldKeyPem: string | null
+): Promise<void> {
+    // Find all SSL-enabled HTTP site resources that use this cert's domain
+    let affectedResources: SiteResource[] = [];
+
+    if (domainId) {
+        affectedResources = await db
+            .select()
+            .from(siteResources)
+            .where(
+                and(
+                    eq(siteResources.domainId, domainId),
+                    eq(siteResources.ssl, true)
+                )
+            );
+    } else {
+        // Fallback: match by exact fullDomain when no domainId is available
+        affectedResources = await db
+            .select()
+            .from(siteResources)
+            .where(
+                and(
+                    eq(siteResources.fullDomain, domain),
+                    eq(siteResources.ssl, true)
+                )
+            );
+    }
+
+    if (affectedResources.length === 0) {
+        logger.debug(
+            `acmeCertSync: no affected site resources for cert domain "${domain}"`
+        );
+        return;
+    }
+
+    logger.info(
+        `acmeCertSync: pushing cert update to ${affectedResources.length} affected site resource(s) for domain "${domain}"`
+    );
+
+    for (const resource of affectedResources) {
+        try {
+            // Get all sites for this resource via siteNetworks
+            const resourceSiteRows = resource.networkId
+                ? await db
+                      .select({ siteId: siteNetworks.siteId })
+                      .from(siteNetworks)
+                      .where(eq(siteNetworks.networkId, resource.networkId))
+                : [];
+
+            if (resourceSiteRows.length === 0) {
+                logger.debug(
+                    `acmeCertSync: no sites for resource ${resource.siteResourceId}, skipping`
+                );
+                continue;
+            }
+
+            // Get all clients with access to this resource
+            const resourceClients = await db
+                .select({
+                    clientId: clients.clientId,
+                    pubKey: clients.pubKey,
+                    subnet: clients.subnet
+                })
+                .from(clients)
+                .innerJoin(
+                    clientSiteResourcesAssociationsCache,
+                    eq(
+                        clients.clientId,
+                        clientSiteResourcesAssociationsCache.clientId
+                    )
+                )
+                .where(
+                    eq(
+                        clientSiteResourcesAssociationsCache.siteResourceId,
+                        resource.siteResourceId
+                    )
+                );
+
+            if (resourceClients.length === 0) {
+                logger.debug(
+                    `acmeCertSync: no clients for resource ${resource.siteResourceId}, skipping`
+                );
+                continue;
+            }
+
+            // Invalidate the cert cache so generateSubnetProxyTargetV2 fetches fresh data
+            if (resource.fullDomain) {
+                await cache.del(`cert:${resource.fullDomain}`);
+            }
+
+            // Generate target once — same cert applies to all sites for this resource
+            const newTargets = await generateSubnetProxyTargetV2(
+                resource,
+                resourceClients
+            );
+
+            if (!newTargets) {
+                logger.debug(
+                    `acmeCertSync: could not generate target for resource ${resource.siteResourceId}, skipping`
+                );
+                continue;
+            }
+
+            // Construct the old targets — same routing shape but with the previous cert/key.
+            // The newt only uses destPrefix/sourcePrefixes for removal, but we keep the
+            // semantics correct so the update message accurately reflects what changed.
+            const oldTargets: SubnetProxyTargetV2[] = newTargets.map((t) => ({
+                ...t,
+                tlsCert: oldCertPem ?? undefined,
+                tlsKey: oldKeyPem ?? undefined
+            }));
+
+            // Push update to each site's newt
+            for (const { siteId } of resourceSiteRows) {
+                const [newt] = await db
+                    .select()
+                    .from(newts)
+                    .where(eq(newts.siteId, siteId))
+                    .limit(1);
+
+                if (!newt) {
+                    logger.debug(
+                        `acmeCertSync: no newt found for site ${siteId}, skipping resource ${resource.siteResourceId}`
+                    );
+                    continue;
+                }
+
+                await updateTargets(
+                    newt.newtId,
+                    { oldTargets: oldTargets, newTargets: newTargets },
+                    newt.version
+                );
+
+                logger.info(
+                    `acmeCertSync: pushed cert update to newt for site ${siteId}, resource ${resource.siteResourceId}`
+                );
+            }
+        } catch (err) {
+            logger.error(
+                `acmeCertSync: error pushing cert update for resource ${resource?.siteResourceId}: ${err}`
+            );
+        }
+    }
+}
+
+async function findDomainId(certDomain: string): Promise<string | null> {
+    // Strip wildcard prefix before lookup (*.example.com -> example.com)
+    const lookupDomain = certDomain.startsWith("*.")
+        ? certDomain.slice(2)
+        : certDomain;
+
+    // 1. Exact baseDomain match (any domain type)
+    const exactMatch = await db
+        .select({ domainId: domains.domainId })
+        .from(domains)
+        .where(eq(domains.baseDomain, lookupDomain))
+        .limit(1);
+
+    if (exactMatch.length > 0) {
+        return exactMatch[0].domainId;
+    }
+
+    // 2. Walk up the domain hierarchy looking for a wildcard-type domain whose
+    //    baseDomain is a suffix of the cert domain. e.g. cert "sub.example.com"
+    //    matches a wildcard domain with baseDomain "example.com".
+    const parts = lookupDomain.split(".");
+    for (let i = 1; i < parts.length; i++) {
+        const candidate = parts.slice(i).join(".");
+        if (!candidate) continue;
+
+        const wildcardMatch = await db
+            .select({ domainId: domains.domainId })
+            .from(domains)
+            .where(
+                and(
+                    eq(domains.baseDomain, candidate),
+                    eq(domains.type, "wildcard")
+                )
+            )
+            .limit(1);
+
+        if (wildcardMatch.length > 0) {
+            return wildcardMatch[0].domainId;
+        }
+    }
+
+    return null;
+}
+
+function extractFirstCert(pemBundle: string): string | null {
+    const match = pemBundle.match(
+        /-----BEGIN CERTIFICATE-----[\s\S]+?-----END CERTIFICATE-----/
+    );
+    return match ? match[0] : null;
+}
+
+async function syncAcmeCerts(
+    acmeJsonPath: string,
+    resolver: string
+): Promise<void> {
+    let raw: string;
+    try {
+        raw = fs.readFileSync(acmeJsonPath, "utf8");
+    } catch (err) {
+        logger.debug(`acmeCertSync: could not read ${acmeJsonPath}: ${err}`);
+        return;
+    }
+
+    let acmeJson: AcmeJson;
+    try {
+        acmeJson = JSON.parse(raw);
+    } catch (err) {
+        logger.debug(`acmeCertSync: could not parse acme.json: ${err}`);
+        return;
+    }
+
+    const resolverData = acmeJson[resolver];
+    if (!resolverData || !Array.isArray(resolverData.Certificates)) {
+        logger.debug(
+            `acmeCertSync: no certificates found for resolver "${resolver}"`
+        );
+        return;
+    }
+
+    for (const cert of resolverData.Certificates) {
+        const domain = cert.domain?.main;
+
+        if (!domain) {
+            logger.debug(`acmeCertSync: skipping cert with missing domain`);
+            continue;
+        }
+
+        if (!cert.certificate || !cert.key) {
+            logger.debug(
+                `acmeCertSync: skipping cert for ${domain} - empty certificate or key field`
+            );
+            continue;
+        }
+
+        const certPem = Buffer.from(cert.certificate, "base64").toString(
+            "utf8"
+        );
+        const keyPem = Buffer.from(cert.key, "base64").toString("utf8");
+
+        if (!certPem.trim() || !keyPem.trim()) {
+            logger.debug(
+                `acmeCertSync: skipping cert for ${domain} - blank PEM after base64 decode`
+            );
+            continue;
+        }
+
+        // Check if cert already exists in DB
+        const existing = await db
+            .select()
+            .from(certificates)
+            .where(eq(certificates.domain, domain))
+            .limit(1);
+
+        let oldCertPem: string | null = null;
+        let oldKeyPem: string | null = null;
+
+        if (existing.length > 0 && existing[0].certFile) {
+            try {
+                const storedCertPem = decrypt(
+                    existing[0].certFile,
+                    config.getRawConfig().server.secret!
+                );
+                if (storedCertPem === certPem) {
+                    logger.debug(
+                        `acmeCertSync: cert for ${domain} is unchanged, skipping`
+                    );
+                    continue;
+                }
+                // Cert has changed; capture old values so we can send a correct
+                // update message to the newt after the DB write.
+                oldCertPem = storedCertPem;
+                if (existing[0].keyFile) {
+                    try {
+                        oldKeyPem = decrypt(
+                            existing[0].keyFile,
+                            config.getRawConfig().server.secret!
+                        );
+                    } catch (keyErr) {
+                        logger.debug(
+                            `acmeCertSync: could not decrypt stored key for ${domain}: ${keyErr}`
+                        );
+                    }
+                }
+            } catch (err) {
+                // Decryption failure means we should proceed with the update
+                logger.debug(
+                    `acmeCertSync: could not decrypt stored cert for ${domain}, will update: ${err}`
+                );
+            }
+        }
+
+        // Parse cert expiry from the first cert in the PEM bundle
+        let expiresAt: number | null = null;
+        const firstCertPem = extractFirstCert(certPem);
+        if (firstCertPem) {
+            try {
+                const x509 = new crypto.X509Certificate(firstCertPem);
+                expiresAt = Math.floor(new Date(x509.validTo).getTime() / 1000);
+            } catch (err) {
+                logger.debug(
+                    `acmeCertSync: could not parse cert expiry for ${domain}: ${err}`
+                );
+            }
+        }
+
+        const wildcard = domain.startsWith("*.");
+        const encryptedCert = encrypt(
+            certPem,
+            config.getRawConfig().server.secret!
+        );
+        const encryptedKey = encrypt(
+            keyPem,
+            config.getRawConfig().server.secret!
+        );
+        const now = Math.floor(Date.now() / 1000);
+
+        const domainId = await findDomainId(domain);
+        if (domainId) {
+            logger.debug(
+                `acmeCertSync: resolved domainId "${domainId}" for cert domain "${domain}"`
+            );
+        } else {
+            logger.debug(
+                `acmeCertSync: no matching domain record found for cert domain "${domain}"`
+            );
+        }
+
+        if (existing.length > 0) {
+            await db
+                .update(certificates)
+                .set({
+                    certFile: encryptedCert,
+                    keyFile: encryptedKey,
+                    status: "valid",
+                    expiresAt,
+                    updatedAt: now,
+                    wildcard,
+                    ...(domainId !== null && { domainId })
+                })
+                .where(eq(certificates.domain, domain));
+
+            logger.info(
+                `acmeCertSync: updated certificate for ${domain} (expires ${expiresAt ? new Date(expiresAt * 1000).toISOString() : "unknown"})`
+            );
+
+            await pushCertUpdateToAffectedNewts(
+                domain,
+                domainId,
+                oldCertPem,
+                oldKeyPem
+            );
+        } else {
+            await db.insert(certificates).values({
+                domain,
+                domainId,
+                certFile: encryptedCert,
+                keyFile: encryptedKey,
+                status: "valid",
+                expiresAt,
+                createdAt: now,
+                updatedAt: now,
+                wildcard
+            });
+
+            logger.info(
+                `acmeCertSync: inserted new certificate for ${domain} (expires ${expiresAt ? new Date(expiresAt * 1000).toISOString() : "unknown"})`
+            );
+
+            // For a brand-new cert, push to any SSL resources that were waiting for it
+            await pushCertUpdateToAffectedNewts(domain, domainId, null, null);
+        }
+    }
+}
+
+export function initAcmeCertSync(): void {
+    if (build == "saas") {
+        logger.debug(`acmeCertSync: skipping ACME cert sync in SaaS build`);
+        return;
+    }
+
+    const privateConfigData = privateConfig.getRawPrivateConfig();
+
+    if (!privateConfigData.flags?.enable_acme_cert_sync) {
+        logger.debug(
+            `acmeCertSync: ACME cert sync is disabled by config flag, skipping`
+        );
+        return;
+    }
+
+    if (privateConfigData.flags.use_pangolin_dns) {
+        logger.debug(
+            `acmeCertSync: ACME cert sync requires use_pangolin_dns flag to be disabled, skipping`
+        );
+        return;
+    }
+
+    const acmeJsonPath =
+        privateConfigData.acme?.acme_json_path ??
+        "config/letsencrypt/acme.json";
+    const resolver = privateConfigData.acme?.resolver ?? "letsencrypt";
+    const intervalMs = privateConfigData.acme?.sync_interval_ms ?? 5000;
+
+    logger.info(
+        `acmeCertSync: starting ACME cert sync from "${acmeJsonPath}" using resolver "${resolver}" every ${intervalMs}ms`
+    );
+
+    // Run immediately on init, then on the configured interval
+    syncAcmeCerts(acmeJsonPath, resolver).catch((err) => {
+        logger.error(`acmeCertSync: error during initial sync: ${err}`);
+    });
+
+    setInterval(() => {
+        syncAcmeCerts(acmeJsonPath, resolver).catch((err) => {
+            logger.error(`acmeCertSync: error during sync: ${err}`);
+        });
+    }, intervalMs);
+}

server/private/lib/certificates.ts

@@ -11,23 +11,15 @@
  * This file is not licensed under the AGPLv3.
  */
 
-import config from "./config";
+import privateConfig from "./config";
+import config from "@server/lib/config";
 import { certificates, db } from "@server/db";
 import { and, eq, isNotNull, or, inArray, sql } from "drizzle-orm";
-import { decryptData } from "@server/lib/encryption";
+import { decrypt } from "@server/lib/crypto";
 import logger from "@server/logger";
 import cache from "#private/lib/cache";
 
-let encryptionKeyHex = "";
-let encryptionKey: Buffer;
-function loadEncryptData() {
-    if (encryptionKey) {
-        return; // already loaded
-    }
 
-    encryptionKeyHex = config.getRawPrivateConfig().server.encryption_key;
-    encryptionKey = Buffer.from(encryptionKeyHex, "hex");
-}
 
 // Define the return type for clarity and type safety
 export type CertificateResult = {
@@ -45,7 +37,7 @@ export async function getValidCertificatesForDomains(
     domains: Set<string>,
     useCache: boolean = true
 ): Promise<Array<CertificateResult>> {
-    loadEncryptData(); // Ensure encryption key is loaded
+
 
     const finalResults: CertificateResult[] = [];
     const domainsToQuery = new Set<string>();
@@ -68,7 +60,7 @@ export async function getValidCertificatesForDomains(
 
     // 2. If all domains were resolved from the cache, return early
     if (domainsToQuery.size === 0) {
-        const decryptedResults = decryptFinalResults(finalResults);
+        const decryptedResults = decryptFinalResults(finalResults, config.getRawConfig().server.secret!);
         return decryptedResults;
     }
 
@@ -173,22 +165,23 @@ export async function getValidCertificatesForDomains(
         }
     }
 
-    const decryptedResults = decryptFinalResults(finalResults);
+    const decryptedResults = decryptFinalResults(finalResults, config.getRawConfig().server.secret!);
     return decryptedResults;
 }
 
 function decryptFinalResults(
-    finalResults: CertificateResult[]
+    finalResults: CertificateResult[],
+    secret: string
 ): CertificateResult[] {
     const validCertsDecrypted = finalResults.map((cert) => {
         // Decrypt and save certificate file
-        const decryptedCert = decryptData(
+        const decryptedCert = decrypt(
             cert.certFile!, // is not null from query
-            encryptionKey
+            secret
         );
 
         // Decrypt and save key file
-        const decryptedKey = decryptData(cert.keyFile!, encryptionKey);
+        const decryptedKey = decrypt(cert.keyFile!, secret);
 
         // Return only the certificate data without org information
         return {

server/private/lib/readConfigFile.ts

@@ -34,10 +34,6 @@ export const privateConfigSchema = z.object({
         }),
     server: z
         .object({
-            encryption_key: z
-                .string()
-                .optional()
-                .transform(getEnvOrYaml("SERVER_ENCRYPTION_KEY")),
             reo_client_id: z
                 .string()
                 .optional()
@@ -95,10 +91,21 @@ export const privateConfigSchema = z.object({
         .object({
             enable_redis: z.boolean().optional().default(false),
             use_pangolin_dns: z.boolean().optional().default(false),
-            use_org_only_idp: z.boolean().optional()
+            use_org_only_idp: z.boolean().optional(),
+            enable_acme_cert_sync: z.boolean().optional().default(true)
         })
         .optional()
         .prefault({}),
+    acme: z
+        .object({
+            acme_json_path: z
+                .string()
+                .optional()
+                .default("config/letsencrypt/acme.json"),
+            resolver: z.string().optional().default("letsencrypt"),
+            sync_interval_ms: z.number().optional().default(5000)
+        })
+        .optional(),
     branding: z
         .object({
             app_name: z.string().optional(),

server/private/lib/traefik/getTraefikConfig.ts

@@ -33,7 +33,7 @@ import {
 } from "drizzle-orm";
 import logger from "@server/logger";
 import config from "@server/lib/config";
-import { orgs, resources, sites, Target, targets } from "@server/db";
+import { orgs, resources, sites, siteNetworks, siteResources, Target, targets } from "@server/db";
 import {
     sanitize,
     encodePath,
@@ -267,6 +267,35 @@ export async function getTraefikConfig(
         });
     });
 
+    // Query siteResources in HTTP mode with SSL enabled and aliases — cert generation / HTTPS edge
+    const siteResourcesWithFullDomain = await db
+        .select({
+            siteResourceId: siteResources.siteResourceId,
+            fullDomain: siteResources.fullDomain,
+            mode: siteResources.mode
+        })
+        .from(siteResources)
+        .innerJoin(siteNetworks, eq(siteResources.networkId, siteNetworks.networkId))
+        .innerJoin(sites, eq(siteNetworks.siteId, sites.siteId))
+        .where(
+            and(
+                eq(siteResources.enabled, true),
+                isNotNull(siteResources.fullDomain),
+                eq(siteResources.mode, "http"),
+                eq(siteResources.ssl, true),
+                or(
+                    eq(sites.exitNodeId, exitNodeId),
+                    and(
+                        isNull(sites.exitNodeId),
+                        sql`(${siteTypes.includes("local") ? 1 : 0} = 1)`,
+                        eq(sites.type, "local"),
+                        sql`(${build != "saas" ? 1 : 0} = 1)`
+                    )
+                ),
+                inArray(sites.type, siteTypes)
+            )
+        );
+
     let validCerts: CertificateResult[] = [];
     if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
         // create a list of all domains to get certs for
@@ -276,6 +305,12 @@ export async function getTraefikConfig(
                 domains.add(resource.fullDomain);
             }
         }
+        // Include siteResource aliases so pangolin-dns also fetches certs for them
+        for (const sr of siteResourcesWithFullDomain) {
+            if (sr.fullDomain) {
+                domains.add(sr.fullDomain);
+            }
+        }
         // get the valid certs for these domains
         validCerts = await getValidCertificatesForDomains(domains, true); // we are caching here because this is called often
         // logger.debug(`Valid certs for domains: ${JSON.stringify(validCerts)}`);
@@ -867,6 +902,139 @@ export async function getTraefikConfig(
         }
     }
 
+    // Add Traefik routes for siteResource aliases (HTTP mode + SSL) so that
+    // Traefik generates TLS certificates for those domains even when no
+    // matching resource exists yet.
+    if (siteResourcesWithFullDomain.length > 0) {
+        // Build a set of domains already covered by normal resources
+        const existingFullDomains = new Set<string>();
+        for (const resource of resourcesMap.values()) {
+            if (resource.fullDomain) {
+                existingFullDomains.add(resource.fullDomain);
+            }
+        }
+
+        for (const sr of siteResourcesWithFullDomain) {
+            if (!sr.fullDomain) continue;
+
+            // Skip if this alias is already handled by a resource router
+            if (existingFullDomains.has(sr.fullDomain)) continue;
+
+            const fullDomain = sr.fullDomain;
+            const srKey = `site-resource-cert-${sr.siteResourceId}`;
+            const siteResourceServiceName = `${srKey}-service`;
+            const siteResourceRouterName = `${srKey}-router`;
+            const siteResourceRewriteMiddlewareName = `${srKey}-rewrite`;
+
+            const maintenancePort = config.getRawConfig().server.next_port;
+            const maintenanceHost =
+                config.getRawConfig().server.internal_hostname;
+
+            if (!config_output.http.routers) {
+                config_output.http.routers = {};
+            }
+            if (!config_output.http.services) {
+                config_output.http.services = {};
+            }
+            if (!config_output.http.middlewares) {
+                config_output.http.middlewares = {};
+            }
+
+            // Service pointing at the internal maintenance/Next.js page
+            config_output.http.services[siteResourceServiceName] = {
+                loadBalancer: {
+                    servers: [
+                        {
+                            url: `http://${maintenanceHost}:${maintenancePort}`
+                        }
+                    ],
+                    passHostHeader: true
+                }
+            };
+
+            // Middleware that rewrites any path to /maintenance-screen
+            config_output.http.middlewares[
+                siteResourceRewriteMiddlewareName
+            ] = {
+                replacePathRegex: {
+                    regex: "^/(.*)",
+                    replacement: "/private-maintenance-screen"
+                }
+            };
+
+            // HTTP -> HTTPS redirect so the ACME challenge can be served
+            config_output.http.routers[
+                `${siteResourceRouterName}-redirect`
+            ] = {
+                entryPoints: [
+                    config.getRawConfig().traefik.http_entrypoint
+                ],
+                middlewares: [redirectHttpsMiddlewareName],
+                service: siteResourceServiceName,
+                rule: `Host(\`${fullDomain}\`)`,
+                priority: 100
+            };
+
+            // Determine TLS / cert-resolver configuration
+            let tls: any = {};
+            if (
+                !privateConfig.getRawPrivateConfig().flags.use_pangolin_dns
+            ) {
+                const domainParts = fullDomain.split(".");
+                const wildCard =
+                    domainParts.length <= 2
+                        ? `*.${domainParts.join(".")}`
+                        : `*.${domainParts.slice(1).join(".")}`;
+
+                const globalDefaultResolver =
+                    config.getRawConfig().traefik.cert_resolver;
+                const globalDefaultPreferWildcard =
+                    config.getRawConfig().traefik.prefer_wildcard_cert;
+
+                tls = {
+                    certResolver: globalDefaultResolver,
+                    ...(globalDefaultPreferWildcard
+                        ? { domains: [{ main: wildCard }] }
+                        : {})
+                };
+            } else {
+                // pangolin-dns: only add route if we already have a valid cert
+                const matchingCert = validCerts.find(
+                    (cert) => cert.queriedDomain === fullDomain
+                );
+                if (!matchingCert) {
+                    logger.debug(
+                        `No matching certificate found for siteResource alias: ${fullDomain}`
+                    );
+                    continue;
+                }
+            }
+
+            // HTTPS router — presence of this entry triggers cert generation
+            config_output.http.routers[siteResourceRouterName] = {
+                entryPoints: [
+                    config.getRawConfig().traefik.https_entrypoint
+                ],
+                service: siteResourceServiceName,
+                middlewares: [siteResourceRewriteMiddlewareName],
+                rule: `Host(\`${fullDomain}\`)`,
+                priority: 100,
+                tls
+            };
+
+            // Assets bypass router — lets Next.js static files load without rewrite
+            config_output.http.routers[`${siteResourceRouterName}-assets`] = {
+                entryPoints: [
+                    config.getRawConfig().traefik.https_entrypoint
+                ],
+                service: siteResourceServiceName,
+                rule: `Host(\`${fullDomain}\`) && (PathPrefix(\`/_next\`) || PathRegexp(\`^/__nextjs*\`))`,
+                priority: 101,
+                tls
+            };
+        }
+    }
+
     if (generateLoginPageRouters) {
         const exitNodeLoginPages = await db
             .select({

server/private/routers/hybrid.ts

@@ -24,14 +24,8 @@ import {
     User,
     certificates,
     exitNodeOrgs,
-    RemoteExitNode,
-    olms,
-    newts,
-    clients,
-    sites,
     domains,
     orgDomains,
-    targets,
     loginPage,
     loginPageOrg,
     LoginPage,
@@ -70,12 +64,9 @@ import {
     updateAndGenerateEndpointDestinations,
     updateSiteBandwidth
 } from "@server/routers/gerbil";
-import * as gerbil from "@server/routers/gerbil";
 import logger from "@server/logger";
-import { decryptData } from "@server/lib/encryption";
+import { decrypt } from "@server/lib/crypto";
 import config from "@server/lib/config";
-import privateConfig from "#private/lib/config";
-import * as fs from "fs";
 import { exchangeSession } from "@server/routers/badger";
 import { validateResourceSessionToken } from "@server/auth/sessions/resource";
 import { checkExitNodeOrg, resolveExitNodes } from "#private/lib/exitNodes";
@@ -298,25 +289,11 @@ hybridRouter.get(
     }
 );
 
-let encryptionKeyHex = "";
-let encryptionKey: Buffer;
-function loadEncryptData() {
-    if (encryptionKey) {
-        return; // already loaded
-    }
-
-    encryptionKeyHex =
-        privateConfig.getRawPrivateConfig().server.encryption_key;
-    encryptionKey = Buffer.from(encryptionKeyHex, "hex");
-}
-
 // Get valid certificates for given domains (supports wildcard certs)
 hybridRouter.get(
     "/certificates/domains",
     async (req: Request, res: Response, next: NextFunction) => {
         try {
-            loadEncryptData(); // Ensure encryption key is loaded
-
             const parsed = getCertificatesByDomainsQuerySchema.safeParse(
                 req.query
             );
@@ -447,13 +424,13 @@ hybridRouter.get(
 
             const result = filtered.map((cert) => {
                 // Decrypt and save certificate file
-                const decryptedCert = decryptData(
+                const decryptedCert = decrypt(
                     cert.certFile!, // is not null from query
-                    encryptionKey
+                    config.getRawConfig().server.secret!
                 );
 
                 // Decrypt and save key file
-                const decryptedKey = decryptData(cert.keyFile!, encryptionKey);
+                const decryptedKey = decrypt(cert.keyFile!, config.getRawConfig().server.secret!);
 
                 // Return only the certificate data without org information
                 return {
@@ -833,9 +810,12 @@ hybridRouter.get(
                     )
                 );
 
-            logger.debug(`User ${userId} has roles in org ${orgId}:`, userOrgRoleRows);
+            logger.debug(
+                `User ${userId} has roles in org ${orgId}:`,
+                userOrgRoleRows
+            );
 
-            return response<{ roleId: number, roleName: string }[]>(res, {
+            return response<{ roleId: number; roleName: string }[]>(res, {
                 data: userOrgRoleRows,
                 success: true,
                 error: false,

server/private/routers/newt/handleConnectionLogMessage.ts

@@ -92,9 +92,14 @@ export const handleConnectionLogMessage: MessageHandler = async (context) => {
         return;
     }
 
-    // Look up the org for this site
+    // Look up the org for this site and check retention settings
     const [site] = await db
-        .select({ orgId: sites.orgId, orgSubnet: orgs.subnet })
+        .select({
+            orgId: sites.orgId,
+            orgSubnet: orgs.subnet,
+            settingsLogRetentionDaysConnection:
+                orgs.settingsLogRetentionDaysConnection
+        })
         .from(sites)
         .innerJoin(orgs, eq(sites.orgId, orgs.orgId))
         .where(eq(sites.siteId, newt.siteId));
@@ -108,6 +113,13 @@ export const handleConnectionLogMessage: MessageHandler = async (context) => {
 
     const orgId = site.orgId;
 
+    if (site.settingsLogRetentionDaysConnection === 0) {
+        logger.debug(
+            `Connection log retention is disabled for org ${orgId}, skipping`
+        );
+        return;
+    }
+
     // Extract the CIDR suffix (e.g. "/16") from the org subnet so we can
     // reconstruct the exact subnet string stored on each client record.
     const cidrSuffix = site.orgSubnet?.includes("/")

server/private/routers/newt/handleRequestLogMessage.ts

@@ -0,0 +1,238 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { db } from "@server/db";
+import { MessageHandler } from "@server/routers/ws";
+import { sites, Newt, orgs, clients, clientSitesAssociationsCache } from "@server/db";
+import { and, eq, inArray } from "drizzle-orm";
+import logger from "@server/logger";
+import { inflate } from "zlib";
+import { promisify } from "util";
+import { logRequestAudit } from "@server/routers/badger/logRequestAudit";
+import { getCountryCodeForIp } from "@server/lib/geoip";
+
+export async function flushRequestLogToDb(): Promise<void> {
+    return;
+}
+
+const zlibInflate = promisify(inflate);
+
+interface HTTPRequestLogData {
+    requestId: string;
+    resourceId: number; // siteResourceId
+    timestamp: string; // ISO 8601
+    method: string;
+    scheme: string; // "http" or "https"
+    host: string;
+    path: string;
+    rawQuery?: string;
+    userAgent?: string;
+    sourceAddr: string; // ip:port
+    tls: boolean;
+}
+
+/**
+ * Decompress a base64-encoded zlib-compressed string into parsed JSON.
+ */
+async function decompressRequestLog(
+    compressed: string
+): Promise<HTTPRequestLogData[]> {
+    const compressedBuffer = Buffer.from(compressed, "base64");
+    const decompressed = await zlibInflate(compressedBuffer);
+    const jsonString = decompressed.toString("utf-8");
+    const parsed = JSON.parse(jsonString);
+
+    if (!Array.isArray(parsed)) {
+        throw new Error("Decompressed request log data is not an array");
+    }
+
+    return parsed;
+}
+
+export const handleRequestLogMessage: MessageHandler = async (context) => {
+    const { message, client } = context;
+    const newt = client as Newt;
+
+    if (!newt) {
+        logger.warn("Request log received but no newt client in context");
+        return;
+    }
+
+    if (!newt.siteId) {
+        logger.warn("Request log received but newt has no siteId");
+        return;
+    }
+
+    if (!message.data?.compressed) {
+        logger.warn("Request log message missing compressed data");
+        return;
+    }
+
+    // Look up the org for this site and check retention settings
+    const [site] = await db
+        .select({
+            orgId: sites.orgId,
+            orgSubnet: orgs.subnet,
+            settingsLogRetentionDaysRequest:
+                orgs.settingsLogRetentionDaysRequest
+        })
+        .from(sites)
+        .innerJoin(orgs, eq(sites.orgId, orgs.orgId))
+        .where(eq(sites.siteId, newt.siteId));
+
+    if (!site) {
+        logger.warn(
+            `Request log received but site ${newt.siteId} not found in database`
+        );
+        return;
+    }
+
+    const orgId = site.orgId;
+
+    if (site.settingsLogRetentionDaysRequest === 0) {
+        logger.debug(
+            `Request log retention is disabled for org ${orgId}, skipping`
+        );
+        return;
+    }
+
+    let entries: HTTPRequestLogData[];
+    try {
+        entries = await decompressRequestLog(message.data.compressed);
+    } catch (error) {
+        logger.error("Failed to decompress request log data:", error);
+        return;
+    }
+
+    if (entries.length === 0) {
+        return;
+    }
+
+    logger.debug(`Request log entries: ${JSON.stringify(entries)}`);
+
+    // Build a map from sourceIp → external endpoint string by joining clients
+    // with clientSitesAssociationsCache. The endpoint is the real-world IP:port
+    // of the client device and is used for GeoIP lookup.
+    const ipToEndpoint = new Map<string, string>();
+
+    const cidrSuffix = site.orgSubnet?.includes("/")
+        ? site.orgSubnet.substring(site.orgSubnet.indexOf("/"))
+        : null;
+
+    if (cidrSuffix) {
+        const uniqueSourceAddrs = new Set<string>();
+        for (const entry of entries) {
+            if (entry.sourceAddr) {
+                uniqueSourceAddrs.add(entry.sourceAddr);
+            }
+        }
+
+        if (uniqueSourceAddrs.size > 0) {
+            const subnetQueries = Array.from(uniqueSourceAddrs).map((addr) => {
+                const ip = addr.includes(":") ? addr.split(":")[0] : addr;
+                return `${ip}${cidrSuffix}`;
+            });
+
+            const matchedClients = await db
+                .select({
+                    subnet: clients.subnet,
+                    endpoint: clientSitesAssociationsCache.endpoint
+                })
+                .from(clients)
+                .innerJoin(
+                    clientSitesAssociationsCache,
+                    and(
+                        eq(
+                            clientSitesAssociationsCache.clientId,
+                            clients.clientId
+                        ),
+                        eq(clientSitesAssociationsCache.siteId, newt.siteId)
+                    )
+                )
+                .where(
+                    and(
+                        eq(clients.orgId, orgId),
+                        inArray(clients.subnet, subnetQueries)
+                    )
+                );
+
+            for (const c of matchedClients) {
+                if (c.endpoint) {
+                    const ip = c.subnet.split("/")[0];
+                    ipToEndpoint.set(ip, c.endpoint);
+                }
+            }
+        }
+    }
+
+    for (const entry of entries) {
+        if (
+            !entry.requestId ||
+            !entry.resourceId ||
+            !entry.method ||
+            !entry.scheme ||
+            !entry.host ||
+            !entry.path ||
+            !entry.sourceAddr
+        ) {
+            logger.debug(
+                `Skipping request log entry with missing required fields: ${JSON.stringify(entry)}`
+            );
+            continue;
+        }
+
+        const originalRequestURL =
+            entry.scheme +
+            "://" +
+            entry.host +
+            entry.path +
+            (entry.rawQuery ? "?" + entry.rawQuery : "");
+
+        // Resolve the client's external endpoint for GeoIP lookup.
+        // sourceAddr is the WireGuard IP (possibly ip:port), so strip the port.
+        const sourceIp = entry.sourceAddr.includes(":")
+            ? entry.sourceAddr.split(":")[0]
+            : entry.sourceAddr;
+        const endpoint = ipToEndpoint.get(sourceIp);
+        let location: string | undefined;
+        if (endpoint) {
+            const endpointIp = endpoint.includes(":")
+                ? endpoint.split(":")[0]
+                : endpoint;
+            location = await getCountryCodeForIp(endpointIp);
+        }
+
+        await logRequestAudit(
+            {
+                action: true,
+                reason: 108,
+                siteResourceId: entry.resourceId,
+                orgId,
+                location
+            },
+            {
+                path: entry.path,
+                originalRequestURL,
+                scheme: entry.scheme,
+                host: entry.host,
+                method: entry.method,
+                tls: entry.tls,
+                requestIp: entry.sourceAddr
+            }
+        );
+    }
+
+    logger.debug(
+        `Buffered ${entries.length} request log entry/entries from newt ${newt.newtId} (site ${newt.siteId})`
+    );
+};

server/private/routers/newt/index.ts

@@ -12,3 +12,4 @@
  */
 
 export * from "./handleConnectionLogMessage";
+export * from "./handleRequestLogMessage";

server/private/routers/ssh/signSshKey.ts

@@ -21,7 +21,7 @@ import {
     roles,
     roundTripMessageTracker,
     siteResources,
-    sites,
+    siteNetworks,
     userOrgs
 } from "@server/db";
 import { logAccessAudit } from "#private/lib/logAccessAudit";
@@ -63,10 +63,12 @@ const bodySchema = z
 
 export type SignSshKeyResponse = {
     certificate: string;
+    messageIds: number[];
     messageId: number;
     sshUsername: string;
     sshHost: string;
     resourceId: number;
+    siteIds: number[];
     siteId: number;
     keyId: string;
     validPrincipals: string[];
@@ -260,10 +262,7 @@ export async function signSshKey(
                 .update(userOrgs)
                 .set({ pamUsername: usernameToUse })
                 .where(
-                    and(
-                        eq(userOrgs.orgId, orgId),
-                        eq(userOrgs.userId, userId)
-                    )
+                    and(eq(userOrgs.orgId, orgId), eq(userOrgs.userId, userId))
                 );
         } else {
             usernameToUse = userOrg.pamUsername;
@@ -395,21 +394,12 @@ export async function signSshKey(
             homedir = roleRows[0].sshCreateHomeDir ?? null;
         }
 
-        // get the site
-        const [newt] = await db
-            .select()
-            .from(newts)
-            .where(eq(newts.siteId, resource.siteId))
-            .limit(1);
+        const sites = await db
+            .select({ siteId: siteNetworks.siteId })
+            .from(siteNetworks)
+            .where(eq(siteNetworks.networkId, resource.networkId!));
 
-        if (!newt) {
-            return next(
-                createHttpError(
-                    HttpCode.INTERNAL_SERVER_ERROR,
-                    "Site associated with resource not found"
-                )
-            );
-        }
+        const siteIds = sites.map((site) => site.siteId);
 
         // Sign the public key
         const now = BigInt(Math.floor(Date.now() / 1000));
@@ -423,43 +413,64 @@ export async function signSshKey(
             validBefore: now + validFor
         });
 
-        const [message] = await db
-            .insert(roundTripMessageTracker)
-            .values({
-                wsClientId: newt.newtId,
-                messageType: `newt/pam/connection`,
-                sentAt: Math.floor(Date.now() / 1000)
-            })
-            .returning();
+        const messageIds: number[] = [];
+        for (const siteId of siteIds) {
+            // get the site
+            const [newt] = await db
+                .select()
+                .from(newts)
+                .where(eq(newts.siteId, siteId))
+                .limit(1);
 
-        if (!message) {
-            return next(
-                createHttpError(
-                    HttpCode.INTERNAL_SERVER_ERROR,
-                    "Failed to create message tracker entry"
-                )
-            );
-        }
-
-        await sendToClient(newt.newtId, {
-            type: `newt/pam/connection`,
-            data: {
-                messageId: message.messageId,
-                orgId: orgId,
-                agentPort: resource.authDaemonPort ?? 22123,
-                externalAuthDaemon: resource.authDaemonMode === "remote",
-                agentHost: resource.destination,
-                caCert: caKeys.publicKeyOpenSSH,
-                username: usernameToUse,
-                niceId: resource.niceId,
-                metadata: {
-                    sudoMode: sudoMode,
-                    sudoCommands: parsedSudoCommands,
-                    homedir: homedir,
-                    groups: parsedGroups
-                }
+            if (!newt) {
+                return next(
+                    createHttpError(
+                        HttpCode.INTERNAL_SERVER_ERROR,
+                        "Site associated with resource not found"
+                    )
+                );
             }
-        });
+
+            const [message] = await db
+                .insert(roundTripMessageTracker)
+                .values({
+                    wsClientId: newt.newtId,
+                    messageType: `newt/pam/connection`,
+                    sentAt: Math.floor(Date.now() / 1000)
+                })
+                .returning();
+
+            if (!message) {
+                return next(
+                    createHttpError(
+                        HttpCode.INTERNAL_SERVER_ERROR,
+                        "Failed to create message tracker entry"
+                    )
+                );
+            }
+
+            messageIds.push(message.messageId);
+
+            await sendToClient(newt.newtId, {
+                type: `newt/pam/connection`,
+                data: {
+                    messageId: message.messageId,
+                    orgId: orgId,
+                    agentPort: resource.authDaemonPort ?? 22123,
+                    externalAuthDaemon: resource.authDaemonMode === "remote",
+                    agentHost: resource.destination,
+                    caCert: caKeys.publicKeyOpenSSH,
+                    username: usernameToUse,
+                    niceId: resource.niceId,
+                    metadata: {
+                        sudoMode: sudoMode,
+                        sudoCommands: parsedSudoCommands,
+                        homedir: homedir,
+                        groups: parsedGroups
+                    }
+                }
+            });
+        }
 
         const expiresIn = Number(validFor); // seconds
 
@@ -480,7 +491,7 @@ export async function signSshKey(
             metadata: JSON.stringify({
                 resourceId: resource.siteResourceId,
                 resource: resource.name,
-                siteId: resource.siteId,
+                siteIds: siteIds
             })
         });
 
@@ -494,7 +505,7 @@ export async function signSshKey(
                 : undefined,
             metadata: {
                 resourceName: resource.name,
-                siteId: resource.siteId,
+                siteId: siteIds[0],
                 sshUsername: usernameToUse,
                 sshHost: sshHost
             },
@@ -505,11 +516,13 @@ export async function signSshKey(
         return response<SignSshKeyResponse>(res, {
             data: {
                 certificate: cert.certificate,
-                messageId: message.messageId,
+                messageIds: messageIds,
+                messageId: messageIds[0], // just pick the first one for backward compatibility
                 sshUsername: usernameToUse,
                 sshHost: sshHost,
                 resourceId: resource.siteResourceId,
-                siteId: resource.siteId,
+                siteIds: siteIds,
+                siteId: siteIds[0], // just pick the first one for backward compatibility
                 keyId: cert.keyId,
                 validPrincipals: cert.validPrincipals,
                 validAfter: cert.validAfter.toISOString(),

server/private/routers/ws/messageHandlers.ts

@@ -18,12 +18,13 @@ import {
 } from "#private/routers/remoteExitNode";
 import { MessageHandler } from "@server/routers/ws";
 import { build } from "@server/build";
-import { handleConnectionLogMessage } from "#private/routers/newt";
+import { handleConnectionLogMessage, handleRequestLogMessage } from "#private/routers/newt";
 
 export const messageHandlers: Record<string, MessageHandler> = {
     "remoteExitNode/register": handleRemoteExitNodeRegisterMessage,
     "remoteExitNode/ping": handleRemoteExitNodePingMessage,
     "newt/access-log": handleConnectionLogMessage,
+    "newt/request-log": handleRequestLogMessage,
 };
 
 if (build != "saas") {