Merge branch 'private-http-ha' into alerting-rules

server/private/routers/hybrid.ts

@@ -24,14 +24,8 @@ import {
     User,
     certificates,
     exitNodeOrgs,
-    RemoteExitNode,
-    olms,
-    newts,
-    clients,
-    sites,
     domains,
     orgDomains,
-    targets,
     loginPage,
     loginPageOrg,
     LoginPage,
@@ -70,12 +64,9 @@ import {
     updateAndGenerateEndpointDestinations,
     updateSiteBandwidth
 } from "@server/routers/gerbil";
-import * as gerbil from "@server/routers/gerbil";
 import logger from "@server/logger";
-import { decryptData } from "@server/lib/encryption";
+import { decrypt } from "@server/lib/crypto";
 import config from "@server/lib/config";
-import privateConfig from "#private/lib/config";
-import * as fs from "fs";
 import { exchangeSession } from "@server/routers/badger";
 import { validateResourceSessionToken } from "@server/auth/sessions/resource";
 import { checkExitNodeOrg, resolveExitNodes } from "#private/lib/exitNodes";
@@ -298,25 +289,11 @@ hybridRouter.get(
     }
 );
 
-let encryptionKeyHex = "";
-let encryptionKey: Buffer;
-function loadEncryptData() {
-    if (encryptionKey) {
-        return; // already loaded
-    }
-
-    encryptionKeyHex =
-        privateConfig.getRawPrivateConfig().server.encryption_key;
-    encryptionKey = Buffer.from(encryptionKeyHex, "hex");
-}
-
 // Get valid certificates for given domains (supports wildcard certs)
 hybridRouter.get(
     "/certificates/domains",
     async (req: Request, res: Response, next: NextFunction) => {
         try {
-            loadEncryptData(); // Ensure encryption key is loaded
-
             const parsed = getCertificatesByDomainsQuerySchema.safeParse(
                 req.query
             );
@@ -447,13 +424,13 @@ hybridRouter.get(
 
             const result = filtered.map((cert) => {
                 // Decrypt and save certificate file
-                const decryptedCert = decryptData(
+                const decryptedCert = decrypt(
                     cert.certFile!, // is not null from query
-                    encryptionKey
+                    config.getRawConfig().server.secret!
                 );
 
                 // Decrypt and save key file
-                const decryptedKey = decryptData(cert.keyFile!, encryptionKey);
+                const decryptedKey = decrypt(cert.keyFile!, config.getRawConfig().server.secret!);
 
                 // Return only the certificate data without org information
                 return {
@@ -833,9 +810,12 @@ hybridRouter.get(
                     )
                 );
 
-            logger.debug(`User ${userId} has roles in org ${orgId}:`, userOrgRoleRows);
+            logger.debug(
+                `User ${userId} has roles in org ${orgId}:`,
+                userOrgRoleRows
+            );
 
-            return response<{ roleId: number, roleName: string }[]>(res, {
+            return response<{ roleId: number; roleName: string }[]>(res, {
                 data: userOrgRoleRows,
                 success: true,
                 error: false,

server/private/routers/newt/handleConnectionLogMessage.ts

@@ -92,9 +92,14 @@ export const handleConnectionLogMessage: MessageHandler = async (context) => {
         return;
     }
 
-    // Look up the org for this site
+    // Look up the org for this site and check retention settings
     const [site] = await db
-        .select({ orgId: sites.orgId, orgSubnet: orgs.subnet })
+        .select({
+            orgId: sites.orgId,
+            orgSubnet: orgs.subnet,
+            settingsLogRetentionDaysConnection:
+                orgs.settingsLogRetentionDaysConnection
+        })
         .from(sites)
         .innerJoin(orgs, eq(sites.orgId, orgs.orgId))
         .where(eq(sites.siteId, newt.siteId));
@@ -108,6 +113,13 @@ export const handleConnectionLogMessage: MessageHandler = async (context) => {
 
     const orgId = site.orgId;
 
+    if (site.settingsLogRetentionDaysConnection === 0) {
+        logger.debug(
+            `Connection log retention is disabled for org ${orgId}, skipping`
+        );
+        return;
+    }
+
     // Extract the CIDR suffix (e.g. "/16") from the org subnet so we can
     // reconstruct the exact subnet string stored on each client record.
     const cidrSuffix = site.orgSubnet?.includes("/")

server/private/routers/newt/handleRequestLogMessage.ts

@@ -0,0 +1,238 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { db } from "@server/db";
+import { MessageHandler } from "@server/routers/ws";
+import { sites, Newt, orgs, clients, clientSitesAssociationsCache } from "@server/db";
+import { and, eq, inArray } from "drizzle-orm";
+import logger from "@server/logger";
+import { inflate } from "zlib";
+import { promisify } from "util";
+import { logRequestAudit } from "@server/routers/badger/logRequestAudit";
+import { getCountryCodeForIp } from "@server/lib/geoip";
+
+export async function flushRequestLogToDb(): Promise<void> {
+    return;
+}
+
+const zlibInflate = promisify(inflate);
+
+interface HTTPRequestLogData {
+    requestId: string;
+    resourceId: number; // siteResourceId
+    timestamp: string; // ISO 8601
+    method: string;
+    scheme: string; // "http" or "https"
+    host: string;
+    path: string;
+    rawQuery?: string;
+    userAgent?: string;
+    sourceAddr: string; // ip:port
+    tls: boolean;
+}
+
+/**
+ * Decompress a base64-encoded zlib-compressed string into parsed JSON.
+ */
+async function decompressRequestLog(
+    compressed: string
+): Promise<HTTPRequestLogData[]> {
+    const compressedBuffer = Buffer.from(compressed, "base64");
+    const decompressed = await zlibInflate(compressedBuffer);
+    const jsonString = decompressed.toString("utf-8");
+    const parsed = JSON.parse(jsonString);
+
+    if (!Array.isArray(parsed)) {
+        throw new Error("Decompressed request log data is not an array");
+    }
+
+    return parsed;
+}
+
+export const handleRequestLogMessage: MessageHandler = async (context) => {
+    const { message, client } = context;
+    const newt = client as Newt;
+
+    if (!newt) {
+        logger.warn("Request log received but no newt client in context");
+        return;
+    }
+
+    if (!newt.siteId) {
+        logger.warn("Request log received but newt has no siteId");
+        return;
+    }
+
+    if (!message.data?.compressed) {
+        logger.warn("Request log message missing compressed data");
+        return;
+    }
+
+    // Look up the org for this site and check retention settings
+    const [site] = await db
+        .select({
+            orgId: sites.orgId,
+            orgSubnet: orgs.subnet,
+            settingsLogRetentionDaysRequest:
+                orgs.settingsLogRetentionDaysRequest
+        })
+        .from(sites)
+        .innerJoin(orgs, eq(sites.orgId, orgs.orgId))
+        .where(eq(sites.siteId, newt.siteId));
+
+    if (!site) {
+        logger.warn(
+            `Request log received but site ${newt.siteId} not found in database`
+        );
+        return;
+    }
+
+    const orgId = site.orgId;
+
+    if (site.settingsLogRetentionDaysRequest === 0) {
+        logger.debug(
+            `Request log retention is disabled for org ${orgId}, skipping`
+        );
+        return;
+    }
+
+    let entries: HTTPRequestLogData[];
+    try {
+        entries = await decompressRequestLog(message.data.compressed);
+    } catch (error) {
+        logger.error("Failed to decompress request log data:", error);
+        return;
+    }
+
+    if (entries.length === 0) {
+        return;
+    }
+
+    logger.debug(`Request log entries: ${JSON.stringify(entries)}`);
+
+    // Build a map from sourceIp → external endpoint string by joining clients
+    // with clientSitesAssociationsCache. The endpoint is the real-world IP:port
+    // of the client device and is used for GeoIP lookup.
+    const ipToEndpoint = new Map<string, string>();
+
+    const cidrSuffix = site.orgSubnet?.includes("/")
+        ? site.orgSubnet.substring(site.orgSubnet.indexOf("/"))
+        : null;
+
+    if (cidrSuffix) {
+        const uniqueSourceAddrs = new Set<string>();
+        for (const entry of entries) {
+            if (entry.sourceAddr) {
+                uniqueSourceAddrs.add(entry.sourceAddr);
+            }
+        }
+
+        if (uniqueSourceAddrs.size > 0) {
+            const subnetQueries = Array.from(uniqueSourceAddrs).map((addr) => {
+                const ip = addr.includes(":") ? addr.split(":")[0] : addr;
+                return `${ip}${cidrSuffix}`;
+            });
+
+            const matchedClients = await db
+                .select({
+                    subnet: clients.subnet,
+                    endpoint: clientSitesAssociationsCache.endpoint
+                })
+                .from(clients)
+                .innerJoin(
+                    clientSitesAssociationsCache,
+                    and(
+                        eq(
+                            clientSitesAssociationsCache.clientId,
+                            clients.clientId
+                        ),
+                        eq(clientSitesAssociationsCache.siteId, newt.siteId)
+                    )
+                )
+                .where(
+                    and(
+                        eq(clients.orgId, orgId),
+                        inArray(clients.subnet, subnetQueries)
+                    )
+                );
+
+            for (const c of matchedClients) {
+                if (c.endpoint) {
+                    const ip = c.subnet.split("/")[0];
+                    ipToEndpoint.set(ip, c.endpoint);
+                }
+            }
+        }
+    }
+
+    for (const entry of entries) {
+        if (
+            !entry.requestId ||
+            !entry.resourceId ||
+            !entry.method ||
+            !entry.scheme ||
+            !entry.host ||
+            !entry.path ||
+            !entry.sourceAddr
+        ) {
+            logger.debug(
+                `Skipping request log entry with missing required fields: ${JSON.stringify(entry)}`
+            );
+            continue;
+        }
+
+        const originalRequestURL =
+            entry.scheme +
+            "://" +
+            entry.host +
+            entry.path +
+            (entry.rawQuery ? "?" + entry.rawQuery : "");
+
+        // Resolve the client's external endpoint for GeoIP lookup.
+        // sourceAddr is the WireGuard IP (possibly ip:port), so strip the port.
+        const sourceIp = entry.sourceAddr.includes(":")
+            ? entry.sourceAddr.split(":")[0]
+            : entry.sourceAddr;
+        const endpoint = ipToEndpoint.get(sourceIp);
+        let location: string | undefined;
+        if (endpoint) {
+            const endpointIp = endpoint.includes(":")
+                ? endpoint.split(":")[0]
+                : endpoint;
+            location = await getCountryCodeForIp(endpointIp);
+        }
+
+        await logRequestAudit(
+            {
+                action: true,
+                reason: 108,
+                siteResourceId: entry.resourceId,
+                orgId,
+                location
+            },
+            {
+                path: entry.path,
+                originalRequestURL,
+                scheme: entry.scheme,
+                host: entry.host,
+                method: entry.method,
+                tls: entry.tls,
+                requestIp: entry.sourceAddr
+            }
+        );
+    }
+
+    logger.debug(
+        `Buffered ${entries.length} request log entry/entries from newt ${newt.newtId} (site ${newt.siteId})`
+    );
+};

server/private/routers/newt/index.ts

@@ -12,3 +12,4 @@
  */
 
 export * from "./handleConnectionLogMessage";
+export * from "./handleRequestLogMessage";

server/private/routers/ssh/signSshKey.ts

@@ -21,7 +21,7 @@ import {
     roles,
     roundTripMessageTracker,
     siteResources,
-    sites,
+    siteNetworks,
     userOrgs
 } from "@server/db";
 import { logAccessAudit } from "#private/lib/logAccessAudit";
@@ -63,10 +63,12 @@ const bodySchema = z
 
 export type SignSshKeyResponse = {
     certificate: string;
+    messageIds: number[];
     messageId: number;
     sshUsername: string;
     sshHost: string;
     resourceId: number;
+    siteIds: number[];
     siteId: number;
     keyId: string;
     validPrincipals: string[];
@@ -260,10 +262,7 @@ export async function signSshKey(
                 .update(userOrgs)
                 .set({ pamUsername: usernameToUse })
                 .where(
-                    and(
-                        eq(userOrgs.orgId, orgId),
-                        eq(userOrgs.userId, userId)
-                    )
+                    and(eq(userOrgs.orgId, orgId), eq(userOrgs.userId, userId))
                 );
         } else {
             usernameToUse = userOrg.pamUsername;
@@ -395,21 +394,12 @@ export async function signSshKey(
             homedir = roleRows[0].sshCreateHomeDir ?? null;
         }
 
-        // get the site
-        const [newt] = await db
-            .select()
-            .from(newts)
-            .where(eq(newts.siteId, resource.siteId))
-            .limit(1);
+        const sites = await db
+            .select({ siteId: siteNetworks.siteId })
+            .from(siteNetworks)
+            .where(eq(siteNetworks.networkId, resource.networkId!));
 
-        if (!newt) {
-            return next(
-                createHttpError(
-                    HttpCode.INTERNAL_SERVER_ERROR,
-                    "Site associated with resource not found"
-                )
-            );
-        }
+        const siteIds = sites.map((site) => site.siteId);
 
         // Sign the public key
         const now = BigInt(Math.floor(Date.now() / 1000));
@@ -423,43 +413,64 @@ export async function signSshKey(
             validBefore: now + validFor
         });
 
-        const [message] = await db
-            .insert(roundTripMessageTracker)
-            .values({
-                wsClientId: newt.newtId,
-                messageType: `newt/pam/connection`,
-                sentAt: Math.floor(Date.now() / 1000)
-            })
-            .returning();
+        const messageIds: number[] = [];
+        for (const siteId of siteIds) {
+            // get the site
+            const [newt] = await db
+                .select()
+                .from(newts)
+                .where(eq(newts.siteId, siteId))
+                .limit(1);
 
-        if (!message) {
-            return next(
-                createHttpError(
-                    HttpCode.INTERNAL_SERVER_ERROR,
-                    "Failed to create message tracker entry"
-                )
-            );
-        }
-
-        await sendToClient(newt.newtId, {
-            type: `newt/pam/connection`,
-            data: {
-                messageId: message.messageId,
-                orgId: orgId,
-                agentPort: resource.authDaemonPort ?? 22123,
-                externalAuthDaemon: resource.authDaemonMode === "remote",
-                agentHost: resource.destination,
-                caCert: caKeys.publicKeyOpenSSH,
-                username: usernameToUse,
-                niceId: resource.niceId,
-                metadata: {
-                    sudoMode: sudoMode,
-                    sudoCommands: parsedSudoCommands,
-                    homedir: homedir,
-                    groups: parsedGroups
-                }
+            if (!newt) {
+                return next(
+                    createHttpError(
+                        HttpCode.INTERNAL_SERVER_ERROR,
+                        "Site associated with resource not found"
+                    )
+                );
             }
-        });
+
+            const [message] = await db
+                .insert(roundTripMessageTracker)
+                .values({
+                    wsClientId: newt.newtId,
+                    messageType: `newt/pam/connection`,
+                    sentAt: Math.floor(Date.now() / 1000)
+                })
+                .returning();
+
+            if (!message) {
+                return next(
+                    createHttpError(
+                        HttpCode.INTERNAL_SERVER_ERROR,
+                        "Failed to create message tracker entry"
+                    )
+                );
+            }
+
+            messageIds.push(message.messageId);
+
+            await sendToClient(newt.newtId, {
+                type: `newt/pam/connection`,
+                data: {
+                    messageId: message.messageId,
+                    orgId: orgId,
+                    agentPort: resource.authDaemonPort ?? 22123,
+                    externalAuthDaemon: resource.authDaemonMode === "remote",
+                    agentHost: resource.destination,
+                    caCert: caKeys.publicKeyOpenSSH,
+                    username: usernameToUse,
+                    niceId: resource.niceId,
+                    metadata: {
+                        sudoMode: sudoMode,
+                        sudoCommands: parsedSudoCommands,
+                        homedir: homedir,
+                        groups: parsedGroups
+                    }
+                }
+            });
+        }
 
         const expiresIn = Number(validFor); // seconds
 
@@ -480,7 +491,7 @@ export async function signSshKey(
             metadata: JSON.stringify({
                 resourceId: resource.siteResourceId,
                 resource: resource.name,
-                siteId: resource.siteId,
+                siteIds: siteIds
             })
         });
 
@@ -494,7 +505,7 @@ export async function signSshKey(
                 : undefined,
             metadata: {
                 resourceName: resource.name,
-                siteId: resource.siteId,
+                siteId: siteIds[0],
                 sshUsername: usernameToUse,
                 sshHost: sshHost
             },
@@ -505,11 +516,13 @@ export async function signSshKey(
         return response<SignSshKeyResponse>(res, {
             data: {
                 certificate: cert.certificate,
-                messageId: message.messageId,
+                messageIds: messageIds,
+                messageId: messageIds[0], // just pick the first one for backward compatibility
                 sshUsername: usernameToUse,
                 sshHost: sshHost,
                 resourceId: resource.siteResourceId,
-                siteId: resource.siteId,
+                siteIds: siteIds,
+                siteId: siteIds[0], // just pick the first one for backward compatibility
                 keyId: cert.keyId,
                 validPrincipals: cert.validPrincipals,
                 validAfter: cert.validAfter.toISOString(),

server/private/routers/ws/messageHandlers.ts

@@ -18,12 +18,13 @@ import {
 } from "#private/routers/remoteExitNode";
 import { MessageHandler } from "@server/routers/ws";
 import { build } from "@server/build";
-import { handleConnectionLogMessage } from "#private/routers/newt";
+import { handleConnectionLogMessage, handleRequestLogMessage } from "#private/routers/newt";
 
 export const messageHandlers: Record<string, MessageHandler> = {
     "remoteExitNode/register": handleRemoteExitNodeRegisterMessage,
     "remoteExitNode/ping": handleRemoteExitNodePingMessage,
     "newt/access-log": handleConnectionLogMessage,
+    "newt/request-log": handleRequestLogMessage,
 };
 
 if (build != "saas") {