From 789b991c569faa7e5513ef3c115fa22540e7c76f Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 12 Apr 2026 15:08:17 -0700 Subject: [PATCH] Logging and http working --- server/db/pg/schema/schema.ts | 1 + server/db/sqlite/schema/schema.ts | 1 + .../newt/handleConnectionLogMessage.ts | 16 +- .../routers/newt/handleRequestLogMessage.ts | 80 +++++++++- .../routers/auditLogs/queryRequestAuditLog.ts | 149 +++++++++++++----- server/routers/auditLogs/types.ts | 1 + server/routers/badger/logRequestAudit.ts | 3 + .../[orgId]/settings/logs/request/page.tsx | 6 +- 8 files changed, 212 insertions(+), 45 deletions(-) diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index aac86c1b9..308a69fc7 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -1004,6 +1004,7 @@ export const requestAuditLog = pgTable( actor: text("actor"), actorId: text("actorId"), resourceId: integer("resourceId"), + siteResourceId: integer("siteResourceId"), ip: text("ip"), location: text("location"), userAgent: text("userAgent"), diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index e58601dc3..fe192014b 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -1104,6 +1104,7 @@ export const requestAuditLog = sqliteTable( actor: text("actor"), actorId: text("actorId"), resourceId: integer("resourceId"), + siteResourceId: integer("siteResourceId"), ip: text("ip"), location: text("location"), userAgent: text("userAgent"), diff --git a/server/private/routers/newt/handleConnectionLogMessage.ts b/server/private/routers/newt/handleConnectionLogMessage.ts index e980f85c9..60a810ee6 100644 --- a/server/private/routers/newt/handleConnectionLogMessage.ts +++ b/server/private/routers/newt/handleConnectionLogMessage.ts @@ -92,9 +92,14 @@ export const handleConnectionLogMessage: MessageHandler = async (context) => { return; } - // Look up the org for this site + // Look up the org for this site and check retention settings const [site] = await db - .select({ orgId: sites.orgId, orgSubnet: orgs.subnet }) + .select({ + orgId: sites.orgId, + orgSubnet: orgs.subnet, + settingsLogRetentionDaysConnection: + orgs.settingsLogRetentionDaysConnection + }) .from(sites) .innerJoin(orgs, eq(sites.orgId, orgs.orgId)) .where(eq(sites.siteId, newt.siteId)); @@ -108,6 +113,13 @@ export const handleConnectionLogMessage: MessageHandler = async (context) => { const orgId = site.orgId; + if (site.settingsLogRetentionDaysConnection === 0) { + logger.debug( + `Connection log retention is disabled for org ${orgId}, skipping` + ); + return; + } + // Extract the CIDR suffix (e.g. "/16") from the org subnet so we can // reconstruct the exact subnet string stored on each client record. const cidrSuffix = site.orgSubnet?.includes("/") diff --git a/server/private/routers/newt/handleRequestLogMessage.ts b/server/private/routers/newt/handleRequestLogMessage.ts index 6cbb18b72..42f1baf2c 100644 --- a/server/private/routers/newt/handleRequestLogMessage.ts +++ b/server/private/routers/newt/handleRequestLogMessage.ts @@ -13,12 +13,13 @@ import { db } from "@server/db"; import { MessageHandler } from "@server/routers/ws"; -import { sites, Newt, orgs } from "@server/db"; -import { eq } from "drizzle-orm"; +import { sites, Newt, orgs, clients, clientSitesAssociationsCache } from "@server/db"; +import { and, eq, inArray } from "drizzle-orm"; import logger from "@server/logger"; import { inflate } from "zlib"; import { promisify } from "util"; import { logRequestAudit } from "@server/routers/badger/logRequestAudit"; +import { getCountryCodeForIp } from "@server/lib/geoip"; export async function flushRequestLogToDb(): Promise { return; @@ -81,6 +82,7 @@ export const handleRequestLogMessage: MessageHandler = async (context) => { const [site] = await db .select({ orgId: sites.orgId, + orgSubnet: orgs.subnet, settingsLogRetentionDaysRequest: orgs.settingsLogRetentionDaysRequest }) @@ -118,6 +120,61 @@ export const handleRequestLogMessage: MessageHandler = async (context) => { logger.debug(`Request log entries: ${JSON.stringify(entries)}`); + // Build a map from sourceIp → external endpoint string by joining clients + // with clientSitesAssociationsCache. The endpoint is the real-world IP:port + // of the client device and is used for GeoIP lookup. + const ipToEndpoint = new Map(); + + const cidrSuffix = site.orgSubnet?.includes("/") + ? site.orgSubnet.substring(site.orgSubnet.indexOf("/")) + : null; + + if (cidrSuffix) { + const uniqueSourceAddrs = new Set(); + for (const entry of entries) { + if (entry.sourceAddr) { + uniqueSourceAddrs.add(entry.sourceAddr); + } + } + + if (uniqueSourceAddrs.size > 0) { + const subnetQueries = Array.from(uniqueSourceAddrs).map((addr) => { + const ip = addr.includes(":") ? addr.split(":")[0] : addr; + return `${ip}${cidrSuffix}`; + }); + + const matchedClients = await db + .select({ + subnet: clients.subnet, + endpoint: clientSitesAssociationsCache.endpoint + }) + .from(clients) + .innerJoin( + clientSitesAssociationsCache, + and( + eq( + clientSitesAssociationsCache.clientId, + clients.clientId + ), + eq(clientSitesAssociationsCache.siteId, newt.siteId) + ) + ) + .where( + and( + eq(clients.orgId, orgId), + inArray(clients.subnet, subnetQueries) + ) + ); + + for (const c of matchedClients) { + if (c.endpoint) { + const ip = c.subnet.split("/")[0]; + ipToEndpoint.set(ip, c.endpoint); + } + } + } + } + for (const entry of entries) { if ( !entry.requestId || @@ -141,12 +198,27 @@ export const handleRequestLogMessage: MessageHandler = async (context) => { entry.path + (entry.rawQuery ? "?" + entry.rawQuery : ""); + // Resolve the client's external endpoint for GeoIP lookup. + // sourceAddr is the WireGuard IP (possibly ip:port), so strip the port. + const sourceIp = entry.sourceAddr.includes(":") + ? entry.sourceAddr.split(":")[0] + : entry.sourceAddr; + const endpoint = ipToEndpoint.get(sourceIp); + let location: string | undefined; + if (endpoint) { + const endpointIp = endpoint.includes(":") + ? endpoint.split(":")[0] + : endpoint; + location = await getCountryCodeForIp(endpointIp); + } + await logRequestAudit( { action: true, reason: 108, - resourceId: entry.resourceId, - orgId + siteResourceId: entry.resourceId, + orgId, + location }, { path: entry.path, diff --git a/server/routers/auditLogs/queryRequestAuditLog.ts b/server/routers/auditLogs/queryRequestAuditLog.ts index 176a9e5d3..000ec9815 100644 --- a/server/routers/auditLogs/queryRequestAuditLog.ts +++ b/server/routers/auditLogs/queryRequestAuditLog.ts @@ -1,8 +1,8 @@ -import { logsDb, primaryLogsDb, requestAuditLog, resources, db, primaryDb } from "@server/db"; +import { logsDb, primaryLogsDb, requestAuditLog, resources, siteResources, db, primaryDb } from "@server/db"; import { registry } from "@server/openApi"; import { NextFunction } from "express"; import { Request, Response } from "express"; -import { eq, gt, lt, and, count, desc, inArray } from "drizzle-orm"; +import { eq, gt, lt, and, count, desc, inArray, isNull, or } from "drizzle-orm"; import { OpenAPITags } from "@server/openApi"; import { z } from "zod"; import createHttpError from "http-errors"; @@ -92,7 +92,10 @@ function getWhere(data: Q) { lt(requestAuditLog.timestamp, data.timeEnd), eq(requestAuditLog.orgId, data.orgId), data.resourceId - ? eq(requestAuditLog.resourceId, data.resourceId) + ? or( + eq(requestAuditLog.resourceId, data.resourceId), + eq(requestAuditLog.siteResourceId, data.resourceId) + ) : undefined, data.actor ? eq(requestAuditLog.actor, data.actor) : undefined, data.method ? eq(requestAuditLog.method, data.method) : undefined, @@ -110,15 +113,16 @@ export function queryRequest(data: Q) { return primaryLogsDb .select({ id: requestAuditLog.id, - timestamp: requestAuditLog.timestamp, - orgId: requestAuditLog.orgId, - action: requestAuditLog.action, - reason: requestAuditLog.reason, - actorType: requestAuditLog.actorType, - actor: requestAuditLog.actor, - actorId: requestAuditLog.actorId, - resourceId: requestAuditLog.resourceId, - ip: requestAuditLog.ip, + timestamp: requestAuditLog.timestamp, + orgId: requestAuditLog.orgId, + action: requestAuditLog.action, + reason: requestAuditLog.reason, + actorType: requestAuditLog.actorType, + actor: requestAuditLog.actor, + actorId: requestAuditLog.actorId, + resourceId: requestAuditLog.resourceId, + siteResourceId: requestAuditLog.siteResourceId, + ip: requestAuditLog.ip, location: requestAuditLog.location, userAgent: requestAuditLog.userAgent, metadata: requestAuditLog.metadata, @@ -137,37 +141,73 @@ export function queryRequest(data: Q) { } async function enrichWithResourceDetails(logs: Awaited>) { - // If logs database is the same as main database, we can do a join - // Otherwise, we need to fetch resource details separately const resourceIds = logs .map(log => log.resourceId) .filter((id): id is number => id !== null && id !== undefined); - if (resourceIds.length === 0) { + const siteResourceIds = logs + .filter(log => log.resourceId == null && log.siteResourceId != null) + .map(log => log.siteResourceId) + .filter((id): id is number => id !== null && id !== undefined); + + if (resourceIds.length === 0 && siteResourceIds.length === 0) { return logs.map(log => ({ ...log, resourceName: null, resourceNiceId: null })); } - // Fetch resource details from main database - const resourceDetails = await primaryDb - .select({ - resourceId: resources.resourceId, - name: resources.name, - niceId: resources.niceId - }) - .from(resources) - .where(inArray(resources.resourceId, resourceIds)); + const resourceMap = new Map(); - // Create a map for quick lookup - const resourceMap = new Map( - resourceDetails.map(r => [r.resourceId, { name: r.name, niceId: r.niceId }]) - ); + if (resourceIds.length > 0) { + const resourceDetails = await primaryDb + .select({ + resourceId: resources.resourceId, + name: resources.name, + niceId: resources.niceId + }) + .from(resources) + .where(inArray(resources.resourceId, resourceIds)); + + for (const r of resourceDetails) { + resourceMap.set(r.resourceId, { name: r.name, niceId: r.niceId }); + } + } + + const siteResourceMap = new Map(); + + if (siteResourceIds.length > 0) { + const siteResourceDetails = await primaryDb + .select({ + siteResourceId: siteResources.siteResourceId, + name: siteResources.name, + niceId: siteResources.niceId + }) + .from(siteResources) + .where(inArray(siteResources.siteResourceId, siteResourceIds)); + + for (const r of siteResourceDetails) { + siteResourceMap.set(r.siteResourceId, { name: r.name, niceId: r.niceId }); + } + } // Enrich logs with resource details - return logs.map(log => ({ - ...log, - resourceName: log.resourceId ? resourceMap.get(log.resourceId)?.name ?? null : null, - resourceNiceId: log.resourceId ? resourceMap.get(log.resourceId)?.niceId ?? null : null - })); + return logs.map(log => { + if (log.resourceId != null) { + const details = resourceMap.get(log.resourceId); + return { + ...log, + resourceName: details?.name ?? null, + resourceNiceId: details?.niceId ?? null + }; + } else if (log.siteResourceId != null) { + const details = siteResourceMap.get(log.siteResourceId); + return { + ...log, + resourceId: log.siteResourceId, + resourceName: details?.name ?? null, + resourceNiceId: details?.niceId ?? null + }; + } + return { ...log, resourceName: null, resourceNiceId: null }; + }); } export function countRequestQuery(data: Q) { @@ -211,7 +251,8 @@ async function queryUniqueFilterAttributes( uniqueLocations, uniqueHosts, uniquePaths, - uniqueResources + uniqueResources, + uniqueSiteResources ] = await Promise.all([ primaryLogsDb .selectDistinct({ actor: requestAuditLog.actor }) @@ -239,6 +280,13 @@ async function queryUniqueFilterAttributes( }) .from(requestAuditLog) .where(baseConditions) + .limit(DISTINCT_LIMIT + 1), + primaryLogsDb + .selectDistinct({ + id: requestAuditLog.siteResourceId + }) + .from(requestAuditLog) + .where(and(baseConditions, isNull(requestAuditLog.resourceId))) .limit(DISTINCT_LIMIT + 1) ]); @@ -259,6 +307,10 @@ async function queryUniqueFilterAttributes( .map(row => row.id) .filter((id): id is number => id !== null); + const siteResourceIds = uniqueSiteResources + .map(row => row.id) + .filter((id): id is number => id !== null); + let resourcesWithNames: Array<{ id: number; name: string | null }> = []; if (resourceIds.length > 0) { @@ -270,10 +322,31 @@ async function queryUniqueFilterAttributes( .from(resources) .where(inArray(resources.resourceId, resourceIds)); - resourcesWithNames = resourceDetails.map(r => ({ - id: r.resourceId, - name: r.name - })); + resourcesWithNames = [ + ...resourcesWithNames, + ...resourceDetails.map(r => ({ + id: r.resourceId, + name: r.name + })) + ]; + } + + if (siteResourceIds.length > 0) { + const siteResourceDetails = await primaryDb + .select({ + siteResourceId: siteResources.siteResourceId, + name: siteResources.name + }) + .from(siteResources) + .where(inArray(siteResources.siteResourceId, siteResourceIds)); + + resourcesWithNames = [ + ...resourcesWithNames, + ...siteResourceDetails.map(r => ({ + id: r.siteResourceId, + name: r.name + })) + ]; } return { diff --git a/server/routers/auditLogs/types.ts b/server/routers/auditLogs/types.ts index 4c278cba5..972eebfe3 100644 --- a/server/routers/auditLogs/types.ts +++ b/server/routers/auditLogs/types.ts @@ -28,6 +28,7 @@ export type QueryRequestAuditLogResponse = { actor: string | null; actorId: string | null; resourceId: number | null; + siteResourceId: number | null; resourceNiceId: string | null; resourceName: string | null; ip: string | null; diff --git a/server/routers/badger/logRequestAudit.ts b/server/routers/badger/logRequestAudit.ts index db4c17939..884fb7ae4 100644 --- a/server/routers/badger/logRequestAudit.ts +++ b/server/routers/badger/logRequestAudit.ts @@ -39,6 +39,7 @@ const auditLogBuffer: Array<{ metadata: any; action: boolean; resourceId?: number; + siteResourceId?: number; reason: number; location?: string; originalRequestURL: string; @@ -187,6 +188,7 @@ export async function logRequestAudit( action: boolean; reason: number; resourceId?: number; + siteResourceId?: number; orgId?: string; location?: string; user?: { username: string; userId: string }; @@ -263,6 +265,7 @@ export async function logRequestAudit( metadata: sanitizeString(metadata), action: data.action, resourceId: data.resourceId, + siteResourceId: data.siteResourceId, reason: data.reason, location: sanitizeString(data.location), originalRequestURL: sanitizeString(body.originalRequestURL) ?? "", diff --git a/src/app/[orgId]/settings/logs/request/page.tsx b/src/app/[orgId]/settings/logs/request/page.tsx index 061995811..c57914c8a 100644 --- a/src/app/[orgId]/settings/logs/request/page.tsx +++ b/src/app/[orgId]/settings/logs/request/page.tsx @@ -512,7 +512,11 @@ export default function GeneralPage() { cell: ({ row }) => { return ( e.stopPropagation()} >