Try to route logs requests to a different database

server/routers/auditLogs/queryRequestAnalytics.ts

@@ -1,4 +1,4 @@
-import { db, requestAuditLog, driver, primaryDb } from "@server/db";
+import { logsDb, requestAuditLog, driver, primaryLogsDb } from "@server/db";
 import { registry } from "@server/openApi";
 import { NextFunction } from "express";
 import { Request, Response } from "express";
@@ -74,12 +74,12 @@ async function query(query: Q) {
         );
     }
 
-    const [all] = await primaryDb
+    const [all] = await primaryLogsDb
         .select({ total: count() })
         .from(requestAuditLog)
         .where(baseConditions);
 
-    const [blocked] = await primaryDb
+    const [blocked] = await primaryLogsDb
         .select({ total: count() })
         .from(requestAuditLog)
         .where(and(baseConditions, eq(requestAuditLog.action, false)));
@@ -90,7 +90,7 @@ async function query(query: Q) {
 
     const DISTINCT_LIMIT = 500;
 
-    const requestsPerCountry = await primaryDb
+    const requestsPerCountry = await primaryLogsDb
         .selectDistinct({
             code: requestAuditLog.location,
             count: totalQ
@@ -118,7 +118,7 @@ async function query(query: Q) {
     const booleanTrue = driver === "pg" ? sql`true` : sql`1`;
     const booleanFalse = driver === "pg" ? sql`false` : sql`0`;
 
-    const requestsPerDay = await primaryDb
+    const requestsPerDay = await primaryLogsDb
         .select({
             day: groupByDayFunction.as("day"),
             allowedCount:

server/routers/auditLogs/queryRequestAuditLog.ts

@@ -1,8 +1,8 @@
-import { db, primaryDb, requestAuditLog, resources } from "@server/db";
+import { logsDb, primaryLogsDb, requestAuditLog, resources, db, primaryDb } from "@server/db";
 import { registry } from "@server/openApi";
 import { NextFunction } from "express";
 import { Request, Response } from "express";
-import { eq, gt, lt, and, count, desc } from "drizzle-orm";
+import { eq, gt, lt, and, count, desc, inArray } from "drizzle-orm";
 import { OpenAPITags } from "@server/openApi";
 import { z } from "zod";
 import createHttpError from "http-errors";
@@ -107,7 +107,7 @@ function getWhere(data: Q) {
 }
 
 export function queryRequest(data: Q) {
-    return primaryDb
+    return primaryLogsDb
         .select({
             id: requestAuditLog.id,
             timestamp: requestAuditLog.timestamp,
@@ -129,21 +129,49 @@ export function queryRequest(data: Q) {
             host: requestAuditLog.host,
             path: requestAuditLog.path,
             method: requestAuditLog.method,
-            tls: requestAuditLog.tls,
-            resourceName: resources.name,
-            resourceNiceId: resources.niceId
+            tls: requestAuditLog.tls
         })
         .from(requestAuditLog)
-        .leftJoin(
-            resources,
-            eq(requestAuditLog.resourceId, resources.resourceId)
-        ) // TODO: Is this efficient?
         .where(getWhere(data))
         .orderBy(desc(requestAuditLog.timestamp));
 }
 
+async function enrichWithResourceDetails(logs: Awaited<ReturnType<typeof queryRequest>>) {
+    // If logs database is the same as main database, we can do a join
+    // Otherwise, we need to fetch resource details separately
+    const resourceIds = logs
+        .map(log => log.resourceId)
+        .filter((id): id is number => id !== null && id !== undefined);
+
+    if (resourceIds.length === 0) {
+        return logs.map(log => ({ ...log, resourceName: null, resourceNiceId: null }));
+    }
+
+    // Fetch resource details from main database
+    const resourceDetails = await primaryDb
+        .select({
+            resourceId: resources.resourceId,
+            name: resources.name,
+            niceId: resources.niceId
+        })
+        .from(resources)
+        .where(inArray(resources.resourceId, resourceIds));
+
+    // Create a map for quick lookup
+    const resourceMap = new Map(
+        resourceDetails.map(r => [r.resourceId, { name: r.name, niceId: r.niceId }])
+    );
+
+    // Enrich logs with resource details
+    return logs.map(log => ({
+        ...log,
+        resourceName: log.resourceId ? resourceMap.get(log.resourceId)?.name ?? null : null,
+        resourceNiceId: log.resourceId ? resourceMap.get(log.resourceId)?.niceId ?? null : null
+    }));
+}
+
 export function countRequestQuery(data: Q) {
-    const countQuery = primaryDb
+    const countQuery = primaryLogsDb
         .select({ count: count() })
         .from(requestAuditLog)
         .where(getWhere(data));
@@ -185,36 +213,31 @@ async function queryUniqueFilterAttributes(
         uniquePaths,
         uniqueResources
     ] = await Promise.all([
-        primaryDb
+        primaryLogsDb
             .selectDistinct({ actor: requestAuditLog.actor })
             .from(requestAuditLog)
             .where(baseConditions)
             .limit(DISTINCT_LIMIT + 1),
-        primaryDb
+        primaryLogsDb
             .selectDistinct({ locations: requestAuditLog.location })
             .from(requestAuditLog)
             .where(baseConditions)
             .limit(DISTINCT_LIMIT + 1),
-        primaryDb
+        primaryLogsDb
             .selectDistinct({ hosts: requestAuditLog.host })
             .from(requestAuditLog)
             .where(baseConditions)
             .limit(DISTINCT_LIMIT + 1),
-        primaryDb
+        primaryLogsDb
             .selectDistinct({ paths: requestAuditLog.path })
             .from(requestAuditLog)
             .where(baseConditions)
             .limit(DISTINCT_LIMIT + 1),
-        primaryDb
+        primaryLogsDb
             .selectDistinct({
-                id: requestAuditLog.resourceId,
-                name: resources.name
+                id: requestAuditLog.resourceId
             })
             .from(requestAuditLog)
-            .leftJoin(
-                resources,
-                eq(requestAuditLog.resourceId, resources.resourceId)
-            )
             .where(baseConditions)
             .limit(DISTINCT_LIMIT + 1)
     ]);
@@ -231,13 +254,33 @@ async function queryUniqueFilterAttributes(
     //     throw new Error("Too many distinct filter attributes to retrieve. Please refine your time range.");
     // }
 
+    // Fetch resource names from main database for the unique resource IDs
+    const resourceIds = uniqueResources
+        .map(row => row.id)
+        .filter((id): id is number => id !== null);
+
+    let resourcesWithNames: Array<{ id: number; name: string | null }> = [];
+
+    if (resourceIds.length > 0) {
+        const resourceDetails = await primaryDb
+            .select({
+                resourceId: resources.resourceId,
+                name: resources.name
+            })
+            .from(resources)
+            .where(inArray(resources.resourceId, resourceIds));
+
+        resourcesWithNames = resourceDetails.map(r => ({
+            id: r.resourceId,
+            name: r.name
+        }));
+    }
+
     return {
         actors: uniqueActors
             .map((row) => row.actor)
             .filter((actor): actor is string => actor !== null),
-        resources: uniqueResources.filter(
-            (row): row is { id: number; name: string | null } => row.id !== null
-        ),
+        resources: resourcesWithNames,
         locations: uniqueLocations
             .map((row) => row.locations)
             .filter((location): location is string => location !== null),
@@ -280,7 +323,10 @@ export async function queryRequestAuditLogs(
 
         const baseQuery = queryRequest(data);
 
-        const log = await baseQuery.limit(data.limit).offset(data.offset);
+        const logsRaw = await baseQuery.limit(data.limit).offset(data.offset);
+
+        // Enrich with resource details (handles cross-database scenario)
+        const log = await enrichWithResourceDetails(logsRaw);
 
         const totalCountResult = await countRequestQuery(data);
         const totalCount = totalCountResult[0].count;

server/routers/badger/logRequestAudit.ts

@@ -1,4 +1,4 @@
-import { db, orgs, requestAuditLog } from "@server/db";
+import { logsDb, primaryLogsDb, db, orgs, requestAuditLog } from "@server/db";
 import logger from "@server/logger";
 import { and, eq, lt, sql } from "drizzle-orm";
 import cache from "@server/lib/cache";
@@ -69,7 +69,7 @@ async function flushAuditLogs() {
     try {
         // Use a transaction to ensure all inserts succeed or fail together
         // This prevents index corruption from partial writes
-        await db.transaction(async (tx) => {
+        await logsDb.transaction(async (tx) => {
             // Batch insert logs in groups of 25 to avoid overwhelming the database
             const BATCH_DB_SIZE = 25;
             for (let i = 0; i < logsToWrite.length; i += BATCH_DB_SIZE) {
@@ -162,7 +162,7 @@ export async function cleanUpOldLogs(orgId: string, retentionDays: number) {
     const cutoffTimestamp = calculateCutoffTimestamp(retentionDays);
 
     try {
-        await db
+        await logsDb
             .delete(requestAuditLog)
             .where(
                 and(