Merge pull request #307 from fosrl/remove_json_group_array

Remove json_group_array from queries

server/routers/newt/handleRegisterMessage.ts

@@ -7,7 +7,7 @@ import {
     Target,
     targets
 } from "@server/db/schema";
-import { eq, and, sql } from "drizzle-orm";
+import { eq, and, sql, inArray } from "drizzle-orm";
 import { addPeer, deletePeer } from "../gerbil/peers";
 import logger from "@server/logger";
 
@@ -75,9 +75,11 @@ export const handleRegisterMessage: MessageHandler = async (context) => {
         allowedIps: [site.subnet]
     });
 
-    const allResources = await db
+    // Improved version
+    const allResources = await db.transaction(async (tx) => {
+        // First get all resources for the site
+        const resourcesList = await tx
             .select({
-            // Resource fields
                 resourceId: resources.resourceId,
                 subdomain: resources.subdomain,
                 fullDomain: resources.fullDomain,
@@ -87,56 +89,70 @@ export const handleRegisterMessage: MessageHandler = async (context) => {
                 emailWhitelistEnabled: resources.emailWhitelistEnabled,
                 http: resources.http,
                 proxyPort: resources.proxyPort,
-            protocol: resources.protocol,
+                protocol: resources.protocol
-            // Targets as a subquery
-            targets: sql<string>`json_group_array(json_object(
-          'targetId', ${targets.targetId},
-          'ip', ${targets.ip},
-          'method', ${targets.method},
-          'port', ${targets.port},
-          'internalPort', ${targets.internalPort},
-          'enabled', ${targets.enabled}
-        ))`.as("targets")
             })
             .from(resources)
-        .leftJoin(
+            .where(eq(resources.siteId, siteId));
-            targets,
+
+        // Get all enabled targets for these resources in a single query
+        const resourceIds = resourcesList.map((r) => r.resourceId);
+        const allTargets =
+            resourceIds.length > 0
+                ? await tx
+                      .select({
+                          resourceId: targets.resourceId,
+                          targetId: targets.targetId,
+                          ip: targets.ip,
+                          method: targets.method,
+                          port: targets.port,
+                          internalPort: targets.internalPort,
+                          enabled: targets.enabled
+                      })
+                      .from(targets)
+                      .where(
                           and(
-                eq(targets.resourceId, resources.resourceId),
+                              inArray(targets.resourceId, resourceIds),
                               eq(targets.enabled, true)
                           )
                       )
-        .where(eq(resources.siteId, siteId))
+                : [];
-        .groupBy(resources.resourceId);
 
-    let tcpTargets: string[] = [];
+        // Combine the data in JS instead of using SQL for the JSON
-    let udpTargets: string[] = [];
+        return resourcesList.map((resource) => ({
+            ...resource,
+            targets: allTargets.filter(
+                (target) => target.resourceId === resource.resourceId
+            )
+        }));
+    });
 
-    for (const resource of allResources) {
+    const { tcpTargets, udpTargets } = allResources.reduce(
-        const targets = JSON.parse(resource.targets);
+        (acc, resource) => {
-        if (!targets || targets.length === 0) {
+            // Skip resources with no targets
-            continue;
+            if (!resource.targets?.length) return acc;
-        }
+
+            // Format valid targets into strings
+            const formattedTargets = resource.targets
+                .filter(
+                    (target: Target) =>
+                        target?.internalPort && target?.ip && target?.port
+                )
+                .map(
+                    (target: Target) =>
+                        `${target.internalPort}:${target.ip}:${target.port}`
+                );
+
+            // Add to the appropriate protocol array
             if (resource.protocol === "tcp") {
-            tcpTargets = tcpTargets.concat(
+                acc.tcpTargets.push(...formattedTargets);
-                targets.map(
-                    (target: Target) =>
-                        `${
-                            target.internalPort ? target.internalPort + ":" : ""
-                        }${target.ip}:${target.port}`
-                )
-            );
             } else {
-            udpTargets = tcpTargets.concat(
+                acc.udpTargets.push(...formattedTargets);
-                targets.map(
+            }
-                    (target: Target) =>
+
-                        `${
+            return acc;
-                            target.internalPort ? target.internalPort + ":" : ""
+        },
-                        }${target.ip}:${target.port}`
+        { tcpTargets: [] as string[], udpTargets: [] as string[] }
-                )
     );
-        }
-    }
 
     return {
         message: {

server/routers/traefik/getTraefikConfig.ts

@@ -1,6 +1,6 @@
 import { Request, Response } from "express";
 import db from "@server/db";
-import { and, eq } from "drizzle-orm";
+import { and, eq, inArray } from "drizzle-orm";
 import logger from "@server/logger";
 import HttpCode from "@server/types/HttpCode";
 import config from "@server/lib/config";
@@ -12,7 +12,10 @@ export async function traefikConfigProvider(
     res: Response
 ): Promise<any> {
     try {
-        const allResources = await db
+        // Get all resources with related data
+        const allResources = await db.transaction(async (tx) => {
+            // First query to get resources with site and org info
+            const resourcesWithRelations = await tx
                 .select({
                     // Resource fields
                     resourceId: resources.resourceId,
@@ -36,28 +39,52 @@ export async function traefikConfigProvider(
                     // Org fields
                     org: {
                         orgId: orgs.orgId
-                },
+                    }
-                // Targets as a subquery
-                targets: sql<string>`json_group_array(json_object(
-          'targetId', ${targets.targetId},
-          'ip', ${targets.ip},
-          'method', ${targets.method},
-          'port', ${targets.port},
-          'internalPort', ${targets.internalPort},
-          'enabled', ${targets.enabled}
-        ))`.as("targets")
                 })
                 .from(resources)
                 .innerJoin(sites, eq(sites.siteId, resources.siteId))
-            .innerJoin(orgs, eq(resources.orgId, orgs.orgId))
+                .innerJoin(orgs, eq(resources.orgId, orgs.orgId));
-            .leftJoin(
+
-                targets,
+            // Get all resource IDs from the first query
+            const resourceIds = resourcesWithRelations.map((r) => r.resourceId);
+
+            // Second query to get all enabled targets for these resources
+            const allTargets =
+                resourceIds.length > 0
+                    ? await tx
+                          .select({
+                              resourceId: targets.resourceId,
+                              targetId: targets.targetId,
+                              ip: targets.ip,
+                              method: targets.method,
+                              port: targets.port,
+                              internalPort: targets.internalPort,
+                              enabled: targets.enabled
+                          })
+                          .from(targets)
+                          .where(
                               and(
-                    eq(targets.resourceId, resources.resourceId),
+                                  inArray(targets.resourceId, resourceIds),
                                   eq(targets.enabled, true)
                               )
                           )
-            .groupBy(resources.resourceId);
+                    : [];
+
+            // Create a map for fast target lookup by resourceId
+            const targetsMap = allTargets.reduce((map, target) => {
+                if (!map.has(target.resourceId)) {
+                    map.set(target.resourceId, []);
+                }
+                map.get(target.resourceId).push(target);
+                return map;
+            }, new Map());
+
+            // Combine the data
+            return resourcesWithRelations.map((resource) => ({
+                ...resource,
+                targets: targetsMap.get(resource.resourceId) || []
+            }));
+        });
 
         if (!allResources.length) {
             return res.status(HttpCode.OK).json({});
@@ -101,7 +128,7 @@ export async function traefikConfigProvider(
         };
 
         for (const resource of allResources) {
-            const targets = JSON.parse(resource.targets);
+            const targets = resource.targets as Target[];
             const site = resource.site;
             const org = resource.org;