Merge branch 'dev' into msg-delivery

server/routers/auth/index.ts

@@ -16,4 +16,4 @@ export * from "./checkResourceSession";
 export * from "./securityKey";
 export * from "./startDeviceWebAuth";
 export * from "./verifyDeviceWebAuth";
-export * from "./pollDeviceWebAuth";
+export * from "./pollDeviceWebAuth";

server/routers/badger/logRequestAudit.ts

@@ -49,27 +49,43 @@ const auditLogBuffer: Array<{
 
 const BATCH_SIZE = 100; // Write to DB every 100 logs
 const BATCH_INTERVAL_MS = 5000; // Or every 5 seconds, whichever comes first
+const MAX_BUFFER_SIZE = 10000; // Prevent unbounded memory growth
 let flushTimer: NodeJS.Timeout | null = null;
+let isFlushInProgress = false;
 
 /**
  * Flush buffered logs to database
  */
 async function flushAuditLogs() {
-    if (auditLogBuffer.length === 0) {
+    if (auditLogBuffer.length === 0 || isFlushInProgress) {
         return;
     }
 
+    isFlushInProgress = true;
+
     // Take all current logs and clear buffer
     const logsToWrite = auditLogBuffer.splice(0, auditLogBuffer.length);
 
     try {
-        // Batch insert all logs at once
-        await db.insert(requestAuditLog).values(logsToWrite);
+        // Batch insert logs in groups of 25 to avoid overwhelming the database
+        const BATCH_DB_SIZE = 25;
+        for (let i = 0; i < logsToWrite.length; i += BATCH_DB_SIZE) {
+            const batch = logsToWrite.slice(i, i + BATCH_DB_SIZE);
+            await db.insert(requestAuditLog).values(batch);
+        }
         logger.debug(`Flushed ${logsToWrite.length} audit logs to database`);
     } catch (error) {
         logger.error("Error flushing audit logs:", error);
         // On error, we lose these logs - consider a fallback strategy if needed
         // (e.g., write to file, or put back in buffer with retry limit)
+    } finally {
+        isFlushInProgress = false;
+        // If buffer filled up while we were flushing, flush again
+        if (auditLogBuffer.length >= BATCH_SIZE) {
+            flushAuditLogs().catch((err) =>
+                logger.error("Error in follow-up flush:", err)
+            );
+        }
     }
 }
 
@@ -95,6 +111,10 @@ export async function shutdownAuditLogger() {
         clearTimeout(flushTimer);
         flushTimer = null;
     }
+    // Force flush even if one is in progress by waiting and retrying
+    while (isFlushInProgress) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+    }
     await flushAuditLogs();
 }
 
@@ -212,6 +232,14 @@ export async function logRequestAudit(
             ? stripPortFromHost(body.requestIp)
             : undefined;
 
+        // Prevent unbounded buffer growth - drop oldest entries if buffer is too large
+        if (auditLogBuffer.length >= MAX_BUFFER_SIZE) {
+            const dropped = auditLogBuffer.splice(0, BATCH_SIZE);
+            logger.warn(
+                `Audit log buffer exceeded max size (${MAX_BUFFER_SIZE}), dropped ${dropped.length} oldest entries`
+            );
+        }
+
         // Add to buffer instead of writing directly to DB
         auditLogBuffer.push({
             timestamp,

server/routers/badger/verifySession.ts

@@ -1035,14 +1035,25 @@ export function isPathAllowed(pattern: string, path: string): boolean {
     logger.debug(`Normalized pattern parts: [${patternParts.join(", ")}]`);
     logger.debug(`Normalized path parts: [${pathParts.join(", ")}]`);
 
+    // Maximum recursion depth to prevent stack overflow and memory issues
+    const MAX_RECURSION_DEPTH = 100;
+
     // Recursive function to try different wildcard matches
-    function matchSegments(patternIndex: number, pathIndex: number): boolean {
-        const indent = "  ".repeat(pathIndex); // Indent based on recursion depth
+    function matchSegments(patternIndex: number, pathIndex: number, depth: number = 0): boolean {
+        // Check recursion depth limit
+        if (depth > MAX_RECURSION_DEPTH) {
+            logger.warn(
+                `Path matching exceeded maximum recursion depth (${MAX_RECURSION_DEPTH}) for pattern "${pattern}" and path "${path}"`
+            );
+            return false;
+        }
+
+        const indent = "  ".repeat(depth); // Indent based on recursion depth
         const currentPatternPart = patternParts[patternIndex];
         const currentPathPart = pathParts[pathIndex];
 
         logger.debug(
-            `${indent}Checking patternIndex=${patternIndex} (${currentPatternPart || "END"}) vs pathIndex=${pathIndex} (${currentPathPart || "END"})`
+            `${indent}Checking patternIndex=${patternIndex} (${currentPatternPart || "END"}) vs pathIndex=${pathIndex} (${currentPathPart || "END"}) [depth=${depth}]`
         );
 
         // If we've consumed all pattern parts, we should have consumed all path parts
@@ -1075,7 +1086,7 @@ export function isPathAllowed(pattern: string, path: string): boolean {
             logger.debug(
                 `${indent}Trying to skip wildcard (consume 0 segments)`
             );
-            if (matchSegments(patternIndex + 1, pathIndex)) {
+            if (matchSegments(patternIndex + 1, pathIndex, depth + 1)) {
                 logger.debug(
                     `${indent}Successfully matched by skipping wildcard`
                 );
@@ -1086,7 +1097,7 @@ export function isPathAllowed(pattern: string, path: string): boolean {
             logger.debug(
                 `${indent}Trying to consume segment "${currentPathPart}" for wildcard`
             );
-            if (matchSegments(patternIndex, pathIndex + 1)) {
+            if (matchSegments(patternIndex, pathIndex + 1, depth + 1)) {
                 logger.debug(
                     `${indent}Successfully matched by consuming segment for wildcard`
                 );
@@ -1114,7 +1125,7 @@ export function isPathAllowed(pattern: string, path: string): boolean {
                 logger.debug(
                     `${indent}Segment with wildcard matches: "${currentPatternPart}" matches "${currentPathPart}"`
                 );
-                return matchSegments(patternIndex + 1, pathIndex + 1);
+                return matchSegments(patternIndex + 1, pathIndex + 1, depth + 1);
             }
 
             logger.debug(
@@ -1135,10 +1146,10 @@ export function isPathAllowed(pattern: string, path: string): boolean {
             `${indent}Segments match: "${currentPatternPart}" = "${currentPathPart}"`
         );
         // Move to next segments in both pattern and path
-        return matchSegments(patternIndex + 1, pathIndex + 1);
+        return matchSegments(patternIndex + 1, pathIndex + 1, depth + 1);
     }
 
-    const result = matchSegments(0, 0);
+    const result = matchSegments(0, 0, 0);
     logger.debug(`Final result: ${result}`);
     return result;
 }

server/routers/client/archiveClient.ts

@@ -0,0 +1,105 @@
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import { db } from "@server/db";
+import { clients } from "@server/db";
+import { eq } from "drizzle-orm";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+import { rebuildClientAssociationsFromClient } from "@server/lib/rebuildClientAssociations";
+import { sendTerminateClient } from "./terminate";
+
+const archiveClientSchema = z.strictObject({
+    clientId: z.string().transform(Number).pipe(z.int().positive())
+});
+
+registry.registerPath({
+    method: "post",
+    path: "/client/{clientId}/archive",
+    description: "Archive a client by its client ID.",
+    tags: [OpenAPITags.Client],
+    request: {
+        params: archiveClientSchema
+    },
+    responses: {}
+});
+
+export async function archiveClient(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = archiveClientSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const { clientId } = parsedParams.data;
+
+        // Check if client exists
+        const [client] = await db
+            .select()
+            .from(clients)
+            .where(eq(clients.clientId, clientId))
+            .limit(1);
+
+        if (!client) {
+            return next(
+                createHttpError(
+                    HttpCode.NOT_FOUND,
+                    `Client with ID ${clientId} not found`
+                )
+            );
+        }
+
+        if (client.archived) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    `Client with ID ${clientId} is already archived`
+                )
+            );
+        }
+
+        await db.transaction(async (trx) => {
+            // Archive the client
+            await trx
+                .update(clients)
+                .set({ archived: true })
+                .where(eq(clients.clientId, clientId));
+
+            // Rebuild associations to clean up related data
+            await rebuildClientAssociationsFromClient(client, trx);
+
+            // Send terminate signal if there's an associated OLM
+            if (client.olmId) {
+                await sendTerminateClient(client.clientId, client.olmId);
+            }
+        });
+
+        return response(res, {
+            data: null,
+            success: true,
+            error: false,
+            message: "Client archived successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(
+                HttpCode.INTERNAL_SERVER_ERROR,
+                "Failed to archive client"
+            )
+        );
+    }
+}

server/routers/client/blockClient.ts

@@ -0,0 +1,101 @@
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import { db } from "@server/db";
+import { clients } from "@server/db";
+import { eq } from "drizzle-orm";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+import { sendTerminateClient } from "./terminate";
+
+const blockClientSchema = z.strictObject({
+    clientId: z.string().transform(Number).pipe(z.int().positive())
+});
+
+registry.registerPath({
+    method: "post",
+    path: "/client/{clientId}/block",
+    description: "Block a client by its client ID.",
+    tags: [OpenAPITags.Client],
+    request: {
+        params: blockClientSchema
+    },
+    responses: {}
+});
+
+export async function blockClient(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = blockClientSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const { clientId } = parsedParams.data;
+
+        // Check if client exists
+        const [client] = await db
+            .select()
+            .from(clients)
+            .where(eq(clients.clientId, clientId))
+            .limit(1);
+
+        if (!client) {
+            return next(
+                createHttpError(
+                    HttpCode.NOT_FOUND,
+                    `Client with ID ${clientId} not found`
+                )
+            );
+        }
+
+        if (client.blocked) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    `Client with ID ${clientId} is already blocked`
+                )
+            );
+        }
+
+        await db.transaction(async (trx) => {
+            // Block the client
+            await trx
+                .update(clients)
+                .set({ blocked: true })
+                .where(eq(clients.clientId, clientId));
+
+            // Send terminate signal if there's an associated OLM and it's connected
+            if (client.olmId && client.online) {
+                await sendTerminateClient(client.clientId, client.olmId);
+            }
+        });
+
+        return response(res, {
+            data: null,
+            success: true,
+            error: false,
+            message: "Client blocked successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(
+                HttpCode.INTERNAL_SERVER_ERROR,
+                "Failed to block client"
+            )
+        );
+    }
+}

server/routers/client/deleteClient.ts

@@ -60,11 +60,12 @@ export async function deleteClient(
             );
         }
 
+        // Only allow deletion of machine clients (clients without userId)
         if (client.userId) {
             return next(
                 createHttpError(
                     HttpCode.BAD_REQUEST,
-                    `Cannot delete a user client with this endpoint`
+                    `Cannot delete a user client. User clients must be archived instead.`
                 )
             );
         }

server/routers/client/getClient.ts

@@ -36,7 +36,7 @@ async function query(clientId?: number, niceId?: string, orgId?: string) {
             .select()
             .from(clients)
             .where(and(eq(clients.niceId, niceId), eq(clients.orgId, orgId)))
-            .leftJoin(olms, eq(olms.clientId, olms.clientId))
+            .leftJoin(olms, eq(clients.clientId, olms.clientId))
             .limit(1);
         return res;
     }

server/routers/client/index.ts

@@ -1,6 +1,10 @@
 export * from "./pickClientDefaults";
 export * from "./createClient";
 export * from "./deleteClient";
+export * from "./archiveClient";
+export * from "./unarchiveClient";
+export * from "./blockClient";
+export * from "./unblockClient";
 export * from "./listClients";
 export * from "./updateClient";
 export * from "./getClient";

server/routers/client/listClients.ts

@@ -136,7 +136,10 @@ function queryClients(
             username: users.username,
             userEmail: users.email,
             niceId: clients.niceId,
-            agent: olms.agent
+            agent: olms.agent,
+            olmArchived: olms.archived,
+            archived: clients.archived,
+            blocked: clients.blocked
         })
         .from(clients)
         .leftJoin(orgs, eq(clients.orgId, orgs.orgId))

server/routers/client/unarchiveClient.ts

@@ -0,0 +1,93 @@
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import { db } from "@server/db";
+import { clients } from "@server/db";
+import { eq } from "drizzle-orm";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+
+const unarchiveClientSchema = z.strictObject({
+    clientId: z.string().transform(Number).pipe(z.int().positive())
+});
+
+registry.registerPath({
+    method: "post",
+    path: "/client/{clientId}/unarchive",
+    description: "Unarchive a client by its client ID.",
+    tags: [OpenAPITags.Client],
+    request: {
+        params: unarchiveClientSchema
+    },
+    responses: {}
+});
+
+export async function unarchiveClient(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = unarchiveClientSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const { clientId } = parsedParams.data;
+
+        // Check if client exists
+        const [client] = await db
+            .select()
+            .from(clients)
+            .where(eq(clients.clientId, clientId))
+            .limit(1);
+
+        if (!client) {
+            return next(
+                createHttpError(
+                    HttpCode.NOT_FOUND,
+                    `Client with ID ${clientId} not found`
+                )
+            );
+        }
+
+        if (!client.archived) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    `Client with ID ${clientId} is not archived`
+                )
+            );
+        }
+
+        // Unarchive the client
+        await db
+            .update(clients)
+            .set({ archived: false })
+            .where(eq(clients.clientId, clientId));
+
+        return response(res, {
+            data: null,
+            success: true,
+            error: false,
+            message: "Client unarchived successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(
+                HttpCode.INTERNAL_SERVER_ERROR,
+                "Failed to unarchive client"
+            )
+        );
+    }
+}

server/routers/client/unblockClient.ts

@@ -0,0 +1,93 @@
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import { db } from "@server/db";
+import { clients } from "@server/db";
+import { eq } from "drizzle-orm";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+
+const unblockClientSchema = z.strictObject({
+    clientId: z.string().transform(Number).pipe(z.int().positive())
+});
+
+registry.registerPath({
+    method: "post",
+    path: "/client/{clientId}/unblock",
+    description: "Unblock a client by its client ID.",
+    tags: [OpenAPITags.Client],
+    request: {
+        params: unblockClientSchema
+    },
+    responses: {}
+});
+
+export async function unblockClient(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = unblockClientSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const { clientId } = parsedParams.data;
+
+        // Check if client exists
+        const [client] = await db
+            .select()
+            .from(clients)
+            .where(eq(clients.clientId, clientId))
+            .limit(1);
+
+        if (!client) {
+            return next(
+                createHttpError(
+                    HttpCode.NOT_FOUND,
+                    `Client with ID ${clientId} not found`
+                )
+            );
+        }
+
+        if (!client.blocked) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    `Client with ID ${clientId} is not blocked`
+                )
+            );
+        }
+
+        // Unblock the client
+        await db
+            .update(clients)
+            .set({ blocked: false })
+            .where(eq(clients.clientId, clientId));
+
+        return response(res, {
+            data: null,
+            success: true,
+            error: false,
+            message: "Client unblocked successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(
+                HttpCode.INTERNAL_SERVER_ERROR,
+                "Failed to unblock client"
+            )
+        );
+    }
+}

server/routers/external.ts

@@ -174,6 +174,38 @@ authenticated.delete(
     client.deleteClient
 );
 
+authenticated.post(
+    "/client/:clientId/archive",
+    verifyClientAccess,
+    verifyUserHasAction(ActionsEnum.archiveClient),
+    logActionAudit(ActionsEnum.archiveClient),
+    client.archiveClient
+);
+
+authenticated.post(
+    "/client/:clientId/unarchive",
+    verifyClientAccess,
+    verifyUserHasAction(ActionsEnum.unarchiveClient),
+    logActionAudit(ActionsEnum.unarchiveClient),
+    client.unarchiveClient
+);
+
+authenticated.post(
+    "/client/:clientId/block",
+    verifyClientAccess,
+    verifyUserHasAction(ActionsEnum.blockClient),
+    logActionAudit(ActionsEnum.blockClient),
+    client.blockClient
+);
+
+authenticated.post(
+    "/client/:clientId/unblock",
+    verifyClientAccess,
+    verifyUserHasAction(ActionsEnum.unblockClient),
+    logActionAudit(ActionsEnum.unblockClient),
+    client.unblockClient
+);
+
 authenticated.post(
     "/client/:clientId",
     verifyClientAccess, // this will check if the user has access to the client
@@ -808,11 +840,18 @@ authenticated.put("/user/:userId/olm", verifyIsLoggedInUser, olm.createUserOlm);
 
 authenticated.get("/user/:userId/olms", verifyIsLoggedInUser, olm.listUserOlms);
 
-authenticated.delete(
-    "/user/:userId/olm/:olmId",
+authenticated.post(
+    "/user/:userId/olm/:olmId/archive",
     verifyIsLoggedInUser,
     verifyOlmAccess,
-    olm.deleteUserOlm
+    olm.archiveUserOlm
+);
+
+authenticated.post(
+    "/user/:userId/olm/:olmId/unarchive",
+    verifyIsLoggedInUser,
+    verifyOlmAccess,
+    olm.unarchiveUserOlm
 );
 
 authenticated.get(

server/routers/integration.ts

@@ -751,9 +751,10 @@ authenticated.post(
 );
 
 authenticated.get(
-    "/idp",
-    verifyApiKeyIsRoot,
-    verifyApiKeyHasAction(ActionsEnum.listIdps),
+    "/idp", // no guards on this because anyone can list idps for login purposes
+    // we do the same for the external api
+    // verifyApiKeyIsRoot,
+    // verifyApiKeyHasAction(ActionsEnum.listIdps),
     idp.listIdps
 );
 
@@ -842,6 +843,38 @@ authenticated.delete(
     client.deleteClient
 );
 
+authenticated.post(
+    "/client/:clientId/archive",
+    verifyApiKeyClientAccess,
+    verifyApiKeyHasAction(ActionsEnum.archiveClient),
+    logActionAudit(ActionsEnum.archiveClient),
+    client.archiveClient
+);
+
+authenticated.post(
+    "/client/:clientId/unarchive",
+    verifyApiKeyClientAccess,
+    verifyApiKeyHasAction(ActionsEnum.unarchiveClient),
+    logActionAudit(ActionsEnum.unarchiveClient),
+    client.unarchiveClient
+);
+
+authenticated.post(
+    "/client/:clientId/block",
+    verifyApiKeyClientAccess,
+    verifyApiKeyHasAction(ActionsEnum.blockClient),
+    logActionAudit(ActionsEnum.blockClient),
+    client.blockClient
+);
+
+authenticated.post(
+    "/client/:clientId/unblock",
+    verifyApiKeyClientAccess,
+    verifyApiKeyHasAction(ActionsEnum.unblockClient),
+    logActionAudit(ActionsEnum.unblockClient),
+    client.unblockClient
+);
+
 authenticated.post(
     "/client/:clientId",
     verifyApiKeyClientAccess,

server/routers/newt/handleReceiveBandwidthMessage.ts

@@ -1,7 +1,7 @@
 import { db } from "@server/db";
 import { MessageHandler } from "@server/routers/ws";
-import { clients, Newt } from "@server/db";
-import { eq } from "drizzle-orm";
+import { clients } from "@server/db";
+import { eq, sql } from "drizzle-orm";
 import logger from "@server/logger";
 
 interface PeerBandwidth {
@@ -10,13 +10,57 @@ interface PeerBandwidth {
     bytesOut: number;
 }
 
+// Retry configuration for deadlock handling
+const MAX_RETRIES = 3;
+const BASE_DELAY_MS = 50;
+
+/**
+ * Check if an error is a deadlock error
+ */
+function isDeadlockError(error: any): boolean {
+    return (
+        error?.code === "40P01" ||
+        error?.cause?.code === "40P01" ||
+        (error?.message && error.message.includes("deadlock"))
+    );
+}
+
+/**
+ * Execute a function with retry logic for deadlock handling
+ */
+async function withDeadlockRetry<T>(
+    operation: () => Promise<T>,
+    context: string
+): Promise<T> {
+    let attempt = 0;
+    while (true) {
+        try {
+            return await operation();
+        } catch (error: any) {
+            if (isDeadlockError(error) && attempt < MAX_RETRIES) {
+                attempt++;
+                const baseDelay = Math.pow(2, attempt - 1) * BASE_DELAY_MS;
+                const jitter = Math.random() * baseDelay;
+                const delay = baseDelay + jitter;
+                logger.warn(
+                    `Deadlock detected in ${context}, retrying attempt ${attempt}/${MAX_RETRIES} after ${delay.toFixed(0)}ms`
+                );
+                await new Promise((resolve) => setTimeout(resolve, delay));
+                continue;
+            }
+            throw error;
+        }
+    }
+}
+
 export const handleReceiveBandwidthMessage: MessageHandler = async (
     context
 ) => {
-    const { message, client, sendToClient } = context;
+    const { message } = context;
 
     if (!message.data.bandwidthData) {
         logger.warn("No bandwidth data provided");
+        return;
     }
 
     const bandwidthData: PeerBandwidth[] = message.data.bandwidthData;
@@ -25,30 +69,40 @@ export const handleReceiveBandwidthMessage: MessageHandler = async (
         throw new Error("Invalid bandwidth data");
     }
 
-    await db.transaction(async (trx) => {
-        for (const peer of bandwidthData) {
-            const { publicKey, bytesIn, bytesOut } = peer;
+    // Sort bandwidth data by publicKey to ensure consistent lock ordering across all instances
+    // This is critical for preventing deadlocks when multiple instances update the same clients
+    const sortedBandwidthData = [...bandwidthData].sort((a, b) =>
+        a.publicKey.localeCompare(b.publicKey)
+    );
 
-            // Find the client by public key
-            const [client] = await trx
-                .select()
-                .from(clients)
-                .where(eq(clients.pubKey, publicKey))
-                .limit(1);
+    const currentTime = new Date().toISOString();
 
-            if (!client) {
-                continue;
-            }
+    // Update each client individually with retry logic
+    // This reduces transaction scope and allows retries per-client
+    for (const peer of sortedBandwidthData) {
+        const { publicKey, bytesIn, bytesOut } = peer;
 
-            // Update the client's bandwidth usage
-            await trx
-                .update(clients)
-                .set({
-                    megabytesOut: (client.megabytesIn || 0) + bytesIn,
-                    megabytesIn: (client.megabytesOut || 0) + bytesOut,
-                    lastBandwidthUpdate: new Date().toISOString()
-                })
-                .where(eq(clients.clientId, client.clientId));
+        try {
+            await withDeadlockRetry(async () => {
+                // Use atomic SQL increment to avoid SELECT then UPDATE pattern
+                // This eliminates the need to read the current value first
+                await db
+                    .update(clients)
+                    .set({
+                        // Note: bytesIn from peer goes to megabytesOut (data sent to client)
+                        // and bytesOut from peer goes to megabytesIn (data received from client)
+                        megabytesOut: sql`COALESCE(${clients.megabytesOut}, 0) + ${bytesIn}`,
+                        megabytesIn: sql`COALESCE(${clients.megabytesIn}, 0) + ${bytesOut}`,
+                        lastBandwidthUpdate: currentTime
+                    })
+                    .where(eq(clients.pubKey, publicKey));
+            }, `update client bandwidth ${publicKey}`);
+        } catch (error) {
+            logger.error(
+                `Failed to update bandwidth for client ${publicKey}:`,
+                error
+            );
+            // Continue with other clients even if one fails
         }
-    });
+    }
 };

server/routers/olm/archiveUserOlm.ts

@@ -0,0 +1,81 @@
+import { NextFunction, Request, Response } from "express";
+import { db } from "@server/db";
+import { olms, clients } from "@server/db";
+import { eq } from "drizzle-orm";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import response from "@server/lib/response";
+import { z } from "zod";
+import { fromError } from "zod-validation-error";
+import logger from "@server/logger";
+import { rebuildClientAssociationsFromClient } from "@server/lib/rebuildClientAssociations";
+import { sendTerminateClient } from "../client/terminate";
+
+const paramsSchema = z
+    .object({
+        userId: z.string(),
+        olmId: z.string()
+    })
+    .strict();
+
+export async function archiveUserOlm(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = paramsSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const { olmId } = parsedParams.data;
+
+        // Archive the OLM and disconnect associated clients in a transaction
+        await db.transaction(async (trx) => {
+            // Find all clients associated with this OLM
+            const associatedClients = await trx
+                .select()
+                .from(clients)
+                .where(eq(clients.olmId, olmId));
+
+            // Disconnect clients from the OLM (set olmId to null)
+            for (const client of associatedClients) {
+                await trx
+                    .update(clients)
+                    .set({ olmId: null })
+                    .where(eq(clients.clientId, client.clientId));
+
+                await rebuildClientAssociationsFromClient(client, trx);
+                await sendTerminateClient(client.clientId, olmId);
+            }
+
+            // Archive the OLM (set archived to true)
+            await trx
+                .update(olms)
+                .set({ archived: true })
+                .where(eq(olms.olmId, olmId));
+        });
+
+        return response(res, {
+            data: null,
+            success: true,
+            error: false,
+            message: "Device archived successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(
+                HttpCode.INTERNAL_SERVER_ERROR,
+                "Failed to archive device"
+            )
+        );
+    }
+}

server/routers/olm/handleOlmPingMessage.ts

@@ -1,7 +1,7 @@
 import { db } from "@server/db";
 import { disconnectClient } from "#dynamic/routers/ws";
 import { getClientConfigVersion, MessageHandler } from "@server/routers/ws";
-import { clients, Olm } from "@server/db";
+import { clients, olms, Olm } from "@server/db";
 import { eq, lt, isNull, and, or } from "drizzle-orm";
 import logger from "@server/logger";
 import { validateSessionToken } from "@server/auth/sessions/app";
@@ -109,29 +109,17 @@ export const handleOlmPingMessage: MessageHandler = async (context) => {
         return;
     }
 
-    if (olm.userId) {
-        // we need to check a user token to make sure its still valid
-        const { session: userSession, user } =
-            await validateSessionToken(userToken);
-        if (!userSession || !user) {
-            logger.warn("Invalid user session for olm ping");
-            return; // by returning here we just ignore the ping and the setInterval will force it to disconnect
-        }
-        if (user.userId !== olm.userId) {
-            logger.warn("User ID mismatch for olm ping");
-            return;
-        }
+    if (!olm.clientId) {
+        logger.warn("Olm has no client ID!");
+        return;
+    }
 
+    try {
         // get the client
         const [client] = await db
             .select()
             .from(clients)
-            .where(
-                and(
-                    eq(clients.olmId, olm.olmId),
-                    eq(clients.userId, olm.userId)
-                )
-            )
+            .where(eq(clients.clientId, olm.clientId))
             .limit(1);
 
         if (!client) {
@@ -139,39 +127,48 @@ export const handleOlmPingMessage: MessageHandler = async (context) => {
             return;
         }
 
-        const sessionId = encodeHexLowerCase(
-            sha256(new TextEncoder().encode(userToken))
-        );
-
-        const policyCheck = await checkOrgAccessPolicy({
-            orgId: client.orgId,
-            userId: olm.userId,
-            sessionId // this is the user token passed in the message
-        });
-
-        if (!policyCheck.allowed) {
-            logger.warn(
-                `Olm user ${olm.userId} does not pass access policies for org ${client.orgId}: ${policyCheck.error}`
+        if (client.blocked) {
+            // NOTE: by returning we dont update the lastPing, so the offline checker will eventually disconnect them
+            logger.debug(
+                `Blocked client ${client.clientId} attempted olm ping`
             );
             return;
         }
-    }
 
-    if (!olm.clientId) {
-        logger.warn("Olm has no client ID!");
-        return;
-    }
+        if (olm.userId) {
+            // we need to check a user token to make sure its still valid
+            const { session: userSession, user } =
+                await validateSessionToken(userToken);
+            if (!userSession || !user) {
+                logger.warn("Invalid user session for olm ping");
+                return; // by returning here we just ignore the ping and the setInterval will force it to disconnect
+            }
+            if (user.userId !== olm.userId) {
+                logger.warn("User ID mismatch for olm ping");
+                return;
+            }
+            if (user.userId !== client.userId) {
+                logger.warn("Client user ID mismatch for olm ping");
+                return;
+            }
 
-    try {
-        // Update the client's last ping timestamp
-        const [client] = await db
-            .update(clients)
-            .set({
-                lastPing: Math.floor(Date.now() / 1000),
-                online: true
-            })
-            .where(eq(clients.clientId, olm.clientId))
-            .returning();
+            const sessionId = encodeHexLowerCase(
+                sha256(new TextEncoder().encode(userToken))
+            );
+
+            const policyCheck = await checkOrgAccessPolicy({
+                orgId: client.orgId,
+                userId: olm.userId,
+                sessionId // this is the user token passed in the message
+            });
+
+            if (!policyCheck.allowed) {
+                logger.warn(
+                    `Olm user ${olm.userId} does not pass access policies for org ${client.orgId}: ${policyCheck.error}`
+                );
+                return;
+            }
+        }
 
         // get the version
         const configVersion = await getClientConfigVersion(olm.olmId);
@@ -182,6 +179,23 @@ export const handleOlmPingMessage: MessageHandler = async (context) => {
             );
             await sendOlmSyncMessage(olm, client);
         }
+
+        // Update the client's last ping timestamp
+        await db
+            .update(clients)
+            .set({
+                lastPing: Math.floor(Date.now() / 1000),
+                online: true,
+                archived: false
+            })
+            .where(eq(clients.clientId, olm.clientId));
+
+        if (olm.archived) {
+            await db
+                .update(olms)
+                .set({ archived: false })
+                .where(eq(olms.olmId, olm.olmId));
+        }
     } catch (error) {
         logger.error("Error handling ping message", { error });
     }

server/routers/olm/handleOlmRegisterMessage.ts

@@ -57,6 +57,11 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => {
         return;
     }
 
+    if (client.blocked) {
+        logger.debug(`Client ${client.clientId} is blocked. Ignoring register.`);
+        return;
+    }
+
     const [org] = await db
         .select()
         .from(orgs)
@@ -114,18 +119,20 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => {
 
     if (
         (olmVersion && olm.version !== olmVersion) ||
-        (olmAgent && olm.agent !== olmAgent)
+        (olmAgent && olm.agent !== olmAgent) ||
+        olm.archived
     ) {
         await db
             .update(olms)
             .set({
                 version: olmVersion,
-                agent: olmAgent
+                agent: olmAgent,
+                archived: false
             })
             .where(eq(olms.olmId, olm.olmId));
     }
 
-    if (client.pubKey !== publicKey) {
+    if (client.pubKey !== publicKey || client.archived) {
         logger.info(
             "Public key mismatch. Updating public key and clearing session info..."
         );
@@ -133,7 +140,8 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => {
         await db
             .update(clients)
             .set({
-                pubKey: publicKey
+                pubKey: publicKey,
+                archived: false,
             })
             .where(eq(clients.clientId, client.clientId));
 

server/routers/olm/index.ts

@@ -3,9 +3,9 @@ export * from "./getOlmToken";
 export * from "./createUserOlm";
 export * from "./handleOlmRelayMessage";
 export * from "./handleOlmPingMessage";
-export * from "./deleteUserOlm";
+export * from "./archiveUserOlm";
+export * from "./unarchiveUserOlm";
 export * from "./listUserOlms";
-export * from "./deleteUserOlm";
 export * from "./getUserOlm";
 export * from "./handleOlmServerPeerAddMessage";
 export * from "./handleOlmUnRelayMessage";

server/routers/olm/listUserOlms.ts

@@ -51,6 +51,7 @@ export type ListUserOlmsResponse = {
         name: string | null;
         clientId: number | null;
         userId: string | null;
+        archived: boolean;
     }>;
     pagination: {
         total: number;
@@ -89,7 +90,7 @@ export async function listUserOlms(
 
         const { userId } = parsedParams.data;
 
-        // Get total count
+        // Get total count (including archived OLMs)
         const [totalCountResult] = await db
             .select({ count: count() })
             .from(olms)
@@ -97,7 +98,7 @@ export async function listUserOlms(
 
         const total = totalCountResult?.count || 0;
 
-        // Get OLMs for the current user
+        // Get OLMs for the current user (including archived OLMs)
         const userOlms = await db
             .select({
                 olmId: olms.olmId,
@@ -105,7 +106,8 @@ export async function listUserOlms(
                 version: olms.version,
                 name: olms.name,
                 clientId: olms.clientId,
-                userId: olms.userId
+                userId: olms.userId,
+                archived: olms.archived
             })
             .from(olms)
             .where(eq(olms.userId, userId))

server/routers/olm/unarchiveUserOlm.ts

@@ -0,0 +1,84 @@
+import { NextFunction, Request, Response } from "express";
+import { db } from "@server/db";
+import { olms } from "@server/db";
+import { eq } from "drizzle-orm";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import response from "@server/lib/response";
+import { z } from "zod";
+import { fromError } from "zod-validation-error";
+import logger from "@server/logger";
+
+const paramsSchema = z
+    .object({
+        userId: z.string(),
+        olmId: z.string()
+    })
+    .strict();
+
+export async function unarchiveUserOlm(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = paramsSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const { olmId } = parsedParams.data;
+
+        // Check if OLM exists and is archived
+        const [olm] = await db
+            .select()
+            .from(olms)
+            .where(eq(olms.olmId, olmId))
+            .limit(1);
+
+        if (!olm) {
+            return next(
+                createHttpError(
+                    HttpCode.NOT_FOUND,
+                    `OLM with ID ${olmId} not found`
+                )
+            );
+        }
+
+        if (!olm.archived) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    `OLM with ID ${olmId} is not archived`
+                )
+            );
+        }
+
+        // Unarchive the OLM (set archived to false)
+        await db
+            .update(olms)
+            .set({ archived: false })
+            .where(eq(olms.olmId, olmId));
+
+        return response(res, {
+            data: null,
+            success: true,
+            error: false,
+            message: "Device unarchived successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(
+                HttpCode.INTERNAL_SERVER_ERROR,
+                "Failed to unarchive device"
+            )
+        );
+    }
+}

server/routers/target/updateTarget.ts

@@ -213,9 +213,11 @@ export async function updateTarget(
 
         // When health check is disabled, reset hcHealth to "unknown"
         // to prevent previously unhealthy targets from being excluded
+        // Also when the site is not a newt, set hcHealth to "unknown"
         const hcHealthValue =
             parsedBody.data.hcEnabled === false ||
-            parsedBody.data.hcEnabled === null
+            parsedBody.data.hcEnabled === null ||
+            site.type !== "newt"
                 ? "unknown"
                 : undefined;