Merge branch 'multi-role' of github.com:fosrl/pangolin into multi-role

thundering herd

license.py

@@ -0,0 +1,115 @@
+import os
+import sys
+
+# --- Configuration ---
+# The header text to be added to the files.
+HEADER_TEXT = """/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+"""
+
+def should_add_header(file_path):
+    """
+    Checks if a file should receive the commercial license header.
+    Returns True if 'private' is in the path or file content.
+    """
+    # Check if 'private' is in the file path (case-insensitive)
+    if 'server/private' in file_path.lower():
+        return True
+
+    # Check if 'private' is in the file content (case-insensitive)
+    # try:
+    #     with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+    #         content = f.read()
+    #         if 'private' in content.lower():
+    #             return True
+    # except Exception as e:
+    #     print(f"Could not read file {file_path}: {e}")
+
+    return False
+
+def process_directory(root_dir):
+    """
+    Recursively scans a directory and adds headers to qualifying .ts or .tsx files,
+    skipping any 'node_modules' directories.
+    """
+    print(f"Scanning directory: {root_dir}")
+    files_processed = 0
+    headers_added = 0
+
+    for root, dirs, files in os.walk(root_dir):
+        # --- MODIFICATION ---
+        # Exclude 'node_modules' directories from the scan to improve performance.
+        if 'node_modules' in dirs:
+            dirs.remove('node_modules')
+
+        for file in files:
+            if file.endswith('.ts') or file.endswith('.tsx'):
+                file_path = os.path.join(root, file)
+                files_processed += 1
+
+                try:
+                    with open(file_path, 'r+', encoding='utf-8') as f:
+                        original_content = f.read()
+                        has_header = original_content.startswith(HEADER_TEXT.strip())
+                        
+                        if should_add_header(file_path):
+                            # Add header only if it's not already there
+                            if not has_header:
+                                f.seek(0, 0) # Go to the beginning of the file
+                                f.write(HEADER_TEXT.strip() + '\n\n' + original_content)
+                                print(f"Added header to: {file_path}")
+                                headers_added += 1
+                            else:
+                                print(f"Header already exists in: {file_path}")
+                        else:
+                            # Remove header if it exists but shouldn't be there
+                            if has_header:
+                                # Find the end of the header and remove it (including following newlines)
+                                header_with_newlines = HEADER_TEXT.strip() + '\n\n'
+                                if original_content.startswith(header_with_newlines):
+                                    content_without_header = original_content[len(header_with_newlines):]
+                                else:
+                                    # Handle case where there might be different newline patterns
+                                    header_end = len(HEADER_TEXT.strip())
+                                    # Skip any newlines after the header
+                                    while header_end < len(original_content) and original_content[header_end] in '\n\r':
+                                        header_end += 1
+                                    content_without_header = original_content[header_end:]
+                                
+                                f.seek(0)
+                                f.write(content_without_header)
+                                f.truncate()
+                                print(f"Removed header from: {file_path}")
+                                headers_added += 1  # Reusing counter for modifications
+
+                except Exception as e:
+                    print(f"Error processing file {file_path}: {e}")
+
+    print("\n--- Scan Complete ---")
+    print(f"Total .ts or .tsx files found: {files_processed}")
+    print(f"Files modified (headers added/removed): {headers_added}")
+
+
+if __name__ == "__main__":
+    # Get the target directory from the command line arguments.
+    # If no directory is provided, it uses the current directory ('.').
+    if len(sys.argv) > 1:
+        target_directory = sys.argv[1]
+    else:
+        target_directory = '.' # Default to current directory
+
+    if not os.path.isdir(target_directory):
+        print(f"Error: Directory '{target_directory}' not found.")
+        sys.exit(1)
+
+    process_directory(os.path.abspath(target_directory))

messages/bg-BG.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "Изтрийте потребител",
     "actionListUsers": "Изброяване на потребители",
     "actionAddUserRole": "Добавяне на роля на потребител",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Генериране на токен за достъп",
     "actionDeleteAccessToken": "Изтриване на токен за достъп",
     "actionListAccessTokens": "Изброяване на токени за достъп",

messages/cs-CZ.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "Odstranit uživatele",
     "actionListUsers": "Seznam uživatelů",
     "actionAddUserRole": "Přidat uživatelskou roli",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Generovat přístupový token",
     "actionDeleteAccessToken": "Odstranit přístupový token",
     "actionListAccessTokens": "Seznam přístupových tokenů",

messages/de-DE.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "Benutzer entfernen",
     "actionListUsers": "Benutzer auflisten",
     "actionAddUserRole": "Benutzerrolle hinzufügen",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Zugriffstoken generieren",
     "actionDeleteAccessToken": "Zugriffstoken löschen",
     "actionListAccessTokens": "Zugriffstoken auflisten",

messages/en-US.json

@@ -512,6 +512,8 @@
     "autoProvisionedDescription": "Allow this user to be automatically managed by identity provider",
     "accessControlsDescription": "Manage what this user can access and do in the organization",
     "accessControlsSubmit": "Save Access Controls",
+    "singleRolePerUserPlanNotice": "Your plan only supports one role per user.",
+    "singleRolePerUserEditionNotice": "This edition only supports one role per user.",
     "roles": "Roles",
     "accessUsersRoles": "Manage Users & Roles",
     "accessUsersRolesDescription": "Invite users and add them to roles to manage access to the organization",
@@ -1150,6 +1152,7 @@
     "actionRemoveUser": "Remove User",
     "actionListUsers": "List Users",
     "actionAddUserRole": "Add User Role",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Generate Access Token",
     "actionDeleteAccessToken": "Delete Access Token",
     "actionListAccessTokens": "List Access Tokens",

messages/es-ES.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "Eliminar usuario",
     "actionListUsers": "Listar usuarios",
     "actionAddUserRole": "Añadir rol de usuario",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Generar token de acceso",
     "actionDeleteAccessToken": "Eliminar token de acceso",
     "actionListAccessTokens": "Lista de Tokens de Acceso",

messages/fr-FR.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "Supprimer un utilisateur",
     "actionListUsers": "Lister les utilisateurs",
     "actionAddUserRole": "Ajouter un rôle utilisateur",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Générer un jeton d'accès",
     "actionDeleteAccessToken": "Supprimer un jeton d'accès",
     "actionListAccessTokens": "Lister les jetons d'accès",

messages/it-IT.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "Rimuovi Utente",
     "actionListUsers": "Elenca Utenti",
     "actionAddUserRole": "Aggiungi Ruolo Utente",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Genera Token di Accesso",
     "actionDeleteAccessToken": "Elimina Token di Accesso",
     "actionListAccessTokens": "Elenca Token di Accesso",

messages/ko-KR.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "사용자 제거",
     "actionListUsers": "사용자 목록",
     "actionAddUserRole": "사용자 역할 추가",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "액세스 토큰 생성",
     "actionDeleteAccessToken": "액세스 토큰 삭제",
     "actionListAccessTokens": "액세스 토큰 목록",

messages/nb-NO.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "Fjern bruker",
     "actionListUsers": "List opp brukere",
     "actionAddUserRole": "Legg til brukerrolle",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Generer tilgangstoken",
     "actionDeleteAccessToken": "Slett tilgangstoken",
     "actionListAccessTokens": "List opp tilgangstokener",

messages/nl-NL.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "Gebruiker verwijderen",
     "actionListUsers": "Gebruikers weergeven",
     "actionAddUserRole": "Gebruikersrol toevoegen",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Genereer Toegangstoken",
     "actionDeleteAccessToken": "Verwijder toegangstoken",
     "actionListAccessTokens": "Lijst toegangstokens",

messages/pl-PL.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "Usuń użytkownika",
     "actionListUsers": "Lista użytkowników",
     "actionAddUserRole": "Dodaj rolę użytkownika",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Wygeneruj token dostępu",
     "actionDeleteAccessToken": "Usuń token dostępu",
     "actionListAccessTokens": "Lista tokenów dostępu",

messages/pt-PT.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "Remover Utilizador",
     "actionListUsers": "Listar Utilizadores",
     "actionAddUserRole": "Adicionar Função ao Utilizador",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Gerar Token de Acesso",
     "actionDeleteAccessToken": "Eliminar Token de Acesso",
     "actionListAccessTokens": "Listar Tokens de Acesso",

messages/ru-RU.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "Удалить пользователя",
     "actionListUsers": "Список пользователей",
     "actionAddUserRole": "Добавить роль пользователя",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Сгенерировать токен доступа",
     "actionDeleteAccessToken": "Удалить токен доступа",
     "actionListAccessTokens": "Список токенов доступа",

messages/tr-TR.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "Kullanıcıyı Kaldır",
     "actionListUsers": "Kullanıcıları Listele",
     "actionAddUserRole": "Kullanıcı Rolü Ekle",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "Erişim Jetonu Oluştur",
     "actionDeleteAccessToken": "Erişim Jetonunu Sil",
     "actionListAccessTokens": "Erişim Jetonlarını Listele",

messages/zh-CN.json

@@ -1148,6 +1148,7 @@
     "actionRemoveUser": "删除用户",
     "actionListUsers": "列出用户",
     "actionAddUserRole": "添加用户角色",
+    "actionSetUserOrgRoles": "Set User Roles",
     "actionGenerateAccessToken": "生成访问令牌",
     "actionDeleteAccessToken": "删除访问令牌",
     "actionListAccessTokens": "访问令牌",

messages/zh-TW.json

@@ -1091,6 +1091,7 @@
   "actionRemoveUser": "刪除用戶",
   "actionListUsers": "列出用戶",
   "actionAddUserRole": "添加用戶角色",
+  "actionSetUserOrgRoles": "Set User Roles",
   "actionGenerateAccessToken": "生成訪問令牌",
   "actionDeleteAccessToken": "刪除訪問令牌",
   "actionListAccessTokens": "訪問令牌",

server/auth/actions.ts

@@ -1,9 +1,10 @@
 import { Request } from "express";
 import { db } from "@server/db";
-import { userActions, roleActions, userOrgs } from "@server/db";
-import { and, eq } from "drizzle-orm";
+import { userActions, roleActions } from "@server/db";
+import { and, eq, inArray } from "drizzle-orm";
 import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export enum ActionsEnum {
     createOrgUser = "createOrgUser",
@@ -53,6 +54,8 @@ export enum ActionsEnum {
     listRoleResources = "listRoleResources",
     // listRoleActions = "listRoleActions",
     addUserRole = "addUserRole",
+    removeUserRole = "removeUserRole",
+    setUserOrgRoles = "setUserOrgRoles",
     // addUserSite = "addUserSite",
     // addUserAction = "addUserAction",
     // removeUserAction = "removeUserAction",
@@ -154,29 +157,16 @@ export async function checkUserActionPermission(
     }
 
     try {
-        let userOrgRoleId = req.userOrgRoleId;
+        let userOrgRoleIds = req.userOrgRoleIds;
 
-        // If userOrgRoleId is not available on the request, fetch it
-        if (userOrgRoleId === undefined) {
-            const userOrgRole = await db
-                .select()
-                .from(userOrgs)
-                .where(
-                    and(
-                        eq(userOrgs.userId, userId),
-                        eq(userOrgs.orgId, req.userOrgId!)
-                    )
-                )
-                .limit(1);
-
-            if (userOrgRole.length === 0) {
+        if (userOrgRoleIds === undefined) {
+            userOrgRoleIds = await getUserOrgRoleIds(userId, req.userOrgId!);
+            if (userOrgRoleIds.length === 0) {
                 throw createHttpError(
                     HttpCode.FORBIDDEN,
                     "User does not have access to this organization"
                 );
             }
-
-            userOrgRoleId = userOrgRole[0].roleId;
         }
 
         // Check if the user has direct permission for the action in the current org
@@ -187,7 +177,7 @@ export async function checkUserActionPermission(
                 and(
                     eq(userActions.userId, userId),
                     eq(userActions.actionId, actionId),
-                    eq(userActions.orgId, req.userOrgId!) // TODO: we cant pass the org id if we are not checking the org
+                    eq(userActions.orgId, req.userOrgId!)
                 )
             )
             .limit(1);
@@ -196,14 +186,14 @@ export async function checkUserActionPermission(
             return true;
         }
 
-        // If no direct permission, check role-based permission
+        // If no direct permission, check role-based permission (any of user's roles)
         const roleActionPermission = await db
             .select()
             .from(roleActions)
             .where(
                 and(
                     eq(roleActions.actionId, actionId),
-                    eq(roleActions.roleId, userOrgRoleId!),
+                    inArray(roleActions.roleId, userOrgRoleIds),
                     eq(roleActions.orgId, req.userOrgId!)
                 )
             )

server/auth/canUserAccessResource.ts

@@ -1,26 +1,29 @@
 import { db } from "@server/db";
-import { and, eq } from "drizzle-orm";
+import { and, eq, inArray } from "drizzle-orm";
 import { roleResources, userResources } from "@server/db";
 
 export async function canUserAccessResource({
     userId,
     resourceId,
-    roleId
+    roleIds
 }: {
     userId: string;
     resourceId: number;
-    roleId: number;
+    roleIds: number[];
 }): Promise<boolean> {
-    const roleResourceAccess = await db
-        .select()
-        .from(roleResources)
-        .where(
-            and(
-                eq(roleResources.resourceId, resourceId),
-                eq(roleResources.roleId, roleId)
-            )
-        )
-        .limit(1);
+    const roleResourceAccess =
+        roleIds.length > 0
+            ? await db
+                  .select()
+                  .from(roleResources)
+                  .where(
+                      and(
+                          eq(roleResources.resourceId, resourceId),
+                          inArray(roleResources.roleId, roleIds)
+                      )
+                  )
+                  .limit(1)
+            : [];
 
     if (roleResourceAccess.length > 0) {
         return true;

server/auth/canUserAccessSiteResource.ts

@@ -1,26 +1,29 @@
 import { db } from "@server/db";
-import { and, eq } from "drizzle-orm";
+import { and, eq, inArray } from "drizzle-orm";
 import { roleSiteResources, userSiteResources } from "@server/db";
 
 export async function canUserAccessSiteResource({
     userId,
     resourceId,
-    roleId
+    roleIds
 }: {
     userId: string;
     resourceId: number;
-    roleId: number;
+    roleIds: number[];
 }): Promise<boolean> {
-    const roleResourceAccess = await db
-        .select()
-        .from(roleSiteResources)
-        .where(
-            and(
-                eq(roleSiteResources.siteResourceId, resourceId),
-                eq(roleSiteResources.roleId, roleId)
-            )
-        )
-        .limit(1);
+    const roleResourceAccess =
+        roleIds.length > 0
+            ? await db
+                  .select()
+                  .from(roleSiteResources)
+                  .where(
+                      and(
+                          eq(roleSiteResources.siteResourceId, resourceId),
+                          inArray(roleSiteResources.roleId, roleIds)
+                      )
+                  )
+                  .limit(1)
+            : [];
 
     if (roleResourceAccess.length > 0) {
         return true;

server/cleanup.ts

@@ -1,8 +1,10 @@
 import { flushBandwidthToDb } from "@server/routers/newt/handleReceiveBandwidthMessage";
 import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth";
+import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator";
 import { cleanup as wsCleanup } from "#dynamic/routers/ws";
 
 async function cleanup() {
+    await stopPingAccumulator();
     await flushBandwidthToDb();
     await flushSiteBandwidthToDb();
     await wsCleanup();

server/db/pg/driver.ts

@@ -1,7 +1,7 @@
 import { drizzle as DrizzlePostgres } from "drizzle-orm/node-postgres";
-import { Pool } from "pg";
 import { readConfigFile } from "@server/lib/readConfigFile";
 import { withReplicas } from "drizzle-orm/pg-core";
+import { createPool } from "./poolConfig";
 
 function createDb() {
     const config = readConfigFile();
@@ -39,12 +39,17 @@ function createDb() {
 
     // Create connection pools instead of individual connections
     const poolConfig = config.postgres.pool;
-    const primaryPool = new Pool({
+    const maxConnections = poolConfig?.max_connections || 20;
+    const idleTimeoutMs = poolConfig?.idle_timeout_ms || 30000;
+    const connectionTimeoutMs = poolConfig?.connection_timeout_ms || 5000;
+
+    const primaryPool = createPool(
         connectionString,
-        max: poolConfig?.max_connections || 20,
-        idleTimeoutMillis: poolConfig?.idle_timeout_ms || 30000,
-        connectionTimeoutMillis: poolConfig?.connection_timeout_ms || 5000
-    });
+        maxConnections,
+        idleTimeoutMs,
+        connectionTimeoutMs,
+        "primary"
+    );
 
     const replicas = [];
 
@@ -55,14 +60,16 @@ function createDb() {
             })
         );
     } else {
+        const maxReplicaConnections =
+            poolConfig?.max_replica_connections || 20;
         for (const conn of replicaConnections) {
-            const replicaPool = new Pool({
-                connectionString: conn.connection_string,
-                max: poolConfig?.max_replica_connections || 20,
-                idleTimeoutMillis: poolConfig?.idle_timeout_ms || 30000,
-                connectionTimeoutMillis:
-                    poolConfig?.connection_timeout_ms || 5000
-            });
+            const replicaPool = createPool(
+                conn.connection_string,
+                maxReplicaConnections,
+                idleTimeoutMs,
+                connectionTimeoutMs,
+                "replica"
+            );
             replicas.push(
                 DrizzlePostgres(replicaPool, {
                     logger: process.env.QUERY_LOGGING == "true"

server/db/pg/logsDriver.ts

@@ -1,9 +1,9 @@
 import { drizzle as DrizzlePostgres } from "drizzle-orm/node-postgres";
-import { Pool } from "pg";
 import { readConfigFile } from "@server/lib/readConfigFile";
 import { withReplicas } from "drizzle-orm/pg-core";
 import { build } from "@server/build";
 import { db as mainDb, primaryDb as mainPrimaryDb } from "./driver";
+import { createPool } from "./poolConfig";
 
 function createLogsDb() {
     // Only use separate logs database in SaaS builds
@@ -42,12 +42,17 @@ function createLogsDb() {
 
     // Create separate connection pool for logs database
     const poolConfig = logsConfig?.pool || config.postgres?.pool;
-    const primaryPool = new Pool({
+    const maxConnections = poolConfig?.max_connections || 20;
+    const idleTimeoutMs = poolConfig?.idle_timeout_ms || 30000;
+    const connectionTimeoutMs = poolConfig?.connection_timeout_ms || 5000;
+
+    const primaryPool = createPool(
         connectionString,
-        max: poolConfig?.max_connections || 20,
-        idleTimeoutMillis: poolConfig?.idle_timeout_ms || 30000,
-        connectionTimeoutMillis: poolConfig?.connection_timeout_ms || 5000
-    });
+        maxConnections,
+        idleTimeoutMs,
+        connectionTimeoutMs,
+        "logs-primary"
+    );
 
     const replicas = [];
 
@@ -58,14 +63,16 @@ function createLogsDb() {
             })
         );
     } else {
+        const maxReplicaConnections =
+            poolConfig?.max_replica_connections || 20;
         for (const conn of replicaConnections) {
-            const replicaPool = new Pool({
-                connectionString: conn.connection_string,
-                max: poolConfig?.max_replica_connections || 20,
-                idleTimeoutMillis: poolConfig?.idle_timeout_ms || 30000,
-                connectionTimeoutMillis:
-                    poolConfig?.connection_timeout_ms || 5000
-            });
+            const replicaPool = createPool(
+                conn.connection_string,
+                maxReplicaConnections,
+                idleTimeoutMs,
+                connectionTimeoutMs,
+                "logs-replica"
+            );
             replicas.push(
                 DrizzlePostgres(replicaPool, {
                     logger: process.env.QUERY_LOGGING == "true"

server/db/pg/poolConfig.ts

@@ -0,0 +1,63 @@
+import { Pool, PoolConfig } from "pg";
+import logger from "@server/logger";
+
+export function createPoolConfig(
+    connectionString: string,
+    maxConnections: number,
+    idleTimeoutMs: number,
+    connectionTimeoutMs: number
+): PoolConfig {
+    return {
+        connectionString,
+        max: maxConnections,
+        idleTimeoutMillis: idleTimeoutMs,
+        connectionTimeoutMillis: connectionTimeoutMs,
+        // TCP keepalive to prevent silent connection drops by NAT gateways,
+        // load balancers, and other intermediate network devices (e.g. AWS
+        // NAT Gateway drops idle TCP connections after ~350s)
+        keepAlive: true,
+        keepAliveInitialDelayMillis: 10000, // send first keepalive after 10s of idle
+        // Allow connections to be released and recreated more aggressively
+        // to avoid stale connections building up
+        allowExitOnIdle: false
+    };
+}
+
+export function attachPoolErrorHandlers(pool: Pool, label: string): void {
+    pool.on("error", (err) => {
+        // This catches errors on idle clients in the pool. Without this
+        // handler an unexpected disconnect would crash the process.
+        logger.error(
+            `Unexpected error on idle ${label} database client: ${err.message}`
+        );
+    });
+
+    pool.on("connect", (client) => {
+        // Set a statement timeout on every new connection so a single slow
+        // query can't block the pool forever
+        client.query("SET statement_timeout = '30s'").catch((err: Error) => {
+            logger.warn(
+                `Failed to set statement_timeout on ${label} client: ${err.message}`
+            );
+        });
+    });
+}
+
+export function createPool(
+    connectionString: string,
+    maxConnections: number,
+    idleTimeoutMs: number,
+    connectionTimeoutMs: number,
+    label: string
+): Pool {
+    const pool = new Pool(
+        createPoolConfig(
+            connectionString,
+            maxConnections,
+            idleTimeoutMs,
+            connectionTimeoutMs
+        )
+    );
+    attachPoolErrorHandlers(pool, label);
+    return pool;
+}

server/db/pg/schema/schema.ts

@@ -9,6 +9,7 @@ import {
     real,
     serial,
     text,
+    unique,
     varchar
 } from "drizzle-orm/pg-core";
 
@@ -335,9 +336,6 @@ export const userOrgs = pgTable("userOrgs", {
             onDelete: "cascade"
         })
         .notNull(),
-    roleId: integer("roleId")
-        .notNull()
-        .references(() => roles.roleId),
     isOwner: boolean("isOwner").notNull().default(false),
     autoProvisioned: boolean("autoProvisioned").default(false),
     pamUsername: varchar("pamUsername") // cleaned username for ssh and such
@@ -386,6 +384,22 @@ export const roles = pgTable("roles", {
     sshUnixGroups: text("sshUnixGroups").default("[]")
 });
 
+export const userOrgRoles = pgTable(
+    "userOrgRoles",
+    {
+        userId: varchar("userId")
+            .notNull()
+            .references(() => users.userId, { onDelete: "cascade" }),
+        orgId: varchar("orgId")
+            .notNull()
+            .references(() => orgs.orgId, { onDelete: "cascade" }),
+        roleId: integer("roleId")
+            .notNull()
+            .references(() => roles.roleId, { onDelete: "cascade" })
+    },
+    (t) => [unique().on(t.userId, t.orgId, t.roleId)]
+);
+
 export const roleActions = pgTable("roleActions", {
     roleId: integer("roleId")
         .notNull()
@@ -1035,6 +1049,7 @@ export type RoleResource = InferSelectModel<typeof roleResources>;
 export type UserResource = InferSelectModel<typeof userResources>;
 export type UserInvite = InferSelectModel<typeof userInvites>;
 export type UserOrg = InferSelectModel<typeof userOrgs>;
+export type UserOrgRole = InferSelectModel<typeof userOrgRoles>;
 export type ResourceSession = InferSelectModel<typeof resourceSessions>;
 export type ResourcePincode = InferSelectModel<typeof resourcePincode>;
 export type ResourcePassword = InferSelectModel<typeof resourcePassword>;

server/db/queries/verifySessionQueries.ts

@@ -12,6 +12,7 @@ import {
     resources,
     roleResources,
     sessions,
+    userOrgRoles,
     userOrgs,
     userResources,
     users,
@@ -104,24 +105,15 @@ export async function getUserSessionWithUser(
 }
 
 /**
- * Get user organization role
+ * Get role name by role ID (for display).
  */
-export async function getUserOrgRole(userId: string, orgId: string) {
-    const userOrgRole = await db
-        .select({
-            userId: userOrgs.userId,
-            orgId: userOrgs.orgId,
-            roleId: userOrgs.roleId,
-            isOwner: userOrgs.isOwner,
-            autoProvisioned: userOrgs.autoProvisioned,
-            roleName: roles.name
-        })
-        .from(userOrgs)
-        .where(and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, orgId)))
-        .leftJoin(roles, eq(userOrgs.roleId, roles.roleId))
+export async function getRoleName(roleId: number): Promise<string | null> {
+    const [row] = await db
+        .select({ name: roles.name })
+        .from(roles)
+        .where(eq(roles.roleId, roleId))
         .limit(1);
-
-    return userOrgRole.length > 0 ? userOrgRole[0] : null;
+    return row?.name ?? null;
 }
 
 /**

server/db/sqlite/schema/schema.ts

@@ -1,6 +1,12 @@
 import { randomUUID } from "crypto";
 import { InferSelectModel } from "drizzle-orm";
-import { index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
+import {
+    index,
+    integer,
+    sqliteTable,
+    text,
+    unique
+} from "drizzle-orm/sqlite-core";
 
 export const domains = sqliteTable("domains", {
     domainId: text("domainId").primaryKey(),
@@ -643,9 +649,6 @@ export const userOrgs = sqliteTable("userOrgs", {
             onDelete: "cascade"
         })
         .notNull(),
-    roleId: integer("roleId")
-        .notNull()
-        .references(() => roles.roleId),
     isOwner: integer("isOwner", { mode: "boolean" }).notNull().default(false),
     autoProvisioned: integer("autoProvisioned", {
         mode: "boolean"
@@ -700,6 +703,22 @@ export const roles = sqliteTable("roles", {
     sshUnixGroups: text("sshUnixGroups").default("[]")
 });
 
+export const userOrgRoles = sqliteTable(
+    "userOrgRoles",
+    {
+        userId: text("userId")
+            .notNull()
+            .references(() => users.userId, { onDelete: "cascade" }),
+        orgId: text("orgId")
+            .notNull()
+            .references(() => orgs.orgId, { onDelete: "cascade" }),
+        roleId: integer("roleId")
+            .notNull()
+            .references(() => roles.roleId, { onDelete: "cascade" })
+    },
+    (t) => [unique().on(t.userId, t.orgId, t.roleId)]
+);
+
 export const roleActions = sqliteTable("roleActions", {
     roleId: integer("roleId")
         .notNull()
@@ -1134,6 +1153,7 @@ export type RoleResource = InferSelectModel<typeof roleResources>;
 export type UserResource = InferSelectModel<typeof userResources>;
 export type UserInvite = InferSelectModel<typeof userInvites>;
 export type UserOrg = InferSelectModel<typeof userOrgs>;
+export type UserOrgRole = InferSelectModel<typeof userOrgRoles>;
 export type ResourceSession = InferSelectModel<typeof resourceSessions>;
 export type ResourcePincode = InferSelectModel<typeof resourcePincode>;
 export type ResourcePassword = InferSelectModel<typeof resourcePassword>;

server/index.ts

@@ -74,7 +74,7 @@ declare global {
             session: Session;
             userOrg?: UserOrg;
             apiKeyOrg?: ApiKeyOrg;
-            userOrgRoleId?: number;
+            userOrgRoleIds?: number[];
             userOrgId?: string;
             userOrgIds?: string[];
             remoteExitNode?: RemoteExitNode;

server/lib/calculateUserClientsForOrgs.ts

@@ -10,6 +10,7 @@ import {
     roles,
     Transaction,
     userClients,
+    userOrgRoles,
     userOrgs
 } from "@server/db";
 import { getUniqueClientName } from "@server/db/names";
@@ -39,20 +40,36 @@ export async function calculateUserClientsForOrgs(
             return;
         }
 
-        // Get all user orgs
-        const allUserOrgs = await transaction
+        // Get all user orgs with all roles (for org list and role-based logic)
+        const userOrgRoleRows = await transaction
             .select()
             .from(userOrgs)
-            .innerJoin(roles, eq(roles.roleId, userOrgs.roleId))
+            .innerJoin(
+                userOrgRoles,
+                and(
+                    eq(userOrgs.userId, userOrgRoles.userId),
+                    eq(userOrgs.orgId, userOrgRoles.orgId)
+                )
+            )
+            .innerJoin(roles, eq(userOrgRoles.roleId, roles.roleId))
             .where(eq(userOrgs.userId, userId));
 
-        const userOrgIds = allUserOrgs.map(({ userOrgs: uo }) => uo.orgId);
+        const userOrgIds = [...new Set(userOrgRoleRows.map((r) => r.userOrgs.orgId))];
+        const orgIdToRoleRows = new Map<
+            string,
+            (typeof userOrgRoleRows)[0][]
+        >();
+        for (const r of userOrgRoleRows) {
+            const list = orgIdToRoleRows.get(r.userOrgs.orgId) ?? [];
+            list.push(r);
+            orgIdToRoleRows.set(r.userOrgs.orgId, list);
+        }
 
         // For each OLM, ensure there's a client in each org the user is in
         for (const olm of userOlms) {
-            for (const userRoleOrg of allUserOrgs) {
-                const { userOrgs: userOrg, roles: role } = userRoleOrg;
-                const orgId = userOrg.orgId;
+            for (const orgId of orgIdToRoleRows.keys()) {
+                const roleRowsForOrg = orgIdToRoleRows.get(orgId)!;
+                const userOrg = roleRowsForOrg[0].userOrgs;
 
                 const [org] = await transaction
                     .select()
@@ -196,7 +213,7 @@ export async function calculateUserClientsForOrgs(
                 const requireApproval =
                     build !== "oss" &&
                     isOrgLicensed &&
-                    role.requireDeviceApproval;
+                    roleRowsForOrg.some((r) => r.roles.requireDeviceApproval);
 
                 const newClientData: InferInsertModel<typeof clients> = {
                     userId,

server/lib/ip.ts

@@ -571,6 +571,129 @@ export function generateSubnetProxyTargets(
     return targets;
 }
 
+export type SubnetProxyTargetV2 = {
+    sourcePrefixes: string[]; // must be cidrs
+    destPrefix: string; // must be a cidr
+    disableIcmp?: boolean;
+    rewriteTo?: string; // must be a cidr
+    portRange?: {
+        min: number;
+        max: number;
+        protocol: "tcp" | "udp";
+    }[];
+};
+
+export function generateSubnetProxyTargetV2(
+    siteResource: SiteResource,
+    clients: {
+        clientId: number;
+        pubKey: string | null;
+        subnet: string | null;
+    }[]
+): SubnetProxyTargetV2 | undefined {
+    if (clients.length === 0) {
+        logger.debug(
+            `No clients have access to site resource ${siteResource.siteResourceId}, skipping target generation.`
+        );
+        return;
+    }
+
+    let target: SubnetProxyTargetV2 | null = null;
+
+    const portRange = [
+        ...parsePortRangeString(siteResource.tcpPortRangeString, "tcp"),
+        ...parsePortRangeString(siteResource.udpPortRangeString, "udp")
+    ];
+    const disableIcmp = siteResource.disableIcmp ?? false;
+
+    if (siteResource.mode == "host") {
+        let destination = siteResource.destination;
+        // check if this is a valid ip
+        const ipSchema = z.union([z.ipv4(), z.ipv6()]);
+        if (ipSchema.safeParse(destination).success) {
+            destination = `${destination}/32`;
+
+            target = {
+                sourcePrefixes: [],
+                destPrefix: destination,
+                portRange,
+                disableIcmp
+            };
+        }
+
+        if (siteResource.alias && siteResource.aliasAddress) {
+            // also push a match for the alias address
+            target = {
+                sourcePrefixes: [],
+                destPrefix: `${siteResource.aliasAddress}/32`,
+                rewriteTo: destination,
+                portRange,
+                disableIcmp
+            };
+        }
+    } else if (siteResource.mode == "cidr") {
+        target = {
+            sourcePrefixes: [],
+            destPrefix: siteResource.destination,
+            portRange,
+            disableIcmp
+        };
+    }
+
+    if (!target) {
+        return;
+    }
+
+    for (const clientSite of clients) {
+        if (!clientSite.subnet) {
+            logger.debug(
+                `Client ${clientSite.clientId} has no subnet, skipping for site resource ${siteResource.siteResourceId}.`
+            );
+            continue;
+        }
+
+        const clientPrefix = `${clientSite.subnet.split("/")[0]}/32`;
+
+        // add client prefix to source prefixes
+        target.sourcePrefixes.push(clientPrefix);
+    }
+
+    // print a nice representation of the targets
+    // logger.debug(
+    //     `Generated subnet proxy targets for: ${JSON.stringify(targets, null, 2)}`
+    // );
+
+    return target;
+}
+
+
+/**
+ * Converts a SubnetProxyTargetV2 to an array of SubnetProxyTarget (v1)
+ * by expanding each source prefix into its own target entry.
+ * @param targetV2 - The v2 target to convert
+ * @returns Array of v1 SubnetProxyTarget objects
+ */
+ export function convertSubnetProxyTargetsV2ToV1(
+     targetsV2: SubnetProxyTargetV2[]
+ ): SubnetProxyTarget[] {
+     return targetsV2.flatMap((targetV2) =>
+         targetV2.sourcePrefixes.map((sourcePrefix) => ({
+             sourcePrefix,
+             destPrefix: targetV2.destPrefix,
+             ...(targetV2.disableIcmp !== undefined && {
+                 disableIcmp: targetV2.disableIcmp
+             }),
+             ...(targetV2.rewriteTo !== undefined && {
+                 rewriteTo: targetV2.rewriteTo
+             }),
+             ...(targetV2.portRange !== undefined && {
+                 portRange: targetV2.portRange
+             })
+         }))
+     );
+ }
+
+
 // Custom schema for validating port range strings
 // Format: "80,443,8000-9000" or "*" for all ports, or empty string
 export const portRangeStringSchema = z

server/lib/readConfigFile.ts

@@ -302,8 +302,8 @@ export const configSchema = z
             .optional()
             .default({
                 block_size: 24,
-                subnet_group: "100.90.128.0/24",
-                utility_subnet_group: "100.96.128.0/24"
+                subnet_group: "100.90.128.0/20",
+                utility_subnet_group: "100.96.128.0/20"
             }),
         rate_limits: z
             .object({

server/lib/rebuildClientAssociations.ts

@@ -14,6 +14,7 @@ import {
     siteResources,
     sites,
     Transaction,
+    userOrgRoles,
     userOrgs,
     userSiteResources
 } from "@server/db";
@@ -32,7 +33,7 @@ import logger from "@server/logger";
 import {
     generateAliasConfig,
     generateRemoteSubnets,
-    generateSubnetProxyTargets,
+    generateSubnetProxyTargetV2,
     parseEndpoint,
     formatEndpoint
 } from "@server/lib/ip";
@@ -77,10 +78,10 @@ export async function getClientSiteResourceAccess(
     // get all of the users in these roles
     const userIdsFromRoles = await trx
         .select({
-            userId: userOrgs.userId
+            userId: userOrgRoles.userId
         })
-        .from(userOrgs)
-        .where(inArray(userOrgs.roleId, roleIds))
+        .from(userOrgRoles)
+        .where(inArray(userOrgRoles.roleId, roleIds))
         .then((rows) => rows.map((row) => row.userId));
 
     const newAllUserIds = Array.from(
@@ -660,19 +661,16 @@ async function handleSubnetProxyTargetUpdates(
         );
 
         if (addedClients.length > 0) {
-            const targetsToAdd = generateSubnetProxyTargets(
+            const targetToAdd = generateSubnetProxyTargetV2(
                 siteResource,
                 addedClients
             );
 
-            if (targetsToAdd.length > 0) {
-                logger.info(
-                    `Adding ${targetsToAdd.length} subnet proxy targets for siteResource ${siteResource.siteResourceId}`
-                );
+            if (targetToAdd) {
                 proxyJobs.push(
                     addSubnetProxyTargets(
                         newt.newtId,
-                        targetsToAdd,
+                        [targetToAdd],
                         newt.version
                     )
                 );
@@ -700,19 +698,16 @@ async function handleSubnetProxyTargetUpdates(
         );
 
         if (removedClients.length > 0) {
-            const targetsToRemove = generateSubnetProxyTargets(
+            const targetToRemove = generateSubnetProxyTargetV2(
                 siteResource,
                 removedClients
             );
 
-            if (targetsToRemove.length > 0) {
-                logger.info(
-                    `Removing ${targetsToRemove.length} subnet proxy targets for siteResource ${siteResource.siteResourceId}`
-                );
+            if (targetToRemove) {
                 proxyJobs.push(
                     removeSubnetProxyTargets(
                         newt.newtId,
-                        targetsToRemove,
+                        [targetToRemove],
                         newt.version
                     )
                 );
@@ -820,12 +815,12 @@ export async function rebuildClientAssociationsFromClient(
 
         // Role-based access
         const roleIds = await trx
-            .select({ roleId: userOrgs.roleId })
-            .from(userOrgs)
+            .select({ roleId: userOrgRoles.roleId })
+            .from(userOrgRoles)
             .where(
                 and(
-                    eq(userOrgs.userId, client.userId),
-                    eq(userOrgs.orgId, client.orgId)
+                    eq(userOrgRoles.userId, client.userId),
+                    eq(userOrgRoles.orgId, client.orgId)
                 )
             ) // this needs to be locked onto this org or else cross-org access could happen
             .then((rows) => rows.map((row) => row.roleId));
@@ -1169,7 +1164,7 @@ async function handleMessagesForClientResources(
             }
 
             for (const resource of resources) {
-                const targets = generateSubnetProxyTargets(resource, [
+                const target = generateSubnetProxyTargetV2(resource, [
                     {
                         clientId: client.clientId,
                         pubKey: client.pubKey,
@@ -1177,11 +1172,11 @@ async function handleMessagesForClientResources(
                     }
                 ]);
 
-                if (targets.length > 0) {
+                if (target) {
                     proxyJobs.push(
                         addSubnetProxyTargets(
                             newt.newtId,
-                            targets,
+                            [target],
                             newt.version
                         )
                     );
@@ -1246,7 +1241,7 @@ async function handleMessagesForClientResources(
             }
 
             for (const resource of resources) {
-                const targets = generateSubnetProxyTargets(resource, [
+                const target = generateSubnetProxyTargetV2(resource, [
                     {
                         clientId: client.clientId,
                         pubKey: client.pubKey,
@@ -1254,11 +1249,11 @@ async function handleMessagesForClientResources(
                     }
                 ]);
 
-                if (targets.length > 0) {
+                if (target) {
                     proxyJobs.push(
                         removeSubnetProxyTargets(
                             newt.newtId,
-                            targets,
+                            [target],
                             newt.version
                         )
                     );

server/lib/sanitize.ts

@@ -0,0 +1,40 @@
+/**
+ * Sanitize a string field before inserting into a database TEXT column.
+ *
+ * Two passes are applied:
+ *
+ * 1. Lone UTF-16 surrogates – JavaScript strings can hold unpaired surrogates
+ *    (e.g. \uD800 without a following \uDC00-\uDFFF codepoint). These are
+ *    valid in JS but cannot be encoded as UTF-8, triggering
+ *    `report_invalid_encoding` in SQLite / Postgres. They are replaced with
+ *    the Unicode replacement character U+FFFD so the data is preserved as a
+ *    visible signal that something was malformed.
+ *
+ * 2. Null bytes and C0 control characters – SQLite stores TEXT as
+ *    null-terminated C strings, so \x00 in a value causes
+ *    `report_invalid_encoding`. Bots and scanners routinely inject null bytes
+ *    into URLs (e.g. `/path\u0000.jpg`). All C0 control characters in the
+ *    range \x00-\x1F are stripped except for the three that are legitimate in
+ *    text payloads: HT (\x09), LF (\x0A), and CR (\x0D). DEL (\x7F) is also
+ *    stripped.
+ */
+export function sanitizeString(value: string): string;
+export function sanitizeString(
+    value: string | null | undefined
+): string | undefined;
+export function sanitizeString(
+    value: string | null | undefined
+): string | undefined {
+    if (value == null) return undefined;
+    return (
+        value
+            // Replace lone high surrogates (not followed by a low surrogate)
+            // and lone low surrogates (not preceded by a high surrogate).
+            .replace(
+                /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g,
+                "\uFFFD"
+            )
+            // Strip null bytes, C0 control chars (except HT/LF/CR), and DEL.
+            .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "")
+    );
+}

server/lib/tokenCache.ts

@@ -0,0 +1,22 @@
+/**
+ * Returns a cached plaintext token from Redis if one exists and decrypts
+ * cleanly, otherwise calls `createSession` to mint a fresh token, stores the
+ * encrypted value in Redis with the given TTL, and returns it.
+ *
+ * Failures at the Redis layer are non-fatal – the function always falls
+ * through to session creation so the caller is never blocked by a Redis outage.
+ *
+ * @param cacheKey   Unique Redis key, e.g. `"newt:token_cache:abc123"`
+ * @param secret     Server secret used for AES encryption/decryption
+ * @param ttlSeconds Cache TTL in seconds (should match session expiry)
+ * @param createSession Factory that mints a new session and returns its raw token
+ */
+export async function getOrCreateCachedToken(
+    cacheKey: string,
+    secret: string,
+    ttlSeconds: number,
+    createSession: () => Promise<string>
+): Promise<string> {
+    const token = await createSession();
+    return token;
+}

server/lib/userOrg.ts

@@ -6,7 +6,7 @@ import {
     siteResources,
     sites,
     Transaction,
-    UserOrg,
+    userOrgRoles,
     userOrgs,
     userResources,
     userSiteResources,
@@ -19,9 +19,15 @@ import { FeatureId } from "@server/lib/billing";
 export async function assignUserToOrg(
     org: Org,
     values: typeof userOrgs.$inferInsert,
+    roleId: number,
     trx: Transaction | typeof db = db
 ) {
     const [userOrg] = await trx.insert(userOrgs).values(values).returning();
+    await trx.insert(userOrgRoles).values({
+        userId: userOrg.userId,
+        orgId: userOrg.orgId,
+        roleId
+    });
 
     // calculate if the user is in any other of the orgs before we count it as an add to the billing org
     if (org.billingOrgId) {
@@ -58,6 +64,14 @@ export async function removeUserFromOrg(
     userId: string,
     trx: Transaction | typeof db = db
 ) {
+    await trx
+        .delete(userOrgRoles)
+        .where(
+            and(
+                eq(userOrgRoles.userId, userId),
+                eq(userOrgRoles.orgId, org.orgId)
+            )
+        );
     await trx
         .delete(userOrgs)
         .where(and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, org.orgId)));

server/lib/userOrgRoles.ts

@@ -0,0 +1,22 @@
+import { db, userOrgRoles } from "@server/db";
+import { and, eq } from "drizzle-orm";
+
+/**
+ * Get all role IDs a user has in an organization.
+ * Returns empty array if the user has no roles in the org (callers must treat as no access).
+ */
+export async function getUserOrgRoleIds(
+    userId: string,
+    orgId: string
+): Promise<number[]> {
+    const rows = await db
+        .select({ roleId: userOrgRoles.roleId })
+        .from(userOrgRoles)
+        .where(
+            and(
+                eq(userOrgRoles.userId, userId),
+                eq(userOrgRoles.orgId, orgId)
+            )
+        );
+    return rows.map((r) => r.roleId);
+}

server/middlewares/getUserOrgs.ts

@@ -21,8 +21,7 @@ export async function getUserOrgs(
     try {
         const userOrganizations = await db
             .select({
-                orgId: userOrgs.orgId,
-                roleId: userOrgs.roleId
+                orgId: userOrgs.orgId
             })
             .from(userOrgs)
             .where(eq(userOrgs.userId, userId));

server/middlewares/index.ts

@@ -17,6 +17,7 @@ export * from "./verifyAccessTokenAccess";
 export * from "./requestTimeout";
 export * from "./verifyClientAccess";
 export * from "./verifyUserHasAction";
+export * from "./verifyUserCanSetUserOrgRoles";
 export * from "./verifyUserIsServerAdmin";
 export * from "./verifyIsLoggedInUser";
 export * from "./verifyIsLoggedInUser";

server/middlewares/integration/index.ts

@@ -1,6 +1,7 @@
 export * from "./verifyApiKey";
 export * from "./verifyApiKeyOrgAccess";
 export * from "./verifyApiKeyHasAction";
+export * from "./verifyApiKeyCanSetUserOrgRoles";
 export * from "./verifyApiKeySiteAccess";
 export * from "./verifyApiKeyResourceAccess";
 export * from "./verifyApiKeyTargetAccess";

server/middlewares/integration/verifyApiKeyCanSetUserOrgRoles.ts

@@ -0,0 +1,74 @@
+import { Request, Response, NextFunction } from "express";
+import createHttpError from "http-errors";
+import HttpCode from "@server/types/HttpCode";
+import logger from "@server/logger";
+import { ActionsEnum } from "@server/auth/actions";
+import { db } from "@server/db";
+import { apiKeyActions } from "@server/db";
+import { and, eq } from "drizzle-orm";
+
+async function apiKeyHasAction(apiKeyId: string, actionId: ActionsEnum) {
+    const [row] = await db
+        .select()
+        .from(apiKeyActions)
+        .where(
+            and(
+                eq(apiKeyActions.apiKeyId, apiKeyId),
+                eq(apiKeyActions.actionId, actionId)
+            )
+        );
+    return !!row;
+}
+
+/**
+ * Allows setUserOrgRoles on the key, or both addUserRole and removeUserRole.
+ */
+export function verifyApiKeyCanSetUserOrgRoles() {
+    return async function (
+        req: Request,
+        res: Response,
+        next: NextFunction
+    ): Promise<any> {
+        try {
+            if (!req.apiKey) {
+                return next(
+                    createHttpError(
+                        HttpCode.UNAUTHORIZED,
+                        "API Key not authenticated"
+                    )
+                );
+            }
+
+            const keyId = req.apiKey.apiKeyId;
+
+            if (await apiKeyHasAction(keyId, ActionsEnum.setUserOrgRoles)) {
+                return next();
+            }
+
+            const hasAdd = await apiKeyHasAction(keyId, ActionsEnum.addUserRole);
+            const hasRemove = await apiKeyHasAction(
+                keyId,
+                ActionsEnum.removeUserRole
+            );
+
+            if (hasAdd && hasRemove) {
+                return next();
+            }
+
+            return next(
+                createHttpError(
+                    HttpCode.FORBIDDEN,
+                    "Key does not have permission perform this action"
+                )
+            );
+        } catch (error) {
+            logger.error("Error verifying API key set user org roles:", error);
+            return next(
+                createHttpError(
+                    HttpCode.INTERNAL_SERVER_ERROR,
+                    "Error verifying key action access"
+                )
+            );
+        }
+    };
+}

server/middlewares/verifyAccessTokenAccess.ts

@@ -6,6 +6,7 @@ import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
 import { canUserAccessResource } from "@server/auth/canUserAccessResource";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifyAccessTokenAccess(
     req: Request,
@@ -93,7 +94,10 @@ export async function verifyAccessTokenAccess(
                 )
             );
         } else {
-            req.userOrgRoleId = req.userOrg.roleId;
+            req.userOrgRoleIds = await getUserOrgRoleIds(
+                req.userOrg.userId,
+                resource[0].orgId!
+            );
             req.userOrgId = resource[0].orgId!;
         }
 
@@ -118,7 +122,7 @@ export async function verifyAccessTokenAccess(
         const resourceAllowed = await canUserAccessResource({
             userId,
             resourceId,
-            roleId: req.userOrgRoleId!
+            roleIds: req.userOrgRoleIds ?? []
         });
 
         if (!resourceAllowed) {

server/middlewares/verifyAdmin.ts

@@ -1,10 +1,11 @@
 import { Request, Response, NextFunction } from "express";
 import { db } from "@server/db";
 import { roles, userOrgs } from "@server/db";
-import { and, eq } from "drizzle-orm";
+import { and, eq, inArray } from "drizzle-orm";
 import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifyAdmin(
     req: Request,
@@ -62,13 +63,29 @@ export async function verifyAdmin(
         }
     }
 
-    const userRole = await db
+    req.userOrgRoleIds = await getUserOrgRoleIds(req.userOrg.userId, orgId!);
+
+    if (req.userOrgRoleIds.length === 0) {
+        return next(
+            createHttpError(
+                HttpCode.FORBIDDEN,
+                "User does not have Admin access"
+            )
+        );
+    }
+
+    const userAdminRoles = await db
         .select()
         .from(roles)
-        .where(eq(roles.roleId, req.userOrg.roleId))
+        .where(
+            and(
+                inArray(roles.roleId, req.userOrgRoleIds),
+                eq(roles.isAdmin, true)
+            )
+        )
         .limit(1);
 
-    if (userRole.length === 0 || !userRole[0].isAdmin) {
+    if (userAdminRoles.length === 0) {
         return next(
             createHttpError(
                 HttpCode.FORBIDDEN,

server/middlewares/verifyApiKeyAccess.ts

@@ -1,10 +1,11 @@
 import { Request, Response, NextFunction } from "express";
 import { db } from "@server/db";
 import { userOrgs, apiKeys, apiKeyOrg } from "@server/db";
-import { and, eq, or } from "drizzle-orm";
+import { and, eq } from "drizzle-orm";
 import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifyApiKeyAccess(
     req: Request,
@@ -103,8 +104,10 @@ export async function verifyApiKeyAccess(
             }
         }
 
-        const userOrgRoleId = req.userOrg.roleId;
-        req.userOrgRoleId = userOrgRoleId;
+        req.userOrgRoleIds = await getUserOrgRoleIds(
+            req.userOrg.userId,
+            orgId
+        );
 
         return next();
     } catch (error) {

server/middlewares/verifyClientAccess.ts

@@ -1,11 +1,12 @@
 import { Request, Response, NextFunction } from "express";
 import { Client, db } from "@server/db";
 import { userOrgs, clients, roleClients, userClients } from "@server/db";
-import { and, eq } from "drizzle-orm";
+import { and, eq, inArray } from "drizzle-orm";
 import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
 import logger from "@server/logger";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifyClientAccess(
     req: Request,
@@ -113,21 +114,30 @@ export async function verifyClientAccess(
             }
         }
 
-        const userOrgRoleId = req.userOrg.roleId;
-        req.userOrgRoleId = userOrgRoleId;
+        req.userOrgRoleIds = await getUserOrgRoleIds(
+            req.userOrg.userId,
+            client.orgId
+        );
         req.userOrgId = client.orgId;
 
-        // Check role-based site access first
-        const [roleClientAccess] = await db
-            .select()
-            .from(roleClients)
-            .where(
-                and(
-                    eq(roleClients.clientId, client.clientId),
-                    eq(roleClients.roleId, userOrgRoleId)
-                )
-            )
-            .limit(1);
+        // Check role-based client access (any of user's roles)
+        const roleClientAccessList =
+            (req.userOrgRoleIds?.length ?? 0) > 0
+                ? await db
+                      .select()
+                      .from(roleClients)
+                      .where(
+                          and(
+                              eq(roleClients.clientId, client.clientId),
+                              inArray(
+                                  roleClients.roleId,
+                                  req.userOrgRoleIds!
+                              )
+                          )
+                      )
+                      .limit(1)
+                : [];
+        const [roleClientAccess] = roleClientAccessList;
 
         if (roleClientAccess) {
             // User has access to the site through their role

server/middlewares/verifyDomainAccess.ts

@@ -1,10 +1,11 @@
 import { Request, Response, NextFunction } from "express";
 import { db, domains, orgDomains } from "@server/db";
-import { userOrgs, apiKeyOrg } from "@server/db";
+import { userOrgs } from "@server/db";
 import { and, eq } from "drizzle-orm";
 import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifyDomainAccess(
     req: Request,
@@ -63,7 +64,7 @@ export async function verifyDomainAccess(
                 .where(
                     and(
                         eq(userOrgs.userId, userId),
-                        eq(userOrgs.orgId, apiKeyOrg.orgId)
+                        eq(userOrgs.orgId, orgId)
                     )
                 )
                 .limit(1);
@@ -97,8 +98,7 @@ export async function verifyDomainAccess(
             }
         }
 
-        const userOrgRoleId = req.userOrg.roleId;
-        req.userOrgRoleId = userOrgRoleId;
+        req.userOrgRoleIds = await getUserOrgRoleIds(req.userOrg.userId, orgId);
 
         return next();
     } catch (error) {

server/middlewares/verifyOrgAccess.ts

@@ -1,10 +1,11 @@
 import { Request, Response, NextFunction } from "express";
-import { db, orgs } from "@server/db";
+import { db } from "@server/db";
 import { userOrgs } from "@server/db";
 import { and, eq } from "drizzle-orm";
 import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifyOrgAccess(
     req: Request,
@@ -64,8 +65,8 @@ export async function verifyOrgAccess(
             }
         }
 
-        // User has access, attach the user's role to the request for potential future use
-        req.userOrgRoleId = req.userOrg.roleId;
+        // User has access, attach the user's role(s) to the request for potential future use
+        req.userOrgRoleIds = await getUserOrgRoleIds(req.userOrg.userId, orgId);
         req.userOrgId = orgId;
 
         return next();

server/middlewares/verifyResourceAccess.ts

@@ -1,10 +1,11 @@
 import { Request, Response, NextFunction } from "express";
 import { db, Resource } from "@server/db";
 import { resources, userOrgs, userResources, roleResources } from "@server/db";
-import { and, eq } from "drizzle-orm";
+import { and, eq, inArray } from "drizzle-orm";
 import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifyResourceAccess(
     req: Request,
@@ -107,20 +108,28 @@ export async function verifyResourceAccess(
             }
         }
 
-        const userOrgRoleId = req.userOrg.roleId;
-        req.userOrgRoleId = userOrgRoleId;
+        req.userOrgRoleIds = await getUserOrgRoleIds(
+            req.userOrg.userId,
+            resource.orgId
+        );
         req.userOrgId = resource.orgId;
 
-        const roleResourceAccess = await db
-            .select()
-            .from(roleResources)
-            .where(
-                and(
-                    eq(roleResources.resourceId, resource.resourceId),
-                    eq(roleResources.roleId, userOrgRoleId)
-                )
-            )
-            .limit(1);
+        const roleResourceAccess =
+            (req.userOrgRoleIds?.length ?? 0) > 0
+                ? await db
+                      .select()
+                      .from(roleResources)
+                      .where(
+                          and(
+                              eq(roleResources.resourceId, resource.resourceId),
+                              inArray(
+                                  roleResources.roleId,
+                                  req.userOrgRoleIds!
+                              )
+                          )
+                      )
+                      .limit(1)
+                : [];
 
         if (roleResourceAccess.length > 0) {
             return next();

server/middlewares/verifyRoleAccess.ts

@@ -6,6 +6,7 @@ import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
 import logger from "@server/logger";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifyRoleAccess(
     req: Request,
@@ -99,7 +100,6 @@ export async function verifyRoleAccess(
         }
 
         if (!req.userOrg) {
-            // get the userORg
             const userOrg = await db
                 .select()
                 .from(userOrgs)
@@ -109,7 +109,7 @@ export async function verifyRoleAccess(
                 .limit(1);
 
             req.userOrg = userOrg[0];
-            req.userOrgRoleId = userOrg[0].roleId;
+            req.userOrgRoleIds = await getUserOrgRoleIds(userId, orgId!);
         }
 
         if (!req.userOrg) {

server/middlewares/verifySiteAccess.ts

@@ -1,10 +1,11 @@
 import { Request, Response, NextFunction } from "express";
 import { db } from "@server/db";
 import { sites, Site, userOrgs, userSites, roleSites, roles } from "@server/db";
-import { and, eq, or } from "drizzle-orm";
+import { and, eq, inArray, or } from "drizzle-orm";
 import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifySiteAccess(
     req: Request,
@@ -112,21 +113,29 @@ export async function verifySiteAccess(
             }
         }
 
-        const userOrgRoleId = req.userOrg.roleId;
-        req.userOrgRoleId = userOrgRoleId;
+        req.userOrgRoleIds = await getUserOrgRoleIds(
+            req.userOrg.userId,
+            site.orgId
+        );
         req.userOrgId = site.orgId;
 
-        // Check role-based site access first
-        const roleSiteAccess = await db
-            .select()
-            .from(roleSites)
-            .where(
-                and(
-                    eq(roleSites.siteId, site.siteId),
-                    eq(roleSites.roleId, userOrgRoleId)
-                )
-            )
-            .limit(1);
+        // Check role-based site access first (any of user's roles)
+        const roleSiteAccess =
+            (req.userOrgRoleIds?.length ?? 0) > 0
+                ? await db
+                      .select()
+                      .from(roleSites)
+                      .where(
+                          and(
+                              eq(roleSites.siteId, site.siteId),
+                              inArray(
+                                  roleSites.roleId,
+                                  req.userOrgRoleIds!
+                              )
+                          )
+                      )
+                      .limit(1)
+                : [];
 
         if (roleSiteAccess.length > 0) {
             // User's role has access to the site

server/middlewares/verifySiteResourceAccess.ts

@@ -1,11 +1,12 @@
 import { Request, Response, NextFunction } from "express";
 import { db, roleSiteResources, userOrgs, userSiteResources } from "@server/db";
 import { siteResources } from "@server/db";
-import { eq, and } from "drizzle-orm";
+import { eq, and, inArray } from "drizzle-orm";
 import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
 import logger from "@server/logger";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifySiteResourceAccess(
     req: Request,
@@ -109,23 +110,34 @@ export async function verifySiteResourceAccess(
             }
         }
 
-        const userOrgRoleId = req.userOrg.roleId;
-        req.userOrgRoleId = userOrgRoleId;
+        req.userOrgRoleIds = await getUserOrgRoleIds(
+            req.userOrg.userId,
+            siteResource.orgId
+        );
         req.userOrgId = siteResource.orgId;
 
         // Attach the siteResource to the request for use in the next middleware/route
         req.siteResource = siteResource;
 
-        const roleResourceAccess = await db
-            .select()
-            .from(roleSiteResources)
-            .where(
-                and(
-                    eq(roleSiteResources.siteResourceId, siteResourceIdNum),
-                    eq(roleSiteResources.roleId, userOrgRoleId)
-                )
-            )
-            .limit(1);
+        const roleResourceAccess =
+            (req.userOrgRoleIds?.length ?? 0) > 0
+                ? await db
+                      .select()
+                      .from(roleSiteResources)
+                      .where(
+                          and(
+                              eq(
+                                  roleSiteResources.siteResourceId,
+                                  siteResourceIdNum
+                              ),
+                              inArray(
+                                  roleSiteResources.roleId,
+                                  req.userOrgRoleIds!
+                              )
+                          )
+                      )
+                      .limit(1)
+                : [];
 
         if (roleResourceAccess.length > 0) {
             return next();

server/middlewares/verifyTargetAccess.ts

@@ -6,6 +6,7 @@ import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
 import { canUserAccessResource } from "../auth/canUserAccessResource";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifyTargetAccess(
     req: Request,
@@ -99,7 +100,10 @@ export async function verifyTargetAccess(
                 )
             );
         } else {
-            req.userOrgRoleId = req.userOrg.roleId;
+            req.userOrgRoleIds = await getUserOrgRoleIds(
+                req.userOrg.userId,
+                resource[0].orgId!
+            );
             req.userOrgId = resource[0].orgId!;
         }
 
@@ -126,7 +130,7 @@ export async function verifyTargetAccess(
         const resourceAllowed = await canUserAccessResource({
             userId,
             resourceId,
-            roleId: req.userOrgRoleId!
+            roleIds: req.userOrgRoleIds ?? []
         });
 
         if (!resourceAllowed) {

server/middlewares/verifyUserCanSetUserOrgRoles.ts

@@ -0,0 +1,54 @@
+import { Request, Response, NextFunction } from "express";
+import createHttpError from "http-errors";
+import HttpCode from "@server/types/HttpCode";
+import logger from "@server/logger";
+import { ActionsEnum, checkUserActionPermission } from "@server/auth/actions";
+
+/**
+ * Allows the new setUserOrgRoles action, or legacy permission pair addUserRole + removeUserRole.
+ */
+export function verifyUserCanSetUserOrgRoles() {
+    return async function (
+        req: Request,
+        res: Response,
+        next: NextFunction
+    ): Promise<any> {
+        try {
+            const canSet = await checkUserActionPermission(
+                ActionsEnum.setUserOrgRoles,
+                req
+            );
+            if (canSet) {
+                return next();
+            }
+
+            const canAdd = await checkUserActionPermission(
+                ActionsEnum.addUserRole,
+                req
+            );
+            const canRemove = await checkUserActionPermission(
+                ActionsEnum.removeUserRole,
+                req
+            );
+
+            if (canAdd && canRemove) {
+                return next();
+            }
+
+            return next(
+                createHttpError(
+                    HttpCode.FORBIDDEN,
+                    "User does not have permission perform this action"
+                )
+            );
+        } catch (error) {
+            logger.error("Error verifying set user org roles access:", error);
+            return next(
+                createHttpError(
+                    HttpCode.INTERNAL_SERVER_ERROR,
+                    "Error verifying role access"
+                )
+            );
+        }
+    };
+}

server/middlewares/verifyUserInRole.ts

@@ -12,7 +12,7 @@ export async function verifyUserInRole(
         const roleId = parseInt(
             req.params.roleId || req.body.roleId || req.query.roleId
         );
-        const userRoleId = req.userOrgRoleId;
+        const userOrgRoleIds = req.userOrgRoleIds ?? [];
 
         if (isNaN(roleId)) {
             return next(
@@ -20,7 +20,7 @@ export async function verifyUserInRole(
             );
         }
 
-        if (!userRoleId) {
+        if (userOrgRoleIds.length === 0) {
             return next(
                 createHttpError(
                     HttpCode.FORBIDDEN,
@@ -29,7 +29,7 @@ export async function verifyUserInRole(
             );
         }
 
-        if (userRoleId !== roleId) {
+        if (!userOrgRoleIds.includes(roleId)) {
             return next(
                 createHttpError(
                     HttpCode.FORBIDDEN,

server/private/cleanup.ts

@@ -15,8 +15,10 @@ import { rateLimitService } from "#private/lib/rateLimit";
 import { cleanup as wsCleanup } from "#private/routers/ws";
 import { flushBandwidthToDb } from "@server/routers/newt/handleReceiveBandwidthMessage";
 import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth";
+import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator";
 
 async function cleanup() {
+    await stopPingAccumulator();
     await flushBandwidthToDb();
     await flushSiteBandwidthToDb();
     await rateLimitService.cleanup();

server/private/lib/cache.ts

@@ -1,3 +1,16 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
 import NodeCache from "node-cache";
 import logger from "@server/logger";
 import { redisManager } from "@server/private/lib/redis";
@@ -24,23 +37,31 @@ setInterval(() => {
  */
 class AdaptiveCache {
     private useRedis(): boolean {
-        return redisManager.isRedisEnabled() && redisManager.getHealthStatus().isHealthy;
+        return (
+            redisManager.isRedisEnabled() &&
+            redisManager.getHealthStatus().isHealthy
+        );
     }
 
     /**
      * Set a value in the cache
      * @param key - Cache key
      * @param value - Value to cache (will be JSON stringified for Redis)
-     * @param ttl - Time to live in seconds (0 = no expiration)
+     * @param ttl - Time to live in seconds (0 = no expiration; omit = 3600s for Redis)
      * @returns boolean indicating success
      */
     async set(key: string, value: any, ttl?: number): Promise<boolean> {
         const effectiveTtl = ttl === 0 ? undefined : ttl;
+        const redisTtl = ttl === 0 ? undefined : (ttl ?? 3600);
 
         if (this.useRedis()) {
             try {
                 const serialized = JSON.stringify(value);
-                const success = await redisManager.set(key, serialized, effectiveTtl);
+                const success = await redisManager.set(
+                    key,
+                    serialized,
+                    redisTtl
+                );
 
                 if (success) {
                     logger.debug(`Set key in Redis: ${key}`);
@@ -48,7 +69,9 @@ class AdaptiveCache {
                 }
 
                 // Redis failed, fall through to local cache
-                logger.debug(`Redis set failed for key ${key}, falling back to local cache`);
+                logger.debug(
+                    `Redis set failed for key ${key}, falling back to local cache`
+                );
             } catch (error) {
                 logger.error(`Redis set error for key ${key}:`, error);
                 // Fall through to local cache
@@ -120,9 +143,14 @@ class AdaptiveCache {
                 }
 
                 // Some Redis deletes failed, fall through to local cache
-                logger.debug(`Some Redis deletes failed, falling back to local cache`);
+                logger.debug(
+                    `Some Redis deletes failed, falling back to local cache`
+                );
             } catch (error) {
-                logger.error(`Redis del error for keys ${keys.join(", ")}:`, error);
+                logger.error(
+                    `Redis del error for keys ${keys.join(", ")}:`,
+                    error
+                );
                 // Fall through to local cache
                 deletedCount = 0;
             }
@@ -195,7 +223,9 @@ class AdaptiveCache {
      */
     async flushAll(): Promise<void> {
         if (this.useRedis()) {
-            logger.warn("Adaptive cache flushAll called - Redis flush not implemented, only local cache will be flushed");
+            logger.warn(
+                "Adaptive cache flushAll called - Redis flush not implemented, only local cache will be flushed"
+            );
         }
 
         localCache.flushAll();
@@ -239,7 +269,9 @@ class AdaptiveCache {
     getTtl(key: string): number {
         // Note: This only works for local cache, Redis TTL is not supported
         if (this.useRedis()) {
-            logger.warn(`getTtl called for key ${key} but Redis TTL lookup is not implemented`);
+            logger.warn(
+                `getTtl called for key ${key} but Redis TTL lookup is not implemented`
+            );
         }
 
         const ttl = localCache.getTtl(key);
@@ -255,7 +287,9 @@ class AdaptiveCache {
      */
     keys(): string[] {
         if (this.useRedis()) {
-            logger.warn("keys() called but Redis keys are not included, only local cache keys returned");
+            logger.warn(
+                "keys() called but Redis keys are not included, only local cache keys returned"
+            );
         }
         return localCache.keys();
     }

server/private/lib/readConfigFile.ts

@@ -57,7 +57,10 @@ export const privateConfigSchema = z.object({
         .object({
             host: z.string(),
             port: portSchema,
-            password: z.string().optional(),
+            password: z
+                .string()
+                .optional()
+                .transform(getEnvOrYaml("REDIS_PASSWORD")),
             db: z.int().nonnegative().optional().default(0),
             replicas: z
                 .array(

server/private/lib/tokenCache.ts

@@ -0,0 +1,77 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import redisManager from "#private/lib/redis";
+import { encrypt, decrypt } from "@server/lib/crypto";
+import logger from "@server/logger";
+
+/**
+ * Returns a cached plaintext token from Redis if one exists and decrypts
+ * cleanly, otherwise calls `createSession` to mint a fresh token, stores the
+ * encrypted value in Redis with the given TTL, and returns it.
+ *
+ * Failures at the Redis layer are non-fatal – the function always falls
+ * through to session creation so the caller is never blocked by a Redis outage.
+ *
+ * @param cacheKey   Unique Redis key, e.g. `"newt:token_cache:abc123"`
+ * @param secret     Server secret used for AES encryption/decryption
+ * @param ttlSeconds Cache TTL in seconds (should match session expiry)
+ * @param createSession Factory that mints a new session and returns its raw token
+ */
+export async function getOrCreateCachedToken(
+    cacheKey: string,
+    secret: string,
+    ttlSeconds: number,
+    createSession: () => Promise<string>
+): Promise<string> {
+    if (redisManager.isRedisEnabled()) {
+        try {
+            const cached = await redisManager.get(cacheKey);
+            if (cached) {
+                const token = decrypt(cached, secret);
+                if (token) {
+                    logger.debug(`Token cache hit for key: ${cacheKey}`);
+                    return token;
+                }
+                // Decryption produced an empty string – treat as a miss
+                logger.warn(
+                    `Token cache decryption returned empty string for key: ${cacheKey}, treating as miss`
+                );
+            }
+        } catch (e) {
+            logger.warn(
+                `Token cache read/decrypt failed for key ${cacheKey}, falling through to session creation:`,
+                e
+            );
+        }
+    }
+
+    const token = await createSession();
+
+    if (redisManager.isRedisEnabled()) {
+        try {
+            const encrypted = encrypt(token, secret);
+            await redisManager.set(cacheKey, encrypted, ttlSeconds);
+            logger.debug(
+                `Token cached in Redis for key: ${cacheKey} (TTL ${ttlSeconds}s)`
+            );
+        } catch (e) {
+            logger.warn(
+                `Token cache write failed for key ${cacheKey} (session was still created):`,
+                e
+            );
+        }
+    }
+
+    return token;
+}

server/private/middlewares/verifyIdpAccess.ts

@@ -13,9 +13,10 @@
 
 import { Request, Response, NextFunction } from "express";
 import { userOrgs, db, idp, idpOrg } from "@server/db";
-import { and, eq, or } from "drizzle-orm";
+import { and, eq } from "drizzle-orm";
 import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifyIdpAccess(
     req: Request,
@@ -84,8 +85,10 @@ export async function verifyIdpAccess(
             );
         }
 
-        const userOrgRoleId = req.userOrg.roleId;
-        req.userOrgRoleId = userOrgRoleId;
+        req.userOrgRoleIds = await getUserOrgRoleIds(
+            req.userOrg.userId,
+            idpRes.idpOrg.orgId
+        );
 
         return next();
     } catch (error) {

server/private/middlewares/verifyRemoteExitNodeAccess.ts

@@ -12,11 +12,12 @@
  */
 
 import { Request, Response, NextFunction } from "express";
-import { db, exitNodeOrgs, exitNodes, remoteExitNodes } from "@server/db";
-import { sites, userOrgs, userSites, roleSites, roles } from "@server/db";
-import { and, eq, or } from "drizzle-orm";
+import { db, exitNodeOrgs, remoteExitNodes } from "@server/db";
+import { userOrgs } from "@server/db";
+import { and, eq } from "drizzle-orm";
 import createHttpError from "http-errors";
 import HttpCode from "@server/types/HttpCode";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 
 export async function verifyRemoteExitNodeAccess(
     req: Request,
@@ -103,8 +104,10 @@ export async function verifyRemoteExitNodeAccess(
             );
         }
 
-        const userOrgRoleId = req.userOrg.roleId;
-        req.userOrgRoleId = userOrgRoleId;
+        req.userOrgRoleIds = await getUserOrgRoleIds(
+            req.userOrg.userId,
+            exitNodeOrg.orgId
+        );
 
         return next();
     } catch (error) {

server/private/routers/external.ts

@@ -26,6 +26,7 @@ import * as misc from "#private/routers/misc";
 import * as reKey from "#private/routers/re-key";
 import * as approval from "#private/routers/approvals";
 import * as ssh from "#private/routers/ssh";
+import * as user from "#private/routers/user";
 
 import {
     verifyOrgAccess,
@@ -33,7 +34,10 @@ import {
     verifyUserIsServerAdmin,
     verifySiteAccess,
     verifyClientAccess,
-    verifyLimits
+    verifyLimits,
+    verifyRoleAccess,
+    verifyUserAccess,
+    verifyUserCanSetUserOrgRoles
 } from "@server/middlewares";
 import { ActionsEnum } from "@server/auth/actions";
 import {
@@ -518,3 +522,33 @@ authenticated.post(
     // logActionAudit(ActionsEnum.signSshKey), // it is handled inside of the function below so we can include more metadata
     ssh.signSshKey
 );
+
+authenticated.post(
+    "/user/:userId/add-role/:roleId",
+    verifyRoleAccess,
+    verifyUserAccess,
+    verifyLimits,
+    verifyUserHasAction(ActionsEnum.addUserRole),
+    logActionAudit(ActionsEnum.addUserRole),
+    user.addUserRole
+);
+
+authenticated.delete(
+    "/user/:userId/remove-role/:roleId",
+    verifyRoleAccess,
+    verifyUserAccess,
+    verifyLimits,
+    verifyUserHasAction(ActionsEnum.removeUserRole),
+    logActionAudit(ActionsEnum.removeUserRole),
+    user.removeUserRole
+);
+
+authenticated.post(
+    "/user/:userId/org/:orgId/roles",
+    verifyOrgAccess,
+    verifyUserAccess,
+    verifyLimits,
+    verifyUserCanSetUserOrgRoles(),
+    logActionAudit(ActionsEnum.setUserOrgRoles),
+    user.setUserOrgRoles
+);

server/private/routers/hybrid.ts

@@ -15,6 +15,7 @@ import { verifySessionRemoteExitNodeMiddleware } from "#private/middlewares/veri
 import { Router } from "express";
 import {
     db,
+    logsDb,
     exitNodes,
     Resource,
     ResourcePassword,
@@ -81,6 +82,7 @@ import { verifyResourceAccessToken } from "@server/auth/verifyResourceAccessToke
 import semver from "semver";
 import { maxmindAsnLookup } from "@server/db/maxmindAsn";
 import { checkOrgAccessPolicy } from "@server/lib/checkOrgAccessPolicy";
+import { sanitizeString } from "@server/lib/sanitize";
 
 // Zod schemas for request validation
 const getResourceByDomainParamsSchema = z.strictObject({
@@ -1859,24 +1861,24 @@ hybridRouter.post(
                 })
                 .map((logEntry) => ({
                     timestamp: logEntry.timestamp,
-                    orgId: logEntry.orgId,
-                    actorType: logEntry.actorType,
-                    actor: logEntry.actor,
-                    actorId: logEntry.actorId,
-                    metadata: logEntry.metadata,
+                    orgId: sanitizeString(logEntry.orgId),
+                    actorType: sanitizeString(logEntry.actorType),
+                    actor: sanitizeString(logEntry.actor),
+                    actorId: sanitizeString(logEntry.actorId),
+                    metadata: sanitizeString(logEntry.metadata),
                     action: logEntry.action,
                     resourceId: logEntry.resourceId,
                     reason: logEntry.reason,
-                    location: logEntry.location,
+                    location: sanitizeString(logEntry.location),
                     // userAgent: data.userAgent, // TODO: add this
                     // headers: data.body.headers,
                     // query: data.body.query,
-                    originalRequestURL: logEntry.originalRequestURL,
-                    scheme: logEntry.scheme,
-                    host: logEntry.host,
-                    path: logEntry.path,
-                    method: logEntry.method,
-                    ip: logEntry.ip,
+                    originalRequestURL: sanitizeString(logEntry.originalRequestURL) ?? "",
+                    scheme: sanitizeString(logEntry.scheme) ?? "",
+                    host: sanitizeString(logEntry.host) ?? "",
+                    path: sanitizeString(logEntry.path) ?? "",
+                    method: sanitizeString(logEntry.method) ?? "",
+                    ip: sanitizeString(logEntry.ip),
                     tls: logEntry.tls
                 }));
 
@@ -1884,7 +1886,7 @@ hybridRouter.post(
             const batchSize = 100;
             for (let i = 0; i < logEntries.length; i += batchSize) {
                 const batch = logEntries.slice(i, i + batchSize);
-                await db.insert(requestAuditLog).values(batch);
+                await logsDb.insert(requestAuditLog).values(batch);
             }
 
             return response(res, {

server/private/routers/integration.ts

@@ -20,8 +20,11 @@ import {
     verifyApiKeyIsRoot,
     verifyApiKeyOrgAccess,
     verifyApiKeyIdpAccess,
+    verifyApiKeyRoleAccess,
+    verifyApiKeyUserAccess,
     verifyLimits
 } from "@server/middlewares";
+import * as user from "#private/routers/user";
 import {
     verifyValidSubscription,
     verifyValidLicense
@@ -140,3 +143,23 @@ authenticated.get(
     verifyApiKeyHasAction(ActionsEnum.listIdps),
     orgIdp.listOrgIdps
 );
+
+authenticated.post(
+    "/user/:userId/add-role/:roleId",
+    verifyApiKeyRoleAccess,
+    verifyApiKeyUserAccess,
+    verifyLimits,
+    verifyApiKeyHasAction(ActionsEnum.addUserRole),
+    logActionAudit(ActionsEnum.addUserRole),
+    user.addUserRole
+);
+
+authenticated.delete(
+    "/user/:userId/remove-role/:roleId",
+    verifyApiKeyRoleAccess,
+    verifyApiKeyUserAccess,
+    verifyLimits,
+    verifyApiKeyHasAction(ActionsEnum.removeUserRole),
+    logActionAudit(ActionsEnum.removeUserRole),
+    user.removeUserRole
+);

server/private/routers/org/sendUsageNotifications.ts

@@ -14,7 +14,7 @@
 import { Request, Response, NextFunction } from "express";
 import { z } from "zod";
 import { db } from "@server/db";
-import { userOrgs, users, roles, orgs } from "@server/db";
+import { userOrgs, userOrgRoles, users, roles, orgs } from "@server/db";
 import { eq, and, or } from "drizzle-orm";
 import response from "@server/lib/response";
 import HttpCode from "@server/types/HttpCode";
@@ -95,7 +95,14 @@ async function getOrgAdmins(orgId: string) {
         })
         .from(userOrgs)
         .innerJoin(users, eq(userOrgs.userId, users.userId))
-        .leftJoin(roles, eq(userOrgs.roleId, roles.roleId))
+        .leftJoin(
+            userOrgRoles,
+            and(
+                eq(userOrgs.userId, userOrgRoles.userId),
+                eq(userOrgs.orgId, userOrgRoles.orgId)
+            )
+        )
+        .leftJoin(roles, eq(userOrgRoles.roleId, roles.roleId))
         .where(
             and(
                 eq(userOrgs.orgId, orgId),
@@ -103,8 +110,11 @@ async function getOrgAdmins(orgId: string) {
             )
         );
 
-    // Filter to only include users with verified emails
-    const orgAdmins = admins.filter(
+    // Dedupe by userId (user may have multiple roles)
+    const byUserId = new Map(
+        admins.map((a) => [a.userId, a])
+    );
+    const orgAdmins = Array.from(byUserId.values()).filter(
         (admin) => admin.email && admin.email.length > 0
     );
 

server/private/routers/remoteExitNode/createRemoteExitNode.ts

@@ -79,7 +79,7 @@ export async function createRemoteExitNode(
 
         const { remoteExitNodeId, secret } = parsedBody.data;
 
-        if (req.user && !req.userOrgRoleId) {
+        if (req.user && (!req.userOrgRoleIds || req.userOrgRoleIds.length === 0)) {
             return next(
                 createHttpError(HttpCode.FORBIDDEN, "User does not have a role")
             );

server/private/routers/remoteExitNode/getRemoteExitNodeToken.ts

@@ -23,8 +23,10 @@ import { z } from "zod";
 import { fromError } from "zod-validation-error";
 import {
     createRemoteExitNodeSession,
-    validateRemoteExitNodeSessionToken
+    validateRemoteExitNodeSessionToken,
+    EXPIRES
 } from "#private/auth/sessions/remoteExitNode";
+import { getOrCreateCachedToken } from "@server/private/lib/tokenCache";
 import { verifyPassword } from "@server/auth/password";
 import logger from "@server/logger";
 import config from "@server/lib/config";
@@ -103,14 +105,23 @@ export async function getRemoteExitNodeToken(
             );
         }
 
-        const resToken = generateSessionToken();
-        await createRemoteExitNodeSession(
-            resToken,
-            existingRemoteExitNode.remoteExitNodeId
+        // Return a cached token if one exists to prevent thundering herd on
+        // simultaneous restarts; falls back to creating a fresh session when
+        // Redis is unavailable or the cache has expired.
+        const resToken = await getOrCreateCachedToken(
+            `remote_exit_node:token_cache:${existingRemoteExitNode.remoteExitNodeId}`,
+            config.getRawConfig().server.secret!,
+            Math.floor(EXPIRES / 1000),
+            async () => {
+                const token = generateSessionToken();
+                await createRemoteExitNodeSession(
+                    token,
+                    existingRemoteExitNode.remoteExitNodeId
+                );
+                return token;
+            }
         );
 
-        // logger.debug(`Created RemoteExitNode token response: ${JSON.stringify(resToken)}`);
-
         return response<{ token: string }>(res, {
             data: {
                 token: resToken

server/private/routers/remoteExitNode/handleRemoteExitNodePingMessage.ts

@@ -38,7 +38,7 @@ export const startRemoteExitNodeOfflineChecker = (): void => {
             );
 
             // Find clients that haven't pinged in the last 2 minutes and mark them as offline
-            const newlyOfflineNodes = await db
+            const offlineNodes = await db
                 .update(exitNodes)
                 .set({ online: false })
                 .where(
@@ -53,32 +53,15 @@ export const startRemoteExitNodeOfflineChecker = (): void => {
                 )
                 .returning();
 
-            // Update the sites to offline if they have not pinged either
-            const exitNodeIds = newlyOfflineNodes.map(
-                (node) => node.exitNodeId
-            );
-
-            const sitesOnNode = await db
-                .select()
-                .from(sites)
-                .where(
-                    and(
-                        eq(sites.online, true),
-                        inArray(sites.exitNodeId, exitNodeIds)
-                    )
+            if (offlineNodes.length > 0) {
+                logger.info(
+                    `checkRemoteExitNodeOffline: Marked ${offlineNodes.length} remoteExitNode client(s) offline due to inactivity`
                 );
 
-            // loop through the sites and process their lastBandwidthUpdate as an iso string and if its more than 1 minute old then mark the site offline
-            for (const site of sitesOnNode) {
-                if (!site.lastBandwidthUpdate) {
-                    continue;
-                }
-                const lastBandwidthUpdate = new Date(site.lastBandwidthUpdate);
-                if (Date.now() - lastBandwidthUpdate.getTime() > 60 * 1000) {
-                    await db
-                        .update(sites)
-                        .set({ online: false })
-                        .where(eq(sites.siteId, site.siteId));
+                for (const offlineClient of offlineNodes) {
+                    logger.debug(
+                        `checkRemoteExitNodeOffline: Client ${offlineClient.exitNodeId} marked offline (lastPing: ${offlineClient.lastPing})`
+                    );
                 }
             }
         } catch (error) {

server/private/routers/ssh/signSshKey.ts

@@ -24,6 +24,7 @@ import {
     sites,
     userOrgs
 } from "@server/db";
+import { logAccessAudit } from "#private/lib/logAccessAudit";
 import { isLicensedOrSubscribed } from "#private/lib/isLicencedOrSubscribed";
 import { tierMatrix } from "@server/lib/billing/tierMatrix";
 import response from "@server/lib/response";
@@ -31,7 +32,7 @@ import HttpCode from "@server/types/HttpCode";
 import createHttpError from "http-errors";
 import logger from "@server/logger";
 import { fromError } from "zod-validation-error";
-import { eq, or, and } from "drizzle-orm";
+import { and, eq, inArray, or } from "drizzle-orm";
 import { canUserAccessSiteResource } from "@server/auth/canUserAccessSiteResource";
 import { signPublicKey, getOrgCAKeys } from "@server/lib/sshCA";
 import config from "@server/lib/config";
@@ -125,7 +126,7 @@ export async function signSshKey(
             resource: resourceQueryString
         } = parsedBody.data;
         const userId = req.user?.userId;
-        const roleId = req.userOrgRoleId!;
+        const roleIds = req.userOrgRoleIds ?? [];
 
         if (!userId) {
             return next(
@@ -133,6 +134,15 @@ export async function signSshKey(
             );
         }
 
+        if (roleIds.length === 0) {
+            return next(
+                createHttpError(
+                    HttpCode.FORBIDDEN,
+                    "User has no role in organization"
+                )
+            );
+        }
+
         const [userOrg] = await db
             .select()
             .from(userOrgs)
@@ -339,7 +349,7 @@ export async function signSshKey(
         const hasAccess = await canUserAccessSiteResource({
             userId: userId,
             resourceId: resource.siteResourceId,
-            roleId: roleId
+            roleIds
         });
 
         if (!hasAccess) {
@@ -351,28 +361,39 @@ export async function signSshKey(
             );
         }
 
-        const [roleRow] = await db
+        const roleRows = await db
             .select()
             .from(roles)
-            .where(eq(roles.roleId, roleId))
-            .limit(1);
+            .where(inArray(roles.roleId, roleIds));
 
-        let parsedSudoCommands: string[] = [];
-        let parsedGroups: string[] = [];
-        try {
-            parsedSudoCommands = JSON.parse(roleRow?.sshSudoCommands ?? "[]");
-            if (!Array.isArray(parsedSudoCommands)) parsedSudoCommands = [];
-        } catch {
-            parsedSudoCommands = [];
+        const parsedSudoCommands: string[] = [];
+        const parsedGroupsSet = new Set<string>();
+        let homedir: boolean | null = null;
+        const sudoModeOrder = { none: 0, commands: 1, all: 2 };
+        let sudoMode: "none" | "commands" | "all" = "none";
+        for (const roleRow of roleRows) {
+            try {
+                const cmds = JSON.parse(roleRow?.sshSudoCommands ?? "[]");
+                if (Array.isArray(cmds)) parsedSudoCommands.push(...cmds);
+            } catch {
+                // skip
+            }
+            try {
+                const grps = JSON.parse(roleRow?.sshUnixGroups ?? "[]");
+                if (Array.isArray(grps)) grps.forEach((g: string) => parsedGroupsSet.add(g));
+            } catch {
+                // skip
+            }
+            if (roleRow?.sshCreateHomeDir === true) homedir = true;
+            const m = roleRow?.sshSudoMode ?? "none";
+            if (sudoModeOrder[m as keyof typeof sudoModeOrder] > sudoModeOrder[sudoMode]) {
+                sudoMode = m as "none" | "commands" | "all";
+            }
         }
-        try {
-            parsedGroups = JSON.parse(roleRow?.sshUnixGroups ?? "[]");
-            if (!Array.isArray(parsedGroups)) parsedGroups = [];
-        } catch {
-            parsedGroups = [];
+        const parsedGroups = Array.from(parsedGroupsSet);
+        if (homedir === null && roleRows.length > 0) {
+            homedir = roleRows[0].sshCreateHomeDir ?? null;
         }
-        const homedir = roleRow?.sshCreateHomeDir ?? null;
-        const sudoMode = roleRow?.sshSudoMode ?? "none";
 
         // get the site
         const [newt] = await db
@@ -463,6 +484,24 @@ export async function signSshKey(
             })
         });
 
+        await logAccessAudit({
+            action: true,
+            type: "ssh",
+            orgId: orgId,
+            resourceId: resource.siteResourceId,
+            user: req.user
+                ? { username: req.user.username ?? "", userId: req.user.userId }
+                : undefined,
+            metadata: {
+                resourceName: resource.name,
+                siteId: resource.siteId,
+                sshUsername: usernameToUse,
+                sshHost: sshHost
+            },
+            userAgent: req.headers["user-agent"],
+            requestIp: req.ip
+        });
+
         return response<SignSshKeyResponse>(res, {
             data: {
                 certificate: cert.certificate,

server/private/routers/user/addUserRole.ts

@@ -1,14 +1,27 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
 import { Request, Response, NextFunction } from "express";
 import { z } from "zod";
-import { clients, db, UserOrg } from "@server/db";
-import { userOrgs, roles } from "@server/db";
+import stoi from "@server/lib/stoi";
+import { clients, db } from "@server/db";
+import { userOrgRoles, userOrgs, roles } from "@server/db";
 import { eq, and } from "drizzle-orm";
 import response from "@server/lib/response";
 import HttpCode from "@server/types/HttpCode";
 import createHttpError from "http-errors";
 import logger from "@server/logger";
 import { fromError } from "zod-validation-error";
-import stoi from "@server/lib/stoi";
 import { OpenAPITags, registry } from "@server/openApi";
 import { rebuildClientAssociationsFromClient } from "@server/lib/rebuildClientAssociations";
 
@@ -17,11 +30,9 @@ const addUserRoleParamsSchema = z.strictObject({
     roleId: z.string().transform(stoi).pipe(z.number())
 });
 
-export type AddUserRoleResponse = z.infer<typeof addUserRoleParamsSchema>;
-
 registry.registerPath({
     method: "post",
-    path: "/role/{roleId}/add/{userId}",
+    path: "/user/{userId}/add-role/{roleId}",
     description: "Add a role to a user.",
     tags: [OpenAPITags.Role, OpenAPITags.User],
     request: {
@@ -111,20 +122,23 @@ export async function addUserRole(
             );
         }
 
-        let newUserRole: UserOrg | null = null;
+        let newUserRole: { userId: string; orgId: string; roleId: number } | null =
+            null;
         await db.transaction(async (trx) => {
-            [newUserRole] = await trx
-                .update(userOrgs)
-                .set({ roleId })
-                .where(
-                    and(
-                        eq(userOrgs.userId, userId),
-                        eq(userOrgs.orgId, role.orgId)
-                    )
-                )
+            const inserted = await trx
+                .insert(userOrgRoles)
+                .values({
+                    userId,
+                    orgId: role.orgId,
+                    roleId
+                })
+                .onConflictDoNothing()
                 .returning();
 
-            // get the client associated with this user in this org
+            if (inserted.length > 0) {
+                newUserRole = inserted[0];
+            }
+
             const orgClients = await trx
                 .select()
                 .from(clients)
@@ -133,17 +147,15 @@ export async function addUserRole(
                         eq(clients.userId, userId),
                         eq(clients.orgId, role.orgId)
                     )
-                )
-                .limit(1);
+                );
 
             for (const orgClient of orgClients) {
-                // we just changed the user's role, so we need to rebuild client associations and what they have access to
                 await rebuildClientAssociationsFromClient(orgClient, trx);
             }
         });
 
         return response(res, {
-            data: newUserRole,
+            data: newUserRole ?? { userId, orgId: role.orgId, roleId },
             success: true,
             error: false,
             message: "Role added to user successfully",

server/private/routers/user/index.ts

@@ -0,0 +1,16 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+export * from "./addUserRole";
+export * from "./removeUserRole";
+export * from "./setUserOrgRoles";

server/private/routers/user/removeUserRole.ts

@@ -0,0 +1,171 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import stoi from "@server/lib/stoi";
+import { db } from "@server/db";
+import { userOrgRoles, userOrgs, roles, clients } from "@server/db";
+import { eq, and } from "drizzle-orm";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+import { rebuildClientAssociationsFromClient } from "@server/lib/rebuildClientAssociations";
+
+const removeUserRoleParamsSchema = z.strictObject({
+    userId: z.string(),
+    roleId: z.string().transform(stoi).pipe(z.number())
+});
+
+registry.registerPath({
+    method: "delete",
+    path: "/user/{userId}/remove-role/{roleId}",
+    description:
+        "Remove a role from a user. User must have at least one role left in the org.",
+    tags: [OpenAPITags.Role, OpenAPITags.User],
+    request: {
+        params: removeUserRoleParamsSchema
+    },
+    responses: {}
+});
+
+export async function removeUserRole(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = removeUserRoleParamsSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const { userId, roleId } = parsedParams.data;
+
+        if (req.user && !req.userOrg) {
+            return next(
+                createHttpError(
+                    HttpCode.FORBIDDEN,
+                    "You do not have access to this organization"
+                )
+            );
+        }
+
+        const [role] = await db
+            .select()
+            .from(roles)
+            .where(eq(roles.roleId, roleId))
+            .limit(1);
+
+        if (!role) {
+            return next(
+                createHttpError(HttpCode.BAD_REQUEST, "Invalid role ID")
+            );
+        }
+
+        const [existingUser] = await db
+            .select()
+            .from(userOrgs)
+            .where(
+                and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, role.orgId))
+            )
+            .limit(1);
+
+        if (!existingUser) {
+            return next(
+                createHttpError(
+                    HttpCode.NOT_FOUND,
+                    "User not found or does not belong to the specified organization"
+                )
+            );
+        }
+
+        if (existingUser.isOwner) {
+            return next(
+                createHttpError(
+                    HttpCode.FORBIDDEN,
+                    "Cannot change the roles of the owner of the organization"
+                )
+            );
+        }
+
+        const remainingRoles = await db
+            .select({ roleId: userOrgRoles.roleId })
+            .from(userOrgRoles)
+            .where(
+                and(
+                    eq(userOrgRoles.userId, userId),
+                    eq(userOrgRoles.orgId, role.orgId)
+                )
+            );
+
+        if (remainingRoles.length <= 1) {
+            const hasThisRole = remainingRoles.some((r) => r.roleId === roleId);
+            if (hasThisRole) {
+                return next(
+                    createHttpError(
+                        HttpCode.FORBIDDEN,
+                        "User must have at least one role in the organization. Remove the last role is not allowed."
+                    )
+                );
+            }
+        }
+
+        await db.transaction(async (trx) => {
+            await trx
+                .delete(userOrgRoles)
+                .where(
+                    and(
+                        eq(userOrgRoles.userId, userId),
+                        eq(userOrgRoles.orgId, role.orgId),
+                        eq(userOrgRoles.roleId, roleId)
+                    )
+                );
+
+            const orgClients = await trx
+                .select()
+                .from(clients)
+                .where(
+                    and(
+                        eq(clients.userId, userId),
+                        eq(clients.orgId, role.orgId)
+                    )
+                );
+
+            for (const orgClient of orgClients) {
+                await rebuildClientAssociationsFromClient(orgClient, trx);
+            }
+        });
+
+        return response(res, {
+            data: { userId, orgId: role.orgId, roleId },
+            success: true,
+            error: false,
+            message: "Role removed from user successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
+        );
+    }
+}

server/private/routers/user/setUserOrgRoles.ts

@@ -0,0 +1,163 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import { clients, db } from "@server/db";
+import { userOrgRoles, userOrgs, roles } from "@server/db";
+import { eq, and, inArray } from "drizzle-orm";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { rebuildClientAssociationsFromClient } from "@server/lib/rebuildClientAssociations";
+
+const setUserOrgRolesParamsSchema = z.strictObject({
+    orgId: z.string(),
+    userId: z.string()
+});
+
+const setUserOrgRolesBodySchema = z.strictObject({
+    roleIds: z.array(z.int().positive()).min(1)
+});
+
+export async function setUserOrgRoles(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = setUserOrgRolesParamsSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const parsedBody = setUserOrgRolesBodySchema.safeParse(req.body);
+        if (!parsedBody.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedBody.error).toString()
+                )
+            );
+        }
+
+        const { orgId, userId } = parsedParams.data;
+        const { roleIds } = parsedBody.data;
+
+        if (req.user && !req.userOrg) {
+            return next(
+                createHttpError(
+                    HttpCode.FORBIDDEN,
+                    "You do not have access to this organization"
+                )
+            );
+        }
+
+        const uniqueRoleIds = [...new Set(roleIds)];
+
+        const [existingUser] = await db
+            .select()
+            .from(userOrgs)
+            .where(and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, orgId)))
+            .limit(1);
+
+        if (!existingUser) {
+            return next(
+                createHttpError(
+                    HttpCode.NOT_FOUND,
+                    "User not found in this organization"
+                )
+            );
+        }
+
+        if (existingUser.isOwner) {
+            return next(
+                createHttpError(
+                    HttpCode.FORBIDDEN,
+                    "Cannot change the roles of the owner of the organization"
+                )
+            );
+        }
+
+        const orgRoles = await db
+            .select({ roleId: roles.roleId })
+            .from(roles)
+            .where(
+                and(
+                    eq(roles.orgId, orgId),
+                    inArray(roles.roleId, uniqueRoleIds)
+                )
+            );
+
+        if (orgRoles.length !== uniqueRoleIds.length) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    "One or more role IDs are invalid for this organization"
+                )
+            );
+        }
+
+        await db.transaction(async (trx) => {
+            await trx
+                .delete(userOrgRoles)
+                .where(
+                    and(
+                        eq(userOrgRoles.userId, userId),
+                        eq(userOrgRoles.orgId, orgId)
+                    )
+                );
+
+            if (uniqueRoleIds.length > 0) {
+                await trx.insert(userOrgRoles).values(
+                    uniqueRoleIds.map((roleId) => ({
+                        userId,
+                        orgId,
+                        roleId
+                    }))
+                );
+            }
+
+            const orgClients = await trx
+                .select()
+                .from(clients)
+                .where(
+                    and(eq(clients.userId, userId), eq(clients.orgId, orgId))
+                );
+
+            for (const orgClient of orgClients) {
+                await rebuildClientAssociationsFromClient(orgClient, trx);
+            }
+        });
+
+        return response(res, {
+            data: { userId, orgId, roleIds: uniqueRoleIds },
+            success: true,
+            error: false,
+            message: "User roles set successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
+        );
+    }
+}

server/private/routers/ws/ws.ts

@@ -19,17 +19,14 @@ import { Socket } from "net";
 import {
     Newt,
     newts,
-    NewtSession,
-    olms,
     Olm,
-    OlmSession,
+    olms,
     RemoteExitNode,
-    RemoteExitNodeSession,
     remoteExitNodes,
-    sites
 } from "@server/db";
 import { eq } from "drizzle-orm";
 import { db } from "@server/db";
+import { recordPing } from "@server/routers/newt/pingAccumulator";
 import { validateNewtSessionToken } from "@server/auth/sessions/newt";
 import { validateOlmSessionToken } from "@server/auth/sessions/olm";
 import logger from "@server/logger";
@@ -197,11 +194,7 @@ const connectedClients: Map<string, AuthenticatedWebSocket[]> = new Map();
 // Config version tracking map (local to this node, resets on server restart)
 const clientConfigVersions: Map<string, number> = new Map();
 
-// Tracks the last Unix timestamp (seconds) at which a ping was flushed to the
-// DB for a given siteId. Resets on server restart which is fine – the first
-// ping after startup will always write, re-establishing the online state.
-const lastPingDbWrite: Map<number, number> = new Map();
-const PING_DB_WRITE_INTERVAL = 45; // seconds
+
 
 // Recovery tracking
 let isRedisRecoveryInProgress = false;
@@ -853,32 +846,16 @@ const setupConnection = async (
         );
     });
 
-    // Handle WebSocket protocol-level pings from older newt clients that do
-    // not send application-level "newt/ping" messages. Update the site's
-    // online state and lastPing timestamp so the offline checker treats them
-    // the same as modern newt clients.
     if (clientType === "newt") {
         const newtClient = client as Newt;
-        ws.on("ping", async () => {
+        ws.on("ping", () => {
             if (!newtClient.siteId) return;
-            const now = Math.floor(Date.now() / 1000);
-            const lastWrite = lastPingDbWrite.get(newtClient.siteId) ?? 0;
-            if (now - lastWrite < PING_DB_WRITE_INTERVAL) return;
-            lastPingDbWrite.set(newtClient.siteId, now);
-            try {
-                await db
-                    .update(sites)
-                    .set({
-                        online: true,
-                        lastPing: now
-                    })
-                    .where(eq(sites.siteId, newtClient.siteId));
-            } catch (error) {
-                logger.error(
-                    "Error updating newt site online state on WS ping",
-                    { error }
-                );
-            }
+            // Record the ping in the accumulator instead of writing to the
+            // database on every WS ping frame. The accumulator flushes all
+            // pending pings in a single batched UPDATE every ~10s, which
+            // prevents connection pool exhaustion under load (especially
+            // with cross-region latency to the database).
+            recordPing(newtClient.siteId);
         });
     }
 

server/routers/accessToken/listAccessTokens.ts

@@ -208,7 +208,7 @@ export async function listAccessTokens(
                 .where(
                     or(
                         eq(userResources.userId, req.user!.userId),
-                        eq(roleResources.roleId, req.userOrgRoleId!)
+                        inArray(roleResources.roleId, req.userOrgRoleIds!)
                     )
                 );
         } else {

server/routers/badger/logRequestAudit.ts

@@ -5,25 +5,7 @@ import cache from "#dynamic/lib/cache";
 import { calculateCutoffTimestamp } from "@server/lib/cleanupLogs";
 import { stripPortFromHost } from "@server/lib/ip";
 
-/**
- * Sanitize a string field by replacing lone UTF-16 surrogates (which cannot
- * be encoded as valid UTF-8) with the Unicode replacement character, and
- * stripping ASCII control characters that are invalid in most text columns.
- */
-function sanitizeString(value: string | undefined | null): string | undefined {
-    if (value == null) return undefined;
-    return (
-        value
-            // Replace lone high surrogates (not followed by a low surrogate)
-            // and lone low surrogates (not preceded by a high surrogate)
-            .replace(
-                /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g,
-                "\uFFFD"
-            )
-            // Strip C0 control characters except HT (\x09), LF (\x0A), CR (\x0D)
-            .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "")
-    );
-}
+import { sanitizeString } from "@server/lib/sanitize";
 
 /**
 

server/routers/badger/verifySession.ts

@@ -3,12 +3,13 @@ import { verifyResourceAccessToken } from "@server/auth/verifyResourceAccessToke
 import {
     getResourceByDomain,
     getResourceRules,
+    getRoleName,
     getRoleResourceAccess,
-    getUserOrgRole,
     getUserResourceAccess,
     getOrgLoginPage,
     getUserSessionWithUser
 } from "@server/db/queries/verifySessionQueries";
+import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
 import {
     LoginPage,
     Org,
@@ -916,9 +917,9 @@ async function isUserAllowedToAccessResource(
         return null;
     }
 
-    const userOrgRole = await getUserOrgRole(user.userId, resource.orgId);
+    const userOrgRoleIds = await getUserOrgRoleIds(user.userId, resource.orgId);
 
-    if (!userOrgRole) {
+    if (!userOrgRoleIds.length) {
         return null;
     }
 
@@ -934,17 +935,23 @@ async function isUserAllowedToAccessResource(
         return null;
     }
 
-    const roleResourceAccess = await getRoleResourceAccess(
-        resource.resourceId,
-        userOrgRole.roleId
-    );
-
-    if (roleResourceAccess) {
+    const roleNames: string[] = [];
+    for (const roleId of userOrgRoleIds) {
+        const roleResourceAccess = await getRoleResourceAccess(
+            resource.resourceId,
+            roleId
+        );
+        if (roleResourceAccess) {
+            const roleName = await getRoleName(roleId);
+            if (roleName) roleNames.push(roleName);
+        }
+    }
+    if (roleNames.length > 0) {
         return {
             username: user.username,
             email: user.email,
             name: user.name,
-            role: userOrgRole.roleName
+            role: roleNames.join(", ")
         };
     }
 
@@ -954,11 +961,15 @@ async function isUserAllowedToAccessResource(
     );
 
     if (userResourceAccess) {
+        const names = await Promise.all(
+            userOrgRoleIds.map((id) => getRoleName(id))
+        );
+        const role = names.filter(Boolean).join(", ") || "";
         return {
             username: user.username,
             email: user.email,
             name: user.name,
-            role: userOrgRole.roleName
+            role
         };
     }
 

server/routers/client/createClient.ts

@@ -92,7 +92,7 @@ export async function createClient(
 
         const { orgId } = parsedParams.data;
 
-        if (req.user && !req.userOrgRoleId) {
+        if (req.user && (!req.userOrgRoleIds || req.userOrgRoleIds.length === 0)) {
             return next(
                 createHttpError(HttpCode.FORBIDDEN, "User does not have a role")
             );
@@ -234,7 +234,7 @@ export async function createClient(
                 clientId: newClient.clientId
             });
 
-            if (req.user && req.userOrgRoleId != adminRole.roleId) {
+            if (req.user && !req.userOrgRoleIds?.includes(adminRole.roleId)) {
                 // make sure the user can access the client
                 trx.insert(userClients).values({
                     userId: req.user.userId,

server/routers/client/listClients.ts

@@ -70,7 +70,7 @@ async function getLatestOlmVersion(): Promise<string | null> {
         tags = tags.filter((version) => !version.name.includes("rc"));
         const latestVersion = tags[0].name;
 
-        olmVersionCache.set("latestOlmVersion", latestVersion);
+        olmVersionCache.set("latestOlmVersion", latestVersion, 3600);
 
         return latestVersion;
     } catch (error: any) {
@@ -297,7 +297,7 @@ export async function listClients(
                 .where(
                     or(
                         eq(userClients.userId, req.user!.userId),
-                        eq(roleClients.roleId, req.userOrgRoleId!)
+                        inArray(roleClients.roleId, req.userOrgRoleIds!)
                     )
                 );
         } else {

server/routers/client/listUserDevices.ts

@@ -71,7 +71,7 @@ async function getLatestOlmVersion(): Promise<string | null> {
         tags = tags.filter((version) => !version.name.includes("rc"));
         const latestVersion = tags[0].name;
 
-        olmVersionCache.set("latestOlmVersion", latestVersion);
+        olmVersionCache.set("latestOlmVersion", latestVersion, 3600);
 
         return latestVersion;
     } catch (error: any) {
@@ -316,7 +316,7 @@ export async function listUserDevices(
                 .where(
                     or(
                         eq(userClients.userId, req.user!.userId),
-                        eq(roleClients.roleId, req.userOrgRoleId!)
+                        inArray(roleClients.roleId, req.userOrgRoleIds!)
                     )
                 );
         } else {

server/routers/client/targets.ts

@@ -1,15 +1,54 @@
 import { sendToClient } from "#dynamic/routers/ws";
-import { db, olms, Transaction } from "@server/db";
+import { db, newts, olms } from "@server/db";
+import {
+    Alias,
+    convertSubnetProxyTargetsV2ToV1,
+    SubnetProxyTarget,
+    SubnetProxyTargetV2
+} from "@server/lib/ip";
 import { canCompress } from "@server/lib/clientVersionChecks";
-import { Alias, SubnetProxyTarget } from "@server/lib/ip";
 import logger from "@server/logger";
 import { eq } from "drizzle-orm";
+import semver from "semver";
+
+const NEWT_V2_TARGETS_VERSION = ">=1.10.3";
+
+export async function convertTargetsIfNessicary(
+    newtId: string,
+    targets: SubnetProxyTarget[] | SubnetProxyTargetV2[]
+) {
+    // get the newt
+    const [newt] = await db
+        .select()
+        .from(newts)
+        .where(eq(newts.newtId, newtId));
+    if (!newt) {
+        throw new Error(`No newt found for id: ${newtId}`);
+    }
+
+    // check the semver
+    if (
+        newt.version &&
+        !semver.satisfies(newt.version, NEWT_V2_TARGETS_VERSION)
+    ) {
+        logger.debug(
+            `addTargets Newt version ${newt.version} does not support targets v2 falling back`
+        );
+        targets = convertSubnetProxyTargetsV2ToV1(
+            targets as SubnetProxyTargetV2[]
+        );
+    }
+
+    return targets;
+}
 
 export async function addTargets(
     newtId: string,
-    targets: SubnetProxyTarget[],
+    targets: SubnetProxyTarget[] | SubnetProxyTargetV2[],
     version?: string | null
 ) {
+    targets = await convertTargetsIfNessicary(newtId, targets);
+
     await sendToClient(
         newtId,
         {
@@ -22,9 +61,11 @@ export async function addTargets(
 
 export async function removeTargets(
     newtId: string,
-    targets: SubnetProxyTarget[],
+    targets: SubnetProxyTarget[] | SubnetProxyTargetV2[],
     version?: string | null
 ) {
+    targets = await convertTargetsIfNessicary(newtId, targets);
+
     await sendToClient(
         newtId,
         {
@@ -38,11 +79,39 @@ export async function removeTargets(
 export async function updateTargets(
     newtId: string,
     targets: {
-        oldTargets: SubnetProxyTarget[];
-        newTargets: SubnetProxyTarget[];
+        oldTargets: SubnetProxyTarget[] | SubnetProxyTargetV2[];
+        newTargets: SubnetProxyTarget[] | SubnetProxyTargetV2[];
     },
     version?: string | null
 ) {
+    // get the newt
+    const [newt] = await db
+        .select()
+        .from(newts)
+        .where(eq(newts.newtId, newtId));
+    if (!newt) {
+        logger.error(`addTargetsL No newt found for id: ${newtId}`);
+        return;
+    }
+
+    // check the semver
+    if (
+        newt.version &&
+        !semver.satisfies(newt.version, NEWT_V2_TARGETS_VERSION)
+    ) {
+        logger.debug(
+            `addTargets Newt version ${newt.version} does not support targets v2 falling back`
+        );
+        targets = {
+            oldTargets: convertSubnetProxyTargetsV2ToV1(
+                targets.oldTargets as SubnetProxyTargetV2[]
+            ),
+            newTargets: convertSubnetProxyTargetsV2ToV1(
+                targets.newTargets as SubnetProxyTargetV2[]
+            )
+        };
+    }
+
     await sendToClient(
         newtId,
         {

server/routers/external.ts

@@ -644,6 +644,7 @@ authenticated.delete(
     logActionAudit(ActionsEnum.deleteRole),
     role.deleteRole
 );
+
 authenticated.post(
     "/role/:roleId/add/:userId",
     verifyRoleAccess,
@@ -651,7 +652,7 @@ authenticated.post(
     verifyLimits,
     verifyUserHasAction(ActionsEnum.addUserRole),
     logActionAudit(ActionsEnum.addUserRole),
-    user.addUserRole
+    user.addUserRoleLegacy
 );
 
 authenticated.post(

server/routers/gerbil/receiveBandwidth.ts

@@ -1,6 +1,5 @@
 import { Request, Response, NextFunction } from "express";
-import { eq, sql } from "drizzle-orm";
-import { sites } from "@server/db";
+import { sql } from "drizzle-orm";
 import { db } from "@server/db";
 import logger from "@server/logger";
 import createHttpError from "http-errors";
@@ -31,7 +30,10 @@ const MAX_RETRIES = 3;
 const BASE_DELAY_MS = 50;
 
 // How often to flush accumulated bandwidth data to the database
-const FLUSH_INTERVAL_MS = 30_000; // 30 seconds
+const FLUSH_INTERVAL_MS = 300_000; // 300 seconds
+
+// Maximum number of sites to include in a single batch UPDATE statement
+const BATCH_CHUNK_SIZE = 250;
 
 // In-memory accumulator: publicKey -> AccumulatorEntry
 let accumulator = new Map<string, AccumulatorEntry>();
@@ -75,13 +77,33 @@ async function withDeadlockRetry<T>(
     }
 }
 
+/**
+ * Execute a raw SQL query that returns rows, in a way that works across both
+ * the PostgreSQL driver (which exposes `execute`) and the SQLite driver (which
+ * exposes `all`).  Drizzle's typed query builder doesn't support bulk
+ * UPDATE … FROM (VALUES …) natively, so we drop to raw SQL here.
+ */
+async function dbQueryRows<T extends Record<string, unknown>>(
+    query: Parameters<(typeof sql)["join"]>[0][number]
+): Promise<T[]> {
+    const anyDb = db as any;
+    if (typeof anyDb.execute === "function") {
+        // PostgreSQL (node-postgres via Drizzle) — returns { rows: [...] } or an array
+        const result = await anyDb.execute(query);
+        return (Array.isArray(result) ? result : (result.rows ?? [])) as T[];
+    }
+    // SQLite (better-sqlite3 via Drizzle) — returns an array directly
+    return (await anyDb.all(query)) as T[];
+}
+
 /**
  * Flush all accumulated site bandwidth data to the database.
  *
  * Swaps out the accumulator before writing so that any bandwidth messages
  * received during the flush are captured in the new accumulator rather than
- * being lost or causing contention. Entries that fail to write are re-queued
- * back into the accumulator so they will be retried on the next flush.
+ * being lost or causing contention. Sites are updated in chunks via a single
+ * batch UPDATE per chunk. Failed chunks are discarded — exact per-flush
+ * accuracy is not critical and re-queuing is not worth the added complexity.
  *
  * This function is exported so that the application's graceful-shutdown
  * cleanup handler can call it before the process exits.
@@ -108,76 +130,76 @@ export async function flushSiteBandwidthToDb(): Promise<void> {
         `Flushing accumulated bandwidth data for ${sortedEntries.length} site(s) to the database`
     );
 
-    // Aggregate billing usage by org, collected during the DB update loop.
+    // Build a lookup so post-processing can reach each entry by publicKey.
+    const snapshotMap = new Map(sortedEntries);
+
+    // Aggregate billing usage by org across all chunks.
     const orgUsageMap = new Map<string, number>();
 
-    for (const [publicKey, { bytesIn, bytesOut, exitNodeId, calcUsage }] of sortedEntries) {
+    // Process in chunks so individual queries stay at a reasonable size.
+    for (let i = 0; i < sortedEntries.length; i += BATCH_CHUNK_SIZE) {
+        const chunk = sortedEntries.slice(i, i + BATCH_CHUNK_SIZE);
+        const chunkEnd = i + chunk.length - 1;
+
+        // Build a parameterised VALUES list: (pubKey, bytesIn, bytesOut), ...
+        // Both PostgreSQL and SQLite (≥ 3.33.0, which better-sqlite3 bundles)
+        // support UPDATE … FROM (VALUES …), letting us update the whole chunk
+        // in a single query instead of N individual round-trips.
+        const valuesList = chunk.map(([publicKey, { bytesIn, bytesOut }]) =>
+            sql`(${publicKey}, ${bytesIn}, ${bytesOut})`
+        );
+        const valuesClause = sql.join(valuesList, sql`, `);
+
+        let rows: { orgId: string; pubKey: string }[] = [];
+
         try {
-            const updatedSite = await withDeadlockRetry(async () => {
-                const [result] = await db
-                    .update(sites)
-                    .set({
-                        megabytesOut: sql`COALESCE(${sites.megabytesOut}, 0) + ${bytesIn}`,
-                        megabytesIn: sql`COALESCE(${sites.megabytesIn}, 0) + ${bytesOut}`,
-                        lastBandwidthUpdate: currentTime,
-                    })
-                    .where(eq(sites.pubKey, publicKey))
-                    .returning({
-                        orgId: sites.orgId,
-                        siteId: sites.siteId
-                    });
-                return result;
-            }, `flush bandwidth for site ${publicKey}`);
-
-            if (updatedSite) {
-                if (exitNodeId) {
-                    const notAllowed = await checkExitNodeOrg(
-                        exitNodeId,
-                        updatedSite.orgId
-                    );
-                    if (notAllowed) {
-                        logger.warn(
-                            `Exit node ${exitNodeId} is not allowed for org ${updatedSite.orgId}`
-                        );
-                        // Skip usage tracking for this site but continue
-                        // processing the rest.
-                        continue;
-                    }
-                }
-
-                if (calcUsage) {
-                    const totalBandwidth = bytesIn + bytesOut;
-                    const current = orgUsageMap.get(updatedSite.orgId) ?? 0;
-                    orgUsageMap.set(updatedSite.orgId, current + totalBandwidth);
-                }
-            }
+            rows = await withDeadlockRetry(async () => {
+                return dbQueryRows<{ orgId: string; pubKey: string }>(sql`
+                    UPDATE sites
+                    SET
+                        "bytesOut"            = COALESCE("bytesOut", 0) + v.bytes_in,
+                        "bytesIn"             = COALESCE("bytesIn", 0)  + v.bytes_out,
+                        "lastBandwidthUpdate" = ${currentTime}
+                    FROM (VALUES ${valuesClause}) AS v(pub_key, bytes_in, bytes_out)
+                    WHERE sites."pubKey" = v.pub_key
+                    RETURNING sites."orgId" AS "orgId", sites."pubKey" AS "pubKey"
+                `);
+            }, `flush bandwidth chunk [${i}–${chunkEnd}]`);
         } catch (error) {
             logger.error(
-                `Failed to flush bandwidth for site ${publicKey}:`,
+                `Failed to flush bandwidth chunk [${i}–${chunkEnd}], discarding ${chunk.length} site(s):`,
                 error
             );
+            // Discard the chunk — exact per-flush accuracy is not critical.
+            continue;
+        }
 
-            // Re-queue the failed entry so it is retried on the next flush
-            // rather than silently dropped.
-            const existing = accumulator.get(publicKey);
-            if (existing) {
-                existing.bytesIn += bytesIn;
-                existing.bytesOut += bytesOut;
-            } else {
-                accumulator.set(publicKey, {
-                    bytesIn,
-                    bytesOut,
-                    exitNodeId,
-                    calcUsage
-                });
+        // Collect billing usage from the returned rows.
+        for (const { orgId, pubKey } of rows) {
+            const entry = snapshotMap.get(pubKey);
+            if (!entry) continue;
+
+            const { bytesIn, bytesOut, exitNodeId, calcUsage } = entry;
+
+            if (exitNodeId) {
+                const notAllowed = await checkExitNodeOrg(exitNodeId, orgId);
+                if (notAllowed) {
+                    logger.warn(
+                        `Exit node ${exitNodeId} is not allowed for org ${orgId}`
+                    );
+                    continue;
+                }
+            }
+
+            if (calcUsage) {
+                const current = orgUsageMap.get(orgId) ?? 0;
+                orgUsageMap.set(orgId, current + bytesIn + bytesOut);
             }
         }
     }
 
-    // Process billing usage updates outside the site-update loop to keep
-    // lock scope small and concerns separated.
+    // Process billing usage updates after all chunks are written.
     if (orgUsageMap.size > 0) {
-        // Sort org IDs for consistent lock ordering.
         const sortedOrgIds = [...orgUsageMap.keys()].sort();
 
         for (const orgId of sortedOrgIds) {

server/routers/idp/validateOidcCallback.ts

@@ -13,6 +13,7 @@ import {
     orgs,
     Role,
     roles,
+    userOrgRoles,
     userOrgs,
     users
 } from "@server/db";
@@ -40,6 +41,7 @@ import {
     assignUserToOrg,
     removeUserFromOrg
 } from "@server/lib/userOrg";
+import { unwrapRoleMapping } from "@app/lib/idpRoleMapping";
 
 const ensureTrailingSlash = (url: string): string => {
     return url;
@@ -366,7 +368,7 @@ export async function validateOidcCallback(
             const defaultRoleMapping = existingIdp.idp.defaultRoleMapping;
             const defaultOrgMapping = existingIdp.idp.defaultOrgMapping;
 
-            const userOrgInfo: { orgId: string; roleId: number }[] = [];
+            const userOrgInfo: { orgId: string; roleIds: number[] }[] = [];
             for (const org of allOrgs) {
                 const [idpOrgRes] = await db
                     .select()
@@ -378,8 +380,6 @@ export async function validateOidcCallback(
                         )
                     );
 
-                let roleId: number | undefined = undefined;
-
                 const orgMapping = idpOrgRes?.orgMapping || defaultOrgMapping;
                 const hydratedOrgMapping = hydrateOrgMapping(
                     orgMapping,
@@ -404,38 +404,47 @@ export async function validateOidcCallback(
                     idpOrgRes?.roleMapping || defaultRoleMapping;
                 if (roleMapping) {
                     logger.debug("Role Mapping", { roleMapping });
-                    const roleName = jmespath.search(claims, roleMapping);
+                    const roleMappingJmes = unwrapRoleMapping(
+                        roleMapping
+                    ).evaluationExpression;
+                    const roleMappingResult = jmespath.search(
+                        claims,
+                        roleMappingJmes
+                    );
+                    const roleNames = normalizeRoleMappingResult(
+                        roleMappingResult
+                    );
 
-                    if (!roleName) {
-                        logger.error("Role name not found in the ID token", {
-                            roleName
+                    if (!roleNames.length) {
+                        logger.error("Role mapping returned no valid roles", {
+                            roleMappingResult
                         });
                         continue;
                     }
 
-                    const [roleRes] = await db
+                    const roleRes = await db
                         .select()
                         .from(roles)
                         .where(
                             and(
                                 eq(roles.orgId, org.orgId),
-                                eq(roles.name, roleName)
+                                inArray(roles.name, roleNames)
                             )
                         );
 
-                    if (!roleRes) {
-                        logger.error("Role not found", {
+                    if (!roleRes.length) {
+                        logger.error("No mapped roles found in organization", {
                             orgId: org.orgId,
-                            roleName
+                            roleNames
                         });
                         continue;
                     }
 
-                    roleId = roleRes.roleId;
+                    const roleIds = [...new Set(roleRes.map((r) => r.roleId))];
 
                     userOrgInfo.push({
                         orgId: org.orgId,
-                        roleId
+                        roleIds
                     });
                 }
             }
@@ -570,31 +579,29 @@ export async function validateOidcCallback(
                     }
                 }
 
-                // Update roles for existing auto-provisioned orgs where the role has changed
-                const orgsToUpdate = autoProvisionedOrgs.filter(
-                    (currentOrg) => {
-                        const newOrg = userOrgInfo.find(
-                            (newOrg) => newOrg.orgId === currentOrg.orgId
+                // Ensure IDP-provided role exists for existing auto-provisioned orgs (add only; never delete other roles)
+                const userRolesInOrgs = await trx
+                    .select()
+                    .from(userOrgRoles)
+                    .where(eq(userOrgRoles.userId, userId!));
+                for (const currentOrg of autoProvisionedOrgs) {
+                    const newRole = userOrgInfo.find(
+                        (newOrg) => newOrg.orgId === currentOrg.orgId
+                    );
+                    if (!newRole) continue;
+                    const currentRolesInOrg = userRolesInOrgs.filter(
+                        (r) => r.orgId === currentOrg.orgId
+                    );
+                    for (const roleId of newRole.roleIds) {
+                        const hasIdpRole = currentRolesInOrg.some(
+                            (r) => r.roleId === roleId
                         );
-                        return newOrg && newOrg.roleId !== currentOrg.roleId;
-                    }
-                );
-
-                if (orgsToUpdate.length > 0) {
-                    for (const org of orgsToUpdate) {
-                        const newRole = userOrgInfo.find(
-                            (newOrg) => newOrg.orgId === org.orgId
-                        );
-                        if (newRole) {
-                            await trx
-                                .update(userOrgs)
-                                .set({ roleId: newRole.roleId })
-                                .where(
-                                    and(
-                                        eq(userOrgs.userId, userId!),
-                                        eq(userOrgs.orgId, org.orgId)
-                                    )
-                                );
+                        if (!hasIdpRole) {
+                            await trx.insert(userOrgRoles).values({
+                                userId: userId!,
+                                orgId: currentOrg.orgId,
+                                roleId
+                            });
                         }
                     }
                 }
@@ -609,6 +616,12 @@ export async function validateOidcCallback(
 
                 if (orgsToAdd.length > 0) {
                     for (const org of orgsToAdd) {
+                        const [initialRoleId, ...additionalRoleIds] =
+                            org.roleIds;
+                        if (!initialRoleId) {
+                            continue;
+                        }
+
                         const [fullOrg] = await trx
                             .select()
                             .from(orgs)
@@ -619,11 +632,19 @@ export async function validateOidcCallback(
                                 {
                                     orgId: org.orgId,
                                     userId: userId!,
-                                    roleId: org.roleId,
                                     autoProvisioned: true,
                                 },
+                                initialRoleId,
                                 trx
                             );
+
+                            for (const roleId of additionalRoleIds) {
+                                await trx.insert(userOrgRoles).values({
+                                    userId: userId!,
+                                    orgId: org.orgId,
+                                    roleId
+                                });
+                            }
                         }
                     }
                 }
@@ -748,3 +769,25 @@ function hydrateOrgMapping(
     }
     return orgMapping.split("{{orgId}}").join(orgId);
 }
+
+function normalizeRoleMappingResult(
+    result: unknown
+): string[] {
+    if (typeof result === "string") {
+        const role = result.trim();
+        return role ? [role] : [];
+    }
+
+    if (Array.isArray(result)) {
+        return [
+            ...new Set(
+                result
+                    .filter((value): value is string => typeof value === "string")
+                    .map((value) => value.trim())
+                    .filter(Boolean)
+            )
+        ];
+    }
+
+    return [];
+}

server/routers/integration.ts

@@ -16,6 +16,7 @@ import {
     verifyApiKey,
     verifyApiKeyOrgAccess,
     verifyApiKeyHasAction,
+    verifyApiKeyCanSetUserOrgRoles,
     verifyApiKeySiteAccess,
     verifyApiKeyResourceAccess,
     verifyApiKeyTargetAccess,
@@ -595,7 +596,7 @@ authenticated.post(
     verifyLimits,
     verifyApiKeyHasAction(ActionsEnum.addUserRole),
     logActionAudit(ActionsEnum.addUserRole),
-    user.addUserRole
+    user.addUserRoleLegacy
 );
 
 authenticated.post(

server/routers/newt/buildConfiguration.ts

@@ -16,8 +16,8 @@ import { eq, and } from "drizzle-orm";
 import config from "@server/lib/config";
 import {
     formatEndpoint,
-    generateSubnetProxyTargets,
-    SubnetProxyTarget
+    generateSubnetProxyTargetV2,
+    SubnetProxyTargetV2
 } from "@server/lib/ip";
 
 export async function buildClientConfigurationForNewtClient(
@@ -143,7 +143,7 @@ export async function buildClientConfigurationForNewtClient(
         .from(siteResources)
         .where(eq(siteResources.siteId, siteId));
 
-    const targetsToSend: SubnetProxyTarget[] = [];
+    const targetsToSend: SubnetProxyTargetV2[] = [];
 
     for (const resource of allSiteResources) {
         // Get clients associated with this specific resource
@@ -168,12 +168,14 @@ export async function buildClientConfigurationForNewtClient(
                 )
             );
 
-        const resourceTargets = generateSubnetProxyTargets(
+        const resourceTarget = generateSubnetProxyTargetV2(
             resource,
             resourceClients
         );
 
-        targetsToSend.push(...resourceTargets);
+        if (resourceTarget) {
+            targetsToSend.push(resourceTarget);
+        }
     }
 
     return {

server/routers/newt/createNewt.ts

@@ -46,7 +46,7 @@ export async function createNewt(
 
         const { newtId, secret } = parsedBody.data;
 
-        if (req.user && !req.userOrgRoleId) {
+        if (req.user && (!req.userOrgRoleIds || req.userOrgRoleIds.length === 0)) {
             return next(
                 createHttpError(HttpCode.FORBIDDEN, "User does not have a role")
             );

server/routers/newt/getNewtToken.ts

@@ -1,6 +1,8 @@
 import { generateSessionToken } from "@server/auth/sessions/app";
-import { db } from "@server/db";
+import { db, newtSessions } from "@server/db";
 import { newts } from "@server/db";
+import { getOrCreateCachedToken } from "#dynamic/lib/tokenCache";
+import { EXPIRES } from "@server/auth/sessions/newt";
 import HttpCode from "@server/types/HttpCode";
 import response from "@server/lib/response";
 import { eq } from "drizzle-orm";
@@ -92,8 +94,19 @@ export async function getNewtToken(
             );
         }
 
-        const resToken = generateSessionToken();
-        await createNewtSession(resToken, existingNewt.newtId);
+        // Return a cached token if one exists to prevent thundering herd on
+        // simultaneous restarts; falls back to creating a fresh session when
+        // Redis is unavailable or the cache has expired.
+        const resToken = await getOrCreateCachedToken(
+            `newt:token_cache:${existingNewt.newtId}`,
+            config.getRawConfig().server.secret!,
+            Math.floor(EXPIRES / 1000),
+            async () => {
+                const token = generateSessionToken();
+                await createNewtSession(token, existingNewt.newtId);
+                return token;
+            }
+        );
 
         return response<{ token: string; serverVersion: string }>(res, {
             data: {

server/routers/newt/handleGetConfigMessage.ts

@@ -6,6 +6,7 @@ import { db, ExitNode, exitNodes, Newt, sites } from "@server/db";
 import { eq } from "drizzle-orm";
 import { sendToExitNode } from "#dynamic/lib/exitNodes";
 import { buildClientConfigurationForNewtClient } from "./buildConfiguration";
+import { convertTargetsIfNessicary } from "../client/targets";
 import { canCompress } from "@server/lib/clientVersionChecks";
 
 const inputSchema = z.object({
@@ -127,13 +128,15 @@ export const handleGetConfigMessage: MessageHandler = async (context) => {
         exitNode
     );
 
+    const targetsToSend = await convertTargetsIfNessicary(newt.newtId, targets);
+
     return {
         message: {
             type: "newt/wg/receive-config",
             data: {
                 ipAddress: site.address,
                 peers,
-                targets
+                targets: targetsToSend
             }
         },
         options: {

server/routers/newt/handleNewtDisconnectingMessage.ts

@@ -6,7 +6,9 @@ import logger from "@server/logger";
 /**
  * Handles disconnecting messages from sites to show disconnected in the ui
  */
-export const handleNewtDisconnectingMessage: MessageHandler = async (context) => {
+export const handleNewtDisconnectingMessage: MessageHandler = async (
+    context
+) => {
     const { message, client: c, sendToClient } = context;
     const newt = c as Newt;
 
@@ -27,7 +29,7 @@ export const handleNewtDisconnectingMessage: MessageHandler = async (context) =>
             .set({
                 online: false
             })
-            .where(eq(sites.siteId, sites.siteId));
+            .where(eq(sites.siteId, newt.siteId));
     } catch (error) {
         logger.error("Error handling disconnecting message", { error });
     }

server/routers/newt/handleNewtPingMessage.ts

@@ -5,6 +5,7 @@ import { Newt } from "@server/db";
 import { eq, lt, isNull, and, or } from "drizzle-orm";
 import logger from "@server/logger";
 import { sendNewtSyncMessage } from "./sync";
+import { recordPing } from "./pingAccumulator";
 
 // Track if the offline checker interval is running
 let offlineCheckerInterval: NodeJS.Timeout | null = null;
@@ -114,18 +115,12 @@ export const handleNewtPingMessage: MessageHandler = async (context) => {
         return;
     }
 
-    try {
-        // Mark the site as online and record the ping timestamp.
-        await db
-            .update(sites)
-            .set({
-                online: true,
-                lastPing: Math.floor(Date.now() / 1000)
-            })
-            .where(eq(sites.siteId, newt.siteId));
-    } catch (error) {
-        logger.error("Error updating online state on newt ping", { error });
-    }
+    // Record the ping in memory; it will be flushed to the database
+    // periodically by the ping accumulator (every ~10s) in a single
+    // batched UPDATE instead of one query per ping. This prevents
+    // connection pool exhaustion under load, especially with
+    // cross-region latency to the database.
+    recordPing(newt.siteId);
 
     // Check config version and sync if stale.
     const configVersion = await getClientConfigVersion(newt.newtId);

server/routers/newt/pingAccumulator.ts

@@ -0,0 +1,382 @@
+import { db } from "@server/db";
+import { sites, clients, olms } from "@server/db";
+import { eq, inArray } from "drizzle-orm";
+import logger from "@server/logger";
+
+/**
+ * Ping Accumulator
+ *
+ * Instead of writing to the database on every single newt/olm ping (which
+ * causes pool exhaustion under load, especially with cross-region latency),
+ * we accumulate pings in memory and flush them to the database periodically
+ * in a single batch.
+ *
+ * This is the same pattern used for bandwidth flushing in
+ * receiveBandwidth.ts and handleReceiveBandwidthMessage.ts.
+ *
+ * Supports two kinds of pings:
+ *   - **Site pings** (from newts): update `sites.online` and `sites.lastPing`
+ *   - **Client pings** (from OLMs): update `clients.online`, `clients.lastPing`,
+ *     `clients.archived`, and optionally reset `olms.archived`
+ */
+
+const FLUSH_INTERVAL_MS = 10_000; // Flush every 10 seconds
+const MAX_RETRIES = 2;
+const BASE_DELAY_MS = 50;
+
+// ── Site (newt) pings ──────────────────────────────────────────────────
+// Map of siteId -> latest ping timestamp (unix seconds)
+const pendingSitePings: Map<number, number> = new Map();
+
+// ── Client (OLM) pings ────────────────────────────────────────────────
+// Map of clientId -> latest ping timestamp (unix seconds)
+const pendingClientPings: Map<number, number> = new Map();
+// Set of olmIds whose `archived` flag should be reset to false
+const pendingOlmArchiveResets: Set<string> = new Set();
+
+let flushTimer: NodeJS.Timeout | null = null;
+
+// ── Public API ─────────────────────────────────────────────────────────
+
+/**
+ * Record a ping for a newt site. This does NOT write to the database
+ * immediately. Instead it stores the latest ping timestamp in memory,
+ * to be flushed periodically by the background timer.
+ */
+export function recordSitePing(siteId: number): void {
+    const now = Math.floor(Date.now() / 1000);
+    pendingSitePings.set(siteId, now);
+}
+
+/** @deprecated Use `recordSitePing` instead. Alias kept for existing call-sites. */
+export const recordPing = recordSitePing;
+
+/**
+ * Record a ping for an OLM client. Batches the `clients` table update
+ * (`online`, `lastPing`, `archived`) and, when `olmArchived` is true,
+ * also queues an `olms` table update to clear the archived flag.
+ */
+export function recordClientPing(
+    clientId: number,
+    olmId: string,
+    olmArchived: boolean
+): void {
+    const now = Math.floor(Date.now() / 1000);
+    pendingClientPings.set(clientId, now);
+    if (olmArchived) {
+        pendingOlmArchiveResets.add(olmId);
+    }
+}
+
+// ── Flush Logic ────────────────────────────────────────────────────────
+
+/**
+ * Flush all accumulated site pings to the database.
+ */
+async function flushSitePingsToDb(): Promise<void> {
+    if (pendingSitePings.size === 0) {
+        return;
+    }
+
+    // Snapshot and clear so new pings arriving during the flush go into a
+    // fresh map for the next cycle.
+    const pingsToFlush = new Map(pendingSitePings);
+    pendingSitePings.clear();
+
+    // Sort by siteId for consistent lock ordering (prevents deadlocks)
+    const sortedEntries = Array.from(pingsToFlush.entries()).sort(
+        ([a], [b]) => a - b
+    );
+
+    const BATCH_SIZE = 50;
+    for (let i = 0; i < sortedEntries.length; i += BATCH_SIZE) {
+        const batch = sortedEntries.slice(i, i + BATCH_SIZE);
+
+        try {
+            await withRetry(async () => {
+                // Group by timestamp for efficient bulk updates
+                const byTimestamp = new Map<number, number[]>();
+                for (const [siteId, timestamp] of batch) {
+                    const group = byTimestamp.get(timestamp) || [];
+                    group.push(siteId);
+                    byTimestamp.set(timestamp, group);
+                }
+
+                if (byTimestamp.size === 1) {
+                    const [timestamp, siteIds] = Array.from(
+                        byTimestamp.entries()
+                    )[0];
+                    await db
+                        .update(sites)
+                        .set({
+                            online: true,
+                            lastPing: timestamp
+                        })
+                        .where(inArray(sites.siteId, siteIds));
+                } else {
+                    await db.transaction(async (tx) => {
+                        for (const [timestamp, siteIds] of byTimestamp) {
+                            await tx
+                                .update(sites)
+                                .set({
+                                    online: true,
+                                    lastPing: timestamp
+                                })
+                                .where(inArray(sites.siteId, siteIds));
+                        }
+                    });
+                }
+            }, "flushSitePingsToDb");
+        } catch (error) {
+            logger.error(
+                `Failed to flush site ping batch (${batch.length} sites), re-queuing for next cycle`,
+                { error }
+            );
+            for (const [siteId, timestamp] of batch) {
+                const existing = pendingSitePings.get(siteId);
+                if (!existing || existing < timestamp) {
+                    pendingSitePings.set(siteId, timestamp);
+                }
+            }
+        }
+    }
+}
+
+/**
+ * Flush all accumulated client (OLM) pings to the database.
+ */
+async function flushClientPingsToDb(): Promise<void> {
+    if (pendingClientPings.size === 0 && pendingOlmArchiveResets.size === 0) {
+        return;
+    }
+
+    // Snapshot and clear
+    const pingsToFlush = new Map(pendingClientPings);
+    pendingClientPings.clear();
+
+    const olmResetsToFlush = new Set(pendingOlmArchiveResets);
+    pendingOlmArchiveResets.clear();
+
+    // ── Flush client pings ─────────────────────────────────────────────
+    if (pingsToFlush.size > 0) {
+        const sortedEntries = Array.from(pingsToFlush.entries()).sort(
+            ([a], [b]) => a - b
+        );
+
+        const BATCH_SIZE = 50;
+        for (let i = 0; i < sortedEntries.length; i += BATCH_SIZE) {
+            const batch = sortedEntries.slice(i, i + BATCH_SIZE);
+
+            try {
+                await withRetry(async () => {
+                    const byTimestamp = new Map<number, number[]>();
+                    for (const [clientId, timestamp] of batch) {
+                        const group = byTimestamp.get(timestamp) || [];
+                        group.push(clientId);
+                        byTimestamp.set(timestamp, group);
+                    }
+
+                    if (byTimestamp.size === 1) {
+                        const [timestamp, clientIds] = Array.from(
+                            byTimestamp.entries()
+                        )[0];
+                        await db
+                            .update(clients)
+                            .set({
+                                lastPing: timestamp,
+                                online: true,
+                                archived: false
+                            })
+                            .where(inArray(clients.clientId, clientIds));
+                    } else {
+                        await db.transaction(async (tx) => {
+                            for (const [timestamp, clientIds] of byTimestamp) {
+                                await tx
+                                    .update(clients)
+                                    .set({
+                                        lastPing: timestamp,
+                                        online: true,
+                                        archived: false
+                                    })
+                                    .where(
+                                        inArray(clients.clientId, clientIds)
+                                    );
+                            }
+                        });
+                    }
+                }, "flushClientPingsToDb");
+            } catch (error) {
+                logger.error(
+                    `Failed to flush client ping batch (${batch.length} clients), re-queuing for next cycle`,
+                    { error }
+                );
+                for (const [clientId, timestamp] of batch) {
+                    const existing = pendingClientPings.get(clientId);
+                    if (!existing || existing < timestamp) {
+                        pendingClientPings.set(clientId, timestamp);
+                    }
+                }
+            }
+        }
+    }
+
+    // ── Flush OLM archive resets ───────────────────────────────────────
+    if (olmResetsToFlush.size > 0) {
+        const olmIds = Array.from(olmResetsToFlush).sort();
+
+        const BATCH_SIZE = 50;
+        for (let i = 0; i < olmIds.length; i += BATCH_SIZE) {
+            const batch = olmIds.slice(i, i + BATCH_SIZE);
+
+            try {
+                await withRetry(async () => {
+                    await db
+                        .update(olms)
+                        .set({ archived: false })
+                        .where(inArray(olms.olmId, batch));
+                }, "flushOlmArchiveResets");
+            } catch (error) {
+                logger.error(
+                    `Failed to flush OLM archive reset batch (${batch.length} olms), re-queuing for next cycle`,
+                    { error }
+                );
+                for (const olmId of batch) {
+                    pendingOlmArchiveResets.add(olmId);
+                }
+            }
+        }
+    }
+}
+
+/**
+ * Flush everything — called by the interval timer and during shutdown.
+ */
+export async function flushPingsToDb(): Promise<void> {
+    await flushSitePingsToDb();
+    await flushClientPingsToDb();
+}
+
+// ── Retry / Error Helpers ──────────────────────────────────────────────
+
+/**
+ * Simple retry wrapper with exponential backoff for transient errors
+ * (connection timeouts, unexpected disconnects).
+ */
+async function withRetry<T>(
+    operation: () => Promise<T>,
+    context: string
+): Promise<T> {
+    let attempt = 0;
+    while (true) {
+        try {
+            return await operation();
+        } catch (error: any) {
+            if (isTransientError(error) && attempt < MAX_RETRIES) {
+                attempt++;
+                const baseDelay = Math.pow(2, attempt - 1) * BASE_DELAY_MS;
+                const jitter = Math.random() * baseDelay;
+                const delay = baseDelay + jitter;
+                logger.warn(
+                    `Transient DB error in ${context}, retrying attempt ${attempt}/${MAX_RETRIES} after ${delay.toFixed(0)}ms`
+                );
+                await new Promise((resolve) => setTimeout(resolve, delay));
+                continue;
+            }
+            throw error;
+        }
+    }
+}
+
+/**
+ * Detect transient connection errors that are safe to retry.
+ */
+function isTransientError(error: any): boolean {
+    if (!error) return false;
+
+    const message = (error.message || "").toLowerCase();
+    const causeMessage = (error.cause?.message || "").toLowerCase();
+    const code = error.code || "";
+
+    // Connection timeout / terminated
+    if (
+        message.includes("connection timeout") ||
+        message.includes("connection terminated") ||
+        message.includes("timeout exceeded when trying to connect") ||
+        causeMessage.includes("connection terminated unexpectedly") ||
+        causeMessage.includes("connection timeout")
+    ) {
+        return true;
+    }
+
+    // PostgreSQL deadlock
+    if (code === "40P01" || message.includes("deadlock")) {
+        return true;
+    }
+
+    // ECONNRESET, ECONNREFUSED, EPIPE
+    if (
+        code === "ECONNRESET" ||
+        code === "ECONNREFUSED" ||
+        code === "EPIPE" ||
+        code === "ETIMEDOUT"
+    ) {
+        return true;
+    }
+
+    return false;
+}
+
+// ── Lifecycle ──────────────────────────────────────────────────────────
+
+/**
+ * Start the background flush timer. Call this once at server startup.
+ */
+export function startPingAccumulator(): void {
+    if (flushTimer) {
+        return; // Already running
+    }
+
+    flushTimer = setInterval(async () => {
+        try {
+            await flushPingsToDb();
+        } catch (error) {
+            logger.error("Unhandled error in ping accumulator flush", {
+                error
+            });
+        }
+    }, FLUSH_INTERVAL_MS);
+
+    // Don't prevent the process from exiting
+    flushTimer.unref();
+
+    logger.info(
+        `Ping accumulator started (flush interval: ${FLUSH_INTERVAL_MS}ms)`
+    );
+}
+
+/**
+ * Stop the background flush timer and perform a final flush.
+ * Call this during graceful shutdown.
+ */
+export async function stopPingAccumulator(): Promise<void> {
+    if (flushTimer) {
+        clearInterval(flushTimer);
+        flushTimer = null;
+    }
+
+    // Final flush to persist any remaining pings
+    try {
+        await flushPingsToDb();
+    } catch (error) {
+        logger.error("Error during final ping accumulator flush", { error });
+    }
+
+    logger.info("Ping accumulator stopped");
+}
+
+/**
+ * Get the number of pending (unflushed) pings. Useful for monitoring.
+ */
+export function getPendingPingCount(): number {
+    return pendingSitePings.size + pendingClientPings.size;
+}

server/routers/olm/createOlm.ts

@@ -46,7 +46,7 @@ export async function createNewt(
 
         const { newtId, secret } = parsedBody.data;
 
-        if (req.user && !req.userOrgRoleId) {
+        if (req.user && (!req.userOrgRoleIds || req.userOrgRoleIds.length === 0)) {
             return next(
                 createHttpError(HttpCode.FORBIDDEN, "User does not have a role")
             );

server/routers/olm/getOlmToken.ts

@@ -8,7 +8,7 @@ import {
     ExitNode,
     exitNodes,
     sites,
-    clientSitesAssociationsCache
+    clientSitesAssociationsCache,
 } from "@server/db";
 import { olms } from "@server/db";
 import HttpCode from "@server/types/HttpCode";
@@ -20,8 +20,10 @@ import { z } from "zod";
 import { fromError } from "zod-validation-error";
 import {
     createOlmSession,
-    validateOlmSessionToken
+    validateOlmSessionToken,
+    EXPIRES
 } from "@server/auth/sessions/olm";
+import { getOrCreateCachedToken } from "#dynamic/lib/tokenCache";
 import { verifyPassword } from "@server/auth/password";
 import logger from "@server/logger";
 import config from "@server/lib/config";
@@ -132,8 +134,19 @@ export async function getOlmToken(
 
         logger.debug("Creating new olm session token");
 
-        const resToken = generateSessionToken();
-        await createOlmSession(resToken, existingOlm.olmId);
+        // Return a cached token if one exists to prevent thundering herd on
+        // simultaneous restarts; falls back to creating a fresh session when
+        // Redis is unavailable or the cache has expired.
+        const resToken = await getOrCreateCachedToken(
+            `olm:token_cache:${existingOlm.olmId}`,
+            config.getRawConfig().server.secret!,
+            Math.floor(EXPIRES / 1000),
+            async () => {
+                const token = generateSessionToken();
+                await createOlmSession(token, existingOlm.olmId);
+                return token;
+            }
+        );
 
         let clientIdToUse;
         if (orgId) {

server/routers/olm/handleOlmPingMessage.ts

@@ -3,6 +3,7 @@ import { db } from "@server/db";
 import { MessageHandler } from "@server/routers/ws";
 import { clients, olms, Olm } from "@server/db";
 import { eq, lt, isNull, and, or } from "drizzle-orm";
+import { recordClientPing } from "@server/routers/newt/pingAccumulator";
 import logger from "@server/logger";
 import { validateSessionToken } from "@server/auth/sessions/app";
 import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
@@ -201,22 +202,12 @@ export const handleOlmPingMessage: MessageHandler = async (context) => {
             await sendOlmSyncMessage(olm, client);
         }
 
-        // Update the client's last ping timestamp
-        await db
-            .update(clients)
-            .set({
-                lastPing: Math.floor(Date.now() / 1000),
-                online: true,
-                archived: false
-            })
-            .where(eq(clients.clientId, olm.clientId));
-
-        if (olm.archived) {
-            await db
-                .update(olms)
-                .set({ archived: false })
-                .where(eq(olms.olmId, olm.olmId));
-        }
+        // Record the ping in memory; it will be flushed to the database
+        // periodically by the ping accumulator (every ~10s) in a single
+        // batched UPDATE instead of one query per ping. This prevents
+        // connection pool exhaustion under load, especially with
+        // cross-region latency to the database.
+        recordClientPing(olm.clientId, olm.olmId, !!olm.archived);
     } catch (error) {
         logger.error("Error handling ping message", { error });
     }

server/routers/org/checkOrgUserAccess.ts

@@ -1,7 +1,7 @@
 import { Request, Response, NextFunction } from "express";
 import { z } from "zod";
 import { db, idp, idpOidcConfig } from "@server/db";
-import { roles, userOrgs, users } from "@server/db";
+import { roles, userOrgRoles, userOrgs, users } from "@server/db";
 import { and, eq } from "drizzle-orm";
 import response from "@server/lib/response";
 import HttpCode from "@server/types/HttpCode";
@@ -14,7 +14,7 @@ import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
 import { CheckOrgAccessPolicyResult } from "@server/lib/checkOrgAccessPolicy";
 
 async function queryUser(orgId: string, userId: string) {
-    const [user] = await db
+    const [userRow] = await db
         .select({
             orgId: userOrgs.orgId,
             userId: users.userId,
@@ -22,10 +22,7 @@ async function queryUser(orgId: string, userId: string) {
             username: users.username,
             name: users.name,
             type: users.type,
-            roleId: userOrgs.roleId,
-            roleName: roles.name,
             isOwner: userOrgs.isOwner,
-            isAdmin: roles.isAdmin,
             twoFactorEnabled: users.twoFactorEnabled,
             autoProvisioned: userOrgs.autoProvisioned,
             idpId: users.idpId,
@@ -35,13 +32,40 @@ async function queryUser(orgId: string, userId: string) {
             idpAutoProvision: idp.autoProvision
         })
         .from(userOrgs)
-        .leftJoin(roles, eq(userOrgs.roleId, roles.roleId))
         .leftJoin(users, eq(userOrgs.userId, users.userId))
         .leftJoin(idp, eq(users.idpId, idp.idpId))
         .leftJoin(idpOidcConfig, eq(idp.idpId, idpOidcConfig.idpId))
         .where(and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, orgId)))
         .limit(1);
-    return user;
+
+    if (!userRow) return undefined;
+
+    const roleRows = await db
+        .select({
+            roleId: userOrgRoles.roleId,
+            roleName: roles.name,
+            isAdmin: roles.isAdmin
+        })
+        .from(userOrgRoles)
+        .leftJoin(roles, eq(userOrgRoles.roleId, roles.roleId))
+        .where(
+            and(
+                eq(userOrgRoles.userId, userId),
+                eq(userOrgRoles.orgId, orgId)
+            )
+        );
+
+    const isAdmin = roleRows.some((r) => r.isAdmin);
+
+    return {
+        ...userRow,
+        isAdmin,
+        roleIds: roleRows.map((r) => r.roleId),
+        roles: roleRows.map((r) => ({
+            roleId: r.roleId,
+            name: r.roleName ?? ""
+        }))
+    };
 }
 
 export type CheckOrgUserAccessResponse = CheckOrgAccessPolicyResult;

server/routers/org/createOrg.ts

@@ -9,6 +9,7 @@ import {
     orgs,
     roleActions,
     roles,
+    userOrgRoles,
     userOrgs,
     users,
     actions
@@ -312,9 +313,13 @@ export async function createOrg(
                 await trx.insert(userOrgs).values({
                     userId: req.user!.userId,
                     orgId: newOrg[0].orgId,
-                    roleId: roleId,
                     isOwner: true
                 });
+                await trx.insert(userOrgRoles).values({
+                    userId: req.user!.userId,
+                    orgId: newOrg[0].orgId,
+                    roleId
+                });
                 ownerUserId = req.user!.userId;
             } else {
                 // if org created by root api key, set the server admin as the owner
@@ -332,9 +337,13 @@ export async function createOrg(
                 await trx.insert(userOrgs).values({
                     userId: serverAdmin.userId,
                     orgId: newOrg[0].orgId,
-                    roleId: roleId,
                     isOwner: true
                 });
+                await trx.insert(userOrgRoles).values({
+                    userId: serverAdmin.userId,
+                    orgId: newOrg[0].orgId,
+                    roleId
+                });
                 ownerUserId = serverAdmin.userId;
             }
 

server/routers/org/getOrgOverview.ts

@@ -117,20 +117,26 @@ export async function getOrgOverview(
             .from(userOrgs)
             .where(eq(userOrgs.orgId, orgId));
 
-        const [role] = await db
-            .select()
-            .from(roles)
-            .where(eq(roles.roleId, req.userOrg.roleId));
+        const roleIds = req.userOrgRoleIds ?? [];
+        const roleRows =
+            roleIds.length > 0
+                ? await db
+                      .select({ name: roles.name, isAdmin: roles.isAdmin })
+                      .from(roles)
+                      .where(inArray(roles.roleId, roleIds))
+                : [];
+        const userRoleName = roleRows.map((r) => r.name ?? "").join(", ") ?? "";
+        const isAdmin = roleRows.some((r) => r.isAdmin === true);
 
         return response<GetOrgOverviewResponse>(res, {
             data: {
                 orgName: org[0].name,
                 orgId: org[0].orgId,
-                userRoleName: role.name,
+                userRoleName,
                 numSites,
                 numUsers,
                 numResources,
-                isAdmin: role.isAdmin || false,
+                isAdmin,
                 isOwner: req.userOrg?.isOwner || false
             },
             success: true,

server/routers/org/listUserOrgs.ts

@@ -1,7 +1,7 @@
 import { Request, Response, NextFunction } from "express";
 import { z } from "zod";
 import { db, roles } from "@server/db";
-import { Org, orgs, userOrgs } from "@server/db";
+import { Org, orgs, userOrgRoles, userOrgs } from "@server/db";
 import response from "@server/lib/response";
 import HttpCode from "@server/types/HttpCode";
 import createHttpError from "http-errors";
@@ -82,10 +82,7 @@ export async function listUserOrgs(
         const { userId } = parsedParams.data;
 
         const userOrganizations = await db
-            .select({
-                orgId: userOrgs.orgId,
-                roleId: userOrgs.roleId
-            })
+            .select({ orgId: userOrgs.orgId })
             .from(userOrgs)
             .where(eq(userOrgs.userId, userId));
 
@@ -116,10 +113,27 @@ export async function listUserOrgs(
                 userOrgs,
                 and(eq(userOrgs.orgId, orgs.orgId), eq(userOrgs.userId, userId))
             )
-            .leftJoin(roles, eq(userOrgs.roleId, roles.roleId))
             .limit(limit)
             .offset(offset);
 
+        const roleRows = await db
+            .select({
+                orgId: userOrgRoles.orgId,
+                isAdmin: roles.isAdmin
+            })
+            .from(userOrgRoles)
+            .leftJoin(roles, eq(userOrgRoles.roleId, roles.roleId))
+            .where(
+                and(
+                    eq(userOrgRoles.userId, userId),
+                    inArray(userOrgRoles.orgId, userOrgIds)
+                )
+            );
+
+        const orgHasAdmin = new Set(
+            roleRows.filter((r) => r.isAdmin).map((r) => r.orgId)
+        );
+
         const totalCountResult = await db
             .select({ count: sql<number>`cast(count(*) as integer)` })
             .from(orgs)
@@ -133,8 +147,8 @@ export async function listUserOrgs(
             if (val.userOrgs && val.userOrgs.isOwner) {
                 res.isOwner = val.userOrgs.isOwner;
             }
-            if (val.roles && val.roles.isAdmin) {
-                res.isAdmin = val.roles.isAdmin;
+            if (val.orgs && orgHasAdmin.has(val.orgs.orgId)) {
+                res.isAdmin = true;
             }
             if (val.userOrgs?.isOwner && val.orgs?.isBillingOrg) {
                 res.isPrimaryOrg = val.orgs.isBillingOrg;

server/routers/resource/createResource.ts

@@ -112,7 +112,7 @@ export async function createResource(
 
         const { orgId } = parsedParams.data;
 
-        if (req.user && !req.userOrgRoleId) {
+        if (req.user && (!req.userOrgRoleIds || req.userOrgRoleIds.length === 0)) {
             return next(
                 createHttpError(HttpCode.FORBIDDEN, "User does not have a role")
             );
@@ -292,7 +292,7 @@ async function createHttpResource(
             resourceId: newResource[0].resourceId
         });
 
-        if (req.user && req.userOrgRoleId != adminRole[0].roleId) {
+        if (req.user && !req.userOrgRoleIds?.includes(adminRole[0].roleId)) {
             // make sure the user can access the resource
             await trx.insert(userResources).values({
                 userId: req.user?.userId!,
@@ -385,7 +385,7 @@ async function createRawResource(
             resourceId: newResource[0].resourceId
         });
 
-        if (req.user && req.userOrgRoleId != adminRole[0].roleId) {
+        if (req.user && !req.userOrgRoleIds?.includes(adminRole[0].roleId)) {
             // make sure the user can access the resource
             await trx.insert(userResources).values({
                 userId: req.user?.userId!,

server/routers/resource/getUserResources.ts

@@ -5,6 +5,7 @@ import {
     resources,
     userResources,
     roleResources,
+    userOrgRoles,
     userOrgs,
     resourcePassword,
     resourcePincode,
@@ -32,22 +33,29 @@ export async function getUserResources(
             );
         }
 
-        // First get the user's role in the organization
-        const userOrgResult = await db
-            .select({
-                roleId: userOrgs.roleId
-            })
+        // Check user is in organization and get their role IDs
+        const [userOrg] = await db
+            .select()
             .from(userOrgs)
             .where(and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, orgId)))
             .limit(1);
 
-        if (userOrgResult.length === 0) {
+        if (!userOrg) {
             return next(
                 createHttpError(HttpCode.FORBIDDEN, "User not in organization")
             );
         }
 
-        const userRoleId = userOrgResult[0].roleId;
+        const userRoleIds = await db
+            .select({ roleId: userOrgRoles.roleId })
+            .from(userOrgRoles)
+            .where(
+                and(
+                    eq(userOrgRoles.userId, userId),
+                    eq(userOrgRoles.orgId, orgId)
+                )
+            )
+            .then((rows) => rows.map((r) => r.roleId));
 
         // Get resources accessible through direct assignment or role assignment
         const directResourcesQuery = db
@@ -55,20 +63,28 @@ export async function getUserResources(
             .from(userResources)
             .where(eq(userResources.userId, userId));
 
-        const roleResourcesQuery = db
-            .select({ resourceId: roleResources.resourceId })
-            .from(roleResources)
-            .where(eq(roleResources.roleId, userRoleId));
+        const roleResourcesQuery =
+            userRoleIds.length > 0
+                ? db
+                      .select({ resourceId: roleResources.resourceId })
+                      .from(roleResources)
+                      .where(inArray(roleResources.roleId, userRoleIds))
+                : Promise.resolve([]);
 
         const directSiteResourcesQuery = db
             .select({ siteResourceId: userSiteResources.siteResourceId })
             .from(userSiteResources)
             .where(eq(userSiteResources.userId, userId));
 
-        const roleSiteResourcesQuery = db
-            .select({ siteResourceId: roleSiteResources.siteResourceId })
-            .from(roleSiteResources)
-            .where(eq(roleSiteResources.roleId, userRoleId));
+        const roleSiteResourcesQuery =
+            userRoleIds.length > 0
+                ? db
+                      .select({
+                          siteResourceId: roleSiteResources.siteResourceId
+                      })
+                      .from(roleSiteResources)
+                      .where(inArray(roleSiteResources.roleId, userRoleIds))
+                : Promise.resolve([]);
 
         const [directResources, roleResourceResults, directSiteResourceResults, roleSiteResourceResults] = await Promise.all([
             directResourcesQuery,

server/routers/resource/listResources.ts

@@ -305,7 +305,7 @@ export async function listResources(
                 .where(
                     or(
                         eq(userResources.userId, req.user!.userId),
-                        eq(roleResources.roleId, req.userOrgRoleId!)
+                        inArray(roleResources.roleId, req.userOrgRoleIds!)
                     )
                 );
         } else {