Merge branch 'dev' into feat/redirect-tables

server/private/lib/blueprints/MtlsSchema.ts

@@ -0,0 +1,19 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { z } from "zod";
+
+export const MtlsSchema = z.object({
+    enabled: z.boolean().optional(),
+    "ca-certificates": z.array(z.string().min(1)).optional()
+});

server/private/lib/dns/server.ts

@@ -384,7 +384,8 @@ export class AuthoritativeDNSServer {
         // Get records from cache or database
         const records = await this.getResourceRecordsByFullDomain(
             queryName,
-            queryType
+            queryType,
+            baseDomain!
         );
 
         if (records.length > 0) {
@@ -475,15 +476,30 @@ export class AuthoritativeDNSServer {
 
         const labels = queryName.replace(/\.$/, "").split(".");
 
-        // Fast path: O(1) in-memory Set lookup — no DB or network I/O
-        if (this.allDomains.size > 0) {
-            for (let i = 0; i < labels.length; i++) {
-                const candidate = labels.slice(i).join(".");
-                if (this.allDomains.has(candidate)) {
-                    this.authoritativeDomainCache.set(cacheKey, candidate);
-                    return candidate;
-                }
+        // Infrastructure zone cuts (e.g. this deployment's own nameserver
+        // zone, cname/site extension zones) that this server is
+        // authoritative for regardless of the customer `domains` table.
+        // Static config, so always available - checked alongside allDomains
+        // below rather than gating a separate fast path on it.
+        const configuredZones = (config.getRawConfig().dns?.zones ?? []).map(
+            (z) => z.toLowerCase()
+        );
+
+        // Fast path: O(1) in-memory Set/array lookup — no DB or network I/O
+        for (let i = 0; i < labels.length; i++) {
+            const candidate = labels.slice(i).join(".");
+            if (
+                this.allDomains.has(candidate) ||
+                configuredZones.includes(candidate)
+            ) {
+                this.authoritativeDomainCache.set(cacheKey, candidate);
+                return candidate;
             }
+        }
+
+        // Once the in-memory customer-domain set has loaded, the checks
+        // above are conclusive - no DB fallback needed.
+        if (this.allDomains.size > 0) {
             this.authoritativeDomainCache.set(cacheKey, null);
             return null;
         }
@@ -582,7 +598,8 @@ export class AuthoritativeDNSServer {
 
     private async getResourceRecordsByFullDomain(
         name: string,
-        queryType: dns.RecordType
+        queryType: dns.RecordType,
+        baseDomain: string
     ): Promise<DNSRecord[]> {
         const cacheKey = `resourceRecords:${name}:${queryType}`;
 
@@ -630,8 +647,14 @@ export class AuthoritativeDNSServer {
                 if (resourceRows.length === 0) {
                     // Resource doesn't exist at all — also pre-populate the domainExists
                     // cache so the subsequent domainExists() call hits memory, not the DB.
+                    // Skip that pre-population when `name` is the zone apex itself: the
+                    // apex always exists (it owns the SOA) even with no resource row, and
+                    // caching `false` here would make domainExists() return that stale
+                    // false instead of ever reaching its own apex check.
                     logger.debug(`No resource found for domain: ${name}`);
-                    this.cache.set(`exists:${name}`, false, 60);
+                    if (name.replace(/\.$/, "") !== baseDomain) {
+                        this.cache.set(`exists:${name}`, false, 60);
+                    }
                     this.cache.set(cacheKey, [], 60);
                     return [];
                 }

server/private/lib/exitNodes/exitNodes.ts

@@ -164,6 +164,7 @@ export async function listExitNodes(
             endpoint: exitNodes.endpoint,
             publicKey: exitNodes.publicKey,
             listenPort: exitNodes.listenPort,
+            relayPort: exitNodes.relayPort,
             reachableAt: exitNodes.reachableAt,
             maxConnections: exitNodes.maxConnections,
             online: exitNodes.online,

server/private/lib/offlineThresholds.ts

@@ -0,0 +1,27 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import type { OfflineThresholds } from "@server/lib/offlineThresholds";
+import privateConfig from "#private/lib/config";
+
+export function getOfflineThresholds(): OfflineThresholds {
+    const offlineChecker = privateConfig.getRawPrivateConfig().offline_checker;
+
+    return {
+        checkIntervalMs: offlineChecker.check_interval_ms,
+        newtMs: offlineChecker.newt_threshold_ms,
+        newtWireguardMs: offlineChecker.newt_wireguard_threshold_ms,
+        olmMs: offlineChecker.olm_threshold_ms,
+        remoteExitNodeMs: offlineChecker.remote_exit_node_threshold_ms
+    };
+}

server/private/lib/readConfigFile.ts

@@ -18,6 +18,7 @@ import { z } from "zod";
 import { colorsSchema } from "@server/lib/colorsSchema";
 import { build } from "@server/build";
 import { getEnvOrYaml } from "@server/lib/getEnvOrYaml";
+import { DEFAULT_OFFLINE_THRESHOLDS } from "@server/lib/offlineThresholds";
 
 const portSchema = z.number().positive().gt(0).lte(65535);
 
@@ -109,6 +110,16 @@ export const privateConfigSchema = z
                     .array(z.string())
                     .optional()
                     .default([]),
+                // Zone names (zone cuts) this server is authoritative for
+                // beyond the customer domains stored in the `domains` table -
+                // e.g. a self-hosted deployment's own nameserver zone or
+                // cname/site extension zones. Any query landing on one of
+                // these names, or an empty non-terminal beneath one, that
+                // doesn't match a more specific record gets NOERROR/NODATA +
+                // SOA (not NXDOMAIN), so QNAME-minimising resolvers (RFC
+                // 9156) don't treat the ancestor as proof nothing exists
+                // below it (RFC 8020). Left for the user to populate.
+                zones: z.array(z.string()).optional().default([]),
                 rate_limit: z
                     .object({
                         enabled: z.boolean().optional().default(true),
@@ -166,6 +177,36 @@ export const privateConfigSchema = z
                     .default("http://gerbil:3004")
             })
             .prefault({}),
+        // How long a newt site / olm client / remote exit node has to go
+        // without a ping before the offline checker marks it offline, and how
+        // often the checkers run.
+        offline_checker: z
+            .object({
+                check_interval_ms: z
+                    .number()
+                    .int()
+                    .positive()
+                    .default(DEFAULT_OFFLINE_THRESHOLDS.checkIntervalMs),
+                newt_threshold_ms: z
+                    .number()
+                    .int()
+                    .default(DEFAULT_OFFLINE_THRESHOLDS.newtMs),
+                // wireguard sites don't ping, so they are judged by their
+                // last bandwidth update instead
+                newt_wireguard_threshold_ms: z
+                    .number()
+                    .int()
+                    .default(DEFAULT_OFFLINE_THRESHOLDS.newtWireguardMs),
+                olm_threshold_ms: z
+                    .number()
+                    .int()
+                    .default(DEFAULT_OFFLINE_THRESHOLDS.olmMs),
+                remote_exit_node_threshold_ms: z
+                    .number()
+                    .int()
+                    .default(DEFAULT_OFFLINE_THRESHOLDS.remoteExitNodeMs)
+            })
+            .prefault({}),
         flags: z
             .object({
                 enable_redis: z.boolean().optional().default(false),

server/private/lib/traefik/getTraefikConfig.ts

@@ -22,6 +22,7 @@ import {
     domains,
     exitNodes,
     loginPage,
+    resourceMtlsCertificates,
     redirects,
     resources,
     siteNetworks,
@@ -54,6 +55,7 @@ import { buildWildcardTls } from "@server/lib/traefik/certResolver";
 import { buildCustomHeadersMiddleware } from "@server/lib/traefik/headersMiddleware";
 import {
     buildHttpLoadBalancerServers,
+    buildLoadBalancingStrategy,
     buildStickySessionCookie,
     buildStickySessionIp,
     buildTcpUdpLoadBalancerServers
@@ -132,6 +134,7 @@ export async function getTraefikConfig(
             domainId: resources.domainId,
             enabled: resources.enabled,
             stickySession: resources.stickySession,
+            loadBalancingStrategy: resources.loadBalancingStrategy,
             tlsServerName: resources.tlsServerName,
             setHostHeader: resources.setHostHeader,
             enableProxy: resources.enableProxy,
@@ -148,6 +151,8 @@ export async function getTraefikConfig(
             maintenanceMessage: resources.maintenanceMessage,
             maintenanceEstimatedTime: resources.maintenanceEstimatedTime,
 
+            mtlsEnabled: resources.mtlsEnabled,
+
             // Target fields
             targetId: targets.targetId,
             targetEnabled: targets.enabled,
@@ -274,6 +279,7 @@ export async function getTraefikConfig(
                 domainId: row.domainId,
                 enabled: row.enabled,
                 stickySession: row.stickySession,
+                loadBalancingStrategy: row.loadBalancingStrategy,
                 tlsServerName: row.tlsServerName,
                 setHostHeader: row.setHostHeader,
                 enableProxy: row.enableProxy,
@@ -295,7 +301,9 @@ export async function getTraefikConfig(
                 maintenanceModeType: row.maintenanceModeType,
                 maintenanceTitle: row.maintenanceTitle,
                 maintenanceMessage: row.maintenanceMessage,
-                maintenanceEstimatedTime: row.maintenanceEstimatedTime
+                maintenanceEstimatedTime: row.maintenanceEstimatedTime,
+
+                mtlsEnabled: row.mtlsEnabled
             });
         }
 
@@ -319,6 +327,33 @@ export async function getTraefikConfig(
         });
     }
 
+    // Trusted client-CA certificates for resources that require mTLS.
+    const mtlsResourceIds = [
+        ...new Set(
+            [...resourcesMap.values()]
+                .filter((r) => r.mode === "http" && r.mtlsEnabled)
+                .map((r) => r.resourceId as number)
+        )
+    ];
+    const mtlsCaCertsByResource = new Map<number, string[]>();
+    if (mtlsResourceIds.length > 0) {
+        const caRows = await db
+            .select({
+                resourceId: resourceMtlsCertificates.resourceId,
+                certificate: resourceMtlsCertificates.certificate
+            })
+            .from(resourceMtlsCertificates)
+            .where(
+                inArray(resourceMtlsCertificates.resourceId, mtlsResourceIds)
+            )
+            .orderBy(resourceMtlsCertificates.mtlsCertificateId);
+        for (const row of caRows) {
+            const list = mtlsCaCertsByResource.get(row.resourceId) ?? [];
+            list.push(row.certificate);
+            mtlsCaCertsByResource.set(row.resourceId, list);
+        }
+    }
+
     // Group browser gateway targets by resource
     const browserGatewayResourcesMap = browserGatewayUiUrl
         ? buildBrowserGatewayResourcesMap(
@@ -628,7 +663,7 @@ export async function getTraefikConfig(
                 resource.pathMatchType
             );
 
-            let tls = {};
+            let tls: Record<string, any> = {};
             if (!pangolinCertModeEnabled) {
                 tls = buildWildcardTls({
                     fullDomain,
@@ -650,9 +685,65 @@ export async function getTraefikConfig(
                 }
             }
 
+            // mTLS: Traefik resolves TLS options per hostname, and routers on
+            // the same host that reference different options make it fall back
+            // to the defaults, so the options are keyed on the resource (not
+            // the per-path router key) and every TLS router below (including
+            // the maintenance ones) references the same name.
+            const mtlsCaCerts =
+                resource.mtlsEnabled && resource.ssl
+                    ? mtlsCaCertsByResource.get(resource.resourceId)
+                    : undefined;
+            if (mtlsCaCerts && mtlsCaCerts.length > 0) {
+                const mtlsOptionsName = `resource-${resource.resourceId}-mtls`;
+                const mtlsHeadersMiddlewareName = `resource-${resource.resourceId}-mtls-headers`;
+
+                if (!config_output.tls) {
+                    config_output.tls = {};
+                }
+                if (!config_output.tls.options) {
+                    config_output.tls.options = {};
+                }
+                // caFiles entries are Traefik FileOrContent values: anything
+                // that isn't an existing file path is read as inline PEM, so
+                // the CA certs travel in this config to any exit node.
+                config_output.tls.options[mtlsOptionsName] = {
+                    clientAuth: {
+                        caFiles: mtlsCaCerts,
+                        clientAuthType: "RequireAndVerifyClientCert"
+                    }
+                };
+
+                config_output.http.middlewares[mtlsHeadersMiddlewareName] = {
+                    passTLSClientCert: {
+                        pem: false,
+                        info: {
+                            notAfter: true,
+                            notBefore: true,
+                            serialNumber: true,
+                            sans: true,
+                            subject: {
+                                commonName: true,
+                                organization: true
+                            },
+                            issuer: {
+                                commonName: true,
+                                organization: true
+                            }
+                        }
+                    }
+                };
+                routerMiddlewares.unshift(mtlsHeadersMiddlewareName);
+
+                tls = { ...tls, options: mtlsOptionsName };
+            }
+
             config_output.http.services![serviceName] = {
                 loadBalancer: {
                     servers: buildHttpLoadBalancerServers(targets),
+                    ...buildLoadBalancingStrategy(
+                        resource.loadBalancingStrategy
+                    ),
                     ...(resource.stickySession
                         ? buildStickySessionCookie(resource.ssl)
                         : {})

server/private/routers/billing/featureLifecycle.ts

@@ -303,6 +303,10 @@ async function disableFeature(
                 await disablemaintenancePage(orgId);
                 break;
 
+            case TierFeature.Mtls:
+                await disableMtls(orgId);
+                break;
+
             case TierFeature.DevicePosture:
                 await disableDevicePosture(orgId);
                 break;
@@ -519,6 +523,18 @@ async function disablemaintenancePage(orgId: string): Promise<void> {
     logger.info(`Disabled maintenance page on all resources for org ${orgId}`);
 }
 
+// The uploaded CA certificates are kept so mTLS can be re-enabled on upgrade.
+async function disableMtls(orgId: string): Promise<void> {
+    await db
+        .update(resources)
+        .set({
+            mtlsEnabled: false
+        })
+        .where(eq(resources.orgId, orgId));
+
+    logger.info(`Disabled mTLS on all resources for org ${orgId}`);
+}
+
 async function disableDevicePosture(orgId: string): Promise<void> {}
 
 async function disableTwoFactorEnforcement(orgId: string): Promise<void> {

server/private/routers/client/addClientRole.ts

@@ -0,0 +1,161 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import { clientOrgRoles, clients, db, roles } from "@server/db";
+import { eq } from "drizzle-orm";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+import {
+    rebuildClientAssociationsFromClient,
+    isOrgRebuildRateLimited
+} from "@server/lib/rebuildClientAssociations";
+
+const addClientRoleParamsSchema = z.strictObject({
+    clientId: z.coerce.number().int().positive(),
+    roleId: z.coerce.number().int().positive()
+});
+
+registry.registerPath({
+    method: "post",
+    path: "/client/{clientId}/add-role/{roleId}",
+    description:
+        "Add a role to a machine client. The client is granted access to the private resources assigned to the role. Clients with a userId cannot be added to roles; they inherit the roles of their user.",
+    tags: [OpenAPITags.Role, OpenAPITags.Client],
+    request: {
+        params: addClientRoleParamsSchema
+    },
+    responses: {
+        200: {
+            description: "Successful response",
+            content: {
+                "application/json": {
+                    schema: z.object({
+                        data: z.record(z.string(), z.any()).nullable(),
+                        success: z.boolean(),
+                        error: z.boolean(),
+                        message: z.string(),
+                        status: z.number()
+                    })
+                }
+            }
+        }
+    }
+});
+
+export async function addClientRole(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = addClientRoleParamsSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const { clientId, roleId } = parsedParams.data;
+
+        if (req.user && !req.userOrg) {
+            return next(
+                createHttpError(
+                    HttpCode.FORBIDDEN,
+                    "You do not have access to this organization"
+                )
+            );
+        }
+
+        const [role] = await db
+            .select()
+            .from(roles)
+            .where(eq(roles.roleId, roleId))
+            .limit(1);
+
+        if (!role) {
+            return next(
+                createHttpError(HttpCode.BAD_REQUEST, "Invalid role ID")
+            );
+        }
+
+        const [client] = await db
+            .select()
+            .from(clients)
+            .where(eq(clients.clientId, clientId))
+            .limit(1);
+
+        if (!client || client.orgId !== role.orgId) {
+            return next(
+                createHttpError(
+                    HttpCode.NOT_FOUND,
+                    "Client not found or does not belong to the specified organization"
+                )
+            );
+        }
+
+        if (client.userId !== null) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    "Cannot add roles to clients that are associated with a user"
+                )
+            );
+        }
+
+        if (await isOrgRebuildRateLimited(role.orgId)) {
+            return next(
+                createHttpError(
+                    HttpCode.TOO_MANY_REQUESTS,
+                    "Too many concurrent rebuild operations for this organization. Please retry after a moment."
+                )
+            );
+        }
+
+        await db
+            .insert(clientOrgRoles)
+            .values({
+                clientId,
+                orgId: role.orgId,
+                roleId
+            })
+            .onConflictDoNothing();
+
+        rebuildClientAssociationsFromClient(client).catch((e) => {
+            logger.error(
+                `Failed to rebuild client associations for client ${clientId} after adding role: ${e}`
+            );
+        });
+
+        return response(res, {
+            data: { clientId, orgId: role.orgId, roleId },
+            success: true,
+            error: false,
+            message: "Role added to client successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
+        );
+    }
+}

server/private/routers/client/index.ts

@@ -0,0 +1,16 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+export * from "./addClientRole";
+export * from "./removeClientRole";
+export * from "./setClientRoles";

server/private/routers/client/removeClientRole.ts

@@ -0,0 +1,153 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import { clientOrgRoles, clients, db, roles } from "@server/db";
+import { and, eq } from "drizzle-orm";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+import {
+    rebuildClientAssociationsFromClient,
+    isOrgRebuildRateLimited
+} from "@server/lib/rebuildClientAssociations";
+
+const removeClientRoleParamsSchema = z.strictObject({
+    clientId: z.coerce.number().int().positive(),
+    roleId: z.coerce.number().int().positive()
+});
+
+registry.registerPath({
+    method: "delete",
+    path: "/client/{clientId}/remove-role/{roleId}",
+    description:
+        "Remove a role from a machine client. Unlike users, machine clients are not required to keep at least one role.",
+    tags: [OpenAPITags.Role, OpenAPITags.Client],
+    request: {
+        params: removeClientRoleParamsSchema
+    },
+    responses: {
+        200: {
+            description: "Successful response",
+            content: {
+                "application/json": {
+                    schema: z.object({
+                        data: z.record(z.string(), z.any()).nullable(),
+                        success: z.boolean(),
+                        error: z.boolean(),
+                        message: z.string(),
+                        status: z.number()
+                    })
+                }
+            }
+        }
+    }
+});
+
+export async function removeClientRole(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = removeClientRoleParamsSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const { clientId, roleId } = parsedParams.data;
+
+        if (req.user && !req.userOrg) {
+            return next(
+                createHttpError(
+                    HttpCode.FORBIDDEN,
+                    "You do not have access to this organization"
+                )
+            );
+        }
+
+        const [role] = await db
+            .select()
+            .from(roles)
+            .where(eq(roles.roleId, roleId))
+            .limit(1);
+
+        if (!role) {
+            return next(
+                createHttpError(HttpCode.BAD_REQUEST, "Invalid role ID")
+            );
+        }
+
+        const [client] = await db
+            .select()
+            .from(clients)
+            .where(eq(clients.clientId, clientId))
+            .limit(1);
+
+        if (!client || client.orgId !== role.orgId) {
+            return next(
+                createHttpError(
+                    HttpCode.NOT_FOUND,
+                    "Client not found or does not belong to the specified organization"
+                )
+            );
+        }
+
+        if (await isOrgRebuildRateLimited(role.orgId)) {
+            return next(
+                createHttpError(
+                    HttpCode.TOO_MANY_REQUESTS,
+                    "Too many concurrent rebuild operations for this organization. Please retry after a moment."
+                )
+            );
+        }
+
+        await db
+            .delete(clientOrgRoles)
+            .where(
+                and(
+                    eq(clientOrgRoles.clientId, clientId),
+                    eq(clientOrgRoles.orgId, role.orgId),
+                    eq(clientOrgRoles.roleId, roleId)
+                )
+            );
+
+        rebuildClientAssociationsFromClient(client).catch((e) => {
+            logger.error(
+                `Failed to rebuild client associations for client ${clientId} after removing role: ${e}`
+            );
+        });
+
+        return response(res, {
+            data: { clientId, orgId: role.orgId, roleId },
+            success: true,
+            error: false,
+            message: "Role removed from client successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
+        );
+    }
+}

server/private/routers/client/setClientRoles.ts

@@ -0,0 +1,204 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025-2026 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import { Request, Response, NextFunction } from "express";
+import { z } from "zod";
+import { clientOrgRoles, clients, db, roles } from "@server/db";
+import { and, eq, inArray } from "drizzle-orm";
+import response from "@server/lib/response";
+import HttpCode from "@server/types/HttpCode";
+import createHttpError from "http-errors";
+import logger from "@server/logger";
+import { fromError } from "zod-validation-error";
+import { OpenAPITags, registry } from "@server/openApi";
+import {
+    rebuildClientAssociationsFromClient,
+    isOrgRebuildRateLimited
+} from "@server/lib/rebuildClientAssociations";
+
+const setClientRolesParamsSchema = z.strictObject({
+    clientId: z.coerce.number().int().positive()
+});
+
+const setClientRolesBodySchema = z.strictObject({
+    roleIds: z.array(z.int().positive())
+});
+
+registry.registerPath({
+    method: "post",
+    path: "/client/{clientId}/roles",
+    description:
+        "Set all roles for a machine client, replacing any existing roles. An empty list removes all roles. Clients with a userId cannot be assigned roles; they inherit the roles of their user.",
+    tags: [OpenAPITags.Role, OpenAPITags.Client],
+    request: {
+        params: setClientRolesParamsSchema,
+        body: {
+            content: {
+                "application/json": {
+                    schema: setClientRolesBodySchema
+                }
+            }
+        }
+    },
+    responses: {
+        200: {
+            description: "Successful response",
+            content: {
+                "application/json": {
+                    schema: z.object({
+                        data: z.record(z.string(), z.any()).nullable(),
+                        success: z.boolean(),
+                        error: z.boolean(),
+                        message: z.string(),
+                        status: z.number()
+                    })
+                }
+            }
+        }
+    }
+});
+
+export async function setClientRoles(
+    req: Request,
+    res: Response,
+    next: NextFunction
+): Promise<any> {
+    try {
+        const parsedParams = setClientRolesParamsSchema.safeParse(req.params);
+        if (!parsedParams.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedParams.error).toString()
+                )
+            );
+        }
+
+        const parsedBody = setClientRolesBodySchema.safeParse(req.body);
+        if (!parsedBody.success) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    fromError(parsedBody.error).toString()
+                )
+            );
+        }
+
+        const { clientId } = parsedParams.data;
+        const { roleIds } = parsedBody.data;
+
+        if (req.user && !req.userOrg) {
+            return next(
+                createHttpError(
+                    HttpCode.FORBIDDEN,
+                    "You do not have access to this organization"
+                )
+            );
+        }
+
+        const uniqueRoleIds = [...new Set(roleIds)];
+
+        const [client] = await db
+            .select()
+            .from(clients)
+            .where(eq(clients.clientId, clientId))
+            .limit(1);
+
+        if (!client) {
+            return next(
+                createHttpError(HttpCode.NOT_FOUND, "Client not found")
+            );
+        }
+
+        if (client.userId !== null) {
+            return next(
+                createHttpError(
+                    HttpCode.BAD_REQUEST,
+                    "Cannot set roles on clients that are associated with a user"
+                )
+            );
+        }
+
+        const orgId = client.orgId;
+
+        if (uniqueRoleIds.length > 0) {
+            const orgRoles = await db
+                .select({ roleId: roles.roleId })
+                .from(roles)
+                .where(
+                    and(
+                        eq(roles.orgId, orgId),
+                        inArray(roles.roleId, uniqueRoleIds)
+                    )
+                );
+
+            if (orgRoles.length !== uniqueRoleIds.length) {
+                return next(
+                    createHttpError(
+                        HttpCode.BAD_REQUEST,
+                        "One or more role IDs are invalid for this organization"
+                    )
+                );
+            }
+        }
+
+        if (await isOrgRebuildRateLimited(orgId)) {
+            return next(
+                createHttpError(
+                    HttpCode.TOO_MANY_REQUESTS,
+                    "Too many concurrent rebuild operations for this organization. Please retry after a moment."
+                )
+            );
+        }
+
+        await db.transaction(async (trx) => {
+            await trx
+                .delete(clientOrgRoles)
+                .where(
+                    and(
+                        eq(clientOrgRoles.clientId, clientId),
+                        eq(clientOrgRoles.orgId, orgId)
+                    )
+                );
+
+            if (uniqueRoleIds.length > 0) {
+                await trx.insert(clientOrgRoles).values(
+                    uniqueRoleIds.map((roleId) => ({
+                        clientId,
+                        orgId,
+                        roleId
+                    }))
+                );
+            }
+        });
+
+        rebuildClientAssociationsFromClient(client).catch((e) => {
+            logger.error(
+                `Failed to rebuild client associations for client ${clientId} after setting roles: ${e}`
+            );
+        });
+
+        return response(res, {
+            data: { clientId, orgId, roleIds: uniqueRoleIds },
+            success: true,
+            error: false,
+            message: "Client roles set successfully",
+            status: HttpCode.OK
+        });
+    } catch (error) {
+        logger.error(error);
+        return next(
+            createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
+        );
+    }
+}

server/private/routers/external.ts

@@ -29,6 +29,7 @@ import * as misc from "#private/routers/misc";
 import * as reKey from "#private/routers/re-key";
 import * as approval from "#private/routers/approvals";
 import * as user from "#private/routers/user";
+import * as client from "#private/routers/client";
 import * as siteProvisioning from "#private/routers/siteProvisioning";
 import * as eventStreamingDestination from "#private/routers/eventStreamingDestination";
 import * as alertRule from "#private/routers/alertRule";
@@ -674,6 +675,36 @@ authenticated.post(
     user.setUserOrgRoles
 );
 
+authenticated.post(
+    "/client/:clientId/add-role/:roleId",
+    verifyClientAccess,
+    verifyRoleAccess,
+    verifyLimits,
+    verifyUserHasAction(ActionsEnum.addClientRole),
+    logActionAudit(ActionsEnum.addClientRole),
+    client.addClientRole
+);
+
+authenticated.delete(
+    "/client/:clientId/remove-role/:roleId",
+    verifyClientAccess,
+    verifyRoleAccess,
+    verifyLimits,
+    verifyUserHasAction(ActionsEnum.removeClientRole),
+    logActionAudit(ActionsEnum.removeClientRole),
+    client.removeClientRole
+);
+
+authenticated.post(
+    "/client/:clientId/roles",
+    verifyClientAccess,
+    verifyRoleAccess,
+    verifyLimits,
+    verifyUserHasAction(ActionsEnum.setClientRoles),
+    logActionAudit(ActionsEnum.setClientRoles),
+    client.setClientRoles
+);
+
 authenticated.put(
     "/org/:orgId/site-provisioning-key",
     verifyValidLicense,

server/private/routers/gerbil/createExitNode.ts

@@ -34,6 +34,7 @@ export async function createExitNode(
             // TODO: eventually we will want to get the next available port so that we can multiple exit nodes
             // const listenPort = await getNextAvailablePort();
             const listenPort = config.getRawConfig().gerbil.start_port;
+            const relayPort = config.getRawConfig().gerbil.clients_start_port;
 
             const exitNodeName =
                 config.getRawConfig().gerbil.exit_node_name ||
@@ -49,6 +50,7 @@ export async function createExitNode(
                         privateConfig.getRawPrivateConfig().app.region || null,
                     address,
                     listenPort,
+                    relayPort,
                     online: true,
                     reachableAt,
                     name: exitNodeName

server/private/routers/integration.ts

@@ -29,11 +29,13 @@ import {
     verifyApiKeyIdpAccess,
     verifyApiKeyRoleAccess,
     verifyApiKeyUserAccess,
+    verifyApiKeyClientAccess,
     verifyApiKeySiteProvisioningKeyAccess,
     verifyApiKeyResourcePolicyAccess,
     verifyLimits
 } from "@server/middlewares";
 import * as user from "#private/routers/user";
+import * as client from "#private/routers/client";
 import {
     verifyValidSubscription,
     verifyValidLicense
@@ -246,6 +248,36 @@ authenticated.delete(
     user.removeUserRole
 );
 
+authenticated.post(
+    "/client/:clientId/add-role/:roleId",
+    verifyApiKeyClientAccess,
+    verifyApiKeyRoleAccess,
+    verifyLimits,
+    verifyApiKeyHasAction(ActionsEnum.addClientRole),
+    logActionAudit(ActionsEnum.addClientRole),
+    client.addClientRole
+);
+
+authenticated.delete(
+    "/client/:clientId/remove-role/:roleId",
+    verifyApiKeyClientAccess,
+    verifyApiKeyRoleAccess,
+    verifyLimits,
+    verifyApiKeyHasAction(ActionsEnum.removeClientRole),
+    logActionAudit(ActionsEnum.removeClientRole),
+    client.removeClientRole
+);
+
+authenticated.post(
+    "/client/:clientId/roles",
+    verifyApiKeyClientAccess,
+    verifyApiKeyRoleAccess,
+    verifyLimits,
+    verifyApiKeyHasAction(ActionsEnum.setClientRoles),
+    logActionAudit(ActionsEnum.setClientRoles),
+    client.setClientRoles
+);
+
 authenticated.put(
     "/org/:orgId/site-provisioning-key",
     verifyValidLicense,

server/private/routers/remoteExitNode/createRemoteExitNode.ts

@@ -201,10 +201,7 @@ export async function createRemoteExitNode(
         let releaseSubnetLock: (() => Promise<void>) | null = null;
         let newExitNodeAddress: string | null = null;
         if (!existingExitNode) {
-            const { value, release } = await getNextAvailableSubnet(
-                db,
-                orgId
-            );
+            const { value, release } = await getNextAvailableSubnet(db, orgId);
             newExitNodeAddress = value;
             releaseSubnetLock = release;
         }
@@ -220,6 +217,7 @@ export async function createRemoteExitNode(
                             endpoint: "",
                             publicKey: "",
                             listenPort: 0,
+                            relayPort: 0,
                             online: false,
                             type: "remoteExitNode"
                         })

server/private/routers/remoteExitNode/offlineChecker.ts

@@ -14,11 +14,10 @@
 import { db, exitNodes } from "@server/db";
 import { eq, lt, isNull, and, or } from "drizzle-orm";
 import logger from "@server/logger";
+import { getOfflineThresholds } from "#private/lib/offlineThresholds";
 
 // Track if the offline checker interval is running
 let offlineCheckerInterval: NodeJS.Timeout | null = null;
-const OFFLINE_CHECK_INTERVAL = 30 * 1000; // Check every 30 seconds
-const OFFLINE_THRESHOLD_MS = 2 * 60 * 1000; // 2 minutes
 
 /**
  * Starts the background interval that checks for clients that haven't pinged recently
@@ -31,11 +30,11 @@ export const startRemoteExitNodeOfflineChecker = (): void => {
 
     offlineCheckerInterval = setInterval(async () => {
         try {
-            const twoMinutesAgo = Math.floor(
-                (Date.now() - OFFLINE_THRESHOLD_MS) / 1000
+            const offlineBefore = Math.floor(
+                (Date.now() - getOfflineThresholds().remoteExitNodeMs) / 1000
             );
 
-            // Find clients that haven't pinged in the last 2 minutes and mark them as offline
+            // Find clients that haven't pinged within the offline threshold and mark them as offline
             const offlineNodes = await db
                 .update(exitNodes)
                 .set({ online: false })
@@ -44,7 +43,7 @@ export const startRemoteExitNodeOfflineChecker = (): void => {
                         eq(exitNodes.online, true),
                         eq(exitNodes.type, "remoteExitNode"),
                         or(
-                            lt(exitNodes.lastPing, twoMinutesAgo),
+                            lt(exitNodes.lastPing, offlineBefore),
                             isNull(exitNodes.lastPing)
                         )
                     )
@@ -65,7 +64,7 @@ export const startRemoteExitNodeOfflineChecker = (): void => {
         } catch (error) {
             logger.error("Error in offline checker interval", { error });
         }
-    }, OFFLINE_CHECK_INTERVAL);
+    }, getOfflineThresholds().checkIntervalMs);
 
     logger.debug("Started offline checker interval");
 };

server/private/routers/ws/ws.ts

@@ -329,6 +329,12 @@ const initializeRedisSubscription = async (): Promise<void> => {
                         redisMessage.message,
                         redisMessage.excludeClientId
                     );
+                } else if (
+                    redisMessage.type === "disconnect" &&
+                    redisMessage.targetClientId
+                ) {
+                    // Close the client's connections if any are on this node
+                    disconnectClientLocal(redisMessage.targetClientId);
                 }
             } catch (error) {
                 logger.error("Error processing Redis message:", error);
@@ -1285,13 +1291,43 @@ if (redisManager.isRedisEnabled()) {
     logger.debug("WebSocket handler initialized in local mode");
 }
 
-// Disconnect a specific client and force them to reconnect
+// Disconnect a specific client on every node and force them to reconnect
 const disconnectClient = async (clientId: string): Promise<boolean> => {
+    const localDisconnected = disconnectClientLocal(clientId);
+
+    if (!redisManager.isRedisEnabled()) {
+        return localDisconnected;
+    }
+
+    try {
+        // Flush queued direct messages first so anything sent to this client
+        // just before (e.g. olm/terminate) reaches the other nodes ahead of
+        // the disconnect on the same channel.
+        await flushPendingRedisDirectMessages();
+
+        const redisMessage: RedisMessage = {
+            type: "disconnect",
+            targetClientId: clientId,
+            fromNodeId: NODE_ID
+        };
+        await redisManager.publish(REDIS_CHANNEL, JSON.stringify(redisMessage));
+        return true;
+    } catch (error) {
+        logger.error(
+            `Failed to publish disconnect for client ID ${clientId} via Redis:`,
+            error
+        );
+        return localDisconnected;
+    }
+};
+
+// Disconnect a specific client's connections on this node only
+const disconnectClientLocal = (clientId: string): boolean => {
     const mapKey = getClientMapKey(clientId);
     const clients = connectedClients.get(mapKey);
 
     if (!clients || clients.length === 0) {
-        logger.debug(`No connections found for client ID: ${clientId}`);
+        logger.debug(`No local connections found for client ID: ${clientId}`);
         return false;
     }