Merge pull request #3870 from TiltedTeapot/fix-pass-relay-port-to-gerbil

Fix: pass relay port to gerbil

server/db/pg/schema/schema.ts

@@ -460,6 +460,7 @@ export const exitNodes = pgTable("exitNodes", {
     endpoint: varchar("endpoint").notNull(),
     publicKey: varchar("publicKey").notNull(),
     listenPort: integer("listenPort").notNull(),
+    relayPort: integer("relayPort").notNull(),
     reachableAt: varchar("reachableAt"),
     maxConnections: integer("maxConnections"),
     online: boolean("online").notNull().default(false),

server/db/sqlite/schema/schema.ts

@@ -130,7 +130,7 @@ export const sites = sqliteTable(
         address: text("address"), // this is the address of the wireguard interface in newt
         endpoint: text("endpoint"), // this is how to reach gerbil externally - gets put into the wireguard config
         localEndpoints: text("localEndpoints"), // JSON encoded list of string ips on the local machine to try to connect to
-        publicKey: text("publicKey"), // TODO: Fix typo in publicKey
+        publicKey: text("publicKey"),
         lastHolePunch: integer("lastHolePunch"),
         listenPort: integer("listenPort"),
         dockerSocketEnabled: integer("dockerSocketEnabled", { mode: "boolean" })
@@ -481,6 +481,7 @@ export const exitNodes = sqliteTable("exitNodes", {
     endpoint: text("endpoint").notNull(), // this is how to reach gerbil externally - gets put into the wireguard config
     publicKey: text("publicKey").notNull(),
     listenPort: integer("listenPort").notNull(),
+    relayPort: integer("relayPort").notNull(),
     reachableAt: text("reachableAt"), // this is the internal address of the gerbil http server for command control
     maxConnections: integer("maxConnections"),
     online: integer("online", { mode: "boolean" }).notNull().default(false),

server/lib/exitNodes/exitNodes.ts

@@ -36,6 +36,7 @@ export async function listExitNodes(
             endpoint: exitNodes.endpoint,
             publicKey: exitNodes.publicKey,
             listenPort: exitNodes.listenPort,
+            relayPort: exitNodes.relayPort,
             reachableAt: exitNodes.reachableAt,
             maxConnections: exitNodes.maxConnections,
             online: exitNodes.online,

server/lib/rebuildClientAssociations.ts

@@ -34,7 +34,6 @@ import {
 import { sendToExitNode } from "#dynamic/lib/exitNodes";
 import { sendToClientsBatch } from "#dynamic/routers/ws";
 import { canCompress } from "@server/lib/clientVersionChecks";
-import config from "@server/lib/config";
 import logger from "@server/logger";
 import {
     generateAliasConfig,
@@ -816,6 +815,7 @@ async function handleMessagesForSiteClients(
             siteId: number;
             exitNode: {
                 publicKey: string;
+                relayPort: number;
                 endpoint: string;
             };
         };
@@ -961,6 +961,7 @@ async function handleMessagesForSiteClients(
                     siteId,
                     exitNode: {
                         publicKey: exitNode.publicKey,
+                        relayPort: exitNode.relayPort,
                         endpoint: exitNode.endpoint
                     }
                 },
@@ -1238,8 +1239,6 @@ async function syncClientExitNodeConnections(
             .map((r) => [r.clientId as number, r])
     );
 
-    const relayPort = config.getRawConfig().gerbil.clients_start_port;
-
     const connectPayloads: {
         clientId: string;
         message: { type: string; data: any };
@@ -1278,7 +1277,7 @@ async function syncClientExitNodeConnections(
                     data: {
                         connect: true,
                         endpoint: `${exitNode.endpoint}:${exitNode.listenPort}`,
-                        relayPort,
+                        relayPort: exitNode.relayPort,
                         publicKey: exitNode.publicKey,
                         serverIP: exitNode.address.split("/")[0],
                         tunnelIP: client.exitNodeSubnet.split("/")[0],
@@ -2451,6 +2450,7 @@ async function handleMessagesForClientSites(
             siteId: number;
             exitNode: {
                 publicKey: string;
+                relayPort: number;
                 endpoint: string;
             };
         };
@@ -2550,6 +2550,7 @@ async function handleMessagesForClientSites(
                     siteId: site.siteId,
                     exitNode: {
                         publicKey: exitNode.publicKey,
+                        relayPort: exitNode.relayPort,
                         endpoint: exitNode.endpoint
                     }
                 },

server/private/routers/gerbil/createExitNode.ts

@@ -34,6 +34,7 @@ export async function createExitNode(
             // TODO: eventually we will want to get the next available port so that we can multiple exit nodes
             // const listenPort = await getNextAvailablePort();
             const listenPort = config.getRawConfig().gerbil.start_port;
+            const relayPort = config.getRawConfig().gerbil.clients_start_port;
 
             const exitNodeName =
                 config.getRawConfig().gerbil.exit_node_name ||
@@ -49,6 +50,7 @@ export async function createExitNode(
                         privateConfig.getRawPrivateConfig().app.region || null,
                     address,
                     listenPort,
+                    relayPort,
                     online: true,
                     reachableAt,
                     name: exitNodeName

server/private/routers/remoteExitNode/createRemoteExitNode.ts

@@ -201,10 +201,7 @@ export async function createRemoteExitNode(
         let releaseSubnetLock: (() => Promise<void>) | null = null;
         let newExitNodeAddress: string | null = null;
         if (!existingExitNode) {
-            const { value, release } = await getNextAvailableSubnet(
+            const { value, release } = await getNextAvailableSubnet(db, orgId);
-                db,
-                orgId
-            );
             newExitNodeAddress = value;
             releaseSubnetLock = release;
         }
@@ -220,6 +217,7 @@ export async function createRemoteExitNode(
                             endpoint: "",
                             publicKey: "",
                             listenPort: 0,
+                            relayPort: 0,
                             online: false,
                             type: "remoteExitNode"
                         })

server/routers/gerbil/createExitNode.ts

@@ -16,6 +16,7 @@ export async function createExitNode(
         const { value: address, release } = await getNextAvailableSubnet();
         try {
             const listenPort = config.getRawConfig().gerbil.start_port;
+            const relayPort = config.getRawConfig().gerbil.clients_start_port;
 
             const exitNodeName =
                 config.getRawConfig().gerbil.exit_node_name ||
@@ -30,6 +31,7 @@ export async function createExitNode(
                     address,
                     online: true,
                     listenPort,
+                    relayPort,
                     reachableAt,
                     name: exitNodeName
                 })

server/routers/gerbil/getConfig.ts

@@ -20,6 +20,7 @@ const getConfigSchema = z.object({
 
 export type GetConfigResponse = {
     listenPort: number;
+    relayPort: number;
     ipAddress: string;
     peers: {
         publicKey: string | null;
@@ -143,6 +144,7 @@ export async function generateGerbilConfig(exitNode: ExitNode) {
 
     const configResponse: GetConfigResponse = {
         listenPort: exitNode.listenPort || 51820,
+        relayPort: exitNode.relayPort || 21820,
         ipAddress: exitNode.address,
         peers
     };

server/routers/newt/buildConfiguration.ts

@@ -96,7 +96,10 @@ export async function buildClientConfigurationForNewtClient(
                         await updatePeer(client.clients.clientId, {
                             siteId: site.siteId,
                             endpoint: site.endpoint!,
-                            relayEndpoint: formatEndpoint(exitNode.endpoint, config.getRawConfig().gerbil.clients_start_port),
+                            relayEndpoint: formatEndpoint(
+                                exitNode.endpoint,
+                                exitNode.relayPort
+                            ),
                             publicKey: site.publicKey!,
                             serverIP: site.address,
                             serverPort: site.listenPort
@@ -121,6 +124,7 @@ export async function buildClientConfigurationForNewtClient(
                                 siteId,
                                 exitNode: {
                                     publicKey: exitNode.publicKey,
+                                    relayPort: exitNode.relayPort,
                                     endpoint: exitNode.endpoint
                                 }
                             }

server/routers/newt/handleNewtGetConfigMessage.ts

@@ -12,7 +12,6 @@ import {
     supportsCertReferences
 } from "@server/lib/clientVersionChecks";
 import { dedupeCertsForTargets } from "@server/lib/ip";
-import config from "@server/lib/config";
 import { waitForSiteRebuildIdle } from "@server/lib/rebuildClientAssociations";
 
 export const handleNewtGetConfigMessage: MessageHandler = async (context) => {
@@ -61,7 +60,7 @@ export const handleNewtGetConfigMessage: MessageHandler = async (context) => {
 
     if (existingSite.lastHolePunch && now - existingSite.lastHolePunch > 12) {
         logger.warn(
-            `Site last hole punch is too old; skipping this register. The site is failing to hole punch and identify its network address with the server. Can the site reach the server on UDP port ${config.getRawConfig().gerbil.clients_start_port}?`
+            `Site last hole punch is too old; skipping this register. The site is failing to hole punch and identify its network address with the server. Can the site reach the server on its Relay UDP port?`
         );
         return;
     }

server/routers/olm/buildConfiguration.ts

@@ -18,7 +18,6 @@ import {
 import logger from "@server/logger";
 import { and, eq, inArray } from "drizzle-orm";
 import { addPeer, deletePeer } from "../newt/peers";
-import config from "@server/lib/config";
 import { SiR } from "react-icons/si";
 
 export async function buildSiteConfigurationForOlmClient(
@@ -129,7 +128,6 @@ export async function buildSiteConfigurationForOlmClient(
         }
     }
 
-    const clientsStartPort = config.getRawConfig().gerbil.clients_start_port;
     const peerOps: Promise<unknown>[] = [];
 
     // Process each site
@@ -216,7 +214,7 @@ export async function buildSiteConfigurationForOlmClient(
                 logger.warn(`Exit node not found for site ${site.siteId}`);
                 continue;
             }
-            relayEndpoint = `${exitNode.endpoint}:${clientsStartPort}`;
+            relayEndpoint = `${exitNode.endpoint}:${exitNode.relayPort}`;
         }
 
         // Add site configuration to the array

server/routers/olm/error.ts

@@ -1,7 +1,4 @@
 import { sendToClient } from "#dynamic/routers/ws";
-import config from "@server/lib/config";
-
-const udpPort = config.getRawConfig().gerbil.clients_start_port;
 
 // Error codes for registration failures
 export const OlmErrorCodes = {
@@ -93,7 +90,7 @@ export const OlmErrorCodes = {
     },
     HOLEPUNCH_MISSING: {
         code: "HOLEPUNCH_MISSING",
-        message: `Unable to coordinate client P2P connection. Please ensure your client can reach the server on UDP port ${udpPort} and try registering again.`
+        message: `Unable to coordinate client P2P connection. Please ensure your client can reach the server on its relay UDP port and try registering again.`
     },
     NO_AVAILABLE_SUBNET: {
         code: "NO_AVAILABLE_SUBNET",

server/routers/olm/getOlmToken.ts

@@ -289,7 +289,7 @@ export async function getOlmToken(
         let exitNodesHpData = allExitNodes.map((exitNode: ExitNode) => {
             return {
                 publicKey: exitNode.publicKey,
-                relayPort: config.getRawConfig().gerbil.clients_start_port,
+                relayPort: exitNode.relayPort,
                 endpoint: exitNode.endpoint,
                 siteIds: exitNodeIdToSiteIds[exitNode.exitNodeId] ?? []
             };
@@ -347,8 +347,7 @@ export async function getOlmToken(
                 exitNodesHpData = [
                     {
                         publicKey: best.publicKey,
-                        relayPort:
+                        relayPort: best.relayPort,
-                            config.getRawConfig().gerbil.clients_start_port,
                         endpoint: best.endpoint,
                         siteIds: []
                         // it should still HP without the site ids but it will get stuck in the client

server/routers/olm/handleOlmRegisterMessage.ts

@@ -20,7 +20,6 @@ import { OlmErrorCodes, sendOlmError } from "./error";
 import { handleFingerprintInsertion } from "./fingerprintingUtils";
 import { build } from "@server/build";
 import { canCompress } from "@server/lib/clientVersionChecks";
-import config from "@server/lib/config";
 import cache from "#dynamic/lib/cache"; // not using regional here because we need this in the register message handler before we know where the client is
 import { waitForClientRebuildIdle } from "@server/lib/rebuildClientAssociations";
 import {
@@ -438,7 +437,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => {
     // TODO: I still think there is a better way to do this rather than locking it out here but ???
     if (now - (client.lastHolePunch || 0) > 12 && sitesCount > 0) {
         logger.warn(
-            `[handleOlmRegisterMessage] Client last hole punch is too old and we have sites to send; skipping this register. The client is failing to hole punch and identify its network address with the server. Can the client reach the server on UDP port ${config.getRawConfig().gerbil.clients_start_port}?`,
+            `[handleOlmRegisterMessage] Client last hole punch is too old and we have sites to send; skipping this register. The client is failing to hole punch and identify its network address with the server. Can the client reach the server on its relay UDP port?`,
             { orgId: client.orgId, clientId: client.clientId }
         );
 

server/routers/olm/handleOlmRelayMessage.ts

@@ -4,7 +4,6 @@ import { clients, clientSitesAssociationsCache, Olm } from "@server/db";
 import { and, eq, inArray } from "drizzle-orm";
 import { updatePeersBatch } from "../newt/peers";
 import logger from "@server/logger";
-import config from "@server/lib/config";
 import { parseSiteChainBatch, resolveNewtIdsBySite } from "./batchUtils";
 
 export const handleOlmRelayMessage: MessageHandler = async (context) => {
@@ -77,6 +76,7 @@ export const handleOlmRelayMessage: MessageHandler = async (context) => {
         siteId: number;
         chainId?: string;
         relayEndpoint: string;
+        relayPort: number;
     }[] = [];
 
     for (let i = 0; i < siteIds.length; i++) {
@@ -97,7 +97,8 @@ export const handleOlmRelayMessage: MessageHandler = async (context) => {
         valid.push({
             siteId,
             chainId: chainIds[i],
-            relayEndpoint: exitNode.endpoint
+            relayEndpoint: exitNode.endpoint,
+            relayPort: exitNode.relayPort
         });
     }
 
@@ -121,7 +122,9 @@ export const handleOlmRelayMessage: MessageHandler = async (context) => {
         );
 
     // Only ack sites we can actually tell their newt to relay for
-    const newtIdBySiteId = await resolveNewtIdsBySite(valid.map((v) => v.siteId));
+    const newtIdBySiteId = await resolveNewtIdsBySite(
+        valid.map((v) => v.siteId)
+    );
     const pushable = valid.filter((v) => {
         if (!newtIdBySiteId.has(v.siteId)) {
             logger.warn(`Newt not found for site ${v.siteId}`);
@@ -144,8 +147,6 @@ export const handleOlmRelayMessage: MessageHandler = async (context) => {
         }))
     );
 
-    const relayPort = config.getRawConfig().gerbil.clients_start_port;
-
     if (isBatch) {
         return {
             message: {
@@ -153,7 +154,7 @@ export const handleOlmRelayMessage: MessageHandler = async (context) => {
                 data: {
                     siteIds: pushable.map((v) => v.siteId),
                     relayEndpoints: pushable.map((v) => v.relayEndpoint),
-                    relayPort,
+                    relayPort: pushable.map((v) => v.relayPort),
                     chainIds: pushable.map((v) => v.chainId)
                 }
             },
@@ -169,7 +170,7 @@ export const handleOlmRelayMessage: MessageHandler = async (context) => {
             data: {
                 siteId: single.siteId,
                 relayEndpoint: single.relayEndpoint,
-                relayPort,
+                relayPort: single.relayPort,
                 chainId: single.chainId
             }
         },

server/routers/olm/handleOlmServerInitAddPeerHandshake.ts

@@ -216,6 +216,7 @@ export const handleOlmServerInitAddPeerHandshake: MessageHandler = async (
                 siteId: site.siteId,
                 exitNode: {
                     publicKey: exitNode.publicKey,
+                    relayPort: exitNode.relayPort,
                     endpoint: exitNode.endpoint
                 }
             },

server/routers/olm/peers.ts

@@ -1,7 +1,6 @@
 import { sendToClient, sendToClientsBatch } from "#dynamic/routers/ws";
 import { clientSitesAssociationsCache, db, olms } from "@server/db";
 import { canCompress } from "@server/lib/clientVersionChecks";
-import config from "@server/lib/config";
 import logger from "@server/logger";
 import { and, eq, inArray } from "drizzle-orm";
 import { Alias } from "yaml";
@@ -155,6 +154,7 @@ export async function initPeerAddHandshake(
         siteId: number;
         exitNode: {
             publicKey: string;
+            relayPort: number;
             endpoint: string;
         };
     },
@@ -181,7 +181,7 @@ export async function initPeerAddHandshake(
                 siteId: peer.siteId,
                 exitNode: {
                     publicKey: peer.exitNode.publicKey,
-                    relayPort: config.getRawConfig().gerbil.clients_start_port,
+                    relayPort: peer.exitNode.relayPort,
                     endpoint: peer.exitNode.endpoint
                 },
                 chainId
@@ -297,6 +297,7 @@ export async function initPeerAddHandshakeBatch(
             siteId: number;
             exitNode: {
                 publicKey: string;
+                relayPort: number;
                 endpoint: string;
             };
         };
@@ -317,8 +318,7 @@ export async function initPeerAddHandshakeBatch(
                     siteId: item.peer.siteId,
                     exitNode: {
                         publicKey: item.peer.exitNode.publicKey,
-                        relayPort:
+                        relayPort: item.peer.exitNode.relayPort,
-                            config.getRawConfig().gerbil.clients_start_port,
                         endpoint: item.peer.exitNode.endpoint
                     },
                     chainId: item.chainId

server/routers/olm/sync.ts

@@ -12,7 +12,6 @@ import { buildSiteConfigurationForOlmClient } from "./buildConfiguration";
 import { sendToClient } from "#dynamic/routers/ws";
 import logger from "@server/logger";
 import { count, eq, inArray } from "drizzle-orm";
-import config from "@server/lib/config";
 import { canCompress } from "@server/lib/clientVersionChecks";
 import { build } from "@server/build";
 
@@ -125,7 +124,7 @@ export async function sendOlmSyncMessage(olm: Olm, client: Client) {
         exitNodesData = allExitNodes.map((exitNode) => {
             return {
                 publicKey: exitNode.publicKey,
-                relayPort: config.getRawConfig().gerbil.clients_start_port,
+                relayPort: exitNode.relayPort,
                 endpoint: exitNode.endpoint,
                 siteIds: exitNodeIdToSiteIds[exitNode.exitNodeId] ?? []
             };

server/setup/copyInConfig.ts

@@ -11,6 +11,7 @@ export async function copyInConfig() {
 
     const endpoint = config.getRawConfig().gerbil.base_endpoint;
     const listenPort = config.getRawConfig().gerbil.start_port;
+    const relayPort = config.getRawConfig().gerbil.clients_start_port;
 
     if (
         !config.getRawConfig().flags?.disable_config_managed_domains &&
@@ -23,7 +24,7 @@ export async function copyInConfig() {
     if (exitNodeName) {
         await db
             .update(exitNodes)
-            .set({ endpoint, listenPort })
+            .set({ endpoint, listenPort, relayPort })
             .where(eq(exitNodes.name, exitNodeName));
     } else {
         await db
@@ -34,6 +35,10 @@ export async function copyInConfig() {
             .update(exitNodes)
             .set({ listenPort })
             .where(ne(exitNodes.listenPort, listenPort));
+        await db
+            .update(exitNodes)
+            .set({ relayPort })
+            .where(ne(exitNodes.relayPort, relayPort));
     }
 }