From 2f65a8c1249e558e9a097bb38a4da64bfb08367a Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 13 Oct 2026 09:35:09 -0400 Subject: [PATCH] Make offline checking intervals configurable Fixes #3855 --- server/lib/offlineThresholds.ts | 26 ++++++++++++++++ server/private/lib/offlineThresholds.ts | 27 ++++++++++++++++ server/private/lib/readConfigFile.ts | 31 +++++++++++++++++++ .../routers/remoteExitNode/offlineChecker.ts | 13 ++++---- server/routers/newt/offlineChecker.ts | 17 +++++----- server/routers/olm/offlineChecker.ts | 13 ++++---- 6 files changed, 104 insertions(+), 23 deletions(-) create mode 100644 server/lib/offlineThresholds.ts create mode 100644 server/private/lib/offlineThresholds.ts diff --git a/server/lib/offlineThresholds.ts b/server/lib/offlineThresholds.ts new file mode 100644 index 000000000..e3c2d369a --- /dev/null +++ b/server/lib/offlineThresholds.ts @@ -0,0 +1,26 @@ +export type OfflineThresholds = { + // how often the offline checkers run + checkIntervalMs: number; + // newt sites that stop pinging for this long are marked offline + newtMs: number; + // wireguard sites with no bandwidth update for this long are marked offline + newtWireguardMs: number; + // olm clients that stop pinging for this long are marked offline + olmMs: number; + // remote exit nodes that stop pinging for this long are marked offline + remoteExitNodeMs: number; +}; + +export const DEFAULT_OFFLINE_THRESHOLDS: OfflineThresholds = { + checkIntervalMs: 30 * 1000, // 30 seconds + newtMs: 2 * 60 * 1000, // 2 minutes + newtWireguardMs: 8 * 60 * 1000, // 8 minutes + olmMs: 2 * 60 * 1000, // 2 minutes + remoteExitNodeMs: 2 * 60 * 1000 // 2 minutes +}; + +// The thresholds are only configurable in the private config file, so the +// open build always uses the defaults. See server/private/lib/offlineThresholds.ts +export function getOfflineThresholds(): OfflineThresholds { + return DEFAULT_OFFLINE_THRESHOLDS; +} diff --git a/server/private/lib/offlineThresholds.ts b/server/private/lib/offlineThresholds.ts new file mode 100644 index 000000000..2f3d392ac --- /dev/null +++ b/server/private/lib/offlineThresholds.ts @@ -0,0 +1,27 @@ +/* + * This file is part of a proprietary work. + * + * Copyright (c) 2025-2026 Fossorial, Inc. + * All rights reserved. + * + * This file is licensed under the Fossorial Commercial License. + * You may not use this file except in compliance with the License. + * Unauthorized use, copying, modification, or distribution is strictly prohibited. + * + * This file is not licensed under the AGPLv3. + */ + +import type { OfflineThresholds } from "@server/lib/offlineThresholds"; +import privateConfig from "#private/lib/config"; + +export function getOfflineThresholds(): OfflineThresholds { + const offlineChecker = privateConfig.getRawPrivateConfig().offline_checker; + + return { + checkIntervalMs: offlineChecker.check_interval_ms, + newtMs: offlineChecker.newt_threshold_ms, + newtWireguardMs: offlineChecker.newt_wireguard_threshold_ms, + olmMs: offlineChecker.olm_threshold_ms, + remoteExitNodeMs: offlineChecker.remote_exit_node_threshold_ms + }; +} diff --git a/server/private/lib/readConfigFile.ts b/server/private/lib/readConfigFile.ts index a78a16d3b..cd24b053b 100644 --- a/server/private/lib/readConfigFile.ts +++ b/server/private/lib/readConfigFile.ts @@ -18,6 +18,7 @@ import { z } from "zod"; import { colorsSchema } from "@server/lib/colorsSchema"; import { build } from "@server/build"; import { getEnvOrYaml } from "@server/lib/getEnvOrYaml"; +import { DEFAULT_OFFLINE_THRESHOLDS } from "@server/lib/offlineThresholds"; const portSchema = z.number().positive().gt(0).lte(65535); @@ -176,6 +177,36 @@ export const privateConfigSchema = z .default("http://gerbil:3004") }) .prefault({}), + // How long a newt site / olm client / remote exit node has to go + // without a ping before the offline checker marks it offline, and how + // often the checkers run. + offline_checker: z + .object({ + check_interval_ms: z + .number() + .int() + .positive() + .default(DEFAULT_OFFLINE_THRESHOLDS.checkIntervalMs), + newt_threshold_ms: z + .number() + .int() + .default(DEFAULT_OFFLINE_THRESHOLDS.newtMs), + // wireguard sites don't ping, so they are judged by their + // last bandwidth update instead + newt_wireguard_threshold_ms: z + .number() + .int() + .default(DEFAULT_OFFLINE_THRESHOLDS.newtWireguardMs), + olm_threshold_ms: z + .number() + .int() + .default(DEFAULT_OFFLINE_THRESHOLDS.olmMs), + remote_exit_node_threshold_ms: z + .number() + .int() + .default(DEFAULT_OFFLINE_THRESHOLDS.remoteExitNodeMs) + }) + .prefault({}), flags: z .object({ enable_redis: z.boolean().optional().default(false), diff --git a/server/private/routers/remoteExitNode/offlineChecker.ts b/server/private/routers/remoteExitNode/offlineChecker.ts index 7f5e906f8..b916102ea 100644 --- a/server/private/routers/remoteExitNode/offlineChecker.ts +++ b/server/private/routers/remoteExitNode/offlineChecker.ts @@ -14,11 +14,10 @@ import { db, exitNodes } from "@server/db"; import { eq, lt, isNull, and, or } from "drizzle-orm"; import logger from "@server/logger"; +import { getOfflineThresholds } from "#private/lib/offlineThresholds"; // Track if the offline checker interval is running let offlineCheckerInterval: NodeJS.Timeout | null = null; -const OFFLINE_CHECK_INTERVAL = 30 * 1000; // Check every 30 seconds -const OFFLINE_THRESHOLD_MS = 2 * 60 * 1000; // 2 minutes /** * Starts the background interval that checks for clients that haven't pinged recently @@ -31,11 +30,11 @@ export const startRemoteExitNodeOfflineChecker = (): void => { offlineCheckerInterval = setInterval(async () => { try { - const twoMinutesAgo = Math.floor( - (Date.now() - OFFLINE_THRESHOLD_MS) / 1000 + const offlineBefore = Math.floor( + (Date.now() - getOfflineThresholds().remoteExitNodeMs) / 1000 ); - // Find clients that haven't pinged in the last 2 minutes and mark them as offline + // Find clients that haven't pinged within the offline threshold and mark them as offline const offlineNodes = await db .update(exitNodes) .set({ online: false }) @@ -44,7 +43,7 @@ export const startRemoteExitNodeOfflineChecker = (): void => { eq(exitNodes.online, true), eq(exitNodes.type, "remoteExitNode"), or( - lt(exitNodes.lastPing, twoMinutesAgo), + lt(exitNodes.lastPing, offlineBefore), isNull(exitNodes.lastPing) ) ) @@ -65,7 +64,7 @@ export const startRemoteExitNodeOfflineChecker = (): void => { } catch (error) { logger.error("Error in offline checker interval", { error }); } - }, OFFLINE_CHECK_INTERVAL); + }, getOfflineThresholds().checkIntervalMs); logger.debug("Started offline checker interval"); }; diff --git a/server/routers/newt/offlineChecker.ts b/server/routers/newt/offlineChecker.ts index 0d9148509..075800747 100644 --- a/server/routers/newt/offlineChecker.ts +++ b/server/routers/newt/offlineChecker.ts @@ -3,12 +3,10 @@ import { hasActiveConnections } from "#dynamic/routers/ws"; import { eq, lt, isNull, and, or, ne, not, inArray } from "drizzle-orm"; import logger from "@server/logger"; import { fireSiteOfflineAlert, fireSiteOnlineAlert } from "@server/lib/alerts"; +import { getOfflineThresholds } from "#dynamic/lib/offlineThresholds"; // Track if the offline checker interval is running let offlineCheckerInterval: NodeJS.Timeout | null = null; -const OFFLINE_CHECK_INTERVAL = 30 * 1000; // Check every 30 seconds -const OFFLINE_THRESHOLD_MS = 2 * 60 * 1000; // 2 minutes -const OFFLINE_THRESHOLD_BANDWIDTH_MS = 8 * 60 * 1000; // 8 minutes /** * Starts the background interval that checks for newt sites that haven't @@ -24,8 +22,9 @@ export const startNewtOfflineChecker = (): void => { offlineCheckerInterval = setInterval(async () => { try { - const twoMinutesAgo = Math.floor( - (Date.now() - OFFLINE_THRESHOLD_MS) / 1000 + const thresholds = getOfflineThresholds(); + const offlineBefore = Math.floor( + (Date.now() - thresholds.newtMs) / 1000 ); // Find all online newt-type sites that haven't pinged recently @@ -46,7 +45,7 @@ export const startNewtOfflineChecker = (): void => { eq(sites.online, true), eq(sites.type, "newt"), or( - lt(sites.lastPing, twoMinutesAgo), + lt(sites.lastPing, offlineBefore), isNull(sites.lastPing) ) ) @@ -105,7 +104,7 @@ export const startNewtOfflineChecker = (): void => { ); const wireguardOfflineThreshold = Math.floor( - (Date.now() - OFFLINE_THRESHOLD_BANDWIDTH_MS) / 1000 + (Date.now() - thresholds.newtWireguardMs) / 1000 ); // loop over each one. If its offline and there is a new update then mark it online. If its online and there is no update then mark it offline @@ -117,7 +116,7 @@ export const startNewtOfflineChecker = (): void => { site.online ) { logger.info( - `Marking wireguard site ${site.siteId} offline: no bandwidth update in over ${OFFLINE_THRESHOLD_BANDWIDTH_MS / 60000} minutes` + `Marking wireguard site ${site.siteId} offline: no bandwidth update in over ${thresholds.newtWireguardMs / 1000} seconds` ); await db.transaction(async (trx) => { @@ -161,7 +160,7 @@ export const startNewtOfflineChecker = (): void => { } catch (error) { logger.error("Error in newt offline checker interval", { error }); } - }, OFFLINE_CHECK_INTERVAL); + }, getOfflineThresholds().checkIntervalMs); logger.debug("Started newt offline checker interval"); }; diff --git a/server/routers/olm/offlineChecker.ts b/server/routers/olm/offlineChecker.ts index 7dd06a29c..a5558fd24 100644 --- a/server/routers/olm/offlineChecker.ts +++ b/server/routers/olm/offlineChecker.ts @@ -5,11 +5,10 @@ import { eq, lt, isNull, and, or } from "drizzle-orm"; import logger from "@server/logger"; import { sendTerminateClient } from "../client/terminate"; import { OlmErrorCodes } from "./error"; +import { getOfflineThresholds } from "#dynamic/lib/offlineThresholds"; // Track if the offline checker interval is running let offlineCheckerInterval: NodeJS.Timeout | null = null; -const OFFLINE_CHECK_INTERVAL = 30 * 1000; // Check every 30 seconds -const OFFLINE_THRESHOLD_MS = 2 * 60 * 1000; // 2 minutes /** * Starts the background interval that checks for clients that haven't pinged recently @@ -22,13 +21,13 @@ export const startOlmOfflineChecker = (): void => { offlineCheckerInterval = setInterval(async () => { try { - const twoMinutesAgo = Math.floor( - (Date.now() - OFFLINE_THRESHOLD_MS) / 1000 + const offlineBefore = Math.floor( + (Date.now() - getOfflineThresholds().olmMs) / 1000 ); // TODO: WE NEED TO MAKE SURE THIS WORKS WITH DISTRIBUTED NODES ALL DOING THE SAME THING - // Find clients that haven't pinged in the last 2 minutes and mark them as offline + // Find clients that haven't pinged within the offline threshold and mark them as offline const offlineClients = await db .update(clients) .set({ online: false }) @@ -36,7 +35,7 @@ export const startOlmOfflineChecker = (): void => { and( eq(clients.online, true), or( - lt(clients.lastPing, twoMinutesAgo), + lt(clients.lastPing, offlineBefore), isNull(clients.lastPing) ) ) @@ -75,7 +74,7 @@ export const startOlmOfflineChecker = (): void => { } catch (error) { logger.error("Error in offline checker interval", { error }); } - }, OFFLINE_CHECK_INTERVAL); + }, getOfflineThresholds().checkIntervalMs); logger.debug("Started offline checker interval"); };