pangolin_mirror/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-10-14 16:49:05 +02:00
Code Issues Packages Projects Releases Wiki Activity

Make offline checking intervals configurable

Browse Source 
Fixes #3855
This commit is contained in:
Owen
2026-10-13 09:35:09 -04:00
parent 6bf4595969
commit 2f65a8c124
6 changed files with 104 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/lib/offlineThresholds.ts
+26
View File
@@ -0,0 +1,26 @@
export type OfflineThresholds = {
// how often the offline checkers run
checkIntervalMs: number;
// newt sites that stop pinging for this long are marked offline
newtMs: number;
// wireguard sites with no bandwidth update for this long are marked offline
newtWireguardMs: number;
// olm clients that stop pinging for this long are marked offline
olmMs: number;
// remote exit nodes that stop pinging for this long are marked offline
remoteExitNodeMs: number;
};
export const DEFAULT_OFFLINE_THRESHOLDS: OfflineThresholds = {
checkIntervalMs: 30 * 1000, // 30 seconds
newtMs: 2 * 60 * 1000, // 2 minutes
newtWireguardMs: 8 * 60 * 1000, // 8 minutes
olmMs: 2 * 60 * 1000, // 2 minutes
remoteExitNodeMs: 2 * 60 * 1000 // 2 minutes
};
// The thresholds are only configurable in the private config file, so the
// open build always uses the defaults. See server/private/lib/offlineThresholds.ts
export function getOfflineThresholds(): OfflineThresholds {
return DEFAULT_OFFLINE_THRESHOLDS;
}
server/private/lib/offlineThresholds.ts
+27
View File
@@ -0,0 +1,27 @@
/*
* This file is part of a proprietary work.
*
* Copyright (c) 2025-2026 Fossorial, Inc.
* All rights reserved.
*
* This file is licensed under the Fossorial Commercial License.
* You may not use this file except in compliance with the License.
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
*
* This file is not licensed under the AGPLv3.
*/
import type { OfflineThresholds } from "@server/lib/offlineThresholds";
import privateConfig from "#private/lib/config";
export function getOfflineThresholds(): OfflineThresholds {
const offlineChecker = privateConfig.getRawPrivateConfig().offline_checker;
return {
checkIntervalMs: offlineChecker.check_interval_ms,
newtMs: offlineChecker.newt_threshold_ms,
newtWireguardMs: offlineChecker.newt_wireguard_threshold_ms,
olmMs: offlineChecker.olm_threshold_ms,
remoteExitNodeMs: offlineChecker.remote_exit_node_threshold_ms
};
}
server/private/lib/readConfigFile.ts
+31
View File
@@ -18,6 +18,7 @@ import { z } from "zod";
import { colorsSchema } from "@server/lib/colorsSchema";
import { build } from "@server/build";
import { getEnvOrYaml } from "@server/lib/getEnvOrYaml";
import { DEFAULT_OFFLINE_THRESHOLDS } from "@server/lib/offlineThresholds";
const portSchema = z.number().positive().gt(0).lte(65535);
@@ -176,6 +177,36 @@ export const privateConfigSchema = z
.default("http://gerbil:3004")
})
.prefault({}),
// How long a newt site / olm client / remote exit node has to go
// without a ping before the offline checker marks it offline, and how
// often the checkers run.
offline_checker: z
.object({
check_interval_ms: z
.number()
.int()
.positive()
.default(DEFAULT_OFFLINE_THRESHOLDS.checkIntervalMs),
newt_threshold_ms: z
.number()
.int()
.default(DEFAULT_OFFLINE_THRESHOLDS.newtMs),
// wireguard sites don't ping, so they are judged by their
// last bandwidth update instead
newt_wireguard_threshold_ms: z
.number()
.int()
.default(DEFAULT_OFFLINE_THRESHOLDS.newtWireguardMs),
olm_threshold_ms: z
.number()
.int()
.default(DEFAULT_OFFLINE_THRESHOLDS.olmMs),
remote_exit_node_threshold_ms: z
.number()
.int()
.default(DEFAULT_OFFLINE_THRESHOLDS.remoteExitNodeMs)
})
.prefault({}),
flags: z
.object({
enable_redis: z.boolean().optional().default(false),
server/private/routers/remoteExitNode/offlineChecker.ts
+6 -7
View File
@@ -14,11 +14,10 @@
import { db, exitNodes } from "@server/db";
import { eq, lt, isNull, and, or } from "drizzle-orm";
import logger from "@server/logger";
import { getOfflineThresholds } from "#private/lib/offlineThresholds";
// Track if the offline checker interval is running
let offlineCheckerInterval: NodeJS.Timeout | null = null;
const OFFLINE_CHECK_INTERVAL = 30 * 1000; // Check every 30 seconds
const OFFLINE_THRESHOLD_MS = 2 * 60 * 1000; // 2 minutes
/**
* Starts the background interval that checks for clients that haven't pinged recently
@@ -31,11 +30,11 @@ export const startRemoteExitNodeOfflineChecker = (): void => {
offlineCheckerInterval = setInterval(async () => {
try {
const twoMinutesAgo = Math.floor(
(Date.now() - OFFLINE_THRESHOLD_MS) / 1000
const offlineBefore = Math.floor(
(Date.now() - getOfflineThresholds().remoteExitNodeMs) / 1000
);
// Find clients that haven't pinged in the last 2 minutes and mark them as offline
// Find clients that haven't pinged within the offline threshold and mark them as offline
const offlineNodes = await db
.update(exitNodes)
.set({ online: false })
@@ -44,7 +43,7 @@ export const startRemoteExitNodeOfflineChecker = (): void => {
eq(exitNodes.online, true),
eq(exitNodes.type, "remoteExitNode"),
or(
lt(exitNodes.lastPing, twoMinutesAgo),
lt(exitNodes.lastPing, offlineBefore),
isNull(exitNodes.lastPing)
)
)
@@ -65,7 +64,7 @@ export const startRemoteExitNodeOfflineChecker = (): void => {
} catch (error) {
logger.error("Error in offline checker interval", { error });
}
}, OFFLINE_CHECK_INTERVAL);
}, getOfflineThresholds().checkIntervalMs);
logger.debug("Started offline checker interval");
};
server/routers/newt/offlineChecker.ts
+8 -9
View File
@@ -3,12 +3,10 @@ import { hasActiveConnections } from "#dynamic/routers/ws";
import { eq, lt, isNull, and, or, ne, not, inArray } from "drizzle-orm";
import logger from "@server/logger";
import { fireSiteOfflineAlert, fireSiteOnlineAlert } from "@server/lib/alerts";
import { getOfflineThresholds } from "#dynamic/lib/offlineThresholds";
// Track if the offline checker interval is running
let offlineCheckerInterval: NodeJS.Timeout | null = null;
const OFFLINE_CHECK_INTERVAL = 30 * 1000; // Check every 30 seconds
const OFFLINE_THRESHOLD_MS = 2 * 60 * 1000; // 2 minutes
const OFFLINE_THRESHOLD_BANDWIDTH_MS = 8 * 60 * 1000; // 8 minutes
/**
* Starts the background interval that checks for newt sites that haven't
@@ -24,8 +22,9 @@ export const startNewtOfflineChecker = (): void => {
offlineCheckerInterval = setInterval(async () => {
try {
const twoMinutesAgo = Math.floor(
(Date.now() - OFFLINE_THRESHOLD_MS) / 1000
const thresholds = getOfflineThresholds();
const offlineBefore = Math.floor(
(Date.now() - thresholds.newtMs) / 1000
);
// Find all online newt-type sites that haven't pinged recently
@@ -46,7 +45,7 @@ export const startNewtOfflineChecker = (): void => {
eq(sites.online, true),
eq(sites.type, "newt"),
or(
lt(sites.lastPing, twoMinutesAgo),
lt(sites.lastPing, offlineBefore),
isNull(sites.lastPing)
)
)
@@ -105,7 +104,7 @@ export const startNewtOfflineChecker = (): void => {
);
const wireguardOfflineThreshold = Math.floor(
(Date.now() - OFFLINE_THRESHOLD_BANDWIDTH_MS) / 1000
(Date.now() - thresholds.newtWireguardMs) / 1000
);
// loop over each one. If its offline and there is a new update then mark it online. If its online and there is no update then mark it offline
@@ -117,7 +116,7 @@ export const startNewtOfflineChecker = (): void => {
site.online
) {
logger.info(
`Marking wireguard site ${site.siteId} offline: no bandwidth update in over ${OFFLINE_THRESHOLD_BANDWIDTH_MS / 60000} minutes`
`Marking wireguard site ${site.siteId} offline: no bandwidth update in over ${thresholds.newtWireguardMs / 1000} seconds`
);
await db.transaction(async (trx) => {
@@ -161,7 +160,7 @@ export const startNewtOfflineChecker = (): void => {
} catch (error) {
logger.error("Error in newt offline checker interval", { error });
}
}, OFFLINE_CHECK_INTERVAL);
}, getOfflineThresholds().checkIntervalMs);
logger.debug("Started newt offline checker interval");
};
server/routers/olm/offlineChecker.ts
+6 -7
View File
@@ -5,11 +5,10 @@ import { eq, lt, isNull, and, or } from "drizzle-orm";
import logger from "@server/logger";
import { sendTerminateClient } from "../client/terminate";
import { OlmErrorCodes } from "./error";
import { getOfflineThresholds } from "#dynamic/lib/offlineThresholds";
// Track if the offline checker interval is running
let offlineCheckerInterval: NodeJS.Timeout | null = null;
const OFFLINE_CHECK_INTERVAL = 30 * 1000; // Check every 30 seconds
const OFFLINE_THRESHOLD_MS = 2 * 60 * 1000; // 2 minutes
/**
* Starts the background interval that checks for clients that haven't pinged recently
@@ -22,13 +21,13 @@ export const startOlmOfflineChecker = (): void => {
offlineCheckerInterval = setInterval(async () => {
try {
const twoMinutesAgo = Math.floor(
(Date.now() - OFFLINE_THRESHOLD_MS) / 1000
const offlineBefore = Math.floor(
(Date.now() - getOfflineThresholds().olmMs) / 1000
);
// TODO: WE NEED TO MAKE SURE THIS WORKS WITH DISTRIBUTED NODES ALL DOING THE SAME THING
// Find clients that haven't pinged in the last 2 minutes and mark them as offline
// Find clients that haven't pinged within the offline threshold and mark them as offline
const offlineClients = await db
.update(clients)
.set({ online: false })
@@ -36,7 +35,7 @@ export const startOlmOfflineChecker = (): void => {
and(
eq(clients.online, true),
or(
lt(clients.lastPing, twoMinutesAgo),
lt(clients.lastPing, offlineBefore),
isNull(clients.lastPing)
)
)
@@ -75,7 +74,7 @@ export const startOlmOfflineChecker = (): void => {
} catch (error) {
logger.error("Error in offline checker interval", { error });
}
}, OFFLINE_CHECK_INTERVAL);
}, getOfflineThresholds().checkIntervalMs);
logger.debug("Started offline checker interval");
};