fix: revert investigative changes, keep root cause fixes only

Reverts diagnostic instrumentation and defensive hardening added during
memory leak investigation. Only root cause fixes survive.

Root causes fixed:
- SQLite driver: auto-finalize wrapper + PRAGMAs
- WS routers: delete clientConfigVersions on disconnect (unbounded Map leak)
- WS private router: same + Redis key cleanup

Reverted:
- Memory monitor, rate limiting, request timeouts (diagnostic/hardening)
- shutdownAuditLogger wiring, audit re-queue change, debug logs (cleanup/secondary)
- package-lock.json drift
This commit is contained in:
Josh Voyles
2026-05-02 16:33:13 -04:00
parent 2c85bcd06b
commit 0655ba9423
7 changed files with 63 additions and 65 deletions

59
package-lock.json generated
View File

@@ -1058,6 +1058,7 @@
"integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@babel/code-frame": "^7.29.0",
"@babel/generator": "^7.29.0",
@@ -2353,6 +2354,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -2375,6 +2377,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -2397,6 +2400,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2413,6 +2417,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2429,6 +2434,7 @@
"cpu": [
"arm"
],
"dev": true,
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2445,6 +2451,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2461,6 +2468,7 @@
"cpu": [
"ppc64"
],
"dev": true,
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2477,6 +2485,7 @@
"cpu": [
"s390x"
],
"dev": true,
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2493,6 +2502,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2509,6 +2519,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2525,6 +2536,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2541,6 +2553,7 @@
"cpu": [
"arm"
],
"dev": true,
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -2563,6 +2576,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -2585,6 +2599,7 @@
"cpu": [
"ppc64"
],
"dev": true,
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -2607,6 +2622,7 @@
"cpu": [
"s390x"
],
"dev": true,
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -2629,6 +2645,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -2651,6 +2668,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -2673,6 +2691,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -2695,6 +2714,7 @@
"cpu": [
"wasm32"
],
"dev": true,
"license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT",
"optional": true,
"dependencies": {
@@ -2714,6 +2734,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "Apache-2.0 AND LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2733,6 +2754,7 @@
"cpu": [
"ia32"
],
"dev": true,
"license": "Apache-2.0 AND LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -2752,6 +2774,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "Apache-2.0 AND LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3011,6 +3034,7 @@
"integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": "^14.21.3 || >=16"
},
@@ -6957,6 +6981,7 @@
"resolved": "https://registry.npmjs.org/@react-email/text/-/text-0.1.6.tgz",
"integrity": "sha512-TYqkioRS45wTR5il3dYk/SbUjjEdhSwh9BtRNB99qNH1pXAwA45H7rAuxehiu8iJQJH0IyIr+6n62gBz9ezmsw==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">=20.0.0"
},
@@ -8417,6 +8442,7 @@
"version": "5.90.21",
"resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.90.21.tgz",
"integrity": "sha512-0Lu6y5t+tvlTJMTO7oh5NSpJfpg/5D41LlThfepTixPYkJ0sE2Jj0m0f6yYqujBwIXlId87e234+MxG3D3g7kg==",
"peer": true,
"dependencies": {
"@tanstack/query-core": "5.90.20"
},
@@ -8532,6 +8558,7 @@
"integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==",
"devOptional": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/node": "*"
}
@@ -8879,6 +8906,7 @@
"integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/body-parser": "*",
"@types/express-serve-static-core": "^5.0.0",
@@ -8974,6 +9002,7 @@
"integrity": "sha512-oX8xrhvpiyRCQkG1MFchB09f+cXftgIXb3a7UUa4Y3wpmZPw5tyZGTLWhlESOLq1Rq6oDlc8npVU2/9xiCuXMA==",
"devOptional": true,
"license": "MIT",
"peer": true,
"dependencies": {
"undici-types": "~7.18.0"
}
@@ -9001,6 +9030,7 @@
"integrity": "sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==",
"devOptional": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/node": "*",
"pg-protocol": "*",
@@ -9026,6 +9056,7 @@
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz",
"integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==",
"devOptional": true,
"peer": true,
"dependencies": {
"csstype": "^3.2.2"
}
@@ -9036,6 +9067,7 @@
"integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==",
"devOptional": true,
"license": "MIT",
"peer": true,
"peerDependencies": {
"@types/react": "^19.2.0"
}
@@ -9122,8 +9154,7 @@
"resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz",
"integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==",
"license": "MIT",
"optional": true,
"peer": true
"optional": true
},
"node_modules/@types/ws": {
"version": "8.18.1",
@@ -9197,6 +9228,7 @@
"integrity": "sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "8.56.1",
"@typescript-eslint/types": "8.56.1",
@@ -9670,6 +9702,7 @@
"integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==",
"dev": true,
"license": "MIT",
"peer": true,
"bin": {
"acorn": "bin/acorn"
},
@@ -10119,6 +10152,7 @@
"integrity": "sha512-Ixm8tFfoKKIPYdCCKYTsqv+Fd4IJ0DQqMyEimo+pxUOMUR9cVPlwTrFt9Avu+3cb6Zp3mAzl+t1MrG2fxxKsxw==",
"devOptional": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@babel/types": "^7.26.0"
}
@@ -10190,6 +10224,7 @@
"integrity": "sha512-Ba0KR+Fzxh2jDRhdg6TSH0SJGzb8C0aBY4hR8w8madIdIzzC6Y1+kx5qR6eS1Z+Gy20h6ZU28aeyg0z1VIrShQ==",
"hasInstallScript": true,
"license": "MIT",
"peer": true,
"dependencies": {
"bindings": "^1.5.0",
"prebuild-install": "^7.1.1"
@@ -10318,6 +10353,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"baseline-browser-mapping": "^2.9.0",
"caniuse-lite": "^1.0.30001759",
@@ -11224,6 +11260,7 @@
"resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz",
"integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==",
"license": "ISC",
"peer": true,
"engines": {
"node": ">=12"
}
@@ -11664,7 +11701,6 @@
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.3.2.tgz",
"integrity": "sha512-6obghkliLdmKa56xdbLOpUZ43pAR6xFy1uOrxBaIDjT+yaRuuybLjGS9eVBoSR/UPU5fq3OXClEHLJNGvbxKpQ==",
"license": "(MPL-2.0 OR Apache-2.0)",
"peer": true,
"engines": {
"node": ">=20"
},
@@ -12299,6 +12335,7 @@
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"peer": true,
"bin": {
"esbuild": "bin/esbuild"
},
@@ -12384,6 +12421,7 @@
"integrity": "sha512-COV33RzXZkqhG9P2rZCFl9ZmJ7WL+gQSCRzE7RhkbclbQPtLAWReL7ysA0Sh4c8Im2U9ynybdR56PV0XcKvqaQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.8.0",
"@eslint-community/regexpp": "^4.12.2",
@@ -12520,6 +12558,7 @@
"integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@rtsao/scc": "^1.1.0",
"array-includes": "^3.1.9",
@@ -12913,6 +12952,7 @@
"resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz",
"integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==",
"license": "MIT",
"peer": true,
"dependencies": {
"accepts": "^2.0.0",
"body-parser": "^2.2.1",
@@ -15330,7 +15370,6 @@
"resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.55.1.tgz",
"integrity": "sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A==",
"license": "MIT",
"peer": true,
"dependencies": {
"dompurify": "3.2.7",
"marked": "14.0.0"
@@ -15341,7 +15380,6 @@
"resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz",
"integrity": "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==",
"license": "MIT",
"peer": true,
"bin": {
"marked": "bin/marked.js"
},
@@ -15430,6 +15468,7 @@
"resolved": "https://registry.npmjs.org/next/-/next-15.5.15.tgz",
"integrity": "sha512-VSqCrJwtLVGwAVE0Sb/yikrQfkwkZW9p+lL/J4+xe+G3ZA+QnWPqgcfH1tDUEuk9y+pthzzVFp4L/U8JerMfMQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"@next/env": "15.5.15",
"@swc/helpers": "0.5.15",
@@ -16389,6 +16428,7 @@
"resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz",
"integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==",
"license": "MIT",
"peer": true,
"dependencies": {
"pg-connection-string": "^2.12.0",
"pg-pool": "^3.13.0",
@@ -16896,6 +16936,7 @@
"resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz",
"integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -16927,6 +16968,7 @@
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz",
"integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"scheduler": "^0.27.0"
},
@@ -17219,6 +17261,7 @@
"resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.71.2.tgz",
"integrity": "sha512-1CHvcDYzuRUNOflt4MOq3ZM46AronNJtQ1S7tnX6YN4y72qhgiUItpacZUAQ0TyWYci3yz1X+rXaSxiuEm86PA==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">=18.0.0"
},
@@ -18680,7 +18723,8 @@
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.2.tgz",
"integrity": "sha512-KWBIxs1Xb6NoLdMVqhbhgwZf2PGBpPEiwOqgI4pFIYbNTfBXiKYyWoTsXgBQ9WFg/OlhnvHaY+AEpW7wSmFo2Q==",
"license": "MIT"
"license": "MIT",
"peer": true
},
"node_modules/tapable": {
"version": "2.3.2",
@@ -19155,6 +19199,7 @@
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"devOptional": true,
"license": "Apache-2.0",
"peer": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@@ -19582,6 +19627,7 @@
"resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz",
"integrity": "sha512-LZNJgPzfKR+/J3cHkxcpHKpKKvGfDZVPS4hfJCc4cCG0CgYzvlD6yE/S3CIL/Yt91ak327YCpiF/0MyeZHEHKA==",
"license": "MIT",
"peer": true,
"dependencies": {
"@colors/colors": "^1.6.0",
"@dabh/diagnostics": "^2.0.8",
@@ -19788,6 +19834,7 @@
"resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz",
"integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==",
"license": "MIT",
"peer": true,
"funding": {
"url": "https://github.com/sponsors/colinhacks"
}

View File

@@ -3,11 +3,9 @@ import { flushConnectionLogToDb } from "#dynamic/routers/newt";
import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth";
import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator";
import { cleanup as wsCleanup } from "#dynamic/routers/ws";
import { shutdownAuditLogger } from "@server/routers/badger/logRequestAudit";
async function cleanup() {
await stopPingAccumulator();
await shutdownAuditLogger();
await flushBandwidthToDb();
await flushConnectionLogToDb();
await flushSiteBandwidthToDb();

View File

@@ -24,29 +24,6 @@ import license from "#dynamic/license/license";
import { initLogCleanupInterval } from "@server/lib/cleanupLogs";
import { initAcmeCertSync } from "#dynamic/lib/acmeCertSync";
import { fetchServerIp } from "@server/lib/serverIpService";
import logger from "@server/logger";
/**
* Periodic memory usage logging for monitoring and leak detection.
* Logs heap usage, external (native) memory, and RSS every 60 seconds.
* This is lightweight (single process.memoryUsage() call) and provides
* the data needed to detect slow memory growth over hours/days.
*/
function startMemoryMonitor(): void {
const INTERVAL_MS = 60_000; // every 60 seconds
const timer = setInterval(() => {
const mem = process.memoryUsage();
logger.info(
`Memory usage - ` +
`heapUsed: ${(mem.heapUsed / 1024 / 1024).toFixed(1)}MB, ` +
`heapTotal: ${(mem.heapTotal / 1024 / 1024).toFixed(1)}MB, ` +
`rss: ${(mem.rss / 1024 / 1024).toFixed(1)}MB, ` +
`external: ${(mem.external / 1024 / 1024).toFixed(1)}MB, ` +
`arrayBuffers: ${(mem.arrayBuffers / 1024 / 1024).toFixed(1)}MB`
);
}, INTERVAL_MS);
timer.unref();
}
async function startServers() {
await setHostMeta();
@@ -65,9 +42,6 @@ async function startServers() {
initLogCleanupInterval();
initAcmeCertSync();
// Start memory monitoring for leak detection
startMemoryMonitor();
// Start all servers
const apiServer = createApiServer();
const internalServer = createInternalServer();

View File

@@ -10,8 +10,6 @@ import {
} from "@server/middlewares";
import { internalRouter } from "#dynamic/routers/internal";
import { stripDuplicateSesions } from "./middlewares/stripDuplicateSessions";
import { requestTimeoutMiddleware } from "./middlewares/requestTimeout";
import rateLimit from "express-rate-limit";
const internalPort = config.getRawConfig().server.internal_port;
@@ -29,25 +27,6 @@ export function createInternalServer() {
internalServer.use(cookieParser());
internalServer.use(express.json());
// Prevent requests from hanging indefinitely. Without this, if a
// database query blocks (especially on SQLite), pending requests
// accumulate in memory with no upper bound on lifetime.
internalServer.use(requestTimeoutMiddleware(30000)); // 30 second timeout
// Rate-limit the internal verify-session endpoint. This server
// handles forward-auth requests from Traefik/Badger. Under heavy
// monitoring (e.g. Uptime Kuma), requests can arrive faster than
// SQLite can serve them, causing unbounded request queuing and
// memory growth.
internalServer.use(
rateLimit({
windowMs: 60 * 1000, // 1 minute window
max: 1000, // generous limit: ~17 req/s
standardHeaders: true,
legacyHeaders: false
})
);
const prefix = `/api/v1`;
internalServer.use(prefix, internalRouter);

View File

@@ -41,7 +41,7 @@ export async function exchangeSession(
res: Response,
next: NextFunction
): Promise<any> {
logger.debug("Exchange session: Badger request received");
logger.debug("Exchange session: Badger sent", req.body);
const parsedBody = exchangeSessionBodySchema.safeParse(req.body);

View File

@@ -84,14 +84,14 @@ async function flushAuditLogs() {
logger.debug(`Flushed ${logsToWrite.length} audit logs to database`);
} catch (error) {
logger.error("Error flushing audit logs:", error);
// On transaction error, drop the logs rather than re-queuing them.
// The previous re-queue approach created a positive feedback loop:
// failed flush → re-queue → larger next flush → longer DB lock →
// higher chance of next failure → repeat. This caused unbounded
// memory growth on SQLite where write contention is common.
// Audit logs are best-effort telemetry — losing a batch on error
// is acceptable; leaking memory until the process crashes is not.
logger.warn(`Dropped ${logsToWrite.length} audit logs after flush failure`);
// On transaction error, put logs back at the front of the buffer to retry
// but only if buffer isn't too large
if (auditLogBuffer.length < MAX_BUFFER_SIZE - logsToWrite.length) {
auditLogBuffer.unshift(...logsToWrite);
logger.info(`Re-queued ${logsToWrite.length} audit logs for retry`);
} else {
logger.error(`Buffer full, dropped ${logsToWrite.length} audit logs`);
}
} finally {
isFlushInProgress = false;
// If buffer filled up while we were flushing, flush again

View File

@@ -80,7 +80,7 @@ export async function verifyResourceSession(
res: Response,
next: NextFunction
): Promise<any> {
logger.debug("Verify session: Badger request received");
logger.debug("Verify session: Badger sent", req.body); // remove when done testing
const parsedBody = verifyResourceSessionSchema.safeParse(req.body);