From ad15b7c3c6742d384f103fa9b69d89af1fba6ecb Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 15 Apr 2026 16:31:15 -0700 Subject: [PATCH] Add new intervals and tcp mode to health checks --- messages/en-US.json | 8 + server/db/pg/schema/schema.ts | 4 +- server/db/sqlite/schema/schema.ts | 4 +- server/lib/blueprints/proxyResources.ts | 10 +- server/lib/blueprints/types.ts | 8 +- server/routers/newt/buildConfiguration.ts | 25 +- server/routers/newt/handleNewtPingMessage.ts | 5 + server/routers/newt/targets.ts | 21 +- server/routers/target/createTarget.ts | 6 +- server/routers/target/updateTarget.ts | 4 + src/components/HealthCheckDialog.tsx | 841 ++++++++++++------- 11 files changed, 614 insertions(+), 322 deletions(-) diff --git a/messages/en-US.json b/messages/en-US.json index db140de0a..7146bac3a 100644 --- a/messages/en-US.json +++ b/messages/en-US.json @@ -1844,6 +1844,14 @@ "healthCheckIntervalMin": "Check interval must be at least 5 seconds", "healthCheckTimeoutMin": "Timeout must be at least 1 second", "healthCheckRetryMin": "Retry attempts must be at least 1", + "healthCheckMode": "Check Mode", + "healthCheckModeDescription": "TCP mode verifies connectivity only. HTTP mode validates the HTTP response.", + "healthyThreshold": "Healthy Threshold", + "healthyThresholdDescription": "Consecutive successes required before marking as healthy.", + "unhealthyThreshold": "Unhealthy Threshold", + "unhealthyThresholdDescription": "Consecutive failures required before marking as unhealthy.", + "healthCheckHealthyThresholdMin": "Healthy threshold must be at least 1", + "healthCheckUnhealthyThresholdMin": "Unhealthy threshold must be at least 1", "httpMethod": "HTTP Method", "selectHttpMethod": "Select HTTP method", "domainPickerSubdomainLabel": "Subdomain", diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 25a848f9f..c542af33b 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -205,7 +205,9 @@ export const targetHealthCheck = pgTable("targetHealthCheck", { hcHealth: text("hcHealth") .$type<"unknown" | "healthy" | "unhealthy">() .default("unknown"), // "unknown", "healthy", "unhealthy" - hcTlsServerName: text("hcTlsServerName") + hcTlsServerName: text("hcTlsServerName"), + hcHealthyThreshold: integer("hcHealthyThreshold").default(1), + hcUnhealthyThreshold: integer("hcUnhealthyThreshold").default(1) }); export const exitNodes = pgTable("exitNodes", { diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 5c9d57e6d..5ec932c3c 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -232,7 +232,9 @@ export const targetHealthCheck = sqliteTable("targetHealthCheck", { hcHealth: text("hcHealth") .$type<"unknown" | "healthy" | "unhealthy">() .default("unknown"), // "unknown", "healthy", "unhealthy" - hcTlsServerName: text("hcTlsServerName") + hcTlsServerName: text("hcTlsServerName"), + hcHealthyThreshold: integer("hcHealthyThreshold").default(1), + hcUnhealthyThreshold: integer("hcUnhealthyThreshold").default(1) }); export const exitNodes = sqliteTable("exitNodes", { diff --git a/server/lib/blueprints/proxyResources.ts b/server/lib/blueprints/proxyResources.ts index 4d78e946d..b75914021 100644 --- a/server/lib/blueprints/proxyResources.ts +++ b/server/lib/blueprints/proxyResources.ts @@ -158,7 +158,9 @@ export async function updateProxyResources( healthcheckData?.["follow-redirects"], hcMethod: healthcheckData?.method, hcStatus: healthcheckData?.status, - hcHealth: "unknown" + hcHealth: "unknown", + hcHealthyThreshold: healthcheckData?.["healthy-threshold"], + hcUnhealthyThreshold: healthcheckData?.["unhealthy-threshold"] }) .returning(); @@ -522,7 +524,9 @@ export async function updateProxyResources( healthcheckData?.followRedirects || healthcheckData?.["follow-redirects"], hcMethod: healthcheckData?.method, - hcStatus: healthcheckData?.status + hcStatus: healthcheckData?.status, + hcHealthyThreshold: healthcheckData?.["healthy-threshold"], + hcUnhealthyThreshold: healthcheckData?.["unhealthy-threshold"] }) .where( eq( @@ -1081,6 +1085,8 @@ function checkIfHealthcheckChanged( JSON.stringify(incoming.hcHeaders) ) return true; + if (existing.hcHealthyThreshold !== incoming.hcHealthyThreshold) return true; + if (existing.hcUnhealthyThreshold !== incoming.hcUnhealthyThreshold) return true; return false; } diff --git a/server/lib/blueprints/types.ts b/server/lib/blueprints/types.ts index 8269e7b65..913cf31ed 100644 --- a/server/lib/blueprints/types.ts +++ b/server/lib/blueprints/types.ts @@ -12,7 +12,7 @@ export const TargetHealthCheckSchema = z.object({ hostname: z.string(), port: z.int().min(1).max(65535), enabled: z.boolean().optional().default(true), - path: z.string().optional().default("/"), + path: z.string().optional(), scheme: z.string().optional(), mode: z.string().default("http"), interval: z.int().default(30), @@ -26,8 +26,10 @@ export const TargetHealthCheckSchema = z.object({ .default(null), "follow-redirects": z.boolean().default(true), followRedirects: z.boolean().optional(), // deprecated alias - method: z.string().default("GET"), - status: z.int().optional() + method: z.string().optional(), + status: z.int().optional(), + "healthy-threshold": z.int().min(1).optional().default(1), + "unhealthy-threshold": z.int().min(1).optional().default(1) }); // Schema for individual target within a resource diff --git a/server/routers/newt/buildConfiguration.ts b/server/routers/newt/buildConfiguration.ts index 6f0bad6f1..fc0abd9cf 100644 --- a/server/routers/newt/buildConfiguration.ts +++ b/server/routers/newt/buildConfiguration.ts @@ -213,7 +213,9 @@ export async function buildTargetConfigurationForNewtClient(siteId: number) { hcHeaders: targetHealthCheck.hcHeaders, hcMethod: targetHealthCheck.hcMethod, hcTlsServerName: targetHealthCheck.hcTlsServerName, - hcStatus: targetHealthCheck.hcStatus + hcStatus: targetHealthCheck.hcStatus, + hcHealthyThreshold: targetHealthCheck.hcHealthyThreshold, + hcUnhealthyThreshold: targetHealthCheck.hcUnhealthyThreshold }) .from(targets) .innerJoin(resources, eq(targets.resourceId, resources.resourceId)) @@ -247,17 +249,12 @@ export async function buildTargetConfigurationForNewtClient(siteId: number) { const healthCheckTargets = allTargets.map((target) => { // make sure the stuff is defined - if ( - !target.hcPath || - !target.hcHostname || - !target.hcPort || - !target.hcInterval || - !target.hcMethod - ) { - // logger.debug( - // `Skipping adding target health check ${target.targetId} due to missing health check fields` - // ); - return null; // Skip targets with missing health check fields + const isTCP = target.hcMode?.toLowerCase() === "tcp"; + if (!target.hcHostname || !target.hcPort || !target.hcInterval) { + return null; + } + if (!isTCP && (!target.hcPath || !target.hcMethod)) { + return null; } // parse headers @@ -287,7 +284,9 @@ export async function buildTargetConfigurationForNewtClient(siteId: number) { hcHeaders: hcHeadersSend, hcMethod: target.hcMethod, hcTlsServerName: target.hcTlsServerName, - hcStatus: target.hcStatus + hcStatus: target.hcStatus, + hcHealthyThreshold: target.hcHealthyThreshold, + hcUnhealthyThreshold: target.hcUnhealthyThreshold }; }); diff --git a/server/routers/newt/handleNewtPingMessage.ts b/server/routers/newt/handleNewtPingMessage.ts index 32f665758..e8a1d341e 100644 --- a/server/routers/newt/handleNewtPingMessage.ts +++ b/server/routers/newt/handleNewtPingMessage.ts @@ -9,6 +9,7 @@ import { eq, lt, isNull, and, or, ne, not } from "drizzle-orm"; import logger from "@server/logger"; import { sendNewtSyncMessage } from "./sync"; import { recordPing } from "./pingAccumulator"; +import { fireSiteOfflineAlert } from "#dynamic/lib/alerts"; // Track if the offline checker interval is running let offlineCheckerInterval: NodeJS.Timeout | null = null; @@ -40,6 +41,8 @@ export const startNewtOfflineChecker = (): void => { const staleSites = await db .select({ siteId: sites.siteId, + orgId: sites.orgId, + name: sites.name, newtId: newts.newtId, lastPing: sites.lastPing }) @@ -104,6 +107,8 @@ export const startNewtOfflineChecker = (): void => { ) ); } + + await fireSiteOfflineAlert(staleSite.orgId, staleSite.siteId, staleSite.name); } // this part only effects self hosted. Its not efficient but we dont expect people to have very many wireguard sites diff --git a/server/routers/newt/targets.ts b/server/routers/newt/targets.ts index 6a523ebe9..afc983472 100644 --- a/server/routers/newt/targets.ts +++ b/server/routers/newt/targets.ts @@ -43,17 +43,18 @@ export async function addTargets( } // Ensure all necessary fields are present - if ( - !hc.hcPath || - !hc.hcHostname || - !hc.hcPort || - !hc.hcInterval || - !hc.hcMethod - ) { + const isTCP = hc.hcMode?.toLowerCase() === "tcp"; + if (!hc.hcHostname || !hc.hcPort || !hc.hcInterval) { logger.debug( `Skipping target ${target.targetId} due to missing health check fields` ); - return null; // Skip targets with missing health check fields + return null; + } + if (!isTCP && (!hc.hcPath || !hc.hcMethod)) { + logger.debug( + `Skipping target ${target.targetId} due to missing HTTP health check fields` + ); + return null; } const hcHeadersParse = hc.hcHeaders ? JSON.parse(hc.hcHeaders) : null; @@ -90,7 +91,9 @@ export async function addTargets( hcHeaders: hcHeadersSend, hcMethod: hc.hcMethod, hcStatus: hcStatus, - hcTlsServerName: hc.hcTlsServerName + hcTlsServerName: hc.hcTlsServerName, + hcHealthyThreshold: hc.hcHealthyThreshold, + hcUnhealthyThreshold: hc.hcUnhealthyThreshold }; }); diff --git a/server/routers/target/createTarget.ts b/server/routers/target/createTarget.ts index ba52d85a1..129a70abf 100644 --- a/server/routers/target/createTarget.ts +++ b/server/routers/target/createTarget.ts @@ -42,6 +42,8 @@ const createTargetSchema = z.strictObject({ hcMethod: z.string().min(1).optional().nullable(), hcStatus: z.int().optional().nullable(), hcTlsServerName: z.string().optional().nullable(), + hcHealthyThreshold: z.int().positive().min(1).optional().nullable(), + hcUnhealthyThreshold: z.int().positive().min(1).optional().nullable(), path: z.string().optional().nullable(), pathMatchType: z.enum(["exact", "prefix", "regex"]).optional().nullable(), rewritePath: z.string().optional().nullable(), @@ -241,7 +243,9 @@ export async function createTarget( hcMethod: targetData.hcMethod ?? null, hcStatus: targetData.hcStatus ?? null, hcHealth: "unknown", - hcTlsServerName: targetData.hcTlsServerName ?? null + hcTlsServerName: targetData.hcTlsServerName ?? null, + hcHealthyThreshold: targetData.hcHealthyThreshold ?? null, + hcUnhealthyThreshold: targetData.hcUnhealthyThreshold ?? null }) .returning(); diff --git a/server/routers/target/updateTarget.ts b/server/routers/target/updateTarget.ts index 1f9eff716..e42ce98a1 100644 --- a/server/routers/target/updateTarget.ts +++ b/server/routers/target/updateTarget.ts @@ -43,6 +43,8 @@ const updateTargetBodySchema = z hcMethod: z.string().min(1).optional().nullable(), hcStatus: z.int().optional().nullable(), hcTlsServerName: z.string().optional().nullable(), + hcHealthyThreshold: z.int().positive().min(1).optional().nullable(), + hcUnhealthyThreshold: z.int().positive().min(1).optional().nullable(), path: z.string().optional().nullable(), pathMatchType: z .enum(["exact", "prefix", "regex"]) @@ -240,6 +242,8 @@ export async function updateTarget( hcMethod: parsedBody.data.hcMethod, hcStatus: parsedBody.data.hcStatus, hcTlsServerName: parsedBody.data.hcTlsServerName, + hcHealthyThreshold: parsedBody.data.hcHealthyThreshold, + hcUnhealthyThreshold: parsedBody.data.hcUnhealthyThreshold, ...(hcHealthValue !== undefined && { hcHealth: hcHealthValue }) }) .where(eq(targetHealthCheck.targetId, targetId)) diff --git a/src/components/HealthCheckDialog.tsx b/src/components/HealthCheckDialog.tsx index c95908025..abdc32d4a 100644 --- a/src/components/HealthCheckDialog.tsx +++ b/src/components/HealthCheckDialog.tsx @@ -52,6 +52,8 @@ type HealthCheckConfig = { hcMode: string; hcUnhealthyInterval: number; hcTlsServerName: string; + hcHealthyThreshold: number; + hcUnhealthyThreshold: number; }; type HealthCheckDialogProps = { @@ -75,44 +77,73 @@ export default function HealthCheckDialog({ }: HealthCheckDialogProps) { const t = useTranslations(); - const healthCheckSchema = z.object({ - hcEnabled: z.boolean(), - hcPath: z.string().min(1, { message: t("healthCheckPathRequired") }), - hcMethod: z - .string() - .min(1, { message: t("healthCheckMethodRequired") }), - hcInterval: z - .int() - .positive() - .min(5, { message: t("healthCheckIntervalMin") }), - hcTimeout: z - .int() - .positive() - .min(1, { message: t("healthCheckTimeoutMin") }), - hcStatus: z.int().positive().min(100).optional().nullable(), - hcHeaders: z - .array(z.object({ name: z.string(), value: z.string() })) - .nullable() - .optional(), - hcScheme: z.string().optional(), - hcHostname: z.string(), - hcPort: z - .string() - .min(1, { message: t("healthCheckPortInvalid") }) - .refine( - (val) => { - const port = parseInt(val); - return port > 0 && port <= 65535; - }, - { - message: t("healthCheckPortInvalid") + const healthCheckSchema = z + .object({ + hcEnabled: z.boolean(), + hcPath: z.string().optional(), + hcMethod: z.string().optional(), + hcInterval: z + .int() + .positive() + .min(5, { message: t("healthCheckIntervalMin") }), + hcTimeout: z + .int() + .positive() + .min(1, { message: t("healthCheckTimeoutMin") }), + hcStatus: z.int().positive().min(100).optional().nullable(), + hcHeaders: z + .array(z.object({ name: z.string(), value: z.string() })) + .nullable() + .optional(), + hcScheme: z.string().optional(), + hcHostname: z.string(), + hcPort: z + .string() + .min(1, { message: t("healthCheckPortInvalid") }) + .refine( + (val) => { + const port = parseInt(val); + return port > 0 && port <= 65535; + }, + { + message: t("healthCheckPortInvalid") + } + ), + hcFollowRedirects: z.boolean(), + hcMode: z.string(), + hcUnhealthyInterval: z.int().positive().min(5), + hcTlsServerName: z.string(), + hcHealthyThreshold: z + .int() + .positive() + .min(1, { + message: t("healthCheckHealthyThresholdMin") + }), + hcUnhealthyThreshold: z + .int() + .positive() + .min(1, { + message: t("healthCheckUnhealthyThresholdMin") + }) + }) + .superRefine((data, ctx) => { + if (data.hcMode !== "tcp") { + if (!data.hcPath || data.hcPath.length < 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: t("healthCheckPathRequired"), + path: ["hcPath"] + }); } - ), - hcFollowRedirects: z.boolean(), - hcMode: z.string(), - hcUnhealthyInterval: z.int().positive().min(5), - hcTlsServerName: z.string() - }); + if (!data.hcMethod || data.hcMethod.length < 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: t("healthCheckMethodRequired"), + path: ["hcMethod"] + }); + } + } + }); const form = useForm>({ resolver: zodResolver(healthCheckSchema), @@ -122,12 +153,10 @@ export default function HealthCheckDialog({ useEffect(() => { if (!open) return; - // Determine default scheme from target method const getDefaultScheme = () => { if (initialConfig?.hcScheme) { return initialConfig.hcScheme; } - // Default to target method if it's http or https, otherwise default to http if (targetMethod === "https") { return "https"; } @@ -148,24 +177,30 @@ export default function HealthCheckDialog({ ? initialConfig.hcPort.toString() : "", hcFollowRedirects: initialConfig?.hcFollowRedirects, - hcMode: initialConfig?.hcMode, + hcMode: initialConfig?.hcMode ?? "http", hcUnhealthyInterval: initialConfig?.hcUnhealthyInterval, - hcTlsServerName: initialConfig?.hcTlsServerName ?? "" + hcTlsServerName: initialConfig?.hcTlsServerName ?? "", + hcHealthyThreshold: initialConfig?.hcHealthyThreshold ?? 1, + hcUnhealthyThreshold: initialConfig?.hcUnhealthyThreshold ?? 1 }); }, [open]); const watchedEnabled = form.watch("hcEnabled"); + const watchedMode = form.watch("hcMode"); const handleFieldChange = async (fieldName: string, value: any) => { try { const currentValues = form.getValues(); const updatedValues = { ...currentValues, [fieldName]: value }; - // Convert hcPort from string to number before passing to parent const configToSend: HealthCheckConfig = { ...updatedValues, + hcPath: updatedValues.hcPath ?? "", + hcMethod: updatedValues.hcMethod ?? "", hcPort: parseInt(updatedValues.hcPort), - hcStatus: updatedValues.hcStatus || null + hcStatus: updatedValues.hcStatus || null, + hcHealthyThreshold: updatedValues.hcHealthyThreshold, + hcUnhealthyThreshold: updatedValues.hcUnhealthyThreshold }; await onChanges(configToSend); @@ -226,14 +261,262 @@ export default function HealthCheckDialog({ {watchedEnabled && (
-
+ {/* Mode */} + ( + + + {t("healthCheckMode")} + + + + {t( + "healthCheckModeDescription" + )} + + + + )} + /> + + {/* Connection fields */} + {watchedMode === "tcp" ? ( +
+ ( + + + {t("healthHostname")} + + + { + field.onChange( + e + ); + handleFieldChange( + "hcHostname", + e.target + .value + ); + }} + /> + + + + )} + /> + ( + + + {t("healthPort")} + + + { + const value = + e.target + .value; + field.onChange( + value + ); + handleFieldChange( + "hcPort", + value + ); + }} + /> + + + + )} + /> +
+ ) : ( +
+ ( + + + {t("healthScheme")} + + + + + )} + /> + ( + + + {t("healthHostname")} + + + { + field.onChange( + e + ); + handleFieldChange( + "hcHostname", + e.target + .value + ); + }} + /> + + + + )} + /> + ( + + + {t("healthPort")} + + + { + const value = + e.target + .value; + field.onChange( + value + ); + handleFieldChange( + "hcPort", + value + ); + }} + /> + + + + )} + /> + ( + + + {t("healthCheckPath")} + + + { + field.onChange( + e + ); + handleFieldChange( + "hcPath", + e.target + .value + ); + }} + /> + + + + )} + /> +
+ )} + + {/* HTTP Method */} + {watchedMode !== "tcp" && ( ( - {t("healthScheme")} + {t("httpMethod")} @@ -273,143 +565,9 @@ export default function HealthCheckDialog({ )} /> - ( - - - {t("healthHostname")} - - - { - field.onChange( - e - ); - handleFieldChange( - "hcHostname", - e.target - .value - ); - }} - /> - - - - )} - /> - ( - - - {t("healthPort")} - - - { - const value = - e.target - .value; - field.onChange( - value - ); - handleFieldChange( - "hcPort", - value - ); - }} - /> - - - - )} - /> - ( - - - {t("healthCheckPath")} - - - { - field.onChange( - e - ); - handleFieldChange( - "hcPath", - e.target - .value - ); - }} - /> - - - - )} - /> -
+ )} - {/* HTTP Method */} - ( - - - {t("httpMethod")} - - - - - )} - /> - - {/* Check Interval, Timeout, and Retry Attempts */} + {/* Check Interval, Unhealthy Interval, and Timeout */}
- {/* Expected Response Codes */} - ( - - - {t("expectedResponseCodes")} - - - { - const value = - parseInt( - e.target - .value + {/* Healthy and Unhealthy Thresholds */} +
+ ( + + + {t("healthyThreshold")} + + + { + const value = + parseInt( + e.target + .value + ); + field.onChange( + value ); - field.onChange( - value - ); - handleFieldChange( - "hcStatus", - value - ); - }} - /> - - - {t( - "expectedResponseCodesDescription" - )} - - - - )} - /> + handleFieldChange( + "hcHealthyThreshold", + value + ); + }} + /> + + + {t( + "healthyThresholdDescription" + )} + + + + )} + /> - {/*TLS Server Name (SNI)*/} - ( - - - {t("tlsServerName")} - - - { - field.onChange(e); - handleFieldChange( - "hcTlsServerName", - e.target.value - ); - }} - /> - - - {t( - "tlsServerNameDescription" - )} - - - - )} - /> + ( + + + {t("unhealthyThreshold")} + + + { + const value = + parseInt( + e.target + .value + ); + field.onChange( + value + ); + handleFieldChange( + "hcUnhealthyThreshold", + value + ); + }} + /> + + + {t( + "unhealthyThresholdDescription" + )} + + + + )} + /> +
- {/* Custom Headers */} - ( - - - {t("customHeaders")} - - - { - field.onChange( - value - ); - handleFieldChange( - "hcHeaders", - value - ); - }} - rows={4} - /> - - - {t( - "customHeadersDescription" - )} - - - - )} - /> + {/* HTTP-only fields */} + {watchedMode !== "tcp" && ( + <> + {/* Expected Response Codes */} + ( + + + {t( + "expectedResponseCodes" + )} + + + { + const value = + parseInt( + e + .target + .value + ); + field.onChange( + value + ); + handleFieldChange( + "hcStatus", + value + ); + }} + /> + + + {t( + "expectedResponseCodesDescription" + )} + + + + )} + /> + + {/* TLS Server Name (SNI) */} + ( + + + {t("tlsServerName")} + + + { + field.onChange( + e + ); + handleFieldChange( + "hcTlsServerName", + e.target + .value + ); + }} + /> + + + {t( + "tlsServerNameDescription" + )} + + + + )} + /> + + {/* Custom Headers */} + ( + + + {t("customHeaders")} + + + { + field.onChange( + value + ); + handleFieldChange( + "hcHeaders", + value + ); + }} + rows={4} + /> + + + {t( + "customHeadersDescription" + )} + + + + )} + /> + + )}
)} @@ -632,4 +889,4 @@ export default function HealthCheckDialog({ ); -} +} \ No newline at end of file