Add new intervals and tcp mode to health checks

This commit is contained in:
Owen
2026-04-15 16:31:15 -07:00
parent b070570cb6
commit ad15b7c3c6
11 changed files with 614 additions and 322 deletions

View File

@@ -205,7 +205,9 @@ export const targetHealthCheck = pgTable("targetHealthCheck", {
hcHealth: text("hcHealth")
.$type<"unknown" | "healthy" | "unhealthy">()
.default("unknown"), // "unknown", "healthy", "unhealthy"
hcTlsServerName: text("hcTlsServerName")
hcTlsServerName: text("hcTlsServerName"),
hcHealthyThreshold: integer("hcHealthyThreshold").default(1),
hcUnhealthyThreshold: integer("hcUnhealthyThreshold").default(1)
});
export const exitNodes = pgTable("exitNodes", {

View File

@@ -232,7 +232,9 @@ export const targetHealthCheck = sqliteTable("targetHealthCheck", {
hcHealth: text("hcHealth")
.$type<"unknown" | "healthy" | "unhealthy">()
.default("unknown"), // "unknown", "healthy", "unhealthy"
hcTlsServerName: text("hcTlsServerName")
hcTlsServerName: text("hcTlsServerName"),
hcHealthyThreshold: integer("hcHealthyThreshold").default(1),
hcUnhealthyThreshold: integer("hcUnhealthyThreshold").default(1)
});
export const exitNodes = sqliteTable("exitNodes", {

View File

@@ -158,7 +158,9 @@ export async function updateProxyResources(
healthcheckData?.["follow-redirects"],
hcMethod: healthcheckData?.method,
hcStatus: healthcheckData?.status,
hcHealth: "unknown"
hcHealth: "unknown",
hcHealthyThreshold: healthcheckData?.["healthy-threshold"],
hcUnhealthyThreshold: healthcheckData?.["unhealthy-threshold"]
})
.returning();
@@ -522,7 +524,9 @@ export async function updateProxyResources(
healthcheckData?.followRedirects ||
healthcheckData?.["follow-redirects"],
hcMethod: healthcheckData?.method,
hcStatus: healthcheckData?.status
hcStatus: healthcheckData?.status,
hcHealthyThreshold: healthcheckData?.["healthy-threshold"],
hcUnhealthyThreshold: healthcheckData?.["unhealthy-threshold"]
})
.where(
eq(
@@ -1081,6 +1085,8 @@ function checkIfHealthcheckChanged(
JSON.stringify(incoming.hcHeaders)
)
return true;
if (existing.hcHealthyThreshold !== incoming.hcHealthyThreshold) return true;
if (existing.hcUnhealthyThreshold !== incoming.hcUnhealthyThreshold) return true;
return false;
}

View File

@@ -12,7 +12,7 @@ export const TargetHealthCheckSchema = z.object({
hostname: z.string(),
port: z.int().min(1).max(65535),
enabled: z.boolean().optional().default(true),
path: z.string().optional().default("/"),
path: z.string().optional(),
scheme: z.string().optional(),
mode: z.string().default("http"),
interval: z.int().default(30),
@@ -26,8 +26,10 @@ export const TargetHealthCheckSchema = z.object({
.default(null),
"follow-redirects": z.boolean().default(true),
followRedirects: z.boolean().optional(), // deprecated alias
method: z.string().default("GET"),
status: z.int().optional()
method: z.string().optional(),
status: z.int().optional(),
"healthy-threshold": z.int().min(1).optional().default(1),
"unhealthy-threshold": z.int().min(1).optional().default(1)
});
// Schema for individual target within a resource

View File

@@ -213,7 +213,9 @@ export async function buildTargetConfigurationForNewtClient(siteId: number) {
hcHeaders: targetHealthCheck.hcHeaders,
hcMethod: targetHealthCheck.hcMethod,
hcTlsServerName: targetHealthCheck.hcTlsServerName,
hcStatus: targetHealthCheck.hcStatus
hcStatus: targetHealthCheck.hcStatus,
hcHealthyThreshold: targetHealthCheck.hcHealthyThreshold,
hcUnhealthyThreshold: targetHealthCheck.hcUnhealthyThreshold
})
.from(targets)
.innerJoin(resources, eq(targets.resourceId, resources.resourceId))
@@ -247,17 +249,12 @@ export async function buildTargetConfigurationForNewtClient(siteId: number) {
const healthCheckTargets = allTargets.map((target) => {
// make sure the stuff is defined
if (
!target.hcPath ||
!target.hcHostname ||
!target.hcPort ||
!target.hcInterval ||
!target.hcMethod
) {
// logger.debug(
// `Skipping adding target health check ${target.targetId} due to missing health check fields`
// );
return null; // Skip targets with missing health check fields
const isTCP = target.hcMode?.toLowerCase() === "tcp";
if (!target.hcHostname || !target.hcPort || !target.hcInterval) {
return null;
}
if (!isTCP && (!target.hcPath || !target.hcMethod)) {
return null;
}
// parse headers
@@ -287,7 +284,9 @@ export async function buildTargetConfigurationForNewtClient(siteId: number) {
hcHeaders: hcHeadersSend,
hcMethod: target.hcMethod,
hcTlsServerName: target.hcTlsServerName,
hcStatus: target.hcStatus
hcStatus: target.hcStatus,
hcHealthyThreshold: target.hcHealthyThreshold,
hcUnhealthyThreshold: target.hcUnhealthyThreshold
};
});

View File

@@ -9,6 +9,7 @@ import { eq, lt, isNull, and, or, ne, not } from "drizzle-orm";
import logger from "@server/logger";
import { sendNewtSyncMessage } from "./sync";
import { recordPing } from "./pingAccumulator";
import { fireSiteOfflineAlert } from "#dynamic/lib/alerts";
// Track if the offline checker interval is running
let offlineCheckerInterval: NodeJS.Timeout | null = null;
@@ -40,6 +41,8 @@ export const startNewtOfflineChecker = (): void => {
const staleSites = await db
.select({
siteId: sites.siteId,
orgId: sites.orgId,
name: sites.name,
newtId: newts.newtId,
lastPing: sites.lastPing
})
@@ -104,6 +107,8 @@ export const startNewtOfflineChecker = (): void => {
)
);
}
await fireSiteOfflineAlert(staleSite.orgId, staleSite.siteId, staleSite.name);
}
// this part only effects self hosted. Its not efficient but we dont expect people to have very many wireguard sites

View File

@@ -43,17 +43,18 @@ export async function addTargets(
}
// Ensure all necessary fields are present
if (
!hc.hcPath ||
!hc.hcHostname ||
!hc.hcPort ||
!hc.hcInterval ||
!hc.hcMethod
) {
const isTCP = hc.hcMode?.toLowerCase() === "tcp";
if (!hc.hcHostname || !hc.hcPort || !hc.hcInterval) {
logger.debug(
`Skipping target ${target.targetId} due to missing health check fields`
);
return null; // Skip targets with missing health check fields
return null;
}
if (!isTCP && (!hc.hcPath || !hc.hcMethod)) {
logger.debug(
`Skipping target ${target.targetId} due to missing HTTP health check fields`
);
return null;
}
const hcHeadersParse = hc.hcHeaders ? JSON.parse(hc.hcHeaders) : null;
@@ -90,7 +91,9 @@ export async function addTargets(
hcHeaders: hcHeadersSend,
hcMethod: hc.hcMethod,
hcStatus: hcStatus,
hcTlsServerName: hc.hcTlsServerName
hcTlsServerName: hc.hcTlsServerName,
hcHealthyThreshold: hc.hcHealthyThreshold,
hcUnhealthyThreshold: hc.hcUnhealthyThreshold
};
});

View File

@@ -42,6 +42,8 @@ const createTargetSchema = z.strictObject({
hcMethod: z.string().min(1).optional().nullable(),
hcStatus: z.int().optional().nullable(),
hcTlsServerName: z.string().optional().nullable(),
hcHealthyThreshold: z.int().positive().min(1).optional().nullable(),
hcUnhealthyThreshold: z.int().positive().min(1).optional().nullable(),
path: z.string().optional().nullable(),
pathMatchType: z.enum(["exact", "prefix", "regex"]).optional().nullable(),
rewritePath: z.string().optional().nullable(),
@@ -241,7 +243,9 @@ export async function createTarget(
hcMethod: targetData.hcMethod ?? null,
hcStatus: targetData.hcStatus ?? null,
hcHealth: "unknown",
hcTlsServerName: targetData.hcTlsServerName ?? null
hcTlsServerName: targetData.hcTlsServerName ?? null,
hcHealthyThreshold: targetData.hcHealthyThreshold ?? null,
hcUnhealthyThreshold: targetData.hcUnhealthyThreshold ?? null
})
.returning();

View File

@@ -43,6 +43,8 @@ const updateTargetBodySchema = z
hcMethod: z.string().min(1).optional().nullable(),
hcStatus: z.int().optional().nullable(),
hcTlsServerName: z.string().optional().nullable(),
hcHealthyThreshold: z.int().positive().min(1).optional().nullable(),
hcUnhealthyThreshold: z.int().positive().min(1).optional().nullable(),
path: z.string().optional().nullable(),
pathMatchType: z
.enum(["exact", "prefix", "regex"])
@@ -240,6 +242,8 @@ export async function updateTarget(
hcMethod: parsedBody.data.hcMethod,
hcStatus: parsedBody.data.hcStatus,
hcTlsServerName: parsedBody.data.hcTlsServerName,
hcHealthyThreshold: parsedBody.data.hcHealthyThreshold,
hcUnhealthyThreshold: parsedBody.data.hcUnhealthyThreshold,
...(hcHealthValue !== undefined && { hcHealth: hcHealthValue })
})
.where(eq(targetHealthCheck.targetId, targetId))